nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

bitmap: Switch from inline to __always_inline

Browse source 
'inline' keyword is only a recommendation for compiler. If it decides to
not inline bitmap functions, the whole small_const_nbits() machinery
doesn't work.

This is how a standard GCC 11.3.0 does for my x86_64 build now. This patch
replaces 'inline' directive with unconditional '__always_inline' to make
sure that there's always a chance for compile-time optimization. It doesn't
change size of kernel image, according to bloat-o-meter.

[[ Brian: split out from:
      Subject: [PATCH 1/3] bitmap: switch from inline to __always_inline
      https://lore.kernel.org/all/20221027043810.350460-2-yury.norov@gmail.com/
   But rewritten, as there were too many conflicts. ]]

Co-developed-by: Brian Norris <briannorris@chromium.org>
Signed-off-by: Brian Norris <briannorris@chromium.org>
Reviewed-by: Kees Cook <kees@kernel.org>
Reviewed-by: Nathan Chancellor <nathan@kernel.org>
Signed-off-by: Yury Norov <yury.norov@gmail.com>
This commit is contained in:
Yury Norov 2024-07-18 17:50:38 -07:00
parent fda1dd3c54
commit ed8cd2b3bd
1 changed files with 76 additions and 64 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

140
include/linux/bitmap.h
View file

@ -203,12 +203,12 @@ unsigned long bitmap_find_next_zero_area_off(unsigned long *map,
* the bit offset of all zero areas this function finds is multiples of that * the bit offset of all zero areas this function finds is multiples of that
* power of 2. A @align_mask of 0 means no alignment is required. * power of 2. A @align_mask of 0 means no alignment is required.
*/ */
static inline unsigned long static __always_inline
bitmap_find_next_zero_area(unsigned long *map, unsigned long bitmap_find_next_zero_area(unsigned long *map,
unsigned long size, unsigned long size,
unsigned long start, unsigned long start,
unsigned int nr, unsigned int nr,
unsigned long align_mask) unsigned long align_mask)
{ {
return bitmap_find_next_zero_area_off(map, size, start, nr, return bitmap_find_next_zero_area_off(map, size, start, nr,
align_mask, 0); align_mask, 0);
@ -228,7 +228,7 @@ void bitmap_fold(unsigned long *dst, const unsigned long *orig,
#define bitmap_size(nbits) (ALIGN(nbits, BITS_PER_LONG) / BITS_PER_BYTE) #define bitmap_size(nbits) (ALIGN(nbits, BITS_PER_LONG) / BITS_PER_BYTE)
static inline void bitmap_zero(unsigned long *dst, unsigned int nbits) static __always_inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
{ {
unsigned int len = bitmap_size(nbits); unsigned int len = bitmap_size(nbits);
@ -238,7 +238,7 @@ static inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
memset(dst, 0, len); memset(dst, 0, len);
} }
static inline void bitmap_fill(unsigned long *dst, unsigned int nbits) static __always_inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
{ {
unsigned int len = bitmap_size(nbits); unsigned int len = bitmap_size(nbits);
@ -248,8 +248,8 @@ static inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
memset(dst, 0xff, len); memset(dst, 0xff, len);
} }
static inline void bitmap_copy(unsigned long *dst, const unsigned long *src, static __always_inline
unsigned int nbits) void bitmap_copy(unsigned long *dst, const unsigned long *src, unsigned int nbits)
{ {
unsigned int len = bitmap_size(nbits); unsigned int len = bitmap_size(nbits);
@ -262,8 +262,8 @@ static inline void bitmap_copy(unsigned long *dst, const unsigned long *src,
/* /*
* Copy bitmap and clear tail bits in last word. * Copy bitmap and clear tail bits in last word.
*/ */
static inline void bitmap_copy_clear_tail(unsigned long *dst, static __always_inline
const unsigned long *src, unsigned int nbits) void bitmap_copy_clear_tail(unsigned long *dst, const unsigned long *src, unsigned int nbits)
{ {
bitmap_copy(dst, src, nbits); bitmap_copy(dst, src, nbits);
if (nbits % BITS_PER_LONG) if (nbits % BITS_PER_LONG)
@ -306,16 +306,18 @@ void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits);
bitmap_copy_clear_tail((unsigned long *)(buf), (const unsigned long *)(bitmap), (nbits)) bitmap_copy_clear_tail((unsigned long *)(buf), (const unsigned long *)(bitmap), (nbits))
#endif #endif
static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1, static __always_inline
const unsigned long *src2, unsigned int nbits) bool bitmap_and(unsigned long *dst, const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{ {
if (small_const_nbits(nbits)) if (small_const_nbits(nbits))
return (*dst = *src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0; return (*dst = *src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0;
return __bitmap_and(dst, src1, src2, nbits); return __bitmap_and(dst, src1, src2, nbits);
} }
static inline void bitmap_or(unsigned long *dst, const unsigned long *src1, static __always_inline
const unsigned long *src2, unsigned int nbits) void bitmap_or(unsigned long *dst, const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{ {
if (small_const_nbits(nbits)) if (small_const_nbits(nbits))
*dst = *src1 | *src2; *dst = *src1 | *src2;
@ -323,8 +325,9 @@ static inline void bitmap_or(unsigned long *dst, const unsigned long *src1,
__bitmap_or(dst, src1, src2, nbits); __bitmap_or(dst, src1, src2, nbits);
} }
static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1, static __always_inline
const unsigned long *src2, unsigned int nbits) void bitmap_xor(unsigned long *dst, const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{ {
if (small_const_nbits(nbits)) if (small_const_nbits(nbits))
*dst = *src1 ^ *src2; *dst = *src1 ^ *src2;
@ -332,16 +335,17 @@ static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1,
__bitmap_xor(dst, src1, src2, nbits); __bitmap_xor(dst, src1, src2, nbits);
} }
static inline bool bitmap_andnot(unsigned long *dst, const unsigned long *src1, static __always_inline
const unsigned long *src2, unsigned int nbits) bool bitmap_andnot(unsigned long *dst, const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{ {
if (small_const_nbits(nbits)) if (small_const_nbits(nbits))
return (*dst = *src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0; return (*dst = *src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
return __bitmap_andnot(dst, src1, src2, nbits); return __bitmap_andnot(dst, src1, src2, nbits);
} }
static inline void bitmap_complement(unsigned long *dst, const unsigned long *src, static __always_inline
unsigned int nbits) void bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int nbits)
{ {
if (small_const_nbits(nbits)) if (small_const_nbits(nbits))
*dst = ~(*src); *dst = ~(*src);
@ -356,8 +360,8 @@ static inline void bitmap_complement(unsigned long *dst, const unsigned long *sr
#endif #endif
#define BITMAP_MEM_MASK (BITMAP_MEM_ALIGNMENT - 1) #define BITMAP_MEM_MASK (BITMAP_MEM_ALIGNMENT - 1)
static inline bool bitmap_equal(const unsigned long *src1, static __always_inline
const unsigned long *src2, unsigned int nbits) bool bitmap_equal(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
{ {
if (small_const_nbits(nbits)) if (small_const_nbits(nbits))
return !((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits)); return !((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits));
@ -376,10 +380,9 @@ static inline bool bitmap_equal(const unsigned long *src1,
* *
* Returns: True if (*@src1 | *@src2) == *@src3, false otherwise * Returns: True if (*@src1 | *@src2) == *@src3, false otherwise
*/ */
static inline bool bitmap_or_equal(const unsigned long *src1, static __always_inline
const unsigned long *src2, bool bitmap_or_equal(const unsigned long *src1, const unsigned long *src2,
const unsigned long *src3, const unsigned long *src3, unsigned int nbits)
unsigned int nbits)
{ {
if (!small_const_nbits(nbits)) if (!small_const_nbits(nbits))
return __bitmap_or_equal(src1, src2, src3, nbits); return __bitmap_or_equal(src1, src2, src3, nbits);
@ -387,9 +390,8 @@ static inline bool bitmap_or_equal(const unsigned long *src1,
return !(((*src1 | *src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits)); return !(((*src1 | *src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits));
} }
static inline bool bitmap_intersects(const unsigned long *src1, static __always_inline
const unsigned long *src2, bool bitmap_intersects(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
unsigned int nbits)
{ {
if (small_const_nbits(nbits)) if (small_const_nbits(nbits))
return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0; return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
@ -397,8 +399,8 @@ static inline bool bitmap_intersects(const unsigned long *src1,
return __bitmap_intersects(src1, src2, nbits); return __bitmap_intersects(src1, src2, nbits);
} }
static inline bool bitmap_subset(const unsigned long *src1, static __always_inline
const unsigned long *src2, unsigned int nbits) bool bitmap_subset(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
{ {
if (small_const_nbits(nbits)) if (small_const_nbits(nbits))
return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits)); return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits));
@ -406,7 +408,8 @@ static inline bool bitmap_subset(const unsigned long *src1,
return __bitmap_subset(src1, src2, nbits); return __bitmap_subset(src1, src2, nbits);
} }
static inline bool bitmap_empty(const unsigned long *src, unsigned nbits) static __always_inline
bool bitmap_empty(const unsigned long *src, unsigned nbits)
{ {
if (small_const_nbits(nbits)) if (small_const_nbits(nbits))
return ! (*src & BITMAP_LAST_WORD_MASK(nbits)); return ! (*src & BITMAP_LAST_WORD_MASK(nbits));
@ -414,7 +417,8 @@ static inline bool bitmap_empty(const unsigned long *src, unsigned nbits)
return find_first_bit(src, nbits) == nbits; return find_first_bit(src, nbits) == nbits;
} }
static inline bool bitmap_full(const unsigned long *src, unsigned int nbits) static __always_inline
bool bitmap_full(const unsigned long *src, unsigned int nbits)
{ {
if (small_const_nbits(nbits)) if (small_const_nbits(nbits))
return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits)); return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));
@ -448,8 +452,8 @@ unsigned long bitmap_weight_andnot(const unsigned long *src1,
return __bitmap_weight_andnot(src1, src2, nbits); return __bitmap_weight_andnot(src1, src2, nbits);
} }
static __always_inline void bitmap_set(unsigned long *map, unsigned int start, static __always_inline
unsigned int nbits) void bitmap_set(unsigned long *map, unsigned int start, unsigned int nbits)
{ {
if (__builtin_constant_p(nbits) && nbits == 1) if (__builtin_constant_p(nbits) && nbits == 1)
__set_bit(start, map); __set_bit(start, map);
@ -464,8 +468,8 @@ static __always_inline void bitmap_set(unsigned long *map, unsigned int start,
__bitmap_set(map, start, nbits); __bitmap_set(map, start, nbits);
} }
static __always_inline void bitmap_clear(unsigned long *map, unsigned int start, static __always_inline
unsigned int nbits) void bitmap_clear(unsigned long *map, unsigned int start, unsigned int nbits)
{ {
if (__builtin_constant_p(nbits) && nbits == 1) if (__builtin_constant_p(nbits) && nbits == 1)
__clear_bit(start, map); __clear_bit(start, map);
@ -480,8 +484,9 @@ static __always_inline void bitmap_clear(unsigned long *map, unsigned int start,
__bitmap_clear(map, start, nbits); __bitmap_clear(map, start, nbits);
} }
static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *src, static __always_inline
unsigned int shift, unsigned int nbits) void bitmap_shift_right(unsigned long *dst, const unsigned long *src,
unsigned int shift, unsigned int nbits)
{ {
if (small_const_nbits(nbits)) if (small_const_nbits(nbits))
*dst = (*src & BITMAP_LAST_WORD_MASK(nbits)) >> shift; *dst = (*src & BITMAP_LAST_WORD_MASK(nbits)) >> shift;
@ -489,8 +494,9 @@ static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *s
__bitmap_shift_right(dst, src, shift, nbits); __bitmap_shift_right(dst, src, shift, nbits);
} }
static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *src, static __always_inline
unsigned int shift, unsigned int nbits) void bitmap_shift_left(unsigned long *dst, const unsigned long *src,
unsigned int shift, unsigned int nbits)
{ {
if (small_const_nbits(nbits)) if (small_const_nbits(nbits))
*dst = (*src << shift) & BITMAP_LAST_WORD_MASK(nbits); *dst = (*src << shift) & BITMAP_LAST_WORD_MASK(nbits);
@ -498,11 +504,12 @@ static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *sr
__bitmap_shift_left(dst, src, shift, nbits); __bitmap_shift_left(dst, src, shift, nbits);
} }
static inline void bitmap_replace(unsigned long *dst, static __always_inline
const unsigned long *old, void bitmap_replace(unsigned long *dst,
const unsigned long *new, const unsigned long *old,
const unsigned long *mask, const unsigned long *new,
unsigned int nbits) const unsigned long *mask,
unsigned int nbits)
{ {
if (small_const_nbits(nbits)) if (small_const_nbits(nbits))
*dst = (*old & ~(*mask)) | (*new & *mask); *dst = (*old & ~(*mask)) | (*new & *mask);
@ -545,8 +552,9 @@ static inline void bitmap_replace(unsigned long *dst,
* bitmap_gather() can be seen as the 'reverse' bitmap_scatter() operation. * bitmap_gather() can be seen as the 'reverse' bitmap_scatter() operation.
* See bitmap_scatter() for details related to this relationship. * See bitmap_scatter() for details related to this relationship.
*/ */
static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src, static __always_inline
const unsigned long *mask, unsigned int nbits) void bitmap_scatter(unsigned long *dst, const unsigned long *src,
const unsigned long *mask, unsigned int nbits)
{ {
unsigned int n = 0; unsigned int n = 0;
unsigned int bit; unsigned int bit;
@ -599,8 +607,9 @@ static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src,
* bitmap_scatter(res, src, mask, n) and a call to * bitmap_scatter(res, src, mask, n) and a call to
* bitmap_scatter(res, result, mask, n) will lead to the same res value. * bitmap_scatter(res, result, mask, n) will lead to the same res value.
*/ */
static inline void bitmap_gather(unsigned long *dst, const unsigned long *src, static __always_inline
const unsigned long *mask, unsigned int nbits) void bitmap_gather(unsigned long *dst, const unsigned long *src,
const unsigned long *mask, unsigned int nbits)
{ {
unsigned int n = 0; unsigned int n = 0;
unsigned int bit; unsigned int bit;
@ -611,9 +620,9 @@ static inline void bitmap_gather(unsigned long *dst, const unsigned long *src,
__assign_bit(n++, dst, test_bit(bit, src)); __assign_bit(n++, dst, test_bit(bit, src));
} }
static inline void bitmap_next_set_region(unsigned long *bitmap, static __always_inline
unsigned int *rs, unsigned int *re, void bitmap_next_set_region(unsigned long *bitmap, unsigned int *rs,
unsigned int end) unsigned int *re, unsigned int end)
{ {
*rs = find_next_bit(bitmap, end, *rs); *rs = find_next_bit(bitmap, end, *rs);
*re = find_next_zero_bit(bitmap, end, *rs + 1); *re = find_next_zero_bit(bitmap, end, *rs + 1);
@ -628,7 +637,8 @@ static inline void bitmap_next_set_region(unsigned long *bitmap,
* This is the complement to __bitmap_find_free_region() and releases * This is the complement to __bitmap_find_free_region() and releases
* the found region (by clearing it in the bitmap). * the found region (by clearing it in the bitmap).
*/ */
static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order) static __always_inline
void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order)
{ {
bitmap_clear(bitmap, pos, BIT(order)); bitmap_clear(bitmap, pos, BIT(order));
} }
@ -644,7 +654,8 @@ static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos
* Returns: 0 on success, or %-EBUSY if specified region wasn't * Returns: 0 on success, or %-EBUSY if specified region wasn't
* free (not all bits were zero). * free (not all bits were zero).
*/ */
static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order) static __always_inline
int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order)
{ {
unsigned int len = BIT(order); unsigned int len = BIT(order);
@ -668,7 +679,8 @@ static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos
* Returns: the bit offset in bitmap of the allocated region, * Returns: the bit offset in bitmap of the allocated region,
* or -errno on failure. * or -errno on failure.
*/ */
static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order) static __always_inline
int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order)
{ {
unsigned int pos, end; /* scans bitmap by regions of size order */ unsigned int pos, end; /* scans bitmap by regions of size order */
@ -722,7 +734,7 @@ static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bi
* That is ``(u32 *)(&val)[0]`` gets the upper 32 bits, * That is ``(u32 *)(&val)[0]`` gets the upper 32 bits,
* but we expect the lower 32-bits of u64. * but we expect the lower 32-bits of u64.
*/ */
static inline void bitmap_from_u64(unsigned long *dst, u64 mask) static __always_inline void bitmap_from_u64(unsigned long *dst, u64 mask)
{ {
bitmap_from_arr64(dst, &mask, 64); bitmap_from_arr64(dst, &mask, 64);
} }
@ -737,9 +749,8 @@ static inline void bitmap_from_u64(unsigned long *dst, u64 mask)
* @map memory region. For @nbits = 0 and @nbits > BITS_PER_LONG the return * @map memory region. For @nbits = 0 and @nbits > BITS_PER_LONG the return
* value is undefined. * value is undefined.
*/ */
static inline unsigned long bitmap_read(const unsigned long *map, static __always_inline
unsigned long start, unsigned long bitmap_read(const unsigned long *map, unsigned long start, unsigned long nbits)
unsigned long nbits)
{ {
size_t index = BIT_WORD(start); size_t index = BIT_WORD(start);
unsigned long offset = start % BITS_PER_LONG; unsigned long offset = start % BITS_PER_LONG;
@ -772,8 +783,9 @@ static inline unsigned long bitmap_read(const unsigned long *map,
* *
* For @nbits == 0 and @nbits > BITS_PER_LONG no writes are performed. * For @nbits == 0 and @nbits > BITS_PER_LONG no writes are performed.
*/ */
static inline void bitmap_write(unsigned long *map, unsigned long value, static __always_inline
unsigned long start, unsigned long nbits) void bitmap_write(unsigned long *map, unsigned long value,
unsigned long start, unsigned long nbits)
{ {
size_t index; size_t index;
unsigned long offset; unsigned long offset;