diff options
Diffstat (limited to 'include/asm-generic')
27 files changed, 666 insertions, 540 deletions
diff --git a/include/asm-generic/4level-fixup.h b/include/asm-generic/4level-fixup.h deleted file mode 100644 index e3667c9a33a5..000000000000 --- a/include/asm-generic/4level-fixup.h +++ /dev/null @@ -1,40 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _4LEVEL_FIXUP_H -#define _4LEVEL_FIXUP_H - -#define __ARCH_HAS_4LEVEL_HACK -#define __PAGETABLE_PUD_FOLDED 1 - -#define PUD_SHIFT PGDIR_SHIFT -#define PUD_SIZE PGDIR_SIZE -#define PUD_MASK PGDIR_MASK -#define PTRS_PER_PUD 1 - -#define pud_t pgd_t - -#define pmd_alloc(mm, pud, address) \ - ((unlikely(pgd_none(*(pud))) && __pmd_alloc(mm, pud, address))? \ - NULL: pmd_offset(pud, address)) - -#define pud_offset(pgd, start) (pgd) -#define pud_none(pud) 0 -#define pud_bad(pud) 0 -#define pud_present(pud) 1 -#define pud_ERROR(pud) do { } while (0) -#define pud_clear(pud) pgd_clear(pud) -#define pud_val(pud) pgd_val(pud) -#define pud_populate(mm, pud, pmd) pgd_populate(mm, pud, pmd) -#define pud_page(pud) pgd_page(pud) -#define pud_page_vaddr(pud) pgd_page_vaddr(pud) - -#undef pud_free_tlb -#define pud_free_tlb(tlb, x, addr) do { } while (0) -#define pud_free(mm, x) do { } while (0) -#define __pud_free_tlb(tlb, x, addr) do { } while (0) - -#undef pud_addr_end -#define pud_addr_end(addr, end) (end) - -#include <asm-generic/5level-fixup.h> - -#endif diff --git a/include/asm-generic/5level-fixup.h b/include/asm-generic/5level-fixup.h index f6947da70d71..4c74b1c1d13b 100644 --- a/include/asm-generic/5level-fixup.h +++ b/include/asm-generic/5level-fixup.h @@ -51,7 +51,6 @@ static inline int p4d_present(p4d_t p4d) #undef p4d_free_tlb #define p4d_free_tlb(tlb, x, addr) do { } while (0) #define p4d_free(mm, x) do { } while (0) -#define __p4d_free_tlb(tlb, x, addr) do { } while (0) #undef p4d_addr_end #define p4d_addr_end(addr, end) (end) diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index 6f4536d70b8e..cd17d50697cc 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -3,3 +3,7 @@ # asm headers that all architectures except um should have # (This file is not included when SRCARCH=um since UML borrows several # asm headers from the host architecutre.) + +mandatory-y += dma-contiguous.h +mandatory-y += msi.h +mandatory-y += simd.h diff --git a/include/asm-generic/bitops-instrumented.h b/include/asm-generic/bitops-instrumented.h deleted file mode 100644 index ddd1c6d9d8db..000000000000 --- a/include/asm-generic/bitops-instrumented.h +++ /dev/null @@ -1,263 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -/* - * This file provides wrappers with sanitizer instrumentation for bit - * operations. - * - * To use this functionality, an arch's bitops.h file needs to define each of - * the below bit operations with an arch_ prefix (e.g. arch_set_bit(), - * arch___set_bit(), etc.). - */ -#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_H -#define _ASM_GENERIC_BITOPS_INSTRUMENTED_H - -#include <linux/kasan-checks.h> - -/** - * set_bit - Atomically set a bit in memory - * @nr: the bit to set - * @addr: the address to start counting from - * - * This is a relaxed atomic operation (no implied memory barriers). - * - * Note that @nr may be almost arbitrarily large; this function is not - * restricted to acting on a single-word quantity. - */ -static inline void set_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch_set_bit(nr, addr); -} - -/** - * __set_bit - Set a bit in memory - * @nr: the bit to set - * @addr: the address to start counting from - * - * Unlike set_bit(), this function is non-atomic. If it is called on the same - * region of memory concurrently, the effect may be that only one operation - * succeeds. - */ -static inline void __set_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch___set_bit(nr, addr); -} - -/** - * clear_bit - Clears a bit in memory - * @nr: Bit to clear - * @addr: Address to start counting from - * - * This is a relaxed atomic operation (no implied memory barriers). - */ -static inline void clear_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch_clear_bit(nr, addr); -} - -/** - * __clear_bit - Clears a bit in memory - * @nr: the bit to clear - * @addr: the address to start counting from - * - * Unlike clear_bit(), this function is non-atomic. If it is called on the same - * region of memory concurrently, the effect may be that only one operation - * succeeds. - */ -static inline void __clear_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch___clear_bit(nr, addr); -} - -/** - * clear_bit_unlock - Clear a bit in memory, for unlock - * @nr: the bit to set - * @addr: the address to start counting from - * - * This operation is atomic and provides release barrier semantics. - */ -static inline void clear_bit_unlock(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch_clear_bit_unlock(nr, addr); -} - -/** - * __clear_bit_unlock - Clears a bit in memory - * @nr: Bit to clear - * @addr: Address to start counting from - * - * This is a non-atomic operation but implies a release barrier before the - * memory operation. It can be used for an unlock if no other CPUs can - * concurrently modify other bits in the word. - */ -static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch___clear_bit_unlock(nr, addr); -} - -/** - * change_bit - Toggle a bit in memory - * @nr: Bit to change - * @addr: Address to start counting from - * - * This is a relaxed atomic operation (no implied memory barriers). - * - * Note that @nr may be almost arbitrarily large; this function is not - * restricted to acting on a single-word quantity. - */ -static inline void change_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch_change_bit(nr, addr); -} - -/** - * __change_bit - Toggle a bit in memory - * @nr: the bit to change - * @addr: the address to start counting from - * - * Unlike change_bit(), this function is non-atomic. If it is called on the same - * region of memory concurrently, the effect may be that only one operation - * succeeds. - */ -static inline void __change_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - arch___change_bit(nr, addr); -} - -/** - * test_and_set_bit - Set a bit and return its old value - * @nr: Bit to set - * @addr: Address to count from - * - * This is an atomic fully-ordered operation (implied full memory barrier). - */ -static inline bool test_and_set_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch_test_and_set_bit(nr, addr); -} - -/** - * __test_and_set_bit - Set a bit and return its old value - * @nr: Bit to set - * @addr: Address to count from - * - * This operation is non-atomic. If two instances of this operation race, one - * can appear to succeed but actually fail. - */ -static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch___test_and_set_bit(nr, addr); -} - -/** - * test_and_set_bit_lock - Set a bit and return its old value, for lock - * @nr: Bit to set - * @addr: Address to count from - * - * This operation is atomic and provides acquire barrier semantics if - * the returned value is 0. - * It can be used to implement bit locks. - */ -static inline bool test_and_set_bit_lock(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch_test_and_set_bit_lock(nr, addr); -} - -/** - * test_and_clear_bit - Clear a bit and return its old value - * @nr: Bit to clear - * @addr: Address to count from - * - * This is an atomic fully-ordered operation (implied full memory barrier). - */ -static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch_test_and_clear_bit(nr, addr); -} - -/** - * __test_and_clear_bit - Clear a bit and return its old value - * @nr: Bit to clear - * @addr: Address to count from - * - * This operation is non-atomic. If two instances of this operation race, one - * can appear to succeed but actually fail. - */ -static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch___test_and_clear_bit(nr, addr); -} - -/** - * test_and_change_bit - Change a bit and return its old value - * @nr: Bit to change - * @addr: Address to count from - * - * This is an atomic fully-ordered operation (implied full memory barrier). - */ -static inline bool test_and_change_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch_test_and_change_bit(nr, addr); -} - -/** - * __test_and_change_bit - Change a bit and return its old value - * @nr: Bit to change - * @addr: Address to count from - * - * This operation is non-atomic. If two instances of this operation race, one - * can appear to succeed but actually fail. - */ -static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch___test_and_change_bit(nr, addr); -} - -/** - * test_bit - Determine whether a bit is set - * @nr: bit number to test - * @addr: Address to start counting from - */ -static inline bool test_bit(long nr, const volatile unsigned long *addr) -{ - kasan_check_read(addr + BIT_WORD(nr), sizeof(long)); - return arch_test_bit(nr, addr); -} - -#if defined(arch_clear_bit_unlock_is_negative_byte) -/** - * clear_bit_unlock_is_negative_byte - Clear a bit in memory and test if bottom - * byte is negative, for unlock. - * @nr: the bit to clear - * @addr: the address to start counting from - * - * This operation is atomic and provides release barrier semantics. - * - * This is a bit of a one-trick-pony for the filemap code, which clears - * PG_locked and tests PG_waiters, - */ -static inline bool -clear_bit_unlock_is_negative_byte(long nr, volatile unsigned long *addr) -{ - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); - return arch_clear_bit_unlock_is_negative_byte(nr, addr); -} -/* Let everybody know we have it. */ -#define clear_bit_unlock_is_negative_byte clear_bit_unlock_is_negative_byte -#endif - -#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_H */ diff --git a/include/asm-generic/bitops/find.h b/include/asm-generic/bitops/find.h index 8a1ee10014de..9fdf21302fdf 100644 --- a/include/asm-generic/bitops/find.h +++ b/include/asm-generic/bitops/find.h @@ -80,4 +80,21 @@ extern unsigned long find_first_zero_bit(const unsigned long *addr, #endif /* CONFIG_GENERIC_FIND_FIRST_BIT */ +/** + * find_next_clump8 - find next 8-bit clump with set bits in a memory region + * @clump: location to store copy of found clump + * @addr: address to base the search on + * @size: bitmap size in number of bits + * @offset: bit offset at which to start searching + * + * Returns the bit offset for the next set clump; the found clump value is + * copied to the location pointed by @clump. If no bits are set, returns @size. + */ +extern unsigned long find_next_clump8(unsigned long *clump, + const unsigned long *addr, + unsigned long size, unsigned long offset); + +#define find_first_clump8(clump, bits, size) \ + find_next_clump8((clump), (bits), (size), 0) + #endif /*_ASM_GENERIC_BITOPS_FIND_H_ */ diff --git a/include/asm-generic/bitops/instrumented-atomic.h b/include/asm-generic/bitops/instrumented-atomic.h new file mode 100644 index 000000000000..18ce3c9e8eec --- /dev/null +++ b/include/asm-generic/bitops/instrumented-atomic.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * This file provides wrappers with sanitizer instrumentation for atomic bit + * operations. + * + * To use this functionality, an arch's bitops.h file needs to define each of + * the below bit operations with an arch_ prefix (e.g. arch_set_bit(), + * arch___set_bit(), etc.). + */ +#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H +#define _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H + +#include <linux/kasan-checks.h> + +/** + * set_bit - Atomically set a bit in memory + * @nr: the bit to set + * @addr: the address to start counting from + * + * This is a relaxed atomic operation (no implied memory barriers). + * + * Note that @nr may be almost arbitrarily large; this function is not + * restricted to acting on a single-word quantity. + */ +static inline void set_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch_set_bit(nr, addr); +} + +/** + * clear_bit - Clears a bit in memory + * @nr: Bit to clear + * @addr: Address to start counting from + * + * This is a relaxed atomic operation (no implied memory barriers). + */ +static inline void clear_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch_clear_bit(nr, addr); +} + +/** + * change_bit - Toggle a bit in memory + * @nr: Bit to change + * @addr: Address to start counting from + * + * This is a relaxed atomic operation (no implied memory barriers). + * + * Note that @nr may be almost arbitrarily large; this function is not + * restricted to acting on a single-word quantity. + */ +static inline void change_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch_change_bit(nr, addr); +} + +/** + * test_and_set_bit - Set a bit and return its old value + * @nr: Bit to set + * @addr: Address to count from + * + * This is an atomic fully-ordered operation (implied full memory barrier). + */ +static inline bool test_and_set_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_and_set_bit(nr, addr); +} + +/** + * test_and_clear_bit - Clear a bit and return its old value + * @nr: Bit to clear + * @addr: Address to count from + * + * This is an atomic fully-ordered operation (implied full memory barrier). + */ +static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_and_clear_bit(nr, addr); +} + +/** + * test_and_change_bit - Change a bit and return its old value + * @nr: Bit to change + * @addr: Address to count from + * + * This is an atomic fully-ordered operation (implied full memory barrier). + */ +static inline bool test_and_change_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_and_change_bit(nr, addr); +} + +#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */ diff --git a/include/asm-generic/bitops/instrumented-lock.h b/include/asm-generic/bitops/instrumented-lock.h new file mode 100644 index 000000000000..ec53fdeea9ec --- /dev/null +++ b/include/asm-generic/bitops/instrumented-lock.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * This file provides wrappers with sanitizer instrumentation for bit + * locking operations. + * + * To use this functionality, an arch's bitops.h file needs to define each of + * the below bit operations with an arch_ prefix (e.g. arch_set_bit(), + * arch___set_bit(), etc.). + */ +#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H +#define _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H + +#include <linux/kasan-checks.h> + +/** + * clear_bit_unlock - Clear a bit in memory, for unlock + * @nr: the bit to set + * @addr: the address to start counting from + * + * This operation is atomic and provides release barrier semantics. + */ +static inline void clear_bit_unlock(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch_clear_bit_unlock(nr, addr); +} + +/** + * __clear_bit_unlock - Clears a bit in memory + * @nr: Bit to clear + * @addr: Address to start counting from + * + * This is a non-atomic operation but implies a release barrier before the + * memory operation. It can be used for an unlock if no other CPUs can + * concurrently modify other bits in the word. + */ +static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch___clear_bit_unlock(nr, addr); +} + +/** + * test_and_set_bit_lock - Set a bit and return its old value, for lock + * @nr: Bit to set + * @addr: Address to count from + * + * This operation is atomic and provides acquire barrier semantics if + * the returned value is 0. + * It can be used to implement bit locks. + */ +static inline bool test_and_set_bit_lock(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_and_set_bit_lock(nr, addr); +} + +#if defined(arch_clear_bit_unlock_is_negative_byte) +/** + * clear_bit_unlock_is_negative_byte - Clear a bit in memory and test if bottom + * byte is negative, for unlock. + * @nr: the bit to clear + * @addr: the address to start counting from + * + * This operation is atomic and provides release barrier semantics. + * + * This is a bit of a one-trick-pony for the filemap code, which clears + * PG_locked and tests PG_waiters, + */ +static inline bool +clear_bit_unlock_is_negative_byte(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch_clear_bit_unlock_is_negative_byte(nr, addr); +} +/* Let everybody know we have it. */ +#define clear_bit_unlock_is_negative_byte clear_bit_unlock_is_negative_byte +#endif + +#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H */ diff --git a/include/asm-generic/bitops/instrumented-non-atomic.h b/include/asm-generic/bitops/instrumented-non-atomic.h new file mode 100644 index 000000000000..95ff28d128a1 --- /dev/null +++ b/include/asm-generic/bitops/instrumented-non-atomic.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * This file provides wrappers with sanitizer instrumentation for non-atomic + * bit operations. + * + * To use this functionality, an arch's bitops.h file needs to define each of + * the below bit operations with an arch_ prefix (e.g. arch_set_bit(), + * arch___set_bit(), etc.). + */ +#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H +#define _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H + +#include <linux/kasan-checks.h> + +/** + * __set_bit - Set a bit in memory + * @nr: the bit to set + * @addr: the address to start counting from + * + * Unlike set_bit(), this function is non-atomic. If it is called on the same + * region of memory concurrently, the effect may be that only one operation + * succeeds. + */ +static inline void __set_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch___set_bit(nr, addr); +} + +/** + * __clear_bit - Clears a bit in memory + * @nr: the bit to clear + * @addr: the address to start counting from + * + * Unlike clear_bit(), this function is non-atomic. If it is called on the same + * region of memory concurrently, the effect may be that only one operation + * succeeds. + */ +static inline void __clear_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch___clear_bit(nr, addr); +} + +/** + * __change_bit - Toggle a bit in memory + * @nr: the bit to change + * @addr: the address to start counting from + * + * Unlike change_bit(), this function is non-atomic. If it is called on the same + * region of memory concurrently, the effect may be that only one operation + * succeeds. + */ +static inline void __change_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + arch___change_bit(nr, addr); +} + +/** + * __test_and_set_bit - Set a bit and return its old value + * @nr: Bit to set + * @addr: Address to count from + * + * This operation is non-atomic. If two instances of this operation race, one + * can appear to succeed but actually fail. + */ +static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch___test_and_set_bit(nr, addr); +} + +/** + * __test_and_clear_bit - Clear a bit and return its old value + * @nr: Bit to clear + * @addr: Address to count from + * + * This operation is non-atomic. If two instances of this operation race, one + * can appear to succeed but actually fail. + */ +static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch___test_and_clear_bit(nr, addr); +} + +/** + * __test_and_change_bit - Change a bit and return its old value + * @nr: Bit to change + * @addr: Address to count from + * + * This operation is non-atomic. If two instances of this operation race, one + * can appear to succeed but actually fail. + */ +static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr) +{ + kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + return arch___test_and_change_bit(nr, addr); +} + +/** + * test_bit - Determine whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static inline bool test_bit(long nr, const volatile unsigned long *addr) +{ + kasan_check_read(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_bit(nr, addr); +} + +#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */ diff --git a/include/asm-generic/bug.h b/include/asm-generic/bug.h index aa6c093d9ce9..384b5c835ced 100644 --- a/include/asm-generic/bug.h +++ b/include/asm-generic/bug.h @@ -10,6 +10,7 @@ #define BUGFLAG_WARNING (1 << 0) #define BUGFLAG_ONCE (1 << 1) #define BUGFLAG_DONE (1 << 2) +#define BUGFLAG_NO_CUT_HERE (1 << 3) /* CUT_HERE already sent */ #define BUGFLAG_TAINT(taint) ((taint) << 8) #define BUG_GET_TAINT(bug) ((bug)->flags >> 8) #endif @@ -61,18 +62,6 @@ struct bug_entry { #define BUG_ON(condition) do { if (unlikely(condition)) BUG(); } while (0) #endif -#ifdef __WARN_FLAGS -#define __WARN_TAINT(taint) __WARN_FLAGS(BUGFLAG_TAINT(taint)) -#define __WARN_ONCE_TAINT(taint) __WARN_FLAGS(BUGFLAG_ONCE|BUGFLAG_TAINT(taint)) - -#define WARN_ON_ONCE(condition) ({ \ - int __ret_warn_on = !!(condition); \ - if (unlikely(__ret_warn_on)) \ - __WARN_ONCE_TAINT(TAINT_WARN); \ - unlikely(__ret_warn_on); \ -}) -#endif - /* * WARN(), WARN_ON(), WARN_ON_ONCE, and so on can be used to report * significant kernel issues that need prompt attention if they should ever @@ -89,27 +78,27 @@ struct bug_entry { * * Use the versions with printk format strings to provide better diagnostics. */ -#ifndef __WARN_TAINT -extern __printf(3, 4) -void warn_slowpath_fmt(const char *file, const int line, - const char *fmt, ...); +#ifndef __WARN_FLAGS extern __printf(4, 5) -void warn_slowpath_fmt_taint(const char *file, const int line, unsigned taint, - const char *fmt, ...); -extern void warn_slowpath_null(const char *file, const int line); -#define WANT_WARN_ON_SLOWPATH -#define __WARN() warn_slowpath_null(__FILE__, __LINE__) -#define __WARN_printf(arg...) warn_slowpath_fmt(__FILE__, __LINE__, arg) -#define __WARN_printf_taint(taint, arg...) \ - warn_slowpath_fmt_taint(__FILE__, __LINE__, taint, arg) +void warn_slowpath_fmt(const char *file, const int line, unsigned taint, + const char *fmt, ...); +#define __WARN() __WARN_printf(TAINT_WARN, NULL) +#define __WARN_printf(taint, arg...) \ + warn_slowpath_fmt(__FILE__, __LINE__, taint, arg) #else extern __printf(1, 2) void __warn_printk(const char *fmt, ...); -#define __WARN() do { \ - printk(KERN_WARNING CUT_HERE); __WARN_TAINT(TAINT_WARN); \ -} while (0) -#define __WARN_printf(arg...) __WARN_printf_taint(TAINT_WARN, arg) -#define __WARN_printf_taint(taint, arg...) \ - do { __warn_printk(arg); __WARN_TAINT(taint); } while (0) +#define __WARN() __WARN_FLAGS(BUGFLAG_TAINT(TAINT_WARN)) +#define __WARN_printf(taint, arg...) do { \ + __warn_printk(arg); \ + __WARN_FLAGS(BUGFLAG_NO_CUT_HERE | BUGFLAG_TAINT(taint));\ + } while (0) +#define WARN_ON_ONCE(condition) ({ \ + int __ret_warn_on = !!(condition); \ + if (unlikely(__ret_warn_on)) \ + __WARN_FLAGS(BUGFLAG_ONCE | \ + BUGFLAG_TAINT(TAINT_WARN)); \ + unlikely(__ret_warn_on); \ +}) #endif /* used internally by panic.c */ @@ -132,7 +121,7 @@ void __warn(const char *file, int line, void *caller, unsigned taint, #define WARN(condition, format...) ({ \ int __ret_warn_on = !!(condition); \ if (unlikely(__ret_warn_on)) \ - __WARN_printf(format); \ + __WARN_printf(TAINT_WARN, format); \ unlikely(__ret_warn_on); \ }) #endif @@ -140,7 +129,7 @@ void __warn(const char *file, int line, void *caller, unsigned taint, #define WARN_TAINT(condition, taint, format...) ({ \ int __ret_warn_on = !!(condition); \ if (unlikely(__ret_warn_on)) \ - __WARN_printf_taint(taint, format); \ + __WARN_printf(taint, format); \ unlikely(__ret_warn_on); \ }) @@ -185,7 +174,7 @@ void __warn(const char *file, int line, void *caller, unsigned taint, #endif #ifndef HAVE_ARCH_BUG_ON -#define BUG_ON(condition) do { if (condition) BUG(); } while (0) +#define BUG_ON(condition) do { if (unlikely(condition)) BUG(); } while (0) #endif #ifndef HAVE_ARCH_WARN_ON diff --git a/include/asm-generic/cacheflush.h b/include/asm-generic/cacheflush.h index a950a22c4890..cac7404b2bdd 100644 --- a/include/asm-generic/cacheflush.h +++ b/include/asm-generic/cacheflush.h @@ -11,71 +11,102 @@ * The cache doesn't need to be flushed when TLB entries change when * the cache is mapped to physical memory, not virtual memory */ +#ifndef flush_cache_all static inline void flush_cache_all(void) { } +#endif +#ifndef flush_cache_mm static inline void flush_cache_mm(struct mm_struct *mm) { } +#endif +#ifndef flush_cache_dup_mm static inline void flush_cache_dup_mm(struct mm_struct *mm) { } +#endif +#ifndef flush_cache_range static inline void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { } +#endif +#ifndef flush_cache_page static inline void flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr, unsigned long pfn) { } +#endif +#ifndef flush_dcache_page static inline void flush_dcache_page(struct page *page) { } +#endif +#ifndef flush_dcache_mmap_lock static inline void flush_dcache_mmap_lock(struct address_space *mapping) { } +#endif +#ifndef flush_dcache_mmap_unlock static inline void flush_dcache_mmap_unlock(struct address_space *mapping) { } +#endif +#ifndef flush_icache_range static inline void flush_icache_range(unsigned long start, unsigned long end) { } +#endif +#ifndef flush_icache_page static inline void flush_icache_page(struct vm_area_struct *vma, struct page *page) { } +#endif +#ifndef flush_icache_user_range static inline void flush_icache_user_range(struct vm_area_struct *vma, struct page *page, unsigned long addr, int len) { } +#endif +#ifndef flush_cache_vmap static inline void flush_cache_vmap(unsigned long start, unsigned long end) { } +#endif +#ifndef flush_cache_vunmap static inline void flush_cache_vunmap(unsigned long start, unsigned long end) { } +#endif -#define copy_to_user_page(vma, page, vaddr, dst, src, len) \ +#ifndef copy_to_user_page +#define copy_to_user_page(vma, page, vaddr, dst, src, len) \ do { \ memcpy(dst, src, len); \ flush_icache_user_range(vma, page, vaddr, len); \ } while (0) +#endif + +#ifndef copy_from_user_page #define copy_from_user_page(vma, page, vaddr, dst, src, len) \ memcpy(dst, src, len) +#endif #endif /* __ASM_CACHEFLUSH_H */ diff --git a/include/asm-generic/div64.h b/include/asm-generic/div64.h index dc9726fdac8f..a3b98c86f077 100644 --- a/include/asm-generic/div64.h +++ b/include/asm-generic/div64.h @@ -28,12 +28,12 @@ /** * do_div - returns 2 values: calculate remainder and update new dividend - * @n: pointer to uint64_t dividend (will be updated) + * @n: uint64_t dividend (will be updated) * @base: uint32_t divisor * * Summary: - * ``uint32_t remainder = *n % base;`` - * ``*n = *n / base;`` + * ``uint32_t remainder = n % base;`` + * ``n = n / base;`` * * Return: (uint32_t)remainder * @@ -178,7 +178,8 @@ static inline uint64_t __arch_xprod_64(const uint64_t m, uint64_t n, bool bias) uint32_t m_hi = m >> 32; uint32_t n_lo = n; uint32_t n_hi = n >> 32; - uint64_t res, tmp; + uint64_t res; + uint32_t res_lo, res_hi, tmp; if (!bias) { res = ((uint64_t)m_lo * n_lo) >> 32; @@ -187,8 +188,9 @@ static inline uint64_t __arch_xprod_64(const uint64_t m, uint64_t n, bool bias) res = (m + (uint64_t)m_lo * n_lo) >> 32; } else { res = m + (uint64_t)m_lo * n_lo; - tmp = (res < m) ? (1ULL << 32) : 0; - res = (res >> 32) + tmp; + res_lo = res >> 32; + res_hi = (res_lo < m_hi); + res = res_lo | ((uint64_t)res_hi << 32); } if (!(m & ((1ULL << 63) | (1ULL << 31)))) { @@ -197,10 +199,12 @@ static inline uint64_t __arch_xprod_64(const uint64_t m, uint64_t n, bool bias) res += (uint64_t)m_hi * n_lo; res >>= 32; } else { - tmp = res += (uint64_t)m_lo * n_hi; + res += (uint64_t)m_lo * n_hi; + tmp = res >> 32; res += (uint64_t)m_hi * n_lo; - tmp = (res < tmp) ? (1ULL << 32) : 0; - res = (res >> 32) + tmp; + res_lo = res >> 32; + res_hi = (res_lo < tmp); + res = res_lo | ((uint64_t)res_hi << 32); } res += (uint64_t)m_hi * n_hi; diff --git a/include/asm-generic/error-injection.h b/include/asm-generic/error-injection.h index 95a159a4137f..80ca61058dd2 100644 --- a/include/asm-generic/error-injection.h +++ b/include/asm-generic/error-injection.h @@ -16,6 +16,8 @@ struct error_injection_entry { int etype; }; +struct pt_regs; + #ifdef CONFIG_FUNCTION_ERROR_INJECTION /* * Whitelist ganerating macro. Specify functions which can be @@ -28,8 +30,12 @@ static struct error_injection_entry __used \ .addr = (unsigned long)fname, \ .etype = EI_ETYPE_##_etype, \ }; + +void override_function_with_return(struct pt_regs *regs); #else #define ALLOW_ERROR_INJECTION(fname, _etype) + +static inline void override_function_with_return(struct pt_regs *regs) { } #endif #endif diff --git a/include/asm-generic/export.h b/include/asm-generic/export.h index 294d6ae785d4..365345f9a9e3 100644 --- a/include/asm-generic/export.h +++ b/include/asm-generic/export.h @@ -1,52 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ #ifndef __ASM_GENERIC_EXPORT_H #define __ASM_GENERIC_EXPORT_H #ifndef KSYM_FUNC #define KSYM_FUNC(x) x #endif -#ifdef CONFIG_64BIT -#ifndef KSYM_ALIGN +#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS +#define KSYM_ALIGN 4 +#elif defined(CONFIG_64BIT) #define KSYM_ALIGN 8 -#endif #else -#ifndef KSYM_ALIGN #define KSYM_ALIGN 4 #endif -#endif #ifndef KCRC_ALIGN #define KCRC_ALIGN 4 #endif .macro __put, val, name #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS - .long \val - ., \name - . + .long \val - ., \name - ., 0 #elif defined(CONFIG_64BIT) - .quad \val, \name + .quad \val, \name, 0 #else - .long \val, \name + .long \val, \name, 0 #endif .endm /* - * note on .section use: @progbits vs %progbits nastiness doesn't matter, - * since we immediately emit into those sections anyway. + * note on .section use: we specify progbits since usage of the "M" (SHF_MERGE) + * section flag requires it. Use '%progbits' instead of '@progbits' since the + * former apparently works on all arches according to the binutils source. */ + .macro ___EXPORT_SYMBOL name,val,sec #ifdef CONFIG_MODULES - .globl __ksymtab_\name .section ___ksymtab\sec+\name,"a" .balign KSYM_ALIGN __ksymtab_\name: __put \val, __kstrtab_\name .previous - .section __ksymtab_strings,"a" + .section __ksymtab_strings,"aMS",%progbits,1 __kstrtab_\name: .asciz "\name" .previous #ifdef CONFIG_MODVERSIONS .section ___kcrctab\sec+\name,"a" .balign KCRC_ALIGN -__kcrctab_\name: #if defined(CONFIG_MODULE_REL_CRCS) .long __crc_\name - . #else @@ -57,7 +56,6 @@ __kcrctab_\name: #endif #endif .endm -#undef __put #if defined(CONFIG_TRIM_UNUSED_KSYMS) diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index b83e2802c969..d39ac997dda8 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -922,39 +922,17 @@ static inline void *phys_to_virt(unsigned long address) /** * DOC: ioremap() and ioremap_*() variants * - * If you have an IOMMU your architecture is expected to have both ioremap() - * and iounmap() implemented otherwise the asm-generic helpers will provide a - * direct mapping. + * Architectures with an MMU are expected to provide ioremap() and iounmap() + * themselves or rely on GENERIC_IOREMAP. For NOMMU architectures we provide + * a default nop-op implementation that expect that the physical address used + * for MMIO are already marked as uncached, and can be used as kernel virtual + * addresses. * - * There are ioremap_*() call variants, if you have no IOMMU we naturally will - * default to direct mapping for all of them, you can override these defaults. - * If you have an IOMMU you are highly encouraged to provide your own - * ioremap variant implementation as there currently is no safe architecture - * agnostic default. To avoid possible improper behaviour default asm-generic - * ioremap_*() variants all return NULL when an IOMMU is available. If you've - * defined your own ioremap_*() variant you must then declare your own - * ioremap_*() variant as defined to itself to avoid the default NULL return. + * ioremap_wc() and ioremap_wt() can provide more relaxed caching attributes + * for specific drivers if the architecture choses to implement them. If they + * are not implemented we fall back to plain ioremap. */ - -#ifdef CONFIG_MMU - -#ifndef ioremap_uc -#define ioremap_uc ioremap_uc -static inline void __iomem *ioremap_uc(phys_addr_t offset, size_t size) -{ - return NULL; -} -#endif - -#else /* !CONFIG_MMU */ - -/* - * Change "struct page" to physical address. - * - * This implementation is for the no-MMU case only... if you have an MMU - * you'll need to provide your own definitions. - */ - +#ifndef CONFIG_MMU #ifndef ioremap #define ioremap ioremap static inline void __iomem *ioremap(phys_addr_t offset, size_t size) @@ -963,53 +941,45 @@ static inline void __iomem *ioremap(phys_addr_t offset, size_t size) } #endif -#ifndef __ioremap -#define __ioremap __ioremap -static inline void __iomem *__ioremap(phys_addr_t offset, size_t size, - unsigned long flags) -{ - return ioremap(offset, size); -} -#endif - #ifndef iounmap #define iounmap iounmap - static inline void iounmap(void __iomem *addr) { } #endif -#endif /* CONFIG_MMU */ -#ifndef ioremap_nocache -void __iomem *ioremap(phys_addr_t phys_addr, size_t size); -#define ioremap_nocache ioremap_nocache -static inline void __iomem *ioremap_nocache(phys_addr_t offset, size_t size) -{ - return ioremap(offset, size); -} -#endif +#elif defined(CONFIG_GENERIC_IOREMAP) +#include <asm/pgtable.h> -#ifndef ioremap_uc -#define ioremap_uc ioremap_uc -static inline void __iomem *ioremap_uc(phys_addr_t offset, size_t size) +void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long prot); +void iounmap(volatile void __iomem *addr); + +static inline void __iomem *ioremap(phys_addr_t addr, size_t size) { - return ioremap_nocache(offset, size); + /* _PAGE_IOREMAP needs to be supplied by the architecture */ + return ioremap_prot(addr, size, _PAGE_IOREMAP); } -#endif +#endif /* !CONFIG_MMU || CONFIG_GENERIC_IOREMAP */ #ifndef ioremap_wc -#define ioremap_wc ioremap_wc -static inline void __iomem *ioremap_wc(phys_addr_t offset, size_t size) -{ - return ioremap_nocache(offset, size); -} +#define ioremap_wc ioremap #endif #ifndef ioremap_wt -#define ioremap_wt ioremap_wt -static inline void __iomem *ioremap_wt(phys_addr_t offset, size_t size) +#define ioremap_wt ioremap +#endif + +/* + * ioremap_uc is special in that we do require an explicit architecture + * implementation. In general you do not want to use this function in a + * driver and use plain ioremap, which is uncached by default. Similarly + * architectures should not implement it unless they have a very good + * reason. + */ +#ifndef ioremap_uc +#define ioremap_uc ioremap_uc +static inline void __iomem *ioremap_uc(phys_addr_t offset, size_t size) { - return ioremap_nocache(offset, size); + return NULL; } #endif diff --git a/include/asm-generic/iomap.h b/include/asm-generic/iomap.h index a008f504a2d0..9d28a5e82f73 100644 --- a/include/asm-generic/iomap.h +++ b/include/asm-generic/iomap.h @@ -94,11 +94,11 @@ extern void ioport_unmap(void __iomem *); #endif #ifndef ARCH_HAS_IOREMAP_WC -#define ioremap_wc ioremap_nocache +#define ioremap_wc ioremap #endif #ifndef ARCH_HAS_IOREMAP_WT -#define ioremap_wt ioremap_nocache +#define ioremap_wt ioremap #endif #ifdef CONFIG_PCI diff --git a/include/asm-generic/mm_hooks.h b/include/asm-generic/mm_hooks.h index 6736ed2f632b..4dbb177d1150 100644 --- a/include/asm-generic/mm_hooks.h +++ b/include/asm-generic/mm_hooks.h @@ -22,11 +22,6 @@ static inline void arch_unmap(struct mm_struct *mm, { } -static inline void arch_bprm_mm_init(struct mm_struct *mm, - struct vm_area_struct *vma) -{ -} - static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write, bool execute, bool foreign) { diff --git a/include/asm-generic/mshyperv.h b/include/asm-generic/mshyperv.h index 0becb7d9704d..b3f1082cc435 100644 --- a/include/asm-generic/mshyperv.h +++ b/include/asm-generic/mshyperv.h @@ -166,9 +166,12 @@ static inline int cpumask_to_vpset(struct hv_vpset *vpset, void hyperv_report_panic(struct pt_regs *regs, long err); void hyperv_report_panic_msg(phys_addr_t pa, size_t size); bool hv_is_hyperv_initialized(void); +bool hv_is_hibernation_supported(void); void hyperv_cleanup(void); +void hv_setup_sched_clock(void *sched_clock); #else /* CONFIG_HYPERV */ static inline bool hv_is_hyperv_initialized(void) { return false; } +static inline bool hv_is_hibernation_supported(void) { return false; } static inline void hyperv_cleanup(void) {} #endif /* CONFIG_HYPERV */ diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h index c2de013b2cf4..35e4a53b83e6 100644 --- a/include/asm-generic/percpu.h +++ b/include/asm-generic/percpu.h @@ -74,7 +74,7 @@ do { \ #define raw_cpu_generic_add_return(pcp, val) \ ({ \ - typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \ + typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ \ *__p += val; \ *__p; \ @@ -82,7 +82,7 @@ do { \ #define raw_cpu_generic_xchg(pcp, nval) \ ({ \ - typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \ + typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ typeof(pcp) __ret; \ __ret = *__p; \ *__p = nval; \ @@ -91,7 +91,7 @@ do { \ #define raw_cpu_generic_cmpxchg(pcp, oval, nval) \ ({ \ - typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \ + typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ typeof(pcp) __ret; \ __ret = *__p; \ if (__ret == (oval)) \ @@ -101,8 +101,8 @@ do { \ #define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ ({ \ - typeof(&(pcp1)) __p1 = raw_cpu_ptr(&(pcp1)); \ - typeof(&(pcp2)) __p2 = raw_cpu_ptr(&(pcp2)); \ + typeof(pcp1) *__p1 = raw_cpu_ptr(&(pcp1)); \ + typeof(pcp2) *__p2 = raw_cpu_ptr(&(pcp2)); \ int __ret = 0; \ if (*__p1 == (oval1) && *__p2 == (oval2)) { \ *__p1 = nval1; \ diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h index 8476175c07e7..73f7421413cb 100644 --- a/include/asm-generic/pgalloc.h +++ b/include/asm-generic/pgalloc.h @@ -49,7 +49,7 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) * @mm: the mm_struct of the current context * @gfp: GFP flags to use for the allocation * - * Allocates a page and runs the pgtable_page_ctor(). + * Allocates a page and runs the pgtable_pte_page_ctor(). * * This function is intended for architectures that need * anything beyond simple page allocation or must have custom GFP flags. @@ -63,7 +63,7 @@ static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp) pte = alloc_page(gfp); if (!pte) return NULL; - if (!pgtable_page_ctor(pte)) { + if (!pgtable_pte_page_ctor(pte)) { __free_page(pte); return NULL; } @@ -76,7 +76,7 @@ static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp) * pte_alloc_one - allocate a page for PTE-level user page table * @mm: the mm_struct of the current context * - * Allocates a page and runs the pgtable_page_ctor(). + * Allocates a page and runs the pgtable_pte_page_ctor(). * * Return: `struct page` initialized as page table or %NULL on error */ @@ -98,15 +98,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm) */ static inline void pte_free(struct mm_struct *mm, struct page *pte_page) { - pgtable_page_dtor(pte_page); + pgtable_pte_page_dtor(pte_page); __free_page(pte_page); } -#else /* CONFIG_MMU */ - -/* This is enough for a nommu architecture */ -#define check_pgt_cache() do { } while (0) - #endif /* CONFIG_MMU */ #endif /* __ASM_GENERIC_PGALLOC_H */ diff --git a/include/asm-generic/pgtable-nop4d.h b/include/asm-generic/pgtable-nop4d.h index aebab905e6cd..ce2cbb3c380f 100644 --- a/include/asm-generic/pgtable-nop4d.h +++ b/include/asm-generic/pgtable-nop4d.h @@ -50,7 +50,7 @@ static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) */ #define p4d_alloc_one(mm, address) NULL #define p4d_free(mm, x) do { } while (0) -#define __p4d_free_tlb(tlb, x, a) do { } while (0) +#define p4d_free_tlb(tlb, x, a) do { } while (0) #undef p4d_addr_end #define p4d_addr_end(addr, end) (end) diff --git a/include/asm-generic/pgtable-nopmd.h b/include/asm-generic/pgtable-nopmd.h index b85b8271a73d..0d9b28cba16d 100644 --- a/include/asm-generic/pgtable-nopmd.h +++ b/include/asm-generic/pgtable-nopmd.h @@ -60,7 +60,7 @@ static inline pmd_t * pmd_offset(pud_t * pud, unsigned long address) static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) { } -#define __pmd_free_tlb(tlb, x, a) do { } while (0) +#define pmd_free_tlb(tlb, x, a) do { } while (0) #undef pmd_addr_end #define pmd_addr_end(addr, end) (end) diff --git a/include/asm-generic/pgtable-nopud.h b/include/asm-generic/pgtable-nopud.h index c77a1d301155..d3776cb494c0 100644 --- a/include/asm-generic/pgtable-nopud.h +++ b/include/asm-generic/pgtable-nopud.h @@ -59,7 +59,7 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) */ #define pud_alloc_one(mm, address) NULL #define pud_free(mm, x) do { } while (0) -#define __pud_free_tlb(tlb, x, a) do { } while (0) +#define pud_free_tlb(tlb, x, a) do { } while (0) #undef pud_addr_end #define pud_addr_end(addr, end) (end) diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h index 75d9d68a6de7..e2e2bef07dd2 100644 --- a/include/asm-generic/pgtable.h +++ b/include/asm-generic/pgtable.h @@ -558,8 +558,19 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) * Do the tests inline, but report and clear the bad entry in mm/memory.c. */ void pgd_clear_bad(pgd_t *); + +#ifndef __PAGETABLE_P4D_FOLDED void p4d_clear_bad(p4d_t *); +#else +#define p4d_clear_bad(p4d) do { } while (0) +#endif + +#ifndef __PAGETABLE_PUD_FOLDED void pud_clear_bad(pud_t *); +#else +#define pud_clear_bad(p4d) do { } while (0) +#endif + void pmd_clear_bad(pmd_t *); static inline int pgd_none_or_clear_bad(pgd_t *pgd) @@ -903,6 +914,21 @@ static inline int pud_write(pud_t pud) } #endif /* pud_write */ +#if !defined(CONFIG_ARCH_HAS_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE) +static inline int pmd_devmap(pmd_t pmd) +{ + return 0; +} +static inline int pud_devmap(pud_t pud) +{ + return 0; +} +static inline int pgd_devmap(pgd_t pgd) +{ + return 0; +} +#endif + #if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \ (defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)) @@ -912,6 +938,31 @@ static inline int pud_trans_huge(pud_t pud) } #endif +/* See pmd_none_or_trans_huge_or_clear_bad for discussion. */ +static inline int pud_none_or_trans_huge_or_dev_or_clear_bad(pud_t *pud) +{ + pud_t pudval = READ_ONCE(*pud); + + if (pud_none(pudval) || pud_trans_huge(pudval) || pud_devmap(pudval)) + return 1; + if (unlikely(pud_bad(pudval))) { + pud_clear_bad(pud); + return 1; + } + return 0; +} + +/* See pmd_trans_unstable for discussion. */ +static inline int pud_trans_unstable(pud_t *pud) +{ +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ + defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) + return pud_none_or_trans_huge_or_dev_or_clear_bad(pud); +#else + return 0; +#endif +} + #ifndef pmd_read_atomic static inline pmd_t pmd_read_atomic(pmd_t *pmdp) { @@ -1002,9 +1053,8 @@ static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd) * need this). If THP is not enabled, the pmd can't go away under the * code even if MADV_DONTNEED runs, but if THP is enabled we need to * run a pmd_trans_unstable before walking the ptes after - * split_huge_page_pmd returns (because it may have run when the pmd - * become null, but then a page fault can map in a THP and not a - * regular page). + * split_huge_pmd returns (because it may have run when the pmd become + * null, but then a page fault can map in a THP and not a regular page). */ static inline int pmd_trans_unstable(pmd_t *pmd) { @@ -1126,7 +1176,7 @@ int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, static inline void init_espfix_bsp(void) { } #endif -extern void __init pgd_cache_init(void); +extern void __init pgtable_cache_init(void); #ifndef __HAVE_ARCH_PFN_MODIFY_ALLOWED static inline bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot) @@ -1188,4 +1238,24 @@ static inline bool arch_has_pfn_modify_check(void) #define mm_pmd_folded(mm) __is_defined(__PAGETABLE_PMD_FOLDED) #endif +/* + * p?d_leaf() - true if this entry is a final mapping to a physical address. + * This differs from p?d_huge() by the fact that they are always available (if + * the architecture supports large pages at the appropriate level) even + * if CONFIG_HUGETLB_PAGE is not defined. + * Only meaningful when called on a valid entry. + */ +#ifndef pgd_leaf +#define pgd_leaf(x) 0 +#endif +#ifndef p4d_leaf +#define p4d_leaf(x) 0 +#endif +#ifndef pud_leaf +#define pud_leaf(x) 0 +#endif +#ifndef pmd_leaf +#define pmd_leaf(x) 0 +#endif + #endif /* _ASM_GENERIC_PGTABLE_H */ diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h index c3046c920063..d683f5e6d791 100644 --- a/include/asm-generic/preempt.h +++ b/include/asm-generic/preempt.h @@ -78,11 +78,11 @@ static __always_inline bool should_resched(int preempt_offset) tif_need_resched()); } -#ifdef CONFIG_PREEMPT +#ifdef CONFIG_PREEMPTION extern asmlinkage void preempt_schedule(void); #define __preempt_schedule() preempt_schedule() extern asmlinkage void preempt_schedule_notrace(void); #define __preempt_schedule_notrace() preempt_schedule_notrace() -#endif /* CONFIG_PREEMPT */ +#endif /* CONFIG_PREEMPTION */ #endif /* __ASM_PREEMPT_H */ diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index 04c0644006fd..f391f6b500b4 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -56,6 +56,15 @@ * Defaults to flushing at tlb_end_vma() to reset the range; helps when * there's large holes between the VMAs. * + * - tlb_remove_table() + * + * tlb_remove_table() is the basic primitive to free page-table directories + * (__p*_free_tlb()). In it's most primitive form it is an alias for + * tlb_remove_page() below, for when page directories are pages and have no + * additional constraints. + * + * See also MMU_GATHER_TABLE_FREE and MMU_GATHER_RCU_TABLE_FREE. + * * - tlb_remove_page() / __tlb_remove_page() * - tlb_remove_page_size() / __tlb_remove_page_size() * @@ -121,65 +130,53 @@ * * Additionally there are a few opt-in features: * - * HAVE_MMU_GATHER_PAGE_SIZE + * MMU_GATHER_PAGE_SIZE * * This ensures we call tlb_flush() every time tlb_change_page_size() actually * changes the size and provides mmu_gather::page_size to tlb_flush(). * - * HAVE_RCU_TABLE_FREE + * This might be useful if your architecture has size specific TLB + * invalidation instructions. + * + * MMU_GATHER_TABLE_FREE * * This provides tlb_remove_table(), to be used instead of tlb_remove_page() - * for page directores (__p*_free_tlb()). This provides separate freeing of - * the page-table pages themselves in a semi-RCU fashion (see comment below). - * Useful if your architecture doesn't use IPIs for remote TLB invalidates - * and therefore doesn't naturally serialize with software page-table walkers. + * for page directores (__p*_free_tlb()). + * + * Useful if your architecture has non-page page directories. * * When used, an architecture is expected to provide __tlb_remove_table() * which does the actual freeing of these pages. * - * HAVE_RCU_TABLE_NO_INVALIDATE + * MMU_GATHER_RCU_TABLE_FREE + * + * Like MMU_GATHER_TABLE_FREE, and adds semi-RCU semantics to the free (see + * comment below). * - * This makes HAVE_RCU_TABLE_FREE avoid calling tlb_flush_mmu_tlbonly() before - * freeing the page-table pages. This can be avoided if you use - * HAVE_RCU_TABLE_FREE and your architecture does _NOT_ use the Linux - * page-tables natively. + * Useful if your architecture doesn't use IPIs for remote TLB invalidates + * and therefore doesn't naturally serialize with software page-table walkers. * * MMU_GATHER_NO_RANGE * * Use this if your architecture lacks an efficient flush_tlb_range(). - */ - -#ifdef CONFIG_HAVE_RCU_TABLE_FREE -/* - * Semi RCU freeing of the page directories. - * - * This is needed by some architectures to implement software pagetable walkers. - * - * gup_fast() and other software pagetable walkers do a lockless page-table - * walk and therefore needs some synchronization with the freeing of the page - * directories. The chosen means to accomplish that is by disabling IRQs over - * the walk. * - * Architectures that use IPIs to flush TLBs will then automagically DTRT, - * since we unlink the page, flush TLBs, free the page. Since the disabling of - * IRQs delays the completion of the TLB flush we can never observe an already - * freed page. + * MMU_GATHER_NO_GATHER * - * Architectures that do not have this (PPC) need to delay the freeing by some - * other means, this is that means. - * - * What we do is batch the freed directory pages (tables) and RCU free them. - * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling - * holds off grace periods. - * - * However, in order to batch these pages we need to allocate storage, this - * allocation is deep inside the MM code and can thus easily fail on memory - * pressure. To guarantee progress we fall back to single table freeing, see - * the implementation of tlb_remove_table_one(). + * If the option is set the mmu_gather will not track individual pages for + * delayed page free anymore. A platform that enables the option needs to + * provide its own implementation of the __tlb_remove_page_size() function to + * free pages. * + * This is useful if your architecture already flushes TLB entries in the + * various ptep_get_and_clear() functions. */ + +#ifdef CONFIG_MMU_GATHER_TABLE_FREE + struct mmu_table_batch { +#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE struct rcu_head rcu; +#endif unsigned int nr; void *tables[0]; }; @@ -189,9 +186,35 @@ struct mmu_table_batch { extern void tlb_remove_table(struct mmu_gather *tlb, void *table); +#else /* !CONFIG_MMU_GATHER_HAVE_TABLE_FREE */ + +/* + * Without MMU_GATHER_TABLE_FREE the architecture is assumed to have page based + * page directories and we can use the normal page batching to free them. + */ +#define tlb_remove_table(tlb, page) tlb_remove_page((tlb), (page)) + +#endif /* CONFIG_MMU_GATHER_TABLE_FREE */ + +#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE +/* + * This allows an architecture that does not use the linux page-tables for + * hardware to skip the TLBI when freeing page tables. + */ +#ifndef tlb_needs_table_invalidate +#define tlb_needs_table_invalidate() (true) +#endif + +#else + +#ifdef tlb_needs_table_invalidate +#error tlb_needs_table_invalidate() requires MMU_GATHER_RCU_TABLE_FREE #endif -#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER +#endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */ + + +#ifndef CONFIG_MMU_GATHER_NO_GATHER /* * If we can't allocate a page to make a big batch of page pointers * to work on, then just handle a few from the on-stack structure. @@ -227,7 +250,7 @@ extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, struct mmu_gather { struct mm_struct *mm; -#ifdef CONFIG_HAVE_RCU_TABLE_FREE +#ifdef CONFIG_MMU_GATHER_TABLE_FREE struct mmu_table_batch *batch; #endif @@ -266,22 +289,18 @@ struct mmu_gather { unsigned int batch_count; -#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER +#ifndef CONFIG_MMU_GATHER_NO_GATHER struct mmu_gather_batch *active; struct mmu_gather_batch local; struct page *__pages[MMU_GATHER_BUNDLE]; -#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE +#ifdef CONFIG_MMU_GATHER_PAGE_SIZE unsigned int page_size; #endif #endif }; -void arch_tlb_gather_mmu(struct mmu_gather *tlb, - struct mm_struct *mm, unsigned long start, unsigned long end); void tlb_flush_mmu(struct mmu_gather *tlb); -void arch_tlb_finish_mmu(struct mmu_gather *tlb, - unsigned long start, unsigned long end, bool force); static inline void __tlb_adjust_range(struct mmu_gather *tlb, unsigned long address, @@ -394,7 +413,12 @@ tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { } static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) { - if (!tlb->end) + /* + * Anything calling __tlb_adjust_range() also sets at least one of + * these bits. + */ + if (!(tlb->freed_tables || tlb->cleared_ptes || tlb->cleared_pmds || + tlb->cleared_puds || tlb->cleared_p4ds)) return; tlb_flush(tlb); @@ -426,9 +450,9 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) static inline void tlb_change_page_size(struct mmu_gather *tlb, unsigned int page_size) { -#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE +#ifdef CONFIG_MMU_GATHER_PAGE_SIZE if (tlb->page_size && tlb->page_size != page_size) { - if (!tlb->fullmm) + if (!tlb->fullmm && !tlb->need_flush_all) tlb_flush_mmu(tlb); } @@ -584,7 +608,6 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm } while (0) #endif -#ifndef __ARCH_HAS_4LEVEL_HACK #ifndef pud_free_tlb #define pud_free_tlb(tlb, pudp, address) \ do { \ @@ -594,9 +617,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm __pud_free_tlb(tlb, pudp, address); \ } while (0) #endif -#endif -#ifndef __ARCH_HAS_5LEVEL_HACK #ifndef p4d_free_tlb #define p4d_free_tlb(tlb, pudp, address) \ do { \ @@ -605,7 +626,6 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm __p4d_free_tlb(tlb, pudp, address); \ } while (0) #endif -#endif #endif /* CONFIG_MMU */ diff --git a/include/asm-generic/vdso/vsyscall.h b/include/asm-generic/vdso/vsyscall.h index e94b19782c92..cec543d9e87b 100644 --- a/include/asm-generic/vdso/vsyscall.h +++ b/include/asm-generic/vdso/vsyscall.h @@ -12,9 +12,9 @@ static __always_inline struct vdso_data *__arch_get_k_vdso_data(void) #endif /* __arch_get_k_vdso_data */ #ifndef __arch_update_vdso_data -static __always_inline int __arch_update_vdso_data(void) +static __always_inline bool __arch_update_vdso_data(void) { - return 0; + return true; } #endif /* __arch_update_vdso_data */ @@ -25,13 +25,6 @@ static __always_inline int __arch_get_clock_mode(struct timekeeper *tk) } #endif /* __arch_get_clock_mode */ -#ifndef __arch_use_vsyscall -static __always_inline int __arch_use_vsyscall(struct vdso_data *vdata) -{ - return 1; -} -#endif /* __arch_use_vsyscall */ - #ifndef __arch_update_vsyscall static __always_inline void __arch_update_vsyscall(struct vdso_data *vdata, struct timekeeper *tk) diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index cd28f63bfbc7..e00f41aa8ec4 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -23,12 +23,11 @@ * _etext = .; * * _sdata = .; - * RO_DATA_SECTION(PAGE_SIZE) - * RW_DATA_SECTION(...) + * RO_DATA(PAGE_SIZE) + * RW_DATA(...) * _edata = .; * * EXCEPTION_TABLE(...) - * NOTES * * BSS_SECTION(0, 0, 0) * _end = .; @@ -54,6 +53,33 @@ #define LOAD_OFFSET 0 #endif +/* + * Only some architectures want to have the .notes segment visible in + * a separate PT_NOTE ELF Program Header. When this happens, it needs + * to be visible in both the kernel text's PT_LOAD and the PT_NOTE + * Program Headers. In this case, though, the PT_LOAD needs to be made + * the default again so that all the following sections don't also end + * up in the PT_NOTE Program Header. + */ +#ifdef EMITS_PT_NOTE +#define NOTES_HEADERS :text :note +#define NOTES_HEADERS_RESTORE __restore_ph : { *(.__restore_ph) } :text +#else +#define NOTES_HEADERS +#define NOTES_HEADERS_RESTORE +#endif + +/* + * Some architectures have non-executable read-only exception tables. + * They can be added to the RO_DATA segment by specifying their desired + * alignment. + */ +#ifdef RO_EXCEPTION_TABLE_ALIGN +#define RO_EXCEPTION_TABLE EXCEPTION_TABLE(RO_EXCEPTION_TABLE_ALIGN) +#else +#define RO_EXCEPTION_TABLE +#endif + /* Align . to a 8 byte boundary equals to maximum function alignment. */ #define ALIGN_FUNCTION() . = ALIGN(8) @@ -110,19 +136,28 @@ #endif #ifdef CONFIG_FTRACE_MCOUNT_RECORD -#ifdef CC_USING_PATCHABLE_FUNCTION_ENTRY -#define MCOUNT_REC() . = ALIGN(8); \ - __start_mcount_loc = .; \ - KEEP(*(__patchable_function_entries)) \ - __stop_mcount_loc = .; -#else +/* + * The ftrace call sites are logged to a section whose name depends on the + * compiler option used. A given kernel image will only use one, AKA + * FTRACE_CALLSITE_SECTION. We capture all of them here to avoid header + * dependencies for FTRACE_CALLSITE_SECTION's definition. + * + * Need to also make ftrace_stub_graph point to ftrace_stub + * so that the same stub location may have different protocols + * and not mess up with C verifiers. + */ #define MCOUNT_REC() . = ALIGN(8); \ __start_mcount_loc = .; \ KEEP(*(__mcount_loc)) \ - __stop_mcount_loc = .; -#endif + KEEP(*(__patchable_function_entries)) \ + __stop_mcount_loc = .; \ + ftrace_stub_graph = ftrace_stub; #else -#define MCOUNT_REC() +# ifdef CONFIG_FUNCTION_TRACER +# define MCOUNT_REC() ftrace_stub_graph = ftrace_stub; +# else +# define MCOUNT_REC() +# endif #endif #ifdef CONFIG_TRACE_BRANCH_PROFILING @@ -215,8 +250,13 @@ __start_lsm_info = .; \ KEEP(*(.lsm_info.init)) \ __end_lsm_info = .; +#define EARLY_LSM_TABLE() . = ALIGN(8); \ + __start_early_lsm_info = .; \ + KEEP(*(.early_lsm_info.init)) \ + __end_early_lsm_info = .; #else #define LSM_TABLE() +#define EARLY_LSM_TABLE() #endif #define ___OF_TABLE(cfg, name) _OF_TABLE_##cfg(name) @@ -343,7 +383,7 @@ /* * Read only Data */ -#define RO_DATA_SECTION(align) \ +#define RO_DATA(align) \ . = ALIGN((align)); \ .rodata : AT(ADDR(.rodata) - LOAD_OFFSET) { \ __start_rodata = .; \ @@ -491,15 +531,13 @@ __start___modver = .; \ KEEP(*(__modver)) \ __stop___modver = .; \ - . = ALIGN((align)); \ - __end_rodata = .; \ } \ - . = ALIGN((align)); - -/* RODATA & RO_DATA provided for backward compatibility. - * All archs are supposed to use RO_DATA() */ -#define RODATA RO_DATA_SECTION(4096) -#define RO_DATA(align) RO_DATA_SECTION(align) + \ + RO_EXCEPTION_TABLE \ + NOTES \ + \ + . = ALIGN((align)); \ + __end_rodata = .; /* * .text section. Map to function alignment to avoid address changes @@ -627,7 +665,8 @@ ACPI_PROBE_TABLE(timer) \ THERMAL_TABLE(governor) \ EARLYCON_TABLE() \ - LSM_TABLE() + LSM_TABLE() \ + EARLY_LSM_TABLE() #define INIT_TEXT \ *(.init.text .init.text.*) \ @@ -784,7 +823,8 @@ __start_notes = .; \ KEEP(*(.note.*)) \ __stop_notes = .; \ - } + } NOTES_HEADERS \ + NOTES_HEADERS_RESTORE #define INIT_SETUP(initsetup_align) \ . = ALIGN(initsetup_align); \ @@ -956,7 +996,7 @@ * matches the requirement of PAGE_ALIGNED_DATA. * * use 0 as page_align if page_aligned data is not used */ -#define RW_DATA_SECTION(cacheline, pagealigned, inittask) \ +#define RW_DATA(cacheline, pagealigned, inittask) \ . = ALIGN(PAGE_SIZE); \ .data : AT(ADDR(.data) - LOAD_OFFSET) { \ INIT_TASK_DATA(inittask) \ |
