Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661/* SPDX-License-Identifier: GPL-2.0 *//* * Describes operations that can be performed on software-defined page table * leaf entries. These are abstracted from the hardware page table entries * themselves by the softleaf_t type, see mm_types.h. */#ifndef _LINUX_LEAFOPS_H#define _LINUX_LEAFOPS_H
#include <linux/mm_types.h>#include <linux/swapops.h>#include <linux/swap.h>
#ifdef CONFIG_MMU
/* Temporary until swp_entry_t eliminated. */#define LEAF_TYPE_SHIFT SWP_TYPE_SHIFT
enum softleaf_type { /* Fundamental types. */ SOFTLEAF_NONE, SOFTLEAF_SWAP, /* Migration types. */ SOFTLEAF_MIGRATION_READ, SOFTLEAF_MIGRATION_READ_EXCLUSIVE, SOFTLEAF_MIGRATION_WRITE, /* Device types. */ SOFTLEAF_DEVICE_PRIVATE_READ, SOFTLEAF_DEVICE_PRIVATE_WRITE, SOFTLEAF_DEVICE_EXCLUSIVE, /* H/W posion types. */ SOFTLEAF_HWPOISON, /* Marker types. */ SOFTLEAF_MARKER,};
/** * softleaf_mk_none() - Create an empty ('none') leaf entry. * Returns: empty leaf entry. */static inline softleaf_t softleaf_mk_none(void){ return ((softleaf_t) { 0 });}
/** * softleaf_from_pte() - Obtain a leaf entry from a PTE entry. * @pte: PTE entry. * * If @pte is present (therefore not a leaf entry) the function returns an empty * leaf entry. Otherwise, it returns a leaf entry. * * Returns: Leaf entry. */static inline softleaf_t softleaf_from_pte(pte_t pte){ softleaf_t arch_entry;
if (pte_present(pte) || pte_none(pte)) return softleaf_mk_none();
pte = pte_swp_clear_flags(pte); arch_entry = __pte_to_swp_entry(pte);
/* Temporary until swp_entry_t eliminated. */ return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));}
/** * softleaf_to_pte() - Obtain a PTE entry from a leaf entry. * @entry: Leaf entry. * * This generates an architecture-specific PTE entry that can be utilised to * encode the metadata the leaf entry encodes. * * Returns: Architecture-specific PTE entry encoding leaf entry. */static inline pte_t softleaf_to_pte(softleaf_t entry){ /* Temporary until swp_entry_t eliminated. */ return swp_entry_to_pte(entry);}
#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION/** * softleaf_from_pmd() - Obtain a leaf entry from a PMD entry. * @pmd: PMD entry. * * If @pmd is present (therefore not a leaf entry) the function returns an empty * leaf entry. Otherwise, it returns a leaf entry. * * Returns: Leaf entry. */static inline softleaf_t softleaf_from_pmd(pmd_t pmd){ softleaf_t arch_entry;
if (pmd_present(pmd) || pmd_none(pmd)) return softleaf_mk_none();
if (pmd_swp_soft_dirty(pmd)) pmd = pmd_swp_clear_soft_dirty(pmd); if (pmd_swp_uffd_wp(pmd)) pmd = pmd_swp_clear_uffd_wp(pmd); arch_entry = __pmd_to_swp_entry(pmd);
/* Temporary until swp_entry_t eliminated. */ return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));}
#else
static inline softleaf_t softleaf_from_pmd(pmd_t pmd){ return softleaf_mk_none();}
#endif
/** * softleaf_is_none() - Is the leaf entry empty? * @entry: Leaf entry. * * Empty entries are typically the result of a 'none' page table leaf entry * being converted to a leaf entry. * * Returns: true if the entry is empty, false otherwise. */static inline bool softleaf_is_none(softleaf_t entry){ return entry.val == 0;}
/** * softleaf_type() - Identify the type of leaf entry. * @entry: Leaf entry. * * Returns: the leaf entry type associated with @entry. */static inline enum softleaf_type softleaf_type(softleaf_t entry){ unsigned int type_num;
if (softleaf_is_none(entry)) return SOFTLEAF_NONE;
type_num = entry.val >> LEAF_TYPE_SHIFT;
if (type_num < MAX_SWAPFILES) return SOFTLEAF_SWAP;
switch (type_num) {#ifdef CONFIG_MIGRATION case SWP_MIGRATION_READ: return SOFTLEAF_MIGRATION_READ; case SWP_MIGRATION_READ_EXCLUSIVE: return SOFTLEAF_MIGRATION_READ_EXCLUSIVE; case SWP_MIGRATION_WRITE: return SOFTLEAF_MIGRATION_WRITE;#endif#ifdef CONFIG_DEVICE_PRIVATE case SWP_DEVICE_WRITE: return SOFTLEAF_DEVICE_PRIVATE_WRITE; case SWP_DEVICE_READ: return SOFTLEAF_DEVICE_PRIVATE_READ; case SWP_DEVICE_EXCLUSIVE: return SOFTLEAF_DEVICE_EXCLUSIVE;#endif#ifdef CONFIG_MEMORY_FAILURE case SWP_HWPOISON: return SOFTLEAF_HWPOISON;#endif case SWP_PTE_MARKER: return SOFTLEAF_MARKER; }
/* Unknown entry type. */ VM_WARN_ON_ONCE(1); return SOFTLEAF_NONE;}
/** * softleaf_is_swap() - Is this leaf entry a swap entry? * @entry: Leaf entry. * * Returns: true if the leaf entry is a swap entry, otherwise false. */static inline bool softleaf_is_swap(softleaf_t entry){ return softleaf_type(entry) == SOFTLEAF_SWAP;}
/** * softleaf_is_migration_write() - Is this leaf entry a writable migration entry? * @entry: Leaf entry. * * Returns: true if the leaf entry is a writable migration entry, otherwise * false. */static inline bool softleaf_is_migration_write(softleaf_t entry){ return softleaf_type(entry) == SOFTLEAF_MIGRATION_WRITE;}
/** * softleaf_is_migration_read() - Is this leaf entry a readable migration entry? * @entry: Leaf entry. * * Returns: true if the leaf entry is a readable migration entry, otherwise * false. */static inline bool softleaf_is_migration_read(softleaf_t entry){ return softleaf_type(entry) == SOFTLEAF_MIGRATION_READ;}
/** * softleaf_is_migration_read_exclusive() - Is this leaf entry an exclusive * readable migration entry? * @entry: Leaf entry. * * Returns: true if the leaf entry is an exclusive readable migration entry, * otherwise false. */static inline bool softleaf_is_migration_read_exclusive(softleaf_t entry){ return softleaf_type(entry) == SOFTLEAF_MIGRATION_READ_EXCLUSIVE;}
/** * softleaf_is_migration() - Is this leaf entry a migration entry? * @entry: Leaf entry. * * Returns: true if the leaf entry is a migration entry, otherwise false. */static inline bool softleaf_is_migration(softleaf_t entry){ switch (softleaf_type(entry)) { case SOFTLEAF_MIGRATION_READ: case SOFTLEAF_MIGRATION_READ_EXCLUSIVE: case SOFTLEAF_MIGRATION_WRITE: return true; default: return false; }}
/** * softleaf_is_device_private_write() - Is this leaf entry a device private * writable entry? * @entry: Leaf entry. * * Returns: true if the leaf entry is a device private writable entry, otherwise * false. */static inline bool softleaf_is_device_private_write(softleaf_t entry){ return softleaf_type(entry) == SOFTLEAF_DEVICE_PRIVATE_WRITE;}
/** * softleaf_is_device_private() - Is this leaf entry a device private entry? * @entry: Leaf entry. * * Returns: true if the leaf entry is a device private entry, otherwise false. */static inline bool softleaf_is_device_private(softleaf_t entry){ switch (softleaf_type(entry)) { case SOFTLEAF_DEVICE_PRIVATE_WRITE: case SOFTLEAF_DEVICE_PRIVATE_READ: return true; default: return false; }}
/** * softleaf_is_device_exclusive() - Is this leaf entry a device-exclusive entry? * @entry: Leaf entry. * * Returns: true if the leaf entry is a device-exclusive entry, otherwise false. */static inline bool softleaf_is_device_exclusive(softleaf_t entry){ return softleaf_type(entry) == SOFTLEAF_DEVICE_EXCLUSIVE;}
/** * softleaf_is_hwpoison() - Is this leaf entry a hardware poison entry? * @entry: Leaf entry. * * Returns: true if the leaf entry is a hardware poison entry, otherwise false. */static inline bool softleaf_is_hwpoison(softleaf_t entry){ return softleaf_type(entry) == SOFTLEAF_HWPOISON;}
/** * softleaf_is_marker() - Is this leaf entry a marker? * @entry: Leaf entry. * * Returns: true if the leaf entry is a marker entry, otherwise false. */static inline bool softleaf_is_marker(softleaf_t entry){ return softleaf_type(entry) == SOFTLEAF_MARKER;}
/** * softleaf_to_marker() - Obtain marker associated with leaf entry. * @entry: Leaf entry, softleaf_is_marker(@entry) must return true. * * Returns: Marker associated with the leaf entry. */static inline pte_marker softleaf_to_marker(softleaf_t entry){ VM_WARN_ON_ONCE(!softleaf_is_marker(entry));
return swp_offset(entry) & PTE_MARKER_MASK;}
/** * softleaf_has_pfn() - Does this leaf entry encode a valid PFN number? * @entry: Leaf entry. * * A pfn swap entry is a special type of swap entry that always has a pfn stored * in the swap offset. They can either be used to represent unaddressable device * memory, to restrict access to a page undergoing migration or to represent a * pfn which has been hwpoisoned and unmapped. * * Returns: true if the leaf entry encodes a PFN, otherwise false. */static inline bool softleaf_has_pfn(softleaf_t entry){ /* Make sure the swp offset can always store the needed fields. */ BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS);
if (softleaf_is_migration(entry)) return true; if (softleaf_is_device_private(entry)) return true; if (softleaf_is_device_exclusive(entry)) return true; if (softleaf_is_hwpoison(entry)) return true;
return false;}
/** * softleaf_to_pfn() - Obtain PFN encoded within leaf entry. * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true. * * Returns: The PFN associated with the leaf entry. */static inline unsigned long softleaf_to_pfn(softleaf_t entry){ VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
/* Temporary until swp_entry_t eliminated. */ return swp_offset(entry) & SWP_PFN_MASK;}
static inline void softleaf_migration_sync(softleaf_t entry, struct folio *folio){ /* * Ensure we do not race with split, which might alter tail pages into new * folios and thus result in observing an unlocked folio. * This matches the write barrier in __split_folio_to_order(). */ smp_rmb();
/* * Any use of migration entries may only occur while the * corresponding page is locked */ VM_WARN_ON_ONCE(!folio_test_locked(folio));}
/** * softleaf_to_page() - Obtains struct page for PFN encoded within leaf entry. * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true. * * Returns: Pointer to the struct page associated with the leaf entry's PFN. */static inline struct page *softleaf_to_page(softleaf_t entry){ struct page *page = pfn_to_page(softleaf_to_pfn(entry));
VM_WARN_ON_ONCE(!softleaf_has_pfn(entry)); if (softleaf_is_migration(entry)) softleaf_migration_sync(entry, page_folio(page));
return page;}
/** * softleaf_to_folio() - Obtains struct folio for PFN encoded within leaf entry. * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true. * * Returns: Pointer to the struct folio associated with the leaf entry's PFN. */static inline struct folio *softleaf_to_folio(softleaf_t entry){ struct folio *folio = pfn_folio(softleaf_to_pfn(entry));
VM_WARN_ON_ONCE(!softleaf_has_pfn(entry)); if (softleaf_is_migration(entry)) softleaf_migration_sync(entry, folio);
return folio;}
/** * softleaf_is_poison_marker() - Is this leaf entry a poison marker? * @entry: Leaf entry. * * The poison marker is set via UFFDIO_POISON. Userfaultfd-specific. * * Returns: true if the leaf entry is a poison marker, otherwise false. */static inline bool softleaf_is_poison_marker(softleaf_t entry){ if (!softleaf_is_marker(entry)) return false;
return softleaf_to_marker(entry) & PTE_MARKER_POISONED;}
/** * softleaf_is_guard_marker() - Is this leaf entry a guard region marker? * @entry: Leaf entry. * * Returns: true if the leaf entry is a guard marker, otherwise false. */static inline bool softleaf_is_guard_marker(softleaf_t entry){ if (!softleaf_is_marker(entry)) return false;
return softleaf_to_marker(entry) & PTE_MARKER_GUARD;}
/** * softleaf_is_uffd_wp_marker() - Is this leaf entry a userfautlfd write protect * marker? * @entry: Leaf entry. * * Userfaultfd-specific. * * Returns: true if the leaf entry is a UFFD WP marker, otherwise false. */static inline bool softleaf_is_uffd_wp_marker(softleaf_t entry){ if (!softleaf_is_marker(entry)) return false;
return softleaf_to_marker(entry) & PTE_MARKER_UFFD_WP;}
#ifdef CONFIG_MIGRATION
/** * softleaf_is_migration_young() - Does this migration entry contain an accessed * bit? * @entry: Leaf entry. * * If the architecture can support storing A/D bits in migration entries, this * determines whether the accessed (or 'young') bit was set on the migrated page * table entry. * * Returns: true if the entry contains an accessed bit, otherwise false. */static inline bool softleaf_is_migration_young(softleaf_t entry){ VM_WARN_ON_ONCE(!softleaf_is_migration(entry));
if (migration_entry_supports_ad()) return swp_offset(entry) & SWP_MIG_YOUNG; /* Keep the old behavior of aging page after migration */ return false;}
/** * softleaf_is_migration_dirty() - Does this migration entry contain a dirty bit? * @entry: Leaf entry. * * If the architecture can support storing A/D bits in migration entries, this * determines whether the dirty bit was set on the migrated page table entry. * * Returns: true if the entry contains a dirty bit, otherwise false. */static inline bool softleaf_is_migration_dirty(softleaf_t entry){ VM_WARN_ON_ONCE(!softleaf_is_migration(entry));
if (migration_entry_supports_ad()) return swp_offset(entry) & SWP_MIG_DIRTY; /* Keep the old behavior of clean page after migration */ return false;}
#else /* CONFIG_MIGRATION */
static inline bool softleaf_is_migration_young(softleaf_t entry){ return false;}
static inline bool softleaf_is_migration_dirty(softleaf_t entry){ return false;}#endif /* CONFIG_MIGRATION */
/** * pte_is_marker() - Does the PTE entry encode a marker leaf entry? * @pte: PTE entry. * * Returns: true if this PTE is a marker leaf entry, otherwise false. */static inline bool pte_is_marker(pte_t pte){ return softleaf_is_marker(softleaf_from_pte(pte));}
/** * pte_is_uffd_wp_marker() - Does this PTE entry encode a userfaultfd write * protect marker leaf entry? * @pte: PTE entry. * * Returns: true if this PTE is a UFFD WP marker leaf entry, otherwise false. */static inline bool pte_is_uffd_wp_marker(pte_t pte){ const softleaf_t entry = softleaf_from_pte(pte);
return softleaf_is_uffd_wp_marker(entry);}
/** * pte_is_uffd_marker() - Does this PTE entry encode a userfault-specific marker * leaf entry? * @pte: PTE entry. * * It's useful to be able to determine which leaf entries encode UFFD-specific * markers so we can handle these correctly. * * Returns: true if this PTE entry is a UFFD-specific marker, otherwise false. */static inline bool pte_is_uffd_marker(pte_t pte){ const softleaf_t entry = softleaf_from_pte(pte);
if (!softleaf_is_marker(entry)) return false;
/* UFFD WP, poisoned swap entries are UFFD-handled. */ if (softleaf_is_uffd_wp_marker(entry)) return true; if (softleaf_is_poison_marker(entry)) return true;
return false;}
#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION)
/** * pmd_is_device_private_entry() - Check if PMD contains a device private swap * entry. * @pmd: The PMD to check. * * Returns true if the PMD contains a swap entry that represents a device private * page mapping. This is used for zone device private pages that have been * swapped out but still need special handling during various memory management * operations. * * Return: true if PMD contains device private entry, false otherwise */static inline bool pmd_is_device_private_entry(pmd_t pmd){ return softleaf_is_device_private(softleaf_from_pmd(pmd));}
#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */
static inline bool pmd_is_device_private_entry(pmd_t pmd){ return false;}
#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */
/** * pmd_is_migration_entry() - Does this PMD entry encode a migration entry? * @pmd: PMD entry. * * Returns: true if the PMD encodes a migration entry, otherwise false. */static inline bool pmd_is_migration_entry(pmd_t pmd){ return softleaf_is_migration(softleaf_from_pmd(pmd));}
/** * softleaf_is_valid_pmd_entry() - Is the specified softleaf entry obtained from * a PMD one that we support at PMD level? * @entry: Entry to check. * Returns: true if the softleaf entry is valid at PMD, otherwise false. */static inline bool softleaf_is_valid_pmd_entry(softleaf_t entry){ /* Only device private, migration entries valid for PMD. */ return softleaf_is_device_private(entry) || softleaf_is_migration(entry);}
/** * pmd_is_valid_softleaf() - Is this PMD entry a valid softleaf entry? * @pmd: PMD entry. * * PMD leaf entries are valid only if they are device private or migration * entries. This function asserts that a PMD leaf entry is valid in this * respect. * * Returns: true if the PMD entry is a valid leaf entry, otherwise false. */static inline bool pmd_is_valid_softleaf(pmd_t pmd){ const softleaf_t entry = softleaf_from_pmd(pmd);
return softleaf_is_valid_pmd_entry(entry);}
/** * pmd_to_softleaf_folio() - Convert the PMD entry to a folio. * @pmd: PMD entry. * * The PMD entry is expected to be a valid PMD softleaf entry. * * Returns: the folio the softleaf entry references if this is a valid softleaf * entry, otherwise NULL. */static inline struct folio *pmd_to_softleaf_folio(pmd_t pmd){ const softleaf_t entry = softleaf_from_pmd(pmd);
if (!softleaf_is_valid_pmd_entry(entry)) { VM_WARN_ON_ONCE(true); return NULL; } return softleaf_to_folio(entry);}
#endif /* CONFIG_MMU */#endif /* _LINUX_LEAFOPS_H */