Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974// SPDX-License-Identifier: GPL-2.0/* * This is a module to test the HMM (Heterogeneous Memory Management) * mirror and zone device private memory migration APIs of the kernel. * Userspace programs can register with the driver to mirror their own address * space and can use the device to read/write any valid virtual address. */#include <linux/init.h>#include <linux/fs.h>#include <linux/mm.h>#include <linux/module.h>#include <linux/kernel.h>#include <linux/cdev.h>#include <linux/device.h>#include <linux/memremap.h>#include <linux/mutex.h>#include <linux/rwsem.h>#include <linux/sched.h>#include <linux/slab.h>#include <linux/highmem.h>#include <linux/delay.h>#include <linux/pagemap.h>#include <linux/hmm.h>#include <linux/vmalloc.h>#include <linux/swap.h>#include <linux/swapops.h>#include <linux/sched/mm.h>#include <linux/platform_device.h>#include <linux/rmap.h>#include <linux/mmu_notifier.h>#include <linux/migrate.h>
#include "test_hmm_uapi.h"
#define DMIRROR_NDEVICES 4#define DMIRROR_RANGE_FAULT_TIMEOUT 1000#define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U)#define DEVMEM_CHUNKS_RESERVE 16
/* * For device_private pages, dpage is just a dummy struct page * representing a piece of device memory. dmirror_devmem_alloc_page * allocates a real system memory page as backing storage to fake a * real device. zone_device_data points to that backing page. But * for device_coherent memory, the struct page represents real * physical CPU-accessible memory that we can use directly. */#define BACKING_PAGE(page) (is_device_private_page((page)) ? \ (page)->zone_device_data : (page))
static unsigned long spm_addr_dev0;module_param(spm_addr_dev0, long, 0644);MODULE_PARM_DESC(spm_addr_dev0, "Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE.");
static unsigned long spm_addr_dev1;module_param(spm_addr_dev1, long, 0644);MODULE_PARM_DESC(spm_addr_dev1, "Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent device type will be used. Make sure spm_addr_dev0 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE.");
static const struct dev_pagemap_ops dmirror_devmem_ops;static const struct mmu_interval_notifier_ops dmirror_min_ops;static dev_t dmirror_dev;
struct dmirror_device;
struct dmirror_bounce { void *ptr; unsigned long size; unsigned long addr; unsigned long cpages;};
#define DPT_XA_TAG_ATOMIC 1UL#define DPT_XA_TAG_WRITE 3UL
/* * Data structure to track address ranges and register for mmu interval * notifier updates. */struct dmirror_interval { struct mmu_interval_notifier notifier; struct dmirror *dmirror;};
/* * Data attached to the open device file. * Note that it might be shared after a fork(). */struct dmirror { struct dmirror_device *mdevice; struct xarray pt; struct mmu_interval_notifier notifier; struct mutex mutex; __u64 flags;};
/* * ZONE_DEVICE pages for migration and simulating device memory. */struct dmirror_chunk { struct dev_pagemap pagemap; struct dmirror_device *mdevice; bool remove;};
/* * Per device data. */struct dmirror_device { struct cdev cdevice; unsigned int zone_device_type; struct device device;
unsigned int devmem_capacity; unsigned int devmem_count; struct dmirror_chunk **devmem_chunks; struct mutex devmem_lock; /* protects the above */
unsigned long calloc; unsigned long cfree; struct page *free_pages; struct folio *free_folios; spinlock_t lock; /* protects the above */};
static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES];
static int dmirror_bounce_init(struct dmirror_bounce *bounce, unsigned long addr, unsigned long size){ bounce->addr = addr; bounce->size = size; bounce->cpages = 0; bounce->ptr = vmalloc(size); if (!bounce->ptr) return -ENOMEM; return 0;}
static bool dmirror_is_private_zone(struct dmirror_device *mdevice){ return (mdevice->zone_device_type == HMM_DMIRROR_MEMORY_DEVICE_PRIVATE);}
static enum migrate_vma_directiondmirror_select_device(struct dmirror *dmirror){ return (dmirror->mdevice->zone_device_type == HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? MIGRATE_VMA_SELECT_DEVICE_PRIVATE : MIGRATE_VMA_SELECT_DEVICE_COHERENT;}
static void dmirror_bounce_fini(struct dmirror_bounce *bounce){ vfree(bounce->ptr);}
static int dmirror_fops_open(struct inode *inode, struct file *filp){ struct cdev *cdev = inode->i_cdev; struct dmirror *dmirror; int ret;
/* Mirror this process address space */ dmirror = kzalloc_obj(*dmirror); if (dmirror == NULL) return -ENOMEM;
dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice); mutex_init(&dmirror->mutex); xa_init(&dmirror->pt);
ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm, 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops); if (ret) { kfree(dmirror); return ret; }
filp->private_data = dmirror; return 0;}
static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk){ unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT; unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT; unsigned long npages = end_pfn - start_pfn + 1; unsigned long i; unsigned long *src_pfns; unsigned long *dst_pfns; unsigned int order = 0;
src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL);
migrate_device_range(src_pfns, start_pfn, npages); for (i = 0; i < npages; i++) { struct page *dpage, *spage;
spage = migrate_pfn_to_page(src_pfns[i]); if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) continue;
if (WARN_ON(!is_device_private_page(spage) && !is_device_coherent_page(spage))) continue;
order = folio_order(page_folio(spage)); spage = BACKING_PAGE(spage); if (src_pfns[i] & MIGRATE_PFN_COMPOUND) { dpage = folio_page(folio_alloc(GFP_HIGHUSER_MOVABLE, order), 0); } else { dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL); order = 0; }
/* TODO Support splitting here */ lock_page(dpage); dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); if (src_pfns[i] & MIGRATE_PFN_WRITE) dst_pfns[i] |= MIGRATE_PFN_WRITE; if (order) dst_pfns[i] |= MIGRATE_PFN_COMPOUND; folio_copy(page_folio(dpage), page_folio(spage)); } migrate_device_pages(src_pfns, dst_pfns, npages); migrate_device_finalize(src_pfns, dst_pfns, npages); kvfree(src_pfns); kvfree(dst_pfns);}
static int dmirror_fops_release(struct inode *inode, struct file *filp){ struct dmirror *dmirror = filp->private_data; struct dmirror_device *mdevice = dmirror->mdevice; int i;
mmu_interval_notifier_remove(&dmirror->notifier);
if (mdevice->devmem_chunks) { for (i = 0; i < mdevice->devmem_count; i++) { struct dmirror_chunk *devmem = mdevice->devmem_chunks[i];
dmirror_device_evict_chunk(devmem); } }
xa_destroy(&dmirror->pt); kfree(dmirror); return 0;}
static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page){ return container_of(page_pgmap(page), struct dmirror_chunk, pagemap);}
static struct dmirror_device *dmirror_page_to_device(struct page *page)
{ return dmirror_page_to_chunk(page)->mdevice;}
static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range){ unsigned long *pfns = range->hmm_pfns; unsigned long pfn;
for (pfn = (range->start >> PAGE_SHIFT); pfn < (range->end >> PAGE_SHIFT); pfn++, pfns++) { struct page *page; void *entry;
/* * Since we asked for hmm_range_fault() to populate pages, * it shouldn't return an error entry on success. */ WARN_ON(*pfns & HMM_PFN_ERROR); WARN_ON(!(*pfns & HMM_PFN_VALID));
page = hmm_pfn_to_page(*pfns); WARN_ON(!page);
entry = page; if (*pfns & HMM_PFN_WRITE) entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); else if (WARN_ON(range->default_flags & HMM_PFN_WRITE)) return -EFAULT; entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); if (xa_is_err(entry)) return xa_err(entry); }
return 0;}
static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, unsigned long end){ unsigned long pfn; void *entry;
/* * The XArray doesn't hold references to pages since it relies on * the mmu notifier to clear page pointers when they become stale. * Therefore, it is OK to just clear the entry. */ xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, end >> PAGE_SHIFT) xa_erase(&dmirror->pt, pfn);}
static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, const struct mmu_notifier_range *range, unsigned long cur_seq){ struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);
/* * Ignore invalidation callbacks for device private pages since * the invalidation is handled as part of the migration process. */ if (range->event == MMU_NOTIFY_MIGRATE && range->owner == dmirror->mdevice) return true;
if (mmu_notifier_range_blockable(range)) mutex_lock(&dmirror->mutex); else if (!mutex_trylock(&dmirror->mutex)) return false;
mmu_interval_set_seq(mni, cur_seq); dmirror_do_update(dmirror, range->start, range->end);
mutex_unlock(&dmirror->mutex); return true;}
static const struct mmu_interval_notifier_ops dmirror_min_ops = { .invalidate = dmirror_interval_invalidate,};
static int dmirror_range_fault(struct dmirror *dmirror, struct hmm_range *range){ struct mm_struct *mm = dmirror->notifier.mm; unsigned long timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); int ret;
while (true) { if (time_after(jiffies, timeout)) { ret = -EBUSY; goto out; }
range->notifier_seq = mmu_interval_read_begin(range->notifier); mmap_read_lock(mm); ret = hmm_range_fault(range); mmap_read_unlock(mm); if (ret) { if (ret == -EBUSY) continue; goto out; }
mutex_lock(&dmirror->mutex); if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) { mutex_unlock(&dmirror->mutex); continue; } break; }
ret = dmirror_do_fault(dmirror, range);
mutex_unlock(&dmirror->mutex);out: return ret;}
static int dmirror_range_fault_unlocked(struct dmirror *dmirror, struct hmm_range *range, unsigned long timeout){ int ret;
while (true) { ret = hmm_range_fault_unlocked_timeout(range, timeout); if (ret) goto out;
mutex_lock(&dmirror->mutex); if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) { mutex_unlock(&dmirror->mutex); continue; } break; }
ret = dmirror_do_fault(dmirror, range);
mutex_unlock(&dmirror->mutex);out: return ret;}
static int dmirror_fault_unlocked(struct dmirror *dmirror, unsigned long start, unsigned long end, bool write, unsigned long timeout){ struct mm_struct *mm = dmirror->notifier.mm; unsigned long addr; unsigned long pfns[32]; struct hmm_range range = { .notifier = &dmirror->notifier, .hmm_pfns = pfns, .pfn_flags_mask = 0, .default_flags = HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), .dev_private_owner = dmirror->mdevice, }; int ret = 0;
if (!mmget_not_zero(mm)) return -EFAULT;
for (addr = start; addr < end; addr = range.end) { range.start = addr; range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
ret = dmirror_range_fault_unlocked(dmirror, &range, timeout); if (ret) break; }
mmput(mm); return ret;}
static int dmirror_fault(struct dmirror *dmirror, unsigned long start, unsigned long end, bool write){ struct mm_struct *mm = dmirror->notifier.mm; unsigned long addr; unsigned long pfns[32]; struct hmm_range range = { .notifier = &dmirror->notifier, .hmm_pfns = pfns, .pfn_flags_mask = 0, .default_flags = HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), .dev_private_owner = dmirror->mdevice, }; int ret = 0;
/* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) return -EFAULT;
for (addr = start; addr < end; addr = range.end) { range.start = addr; range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
ret = dmirror_range_fault(dmirror, &range); if (ret) break; }
mmput(mm); return ret;}
static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, unsigned long end, struct dmirror_bounce *bounce){ unsigned long pfn; void *ptr;
ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { void *entry; struct page *page;
entry = xa_load(&dmirror->pt, pfn); page = xa_untag_pointer(entry); if (!page) return -ENOENT;
memcpy_from_page(ptr, page, 0, PAGE_SIZE);
ptr += PAGE_SIZE; bounce->cpages++; }
return 0;}
static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd){ struct dmirror_bounce bounce; unsigned long start, end; unsigned long size = cmd->npages << PAGE_SHIFT; int ret;
start = cmd->addr; end = start + size; if (end < start) return -EINVAL;
ret = dmirror_bounce_init(&bounce, start, size); if (ret) return ret;
while (1) { mutex_lock(&dmirror->mutex); ret = dmirror_do_read(dmirror, start, end, &bounce); mutex_unlock(&dmirror->mutex); if (ret != -ENOENT) break;
start = cmd->addr + (bounce.cpages << PAGE_SHIFT); ret = dmirror_fault(dmirror, start, end, false); if (ret) break; cmd->faults++; }
if (ret == 0) { if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, bounce.size)) ret = -EFAULT; } cmd->cpages = bounce.cpages; dmirror_bounce_fini(&bounce); return ret;}
static int dmirror_read_unlocked(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd, unsigned long timeout){ struct dmirror_bounce bounce; unsigned long start, end; unsigned long size = cmd->npages << PAGE_SHIFT; int ret;
start = cmd->addr; end = start + size; if (end < start) return -EINVAL;
ret = dmirror_bounce_init(&bounce, start, size); if (ret) return ret;
while (1) { mutex_lock(&dmirror->mutex); ret = dmirror_do_read(dmirror, start, end, &bounce); mutex_unlock(&dmirror->mutex); if (ret != -ENOENT) break;
start = cmd->addr + (bounce.cpages << PAGE_SHIFT); ret = dmirror_fault_unlocked(dmirror, start, end, false, timeout); if (ret) break; cmd->faults++; }
if (ret == 0) { if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, bounce.size)) ret = -EFAULT; } cmd->cpages = bounce.cpages; dmirror_bounce_fini(&bounce); return ret;}
static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, unsigned long end, struct dmirror_bounce *bounce){ unsigned long pfn; void *ptr;
ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { void *entry; struct page *page;
entry = xa_load(&dmirror->pt, pfn); page = xa_untag_pointer(entry); if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE) return -ENOENT;
memcpy_to_page(page, 0, ptr, PAGE_SIZE);
ptr += PAGE_SIZE; bounce->cpages++; }
return 0;}
static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd){ struct dmirror_bounce bounce; unsigned long start, end; unsigned long size = cmd->npages << PAGE_SHIFT; int ret;
start = cmd->addr; end = start + size; if (end < start) return -EINVAL;
ret = dmirror_bounce_init(&bounce, start, size); if (ret) return ret; if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr), bounce.size)) { ret = -EFAULT; goto fini; }
while (1) { mutex_lock(&dmirror->mutex); ret = dmirror_do_write(dmirror, start, end, &bounce); mutex_unlock(&dmirror->mutex); if (ret != -ENOENT) break;
start = cmd->addr + (bounce.cpages << PAGE_SHIFT); ret = dmirror_fault(dmirror, start, end, true); if (ret) break; cmd->faults++; }
fini: cmd->cpages = bounce.cpages; dmirror_bounce_fini(&bounce); return ret;}
static int dmirror_allocate_chunk(struct dmirror_device *mdevice, struct page **ppage, bool is_large){ struct dmirror_chunk *devmem; struct resource *res = NULL; unsigned long pfn; unsigned long pfn_first; unsigned long pfn_last; void *ptr; int ret = -ENOMEM;
devmem = kzalloc_obj(*devmem); if (!devmem) return ret;
switch (mdevice->zone_device_type) { case HMM_DMIRROR_MEMORY_DEVICE_PRIVATE: res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE, "hmm_dmirror"); if (IS_ERR_OR_NULL(res)) goto err_devmem; devmem->pagemap.range.start = res->start; devmem->pagemap.range.end = res->end; devmem->pagemap.type = MEMORY_DEVICE_PRIVATE; break; case HMM_DMIRROR_MEMORY_DEVICE_COHERENT: devmem->pagemap.range.start = (MINOR(mdevice->device.devt) - 2) ? spm_addr_dev0 : spm_addr_dev1; devmem->pagemap.range.end = devmem->pagemap.range.start + DEVMEM_CHUNK_SIZE - 1; devmem->pagemap.type = MEMORY_DEVICE_COHERENT; break; default: ret = -EINVAL; goto err_devmem; }
devmem->pagemap.nr_range = 1; devmem->pagemap.ops = &dmirror_devmem_ops; devmem->pagemap.owner = mdevice;
mutex_lock(&mdevice->devmem_lock);
if (mdevice->devmem_count == mdevice->devmem_capacity) { struct dmirror_chunk **new_chunks; unsigned int new_capacity;
new_capacity = mdevice->devmem_capacity + DEVMEM_CHUNKS_RESERVE; new_chunks = krealloc(mdevice->devmem_chunks, sizeof(new_chunks[0]) * new_capacity, GFP_KERNEL); if (!new_chunks) goto err_release; mdevice->devmem_capacity = new_capacity; mdevice->devmem_chunks = new_chunks; } ptr = memremap_pages(&devmem->pagemap, numa_node_id()); if (IS_ERR_OR_NULL(ptr)) { if (ptr) ret = PTR_ERR(ptr); else ret = -EFAULT; goto err_release; }
devmem->mdevice = mdevice; pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT; pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT); mdevice->devmem_chunks[mdevice->devmem_count++] = devmem;
mutex_unlock(&mdevice->devmem_lock);
pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n", DEVMEM_CHUNK_SIZE / (1024 * 1024), mdevice->devmem_count, mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)), pfn_first, pfn_last);
spin_lock(&mdevice->lock); for (pfn = pfn_first; pfn < pfn_last; ) { struct page *page = pfn_to_page(pfn);
if (is_large && IS_ALIGNED(pfn, HPAGE_PMD_NR) && (pfn + HPAGE_PMD_NR <= pfn_last)) { page->zone_device_data = mdevice->free_folios; mdevice->free_folios = page_folio(page); pfn += HPAGE_PMD_NR; continue; }
page->zone_device_data = mdevice->free_pages; mdevice->free_pages = page; pfn++; }
ret = 0; if (ppage) { if (is_large) { if (!mdevice->free_folios) { ret = -ENOMEM; goto err_unlock; } *ppage = folio_page(mdevice->free_folios, 0); mdevice->free_folios = (*ppage)->zone_device_data; mdevice->calloc += HPAGE_PMD_NR; } else if (mdevice->free_pages) { *ppage = mdevice->free_pages; mdevice->free_pages = (*ppage)->zone_device_data; mdevice->calloc++; } else { ret = -ENOMEM; goto err_unlock; } }err_unlock: spin_unlock(&mdevice->lock);
return ret;
err_release: mutex_unlock(&mdevice->devmem_lock); if (res && devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) release_mem_region(devmem->pagemap.range.start, range_len(&devmem->pagemap.range));err_devmem: kfree(devmem);
return ret;}
static struct page *dmirror_devmem_alloc_page(struct dmirror *dmirror, bool is_large){ struct page *dpage = NULL; struct page *rpage = NULL; unsigned int order = is_large ? HPAGE_PMD_ORDER : 0; struct dmirror_device *mdevice = dmirror->mdevice;
/* * For ZONE_DEVICE private type, this is a fake device so we allocate * real system memory to store our device memory. * For ZONE_DEVICE coherent type we use the actual dpage to store the * data and ignore rpage. */ if (dmirror_is_private_zone(mdevice)) { rpage = folio_page(folio_alloc(GFP_HIGHUSER, order), 0); if (!rpage) return NULL; } spin_lock(&mdevice->lock);
if (is_large && mdevice->free_folios) { dpage = folio_page(mdevice->free_folios, 0); mdevice->free_folios = dpage->zone_device_data; mdevice->calloc += 1 << order; spin_unlock(&mdevice->lock); } else if (!is_large && mdevice->free_pages) { dpage = mdevice->free_pages; mdevice->free_pages = dpage->zone_device_data; mdevice->calloc++; spin_unlock(&mdevice->lock); } else { spin_unlock(&mdevice->lock); if (dmirror_allocate_chunk(mdevice, &dpage, is_large)) goto error; }
zone_device_folio_init(page_folio(dpage), page_pgmap(folio_page(page_folio(dpage), 0)), order); dpage->zone_device_data = rpage; return dpage;
error: if (rpage) __free_pages(rpage, order); return NULL;}
static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, struct dmirror *dmirror){ const unsigned long *src = args->src; unsigned long *dst = args->dst; unsigned long addr;
for (addr = args->start; addr < args->end; ) { struct page *spage; struct page *dpage; struct page *rpage; bool is_large = *src & MIGRATE_PFN_COMPOUND; int write = (*src & MIGRATE_PFN_WRITE) ? MIGRATE_PFN_WRITE : 0; unsigned long nr = 1;
if (!(*src & MIGRATE_PFN_MIGRATE)) goto next;
/* * Note that spage might be NULL which is OK since it is an * unallocated pte_none() or read-only zero page. */ spage = migrate_pfn_to_page(*src); if (WARN(spage && is_zone_device_page(spage), "page already in device spage pfn: 0x%lx\n", page_to_pfn(spage))) goto next;
if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) { dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC; dpage = NULL; } else dpage = dmirror_devmem_alloc_page(dmirror, is_large);
if (!dpage) { struct folio *folio; unsigned long i; unsigned long spfn = *src >> MIGRATE_PFN_SHIFT; struct page *src_page;
if (!is_large) goto next;
if (!spage && is_large) { nr = HPAGE_PMD_NR; } else { folio = page_folio(spage); nr = folio_nr_pages(folio); }
for (i = 0; i < nr && addr < args->end; i++) { dpage = dmirror_devmem_alloc_page(dmirror, false); rpage = BACKING_PAGE(dpage); rpage->zone_device_data = dmirror;
*dst = migrate_pfn(page_to_pfn(dpage)) | write; src_page = pfn_to_page(spfn + i);
if (spage) copy_highpage(rpage, src_page); else clear_highpage(rpage); src++; dst++; addr += PAGE_SIZE; } continue; }
rpage = BACKING_PAGE(dpage);
/* * Normally, a device would use the page->zone_device_data to * point to the mirror but here we use it to hold the page for * the simulated device memory and that page holds the pointer * to the mirror. */ rpage->zone_device_data = dmirror;
pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n", page_to_pfn(spage), page_to_pfn(dpage));
*dst = migrate_pfn(page_to_pfn(dpage)) | write;
if (is_large) { int i; struct folio *folio = page_folio(dpage); *dst |= MIGRATE_PFN_COMPOUND;
if (folio_test_large(folio)) { for (i = 0; i < folio_nr_pages(folio); i++) { struct page *dst_page = pfn_to_page(page_to_pfn(rpage) + i); struct page *src_page = pfn_to_page(page_to_pfn(spage) + i);
if (spage) copy_highpage(dst_page, src_page); else clear_highpage(dst_page); src++; dst++; addr += PAGE_SIZE; } continue; } }
if (spage) copy_highpage(rpage, spage); else clear_highpage(rpage);
next: src++; dst++; addr += PAGE_SIZE; }}
static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start, unsigned long end){ unsigned long pfn;
for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { void *entry;
entry = xa_load(&dmirror->pt, pfn); if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC) return -EPERM; }
return 0;}
static int dmirror_atomic_map(unsigned long addr, struct page *page, struct dmirror *dmirror){ void *entry;
/* Map the migrated pages into the device's page tables. */ mutex_lock(&dmirror->mutex);
entry = xa_tag_pointer(page, DPT_XA_TAG_ATOMIC); entry = xa_store(&dmirror->pt, addr >> PAGE_SHIFT, entry, GFP_ATOMIC); if (xa_is_err(entry)) { mutex_unlock(&dmirror->mutex); return xa_err(entry); }
mutex_unlock(&dmirror->mutex); return 0;}
static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, struct dmirror *dmirror){ unsigned long start = args->start; unsigned long end = args->end; const unsigned long *src = args->src; const unsigned long *dst = args->dst; unsigned long pfn; const unsigned long start_pfn = start >> PAGE_SHIFT; const unsigned long end_pfn = end >> PAGE_SHIFT;
/* Map the migrated pages into the device's page tables. */ mutex_lock(&dmirror->mutex);
for (pfn = start_pfn; pfn < end_pfn; pfn++, src++, dst++) { struct page *dpage; void *entry; int nr, i; struct page *rpage;
if (!(*src & MIGRATE_PFN_MIGRATE)) continue;
dpage = migrate_pfn_to_page(*dst); if (!dpage) continue;
if (*dst & MIGRATE_PFN_COMPOUND) nr = folio_nr_pages(page_folio(dpage)); else nr = 1;
WARN_ON_ONCE(end_pfn < start_pfn + nr);
rpage = BACKING_PAGE(dpage); VM_WARN_ON(folio_nr_pages(page_folio(rpage)) != nr);
for (i = 0; i < nr; i++) { entry = folio_page(page_folio(rpage), i); if (*dst & MIGRATE_PFN_WRITE) entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); entry = xa_store(&dmirror->pt, pfn + i, entry, GFP_ATOMIC); if (xa_is_err(entry)) { mutex_unlock(&dmirror->mutex); return xa_err(entry); } } }
mutex_unlock(&dmirror->mutex); return 0;}
static int dmirror_exclusive(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd){ unsigned long start, end, addr; unsigned long size = cmd->npages << PAGE_SHIFT; struct mm_struct *mm = dmirror->notifier.mm; struct dmirror_bounce bounce; int ret = 0;
start = cmd->addr; end = start + size; if (end < start) return -EINVAL;
/* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) return -EINVAL;
mmap_read_lock(mm); for (addr = start; !ret && addr < end; addr += PAGE_SIZE) { struct folio *folio; struct page *page;
page = make_device_exclusive(mm, addr, NULL, &folio); if (IS_ERR(page)) { ret = PTR_ERR(page); break; }
ret = dmirror_atomic_map(addr, page, dmirror); folio_unlock(folio); folio_put(folio); } mmap_read_unlock(mm); mmput(mm);
if (ret) return ret;
/* Return the migrated data for verification. */ ret = dmirror_bounce_init(&bounce, start, size); if (ret) return ret; mutex_lock(&dmirror->mutex); ret = dmirror_do_read(dmirror, start, end, &bounce); mutex_unlock(&dmirror->mutex); if (ret == 0) { if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, bounce.size)) ret = -EFAULT; }
cmd->cpages = bounce.cpages; dmirror_bounce_fini(&bounce); return ret;}
static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, struct dmirror *dmirror){ const unsigned long *src = args->src; unsigned long *dst = args->dst; unsigned long start = args->start; unsigned long end = args->end; unsigned long addr; unsigned int order = 0; int i;
for (addr = start; addr < end; ) { struct page *dpage, *spage;
spage = migrate_pfn_to_page(*src); if (!spage || !(*src & MIGRATE_PFN_MIGRATE)) { addr += PAGE_SIZE; goto next; }
if (WARN_ON(!is_device_private_page(spage) && !is_device_coherent_page(spage))) { addr += PAGE_SIZE; goto next; }
spage = BACKING_PAGE(spage); order = folio_order(page_folio(spage)); if (order) *dst = MIGRATE_PFN_COMPOUND; if (*src & MIGRATE_PFN_WRITE) *dst |= MIGRATE_PFN_WRITE;
if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) { dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC; *dst &= ~MIGRATE_PFN_COMPOUND; dpage = NULL; } else if (order) { dpage = folio_page(vma_alloc_folio(GFP_HIGHUSER_MOVABLE, order, args->vma, addr), 0); } else { dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); }
if (!dpage && !order) return VM_FAULT_OOM;
if (dpage) { pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n", page_to_pfn(spage), page_to_pfn(dpage)); lock_page(dpage); *dst |= migrate_pfn(page_to_pfn(dpage)); }
for (i = 0; i < (1 << order); i++) { struct page *src_page; struct page *dst_page;
/* Try with smaller pages if large allocation fails */ if (!dpage && order) { dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); if (!dpage) { /* Unlock and free pages already allocated. */ while (i > 0) { struct page *fpage;
fpage = migrate_pfn_to_page(dst[--i]); unlock_page(fpage); __free_page(fpage); } /* Clear remaining dst entries to avoid * migrate_vma_pages/finalize() using * uninitialized values. */ while (i < (1 << order)) { dst[i] = 0; i++; } return VM_FAULT_OOM; } lock_page(dpage); dst[i] = migrate_pfn(page_to_pfn(dpage)); dst_page = pfn_to_page(page_to_pfn(dpage)); dpage = NULL; /* For the next iteration */ } else { dst_page = pfn_to_page(page_to_pfn(dpage) + i); }
src_page = pfn_to_page(page_to_pfn(spage) + i);
xa_erase(&dmirror->pt, addr >> PAGE_SHIFT); addr += PAGE_SIZE; copy_highpage(dst_page, src_page); }next: src += 1 << order; dst += 1 << order; } return 0;}
static unsigned longdmirror_successful_migrated_pages(struct migrate_vma *migrate){ unsigned long cpages = 0; unsigned long i;
for (i = 0; i < migrate->npages; i++) { if (migrate->src[i] & MIGRATE_PFN_VALID && migrate->src[i] & MIGRATE_PFN_MIGRATE) cpages++; } return cpages;}
static int dmirror_migrate_to_system(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd){ unsigned long start, end, addr; unsigned long size = cmd->npages << PAGE_SHIFT; struct mm_struct *mm = dmirror->notifier.mm; struct vm_area_struct *vma; struct migrate_vma args = { 0 }; unsigned long next; int ret; unsigned long *src_pfns; unsigned long *dst_pfns;
start = cmd->addr; end = start + size; if (end < start) return -EINVAL;
/* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) return -EINVAL;
src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL);
cmd->cpages = 0; mmap_read_lock(mm); for (addr = start; addr < end; addr = next) { vma = vma_lookup(mm, addr); if (!vma || !(vma->vm_flags & VM_READ)) { ret = -EINVAL; goto out; } next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT)); if (next > vma->vm_end) next = vma->vm_end;
args.vma = vma; args.src = src_pfns; args.dst = dst_pfns; args.start = addr; args.end = next; args.pgmap_owner = dmirror->mdevice; args.flags = dmirror_select_device(dmirror) | MIGRATE_VMA_SELECT_COMPOUND;
ret = migrate_vma_setup(&args); if (ret) goto out;
pr_debug("Migrating from device mem to sys mem\n"); if (dmirror_devmem_fault_alloc_and_copy(&args, dmirror)) { migrate_vma_finalize(&args); ret = -ENOMEM; goto out; }
migrate_vma_pages(&args); cmd->cpages += dmirror_successful_migrated_pages(&args); migrate_vma_finalize(&args); }out: mmap_read_unlock(mm); mmput(mm); kvfree(src_pfns); kvfree(dst_pfns);
return ret;}
static int dmirror_migrate_to_device(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd){ unsigned long start, end, addr; unsigned long size = cmd->npages << PAGE_SHIFT; struct mm_struct *mm = dmirror->notifier.mm; struct vm_area_struct *vma; struct dmirror_bounce bounce; struct migrate_vma args = { 0 }; unsigned long next; int ret; unsigned long *src_pfns = NULL; unsigned long *dst_pfns = NULL;
start = cmd->addr; end = start + size; if (end < start) return -EINVAL;
/* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) return -EINVAL;
src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL);
ret = 0; mmap_read_lock(mm); for (addr = start; addr < end; addr = next) { vma = vma_lookup(mm, addr); if (!vma || !(vma->vm_flags & VM_READ)) { ret = -EINVAL; goto out; } next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT)); if (next > vma->vm_end) next = vma->vm_end;
args.vma = vma; args.src = src_pfns; args.dst = dst_pfns; args.start = addr; args.end = next; args.pgmap_owner = dmirror->mdevice; args.flags = MIGRATE_VMA_SELECT_SYSTEM | MIGRATE_VMA_SELECT_COMPOUND; ret = migrate_vma_setup(&args); if (ret) goto out;
pr_debug("Migrating from sys mem to device mem\n"); dmirror_migrate_alloc_and_copy(&args, dmirror); migrate_vma_pages(&args); dmirror_migrate_finalize_and_map(&args, dmirror); migrate_vma_finalize(&args); } mmap_read_unlock(mm); mmput(mm);
/* * Return the migrated data for verification. * Only for pages in device zone */ ret = dmirror_bounce_init(&bounce, start, size); if (ret) goto free_mem; mutex_lock(&dmirror->mutex); ret = dmirror_do_read(dmirror, start, end, &bounce); mutex_unlock(&dmirror->mutex); if (ret == 0) { if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, bounce.size)) ret = -EFAULT; } cmd->cpages = bounce.cpages; dmirror_bounce_fini(&bounce); goto free_mem;
out: mmap_read_unlock(mm); mmput(mm);free_mem: kvfree(src_pfns); kvfree(dst_pfns); return ret;}
static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range, unsigned char *perm, unsigned long entry){ struct page *page;
if (entry & HMM_PFN_ERROR) { *perm = HMM_DMIRROR_PROT_ERROR; return; } if (!(entry & HMM_PFN_VALID)) { *perm = HMM_DMIRROR_PROT_NONE; return; }
page = hmm_pfn_to_page(entry); if (is_device_private_page(page)) { /* Is the page migrated to this device or some other? */ if (dmirror->mdevice == dmirror_page_to_device(page)) *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL; else *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE; } else if (is_device_coherent_page(page)) { /* Is the page migrated to this device or some other? */ if (dmirror->mdevice == dmirror_page_to_device(page)) *perm = HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL; else *perm = HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE; } else if (is_zero_pfn(page_to_pfn(page))) *perm = HMM_DMIRROR_PROT_ZERO; else *perm = HMM_DMIRROR_PROT_NONE; if (entry & HMM_PFN_WRITE) *perm |= HMM_DMIRROR_PROT_WRITE; else *perm |= HMM_DMIRROR_PROT_READ; if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT) *perm |= HMM_DMIRROR_PROT_PMD; else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT) *perm |= HMM_DMIRROR_PROT_PUD;}
static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni, const struct mmu_notifier_range *range, unsigned long cur_seq){ struct dmirror_interval *dmi = container_of(mni, struct dmirror_interval, notifier); struct dmirror *dmirror = dmi->dmirror;
if (mmu_notifier_range_blockable(range)) mutex_lock(&dmirror->mutex); else if (!mutex_trylock(&dmirror->mutex)) return false;
/* * Snapshots only need to set the sequence number since any * invalidation in the interval invalidates the whole snapshot. */ mmu_interval_set_seq(mni, cur_seq);
mutex_unlock(&dmirror->mutex); return true;}
static const struct mmu_interval_notifier_ops dmirror_mrn_ops = { .invalidate = dmirror_snapshot_invalidate,};
static int dmirror_range_snapshot(struct dmirror *dmirror, struct hmm_range *range, unsigned char *perm){ struct mm_struct *mm = dmirror->notifier.mm; struct dmirror_interval notifier; unsigned long timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); unsigned long i; unsigned long n; int ret = 0;
notifier.dmirror = dmirror; range->notifier = ¬ifier.notifier;
ret = mmu_interval_notifier_insert(range->notifier, mm, range->start, range->end - range->start, &dmirror_mrn_ops); if (ret) return ret;
while (true) { if (time_after(jiffies, timeout)) { ret = -EBUSY; goto out; }
range->notifier_seq = mmu_interval_read_begin(range->notifier);
mmap_read_lock(mm); ret = hmm_range_fault(range); mmap_read_unlock(mm); if (ret) { if (ret == -EBUSY) continue; goto out; }
mutex_lock(&dmirror->mutex); if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) { mutex_unlock(&dmirror->mutex); continue; } break; }
n = (range->end - range->start) >> PAGE_SHIFT; for (i = 0; i < n; i++) dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);
mutex_unlock(&dmirror->mutex);out: mmu_interval_notifier_remove(range->notifier); return ret;}
static int dmirror_snapshot(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd){ struct mm_struct *mm = dmirror->notifier.mm; unsigned long start, end; unsigned long size = cmd->npages << PAGE_SHIFT; unsigned long addr; unsigned long next; unsigned long pfns[32]; unsigned char perm[32]; char __user *uptr; struct hmm_range range = { .hmm_pfns = pfns, .dev_private_owner = dmirror->mdevice, }; int ret = 0;
start = cmd->addr; end = start + size; if (end < start) return -EINVAL;
/* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) return -EINVAL;
/* * Register a temporary notifier to detect invalidations even if it * overlaps with other mmu_interval_notifiers. */ uptr = u64_to_user_ptr(cmd->ptr); for (addr = start; addr < end; addr = next) { unsigned long n;
next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); range.start = addr; range.end = next;
ret = dmirror_range_snapshot(dmirror, &range, perm); if (ret) break;
n = (range.end - range.start) >> PAGE_SHIFT; if (copy_to_user(uptr, perm, n)) { ret = -EFAULT; break; }
cmd->cpages += n; uptr += n; } mmput(mm);
return ret;}
/* Removes free pages from the free list so they can't be re-allocated */static void dmirror_remove_free_pages(struct dmirror_chunk *devmem){ struct dmirror_device *mdevice = devmem->mdevice; struct page *page; struct folio *folio;
for (folio = mdevice->free_folios; folio; folio = folio_zone_device_data(folio)) if (dmirror_page_to_chunk(folio_page(folio, 0)) == devmem) mdevice->free_folios = folio_zone_device_data(folio); for (page = mdevice->free_pages; page; page = page->zone_device_data) if (dmirror_page_to_chunk(page) == devmem) mdevice->free_pages = page->zone_device_data;}
static void dmirror_device_remove_chunks(struct dmirror_device *mdevice){ unsigned int i;
mutex_lock(&mdevice->devmem_lock); if (mdevice->devmem_chunks) { for (i = 0; i < mdevice->devmem_count; i++) { struct dmirror_chunk *devmem = mdevice->devmem_chunks[i];
spin_lock(&mdevice->lock); devmem->remove = true; dmirror_remove_free_pages(devmem); spin_unlock(&mdevice->lock);
dmirror_device_evict_chunk(devmem); memunmap_pages(&devmem->pagemap); if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) release_mem_region(devmem->pagemap.range.start, range_len(&devmem->pagemap.range)); kfree(devmem); } mdevice->devmem_count = 0; mdevice->devmem_capacity = 0; mdevice->free_pages = NULL; mdevice->free_folios = NULL; kfree(mdevice->devmem_chunks); mdevice->devmem_chunks = NULL; } mutex_unlock(&mdevice->devmem_lock);}
static long dmirror_fops_unlocked_ioctl(struct file *filp, unsigned int command, unsigned long arg){ void __user *uarg = (void __user *)arg; struct hmm_dmirror_cmd cmd; struct dmirror *dmirror; int ret;
dmirror = filp->private_data; if (!dmirror) return -EINVAL;
if (copy_from_user(&cmd, uarg, sizeof(cmd))) return -EFAULT;
if (cmd.addr & ~PAGE_MASK) return -EINVAL; if (cmd.npages > ULONG_MAX >> PAGE_SHIFT) return -EINVAL; if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT))) return -EINVAL;
cmd.cpages = 0; cmd.faults = 0;
switch (command) { case HMM_DMIRROR_READ: ret = dmirror_read(dmirror, &cmd); break;
case HMM_DMIRROR_WRITE: ret = dmirror_write(dmirror, &cmd); break;
case HMM_DMIRROR_MIGRATE_TO_DEV: ret = dmirror_migrate_to_device(dmirror, &cmd); break;
case HMM_DMIRROR_MIGRATE_TO_SYS: ret = dmirror_migrate_to_system(dmirror, &cmd); break;
case HMM_DMIRROR_EXCLUSIVE: ret = dmirror_exclusive(dmirror, &cmd); break;
case HMM_DMIRROR_CHECK_EXCLUSIVE: ret = dmirror_check_atomic(dmirror, cmd.addr, cmd.addr + (cmd.npages << PAGE_SHIFT)); break;
case HMM_DMIRROR_SNAPSHOT: ret = dmirror_snapshot(dmirror, &cmd); break;
case HMM_DMIRROR_RELEASE: dmirror_device_remove_chunks(dmirror->mdevice); ret = 0; break; case HMM_DMIRROR_FLAGS: dmirror->flags = cmd.npages; ret = 0; break; case HMM_DMIRROR_READ_UNLOCKED: ret = dmirror_read_unlocked(dmirror, &cmd, 0); break; default: return -EINVAL; } if (ret) return ret;
if (copy_to_user(uarg, &cmd, sizeof(cmd))) return -EFAULT;
return 0;}
static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma){ unsigned long addr;
for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) { struct page *page; int ret;
page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!page) return -ENOMEM;
ret = vm_insert_page(vma, addr, page); if (ret) { __free_page(page); return ret; } put_page(page); }
return 0;}
static const struct file_operations dmirror_fops = { .open = dmirror_fops_open, .release = dmirror_fops_release, .mmap = dmirror_fops_mmap, .unlocked_ioctl = dmirror_fops_unlocked_ioctl, .llseek = default_llseek, .owner = THIS_MODULE,};
static void dmirror_devmem_free(struct folio *folio){ struct page *page = &folio->page; struct page *rpage = BACKING_PAGE(page); struct dmirror_device *mdevice; struct folio *rfolio = page_folio(rpage); unsigned int order = folio_order(rfolio);
if (rpage != page) { if (order) __free_pages(rpage, order); else __free_page(rpage); rpage = NULL; }
mdevice = dmirror_page_to_device(page); spin_lock(&mdevice->lock);
/* Return page to our allocator if not freeing the chunk */ if (!dmirror_page_to_chunk(page)->remove) { mdevice->cfree += 1 << order; if (order) { page->zone_device_data = mdevice->free_folios; mdevice->free_folios = page_folio(page); } else { page->zone_device_data = mdevice->free_pages; mdevice->free_pages = page; } } spin_unlock(&mdevice->lock);}
static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf){ struct migrate_vma args = { 0 }; struct page *rpage; struct dmirror *dmirror; vm_fault_t ret = 0; unsigned int order, nr;
/* * Normally, a device would use the page->zone_device_data to point to * the mirror but here we use it to hold the page for the simulated * device memory and that page holds the pointer to the mirror. */ rpage = folio_zone_device_data(page_folio(vmf->page)); dmirror = rpage->zone_device_data;
/* FIXME demonstrate how we can adjust migrate range */ order = folio_order(page_folio(vmf->page)); nr = 1 << order;
/* * When folios are partially mapped, we can't rely on the folio * order of vmf->page as the folio might not be fully split yet */ if (vmf->pte) { order = 0; nr = 1; }
/* * Consider a per-cpu cache of src and dst pfns, but with * large number of cpus that might not scale well. */ args.start = ALIGN_DOWN(vmf->address, (PAGE_SIZE << order)); args.vma = vmf->vma; args.end = args.start + (PAGE_SIZE << order);
nr = (args.end - args.start) >> PAGE_SHIFT; args.src = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL); args.dst = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL); args.pgmap_owner = dmirror->mdevice; args.flags = dmirror_select_device(dmirror); args.fault_page = vmf->page;
if (!args.src || !args.dst) { ret = VM_FAULT_OOM; goto err; }
if (order) args.flags |= MIGRATE_VMA_SELECT_COMPOUND;
/* * In practice migrate_vma_setup() should never fail unless the * test is wrong as it just tests some static VMA properties. */ if (migrate_vma_setup(&args)) { ret = VM_FAULT_SIGBUS; goto err; }
ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror); if (ret) { migrate_vma_finalize(&args); goto err; } migrate_vma_pages(&args); /* * No device finalize step is needed since * dmirror_devmem_fault_alloc_and_copy() will have already * invalidated the device page table. */ migrate_vma_finalize(&args);err: kfree(args.src); kfree(args.dst); return ret;}
static void dmirror_devmem_folio_split(struct folio *head, struct folio *tail){ struct page *rpage = BACKING_PAGE(folio_page(head, 0)); struct page *rpage_tail; struct folio *rfolio; unsigned long offset = 0;
if (!rpage) { tail->page.zone_device_data = NULL; return; }
rfolio = page_folio(rpage);
if (tail == NULL) { folio_reset_order(rfolio); rfolio->mapping = NULL; folio_set_count(rfolio, 1); return; }
offset = folio_pfn(tail) - folio_pfn(head);
rpage_tail = folio_page(rfolio, offset); tail->page.zone_device_data = rpage_tail; rpage_tail->zone_device_data = rpage->zone_device_data; clear_compound_head(rpage_tail); rpage_tail->mapping = NULL;
folio_page(tail, 0)->mapping = folio_page(head, 0)->mapping; tail->pgmap = head->pgmap; folio_set_count(page_folio(rpage_tail), 1);}
static const struct dev_pagemap_ops dmirror_devmem_ops = { .folio_free = dmirror_devmem_free, .migrate_to_ram = dmirror_devmem_fault, .folio_split = dmirror_devmem_folio_split,};
static void dmirror_device_release(struct device *dev){ struct dmirror_device *mdevice = container_of(dev, struct dmirror_device, device);
dmirror_device_remove_chunks(mdevice);}
static int dmirror_device_init(struct dmirror_device *mdevice, int id){ dev_t dev; int ret;
dev = MKDEV(MAJOR(dmirror_dev), id); mutex_init(&mdevice->devmem_lock); spin_lock_init(&mdevice->lock);
cdev_init(&mdevice->cdevice, &dmirror_fops); mdevice->cdevice.owner = THIS_MODULE; mdevice->device.release = dmirror_device_release;
device_initialize(&mdevice->device); mdevice->device.devt = dev;
ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id); if (ret) goto put_device;
/* Build a list of free ZONE_DEVICE struct pages */ ret = dmirror_allocate_chunk(mdevice, NULL, false); if (ret) goto put_device;
ret = cdev_device_add(&mdevice->cdevice, &mdevice->device); if (ret) goto put_device;
return 0;
put_device: put_device(&mdevice->device); return ret;}
static void dmirror_device_remove(struct dmirror_device *mdevice){ cdev_device_del(&mdevice->cdevice, &mdevice->device); put_device(&mdevice->device);}
static int __init hmm_dmirror_init(void){ int ret; int id = 0; int ndevices = 0;
ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES, "HMM_DMIRROR"); if (ret) goto err_unreg;
memset(dmirror_devices, 0, DMIRROR_NDEVICES * sizeof(dmirror_devices[0])); dmirror_devices[ndevices++].zone_device_type = HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; dmirror_devices[ndevices++].zone_device_type = HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; if (spm_addr_dev0 && spm_addr_dev1) { dmirror_devices[ndevices++].zone_device_type = HMM_DMIRROR_MEMORY_DEVICE_COHERENT; dmirror_devices[ndevices++].zone_device_type = HMM_DMIRROR_MEMORY_DEVICE_COHERENT; } for (id = 0; id < ndevices; id++) { ret = dmirror_device_init(dmirror_devices + id, id); if (ret) goto err_chrdev; }
pr_info("HMM test module loaded. This is only for testing HMM.\n"); return 0;
err_chrdev: while (--id >= 0) dmirror_device_remove(dmirror_devices + id); unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);err_unreg: return ret;}
static void __exit hmm_dmirror_exit(void){ int id;
for (id = 0; id < DMIRROR_NDEVICES; id++) if (dmirror_devices[id].zone_device_type) dmirror_device_remove(dmirror_devices + id); unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);}
module_init(hmm_dmirror_init);module_exit(hmm_dmirror_exit);MODULE_DESCRIPTION("HMM (Heterogeneous Memory Management) test module");MODULE_LICENSE("GPL");