Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198// SPDX-License-Identifier: GPL-2.0/* * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk> */#include <linux/mm.h>#include <linux/swap.h>#include <linux/bio-integrity.h>#include <linux/blkdev.h>#include <linux/uio.h>#include <linux/iocontext.h>#include <linux/slab.h>#include <linux/init.h>#include <linux/kernel.h>#include <linux/export.h>#include <linux/mempool.h>#include <linux/workqueue.h>#include <linux/cgroup.h>#include <linux/highmem.h>#include <linux/blk-crypto.h>#include <linux/xarray.h>#include <linux/kmemleak.h>
#include <trace/events/block.h>#include "blk.h"#include "blk-rq-qos.h"#include "blk-cgroup.h"
#define ALLOC_CACHE_THRESHOLD 16#define ALLOC_CACHE_MAX 256
struct bio_alloc_cache { struct bio *free_list; struct bio *free_list_irq; unsigned int nr; unsigned int nr_irq;};
#define BIO_INLINE_VECS 4
static struct biovec_slab { int nr_vecs; char *name; struct kmem_cache *slab;} bvec_slabs[] __read_mostly = { { .nr_vecs = 16, .name = "biovec-16" }, { .nr_vecs = 64, .name = "biovec-64" }, { .nr_vecs = 128, .name = "biovec-128" }, { .nr_vecs = BIO_MAX_VECS, .name = "biovec-max" },};
static struct biovec_slab *biovec_slab(unsigned short nr_vecs){ switch (nr_vecs) { /* smaller bios use inline vecs */ case 5 ... 16: return &bvec_slabs[0]; case 17 ... 64: return &bvec_slabs[1]; case 65 ... 128: return &bvec_slabs[2]; case 129 ... BIO_MAX_VECS: return &bvec_slabs[3]; default: BUG(); return NULL; }}
/* * fs_bio_set is the bio_set containing bio and iovec memory pools used by * IO code that does not need private memory pools. */struct bio_set fs_bio_set;EXPORT_SYMBOL(fs_bio_set);
/* * Our slab pool management */struct bio_slab { struct kmem_cache *slab; unsigned int slab_ref; unsigned int slab_size; char name[12];};static DEFINE_MUTEX(bio_slab_lock);static DEFINE_XARRAY(bio_slabs);
static struct bio_slab *create_bio_slab(unsigned int size){ struct bio_slab *bslab = kzalloc_obj(*bslab);
if (!bslab) return NULL;
snprintf(bslab->name, sizeof(bslab->name), "bio-%d", size); bslab->slab = kmem_cache_create(bslab->name, size, ARCH_KMALLOC_MINALIGN, SLAB_HWCACHE_ALIGN | SLAB_TYPESAFE_BY_RCU, NULL); if (!bslab->slab) goto fail_alloc_slab;
bslab->slab_ref = 1; bslab->slab_size = size;
if (!xa_err(xa_store(&bio_slabs, size, bslab, GFP_KERNEL))) return bslab;
kmem_cache_destroy(bslab->slab);
fail_alloc_slab: kfree(bslab); return NULL;}
static inline unsigned int bs_bio_slab_size(struct bio_set *bs){ return bs->front_pad + sizeof(struct bio) + bs->back_pad;}
static inline void *bio_slab_addr(struct bio *bio){ return (void *)bio - bio->bi_pool->front_pad;}
static struct kmem_cache *bio_find_or_create_slab(struct bio_set *bs){ unsigned int size = bs_bio_slab_size(bs); struct bio_slab *bslab;
mutex_lock(&bio_slab_lock); bslab = xa_load(&bio_slabs, size); if (bslab) bslab->slab_ref++; else bslab = create_bio_slab(size); mutex_unlock(&bio_slab_lock);
if (bslab) return bslab->slab; return NULL;}
static void bio_put_slab(struct bio_set *bs){ struct bio_slab *bslab = NULL; unsigned int slab_size = bs_bio_slab_size(bs);
mutex_lock(&bio_slab_lock);
bslab = xa_load(&bio_slabs, slab_size); if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n")) goto out;
WARN_ON_ONCE(bslab->slab != bs->bio_slab);
WARN_ON(!bslab->slab_ref);
if (--bslab->slab_ref) goto out;
xa_erase(&bio_slabs, slab_size);
kmem_cache_destroy(bslab->slab); kfree(bslab);
out: mutex_unlock(&bio_slab_lock);}
/* * Make the first allocation restricted and don't dump info on allocation * failures, since we'll fall back to the mempool in case of failure. */static inline gfp_t try_alloc_gfp(gfp_t gfp){ return (gfp & ~(__GFP_DIRECT_RECLAIM | __GFP_IO)) | __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;}
void bio_uninit(struct bio *bio){#ifdef CONFIG_BLK_CGROUP if (bio->bi_blkg) { blkg_put(bio->bi_blkg); bio->bi_blkg = NULL; }#endif if (bio_integrity(bio)) bio_integrity_free(bio);
bio_crypt_free_ctx(bio);}EXPORT_SYMBOL(bio_uninit);
static void bio_free(struct bio *bio){ struct bio_set *bs = bio->bi_pool; void *p = bio;
WARN_ON_ONCE(!bs); WARN_ON_ONCE(bio->bi_max_vecs > BIO_MAX_VECS);
bio_uninit(bio); if (bio->bi_max_vecs == BIO_MAX_VECS) mempool_free(bio->bi_io_vec, &bs->bvec_pool); else if (bio->bi_max_vecs > BIO_INLINE_VECS) kmem_cache_free(biovec_slab(bio->bi_max_vecs)->slab, bio->bi_io_vec); mempool_free(p - bs->front_pad, &bs->bio_pool);}
/* * Users of this function have their own bio allocation. Subsequently, * they must remember to pair any call to bio_init() with bio_uninit() * when IO has completed, or when the bio is released. */void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, unsigned short max_vecs, blk_opf_t opf){ bio->bi_next = NULL; bio->bi_bdev = bdev; bio->bi_opf = opf; bio->bi_flags = 0; bio->bi_ioprio = 0; bio->bi_write_hint = 0; bio->bi_write_stream = 0; bio->bi_status = 0; bio->bi_bvec_gap_bit = 0; bio->bi_iter.bi_sector = 0; bio->bi_iter.bi_size = 0; bio->bi_iter.bi_idx = 0; bio->bi_iter.bi_offset = 0; bio->bi_end_io = NULL; bio->bi_private = NULL;#ifdef CONFIG_BLK_CGROUP bio->bi_blkg = NULL; bio->issue_time_ns = 0; if (bdev) bio_associate_blkg(bio);#ifdef CONFIG_BLK_CGROUP_IOCOST bio->bi_iocost_cost = 0;#endif#endif#ifdef CONFIG_BLK_INLINE_ENCRYPTION bio->bi_crypt_context = NULL;#endif#ifdef CONFIG_BLK_DEV_INTEGRITY bio->bi_integrity = NULL;#endif bio->bi_vcnt = 0;
atomic_set(&bio->__bi_remaining, 1); atomic_set(&bio->__bi_cnt, 1); bio->bi_cookie = BLK_QC_T_NONE;
bio->bi_max_vecs = max_vecs; bio->bi_io_vec = table; bio->bi_pool = NULL;}EXPORT_SYMBOL(bio_init);
/** * bio_reset - reinitialize a bio * @bio: bio to reset * @bdev: block device to use the bio for * @opf: operation and flags for bio * * Description: * After calling bio_reset(), @bio will be in the same state as a freshly * allocated bio returned bio bio_alloc_bioset() - the only fields that are * preserved are the ones that are initialized by bio_alloc_bioset(). See * comment in struct bio. */void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf){ struct bio_vec *bv = bio->bi_io_vec;
bio_uninit(bio); memset(bio, 0, BIO_RESET_BYTES); atomic_set(&bio->__bi_remaining, 1); bio->bi_io_vec = bv; bio->bi_bdev = bdev; if (bio->bi_bdev) bio_associate_blkg(bio); bio->bi_opf = opf;}EXPORT_SYMBOL(bio_reset);
/** * bio_reuse - reuse a bio with the payload left intact * @bio: bio to reuse * @opf: operation and flags for the next I/O * * Allow reusing an existing bio for another operation with all set up * fields including the payload, device and end_io handler left intact. * * Typically used when @bio is first used to read data which is then written * to another location without modification. @bio must not be in-flight and * owned by the caller. Can't be used for cloned bios. * * Note: Can't be used when @bio has integrity or blk-crypto contexts for now. * Feel free to add that support when you need it, though. */void bio_reuse(struct bio *bio, blk_opf_t opf){ unsigned short vcnt = bio->bi_vcnt, i; bio_end_io_t *end_io = bio->bi_end_io; void *private = bio->bi_private;
WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); WARN_ON_ONCE(bio_integrity(bio)); WARN_ON_ONCE(bio_has_crypt_ctx(bio));
bio_reset(bio, bio->bi_bdev, opf); for (i = 0; i < vcnt; i++) bio->bi_iter.bi_size += bio->bi_io_vec[i].bv_len; bio->bi_vcnt = vcnt; bio->bi_private = private; bio->bi_end_io = end_io;}EXPORT_SYMBOL_GPL(bio_reuse);
/** * bio_prepare_reissue - prepare a bio for reuissing the original I/O * @bio: bio to reuse * @bdev: block device to use the bio for * * Prepare @bio to be resubmitted to retry the original operation. * The caller must reset bio->bi_iter to the original state. */void bio_prepare_reissue(struct bio *bio, struct block_device *bdev){ bio->bi_bdev = bdev; bio_associate_blkg(bio); bio->bi_flags &= (BIO_PAGE_PINNED | BIO_CLONED | BIO_QUIET | BIO_REFFED); bio->bi_status = BLK_STS_OK; bio->bi_bvec_gap_bit = 0; atomic_set(&bio->__bi_remaining, 1);}EXPORT_SYMBOL_GPL(bio_prepare_reissue);
static struct bio *__bio_chain_endio(struct bio *bio){ struct bio *parent = bio->bi_private;
if (bio->bi_status && !parent->bi_status) parent->bi_status = bio->bi_status; bio_put(bio); return parent;}
/* * This function should only be used as a flag and must never be called. * If execution reaches here, it indicates a serious programming error. */static void bio_chain_endio(struct bio *bio){ BUG();}
/** * bio_chain - chain bio completions * @bio: the target bio * @parent: the parent bio of @bio * * The caller won't have a bi_end_io called when @bio completes - instead, * @parent's bi_end_io won't be called until both @parent and @bio have * completed; the chained bio will also be freed when it completes. * * The caller must not set bi_private or bi_end_io in @bio. */void bio_chain(struct bio *bio, struct bio *parent){ BUG_ON(bio->bi_private || bio->bi_end_io);
bio->bi_private = parent; bio->bi_end_io = bio_chain_endio; bio_inc_remaining(parent);}EXPORT_SYMBOL(bio_chain);
/** * bio_chain_and_submit - submit a bio after chaining it to another one * @prev: bio to chain and submit * @new: bio to chain to * * If @prev is non-NULL, chain it to @new and submit it. * * Return: @new. */struct bio *bio_chain_and_submit(struct bio *prev, struct bio *new){ if (prev) { bio_chain(prev, new); submit_bio(prev); } return new;}
struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev, unsigned int nr_pages, blk_opf_t opf, gfp_t gfp){ return bio_chain_and_submit(bio, bio_alloc(bdev, nr_pages, opf, gfp));}EXPORT_SYMBOL_GPL(blk_next_bio);
static void bio_alloc_rescue(struct work_struct *work){ struct bio_set *bs = container_of(work, struct bio_set, rescue_work); struct bio *bio;
while (1) { spin_lock(&bs->rescue_lock); bio = bio_list_pop(&bs->rescue_list); spin_unlock(&bs->rescue_lock);
if (!bio) break;
submit_bio_noacct(bio); }}
/* * submit_bio_noacct() converts recursion to iteration; this means if we're * running beneath it, any bios we allocate and submit will not be submitted * (and thus freed) until after we return. * * This exposes us to a potential deadlock if we allocate multiple bios from the * same bio_set while running underneath submit_bio_noacct(). If we were to * allocate multiple bios (say a stacking block driver that was splitting bios), * we would deadlock if we exhausted the mempool's reserve. * * We solve this, and guarantee forward progress by punting the bios on * current->bio_list to a per bio_set rescuer workqueue before blocking to wait * for elements being returned to the mempool. */static void punt_bios_to_rescuer(struct bio_set *bs){ struct bio_list punt, nopunt; struct bio *bio;
if (!current->bio_list || !bs->rescue_workqueue) return; if (bio_list_empty(¤t->bio_list[0]) && bio_list_empty(¤t->bio_list[1])) return;
/* * In order to guarantee forward progress we must punt only bios that * were allocated from this bio_set; otherwise, if there was a bio on * there for a stacking driver higher up in the stack, processing it * could require allocating bios from this bio_set, and doing that from * our own rescuer would be bad. * * Since bio lists are singly linked, pop them all instead of trying to * remove from the middle of the list: */
bio_list_init(&punt); bio_list_init(&nopunt);
while ((bio = bio_list_pop(¤t->bio_list[0]))) bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio); current->bio_list[0] = nopunt;
bio_list_init(&nopunt); while ((bio = bio_list_pop(¤t->bio_list[1]))) bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio); current->bio_list[1] = nopunt;
spin_lock(&bs->rescue_lock); bio_list_merge(&bs->rescue_list, &punt); spin_unlock(&bs->rescue_lock);
queue_work(bs->rescue_workqueue, &bs->rescue_work);}
static void bio_alloc_irq_cache_splice(struct bio_alloc_cache *cache){ unsigned long flags;
/* cache->free_list must be empty */ if (WARN_ON_ONCE(cache->free_list)) return;
local_irq_save(flags); cache->free_list = cache->free_list_irq; cache->free_list_irq = NULL; cache->nr += cache->nr_irq; cache->nr_irq = 0; local_irq_restore(flags);}
static struct bio *bio_alloc_percpu_cache(struct bio_set *bs){ struct bio_alloc_cache *cache; struct bio *bio;
cache = per_cpu_ptr(bs->cache, get_cpu()); if (!cache->free_list) { if (READ_ONCE(cache->nr_irq) >= ALLOC_CACHE_THRESHOLD) bio_alloc_irq_cache_splice(cache); if (!cache->free_list) { put_cpu(); return NULL; } } bio = cache->free_list; cache->free_list = bio->bi_next; cache->nr--; put_cpu(); bio->bi_pool = bs;
kmemleak_alloc(bio_slab_addr(bio), kmem_cache_size(bs->bio_slab), 1, GFP_NOIO); return bio;}
/** * bio_alloc_bioset - allocate a bio for I/O * @bdev: block device to allocate the bio for (can be %NULL) * @nr_vecs: number of bvecs to pre-allocate * @opf: operation and flags for bio * @gfp: the GFP_* mask given to the slab allocator * @bs: the bio_set to allocate from. * * Allocate a bio from the mempools in @bs. * * If %__GFP_DIRECT_RECLAIM is set then bio_alloc will always be able to * allocate a bio. This is due to the mempool guarantees. To make this work, * callers must never allocate more than 1 bio at a time from the general pool. * Callers that need to allocate more than 1 bio must always submit the * previously allocated bio for IO before attempting to allocate a new one. * Failure to do so can cause deadlocks under memory pressure. * * Note that when running under submit_bio_noacct() (i.e. any block driver), * bios are not submitted until after you return - see the code in * submit_bio_noacct() that converts recursion into iteration, to prevent * stack overflows. * * This would normally mean allocating multiple bios under submit_bio_noacct() * would be susceptible to deadlocks, but we have * deadlock avoidance code that resubmits any blocked bios from a rescuer * thread. * * However, we do not guarantee forward progress for allocations from other * mempools. Doing multiple allocations from the same mempool under * submit_bio_noacct() should be avoided - instead, use bio_set's front_pad * for per bio allocations. * * Returns: Pointer to new bio on success, NULL on failure. */struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp, struct bio_set *bs){ struct bio_vec *bvecs = NULL; struct bio *bio = NULL; gfp_t saved_gfp = gfp; void *p;
/* should not use nobvec bioset for nr_vecs > 0 */ if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_vecs > 0)) return NULL;
if (saved_gfp & __GFP_DIRECT_RECLAIM) gfp = try_alloc_gfp(gfp); if (bs->cache && nr_vecs <= BIO_INLINE_VECS) { /* * Set REQ_ALLOC_CACHE even if no cached bio is available to * return the allocated bio to the percpu cache when done. */ opf |= REQ_ALLOC_CACHE; bio = bio_alloc_percpu_cache(bs); } else { opf &= ~REQ_ALLOC_CACHE; }
/* * For a bioset without a percpu cache, or when the percpu cache was * empty, try a slab allocation with optimistic GFP_ flags before * falling back to the mempool. */ if (!bio) { p = kmem_cache_alloc(bs->bio_slab, gfp); if (p) bio = p + bs->front_pad; }
if (bio && nr_vecs > BIO_INLINE_VECS) { struct biovec_slab *bvs = biovec_slab(nr_vecs);
/* * Upgrade nr_vecs to take full advantage of the allocation. * We also rely on this in bio_free(). */ nr_vecs = bvs->nr_vecs; bvecs = kmem_cache_alloc(bvs->slab, gfp); if (unlikely(!bvecs)) { kmem_cache_free(bs->bio_slab, p); bio = NULL; } }
if (unlikely(!bio)) { /* * Give up if we are not allow to sleep as non-blocking mempool * allocations just go back to the slab allocation. */ if (!(saved_gfp & __GFP_DIRECT_RECLAIM)) return NULL;
punt_bios_to_rescuer(bs);
/* * Don't rob the mempools by returning to the per-CPU cache if * we're tight on memory. */ opf &= ~REQ_ALLOC_CACHE;
p = mempool_alloc(&bs->bio_pool, saved_gfp); bio = p + bs->front_pad; if (nr_vecs > BIO_INLINE_VECS) { nr_vecs = BIO_MAX_VECS; bvecs = mempool_alloc(&bs->bvec_pool, saved_gfp); } }
if (nr_vecs && nr_vecs <= BIO_INLINE_VECS) bio_init_inline(bio, bdev, nr_vecs, opf); else bio_init(bio, bdev, bvecs, nr_vecs, opf); bio->bi_pool = bs; return bio;}EXPORT_SYMBOL(bio_alloc_bioset);
/** * bio_kmalloc - kmalloc a bio * @nr_vecs: number of bio_vecs to allocate * @gfp_mask: the GFP_* mask given to the slab allocator * * Use kmalloc to allocate a bio (including bvecs). The bio must be initialized * using bio_init() before use. To free a bio returned from this function use * kfree() after calling bio_uninit(). A bio returned from this function can * be reused by calling bio_uninit() before calling bio_init() again. * * Note that unlike bio_alloc() or bio_alloc_bioset() allocations from this * function are not backed by a mempool can fail. Do not use this function * for allocations in the file system I/O path. * * Returns: Pointer to new bio on success, NULL on failure. */struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask){ struct bio *bio;
if (nr_vecs > BIO_MAX_INLINE_VECS) return NULL; return kmalloc(sizeof(*bio) + nr_vecs * sizeof(struct bio_vec), gfp_mask);}EXPORT_SYMBOL(bio_kmalloc);
void zero_fill_bio(struct bio *bio){ struct bio_vec bv; struct bvec_iter iter;
bio_for_each_segment(bv, bio, iter) memzero_bvec(&bv);}EXPORT_SYMBOL(zero_fill_bio);
/** * bio_truncate - truncate the bio to small size of @new_size * @bio: the bio to be truncated * @new_size: new size for truncating the bio * * Description: * Truncate the bio to new size of @new_size. If bio_op(bio) is * REQ_OP_READ, zero the truncated part. This function should only * be used for handling corner cases, such as bio eod. */static void bio_truncate(struct bio *bio, unsigned new_size){ struct bio_vec bv; struct bvec_iter iter; unsigned int done = 0; bool truncated = false;
if (new_size >= bio->bi_iter.bi_size) return;
if (bio_op(bio) != REQ_OP_READ) goto exit;
bio_for_each_segment(bv, bio, iter) { if (done + bv.bv_len > new_size) { size_t offset;
if (!truncated) offset = new_size - done; else offset = 0; memzero_page(bv.bv_page, bv.bv_offset + offset, bv.bv_len - offset); truncated = true; } done += bv.bv_len; }
exit: /* * Don't touch bvec table here and make it really immutable, since * fs bio user has to retrieve all pages via bio_for_each_segment_all * in its .end_bio() callback. * * It is enough to truncate bio by updating .bi_size since we can make * correct bvec with the updated .bi_size for drivers. */ bio->bi_iter.bi_size = new_size;}
/** * guard_bio_eod - truncate a BIO to fit the block device * @bio: bio to truncate * * This allows us to do IO even on the odd last sectors of a device, even if the * block size is some multiple of the physical sector size. * * We'll just truncate the bio to the size of the device, and clear the end of * the buffer head manually. Truly out-of-range accesses will turn into actual * I/O errors, this only handles the "we need to be able to do I/O at the final * sector" case. */void guard_bio_eod(struct bio *bio){ sector_t maxsector = bdev_nr_sectors(bio->bi_bdev);
if (!maxsector) return;
/* * If the *whole* IO is past the end of the device, * let it through, and the IO layer will turn it into * an EIO. */ if (unlikely(bio->bi_iter.bi_sector >= maxsector)) return;
maxsector -= bio->bi_iter.bi_sector; if (likely((bio->bi_iter.bi_size >> 9) <= maxsector)) return;
bio_truncate(bio, maxsector << 9);}
static int __bio_alloc_cache_prune(struct bio_alloc_cache *cache, unsigned int nr){ unsigned int i = 0; struct bio *bio;
while ((bio = cache->free_list) != NULL) { cache->free_list = bio->bi_next; cache->nr--; kmemleak_alloc(bio_slab_addr(bio), kmem_cache_size(bio->bi_pool->bio_slab), 1, GFP_KERNEL); bio_free(bio); if (++i == nr) break; } return i;}
static void bio_alloc_cache_prune(struct bio_alloc_cache *cache, unsigned int nr){ nr -= __bio_alloc_cache_prune(cache, nr); if (!READ_ONCE(cache->free_list)) { bio_alloc_irq_cache_splice(cache); __bio_alloc_cache_prune(cache, nr); }}
static int bio_cpu_dead(unsigned int cpu, struct hlist_node *node){ struct bio_set *bs;
bs = hlist_entry_safe(node, struct bio_set, cpuhp_dead); if (bs->cache) { struct bio_alloc_cache *cache = per_cpu_ptr(bs->cache, cpu);
bio_alloc_cache_prune(cache, -1U); } return 0;}
static void bio_alloc_cache_destroy(struct bio_set *bs){ int cpu;
if (!bs->cache) return;
cpuhp_state_remove_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead); for_each_possible_cpu(cpu) { struct bio_alloc_cache *cache;
cache = per_cpu_ptr(bs->cache, cpu); bio_alloc_cache_prune(cache, -1U); } free_percpu(bs->cache); bs->cache = NULL;}
static inline void bio_put_percpu_cache(struct bio *bio){ struct bio_alloc_cache *cache;
cache = per_cpu_ptr(bio->bi_pool->cache, get_cpu()); if (READ_ONCE(cache->nr_irq) + cache->nr > ALLOC_CACHE_MAX) goto out_free;
if (in_task()) { bio_uninit(bio); bio->bi_next = cache->free_list; /* Not necessary but helps not to iopoll already freed bios */ bio->bi_bdev = NULL; cache->free_list = bio; cache->nr++; kmemleak_free(bio_slab_addr(bio)); } else if (in_hardirq()) { lockdep_assert_irqs_disabled();
bio_uninit(bio); bio->bi_next = cache->free_list_irq; cache->free_list_irq = bio; cache->nr_irq++; kmemleak_free(bio_slab_addr(bio)); } else { goto out_free; } put_cpu(); return;out_free: put_cpu(); bio_free(bio);}
/** * bio_put - release a reference to a bio * @bio: bio to release reference to * * Description: * Put a reference to a &struct bio, either one you have gotten with * bio_alloc, bio_get or bio_clone_*. The last put of a bio will free it. **/void bio_put(struct bio *bio){ if (unlikely(bio_flagged(bio, BIO_REFFED))) { BUG_ON(!atomic_read(&bio->__bi_cnt)); if (!atomic_dec_and_test(&bio->__bi_cnt)) return; } if (bio->bi_opf & REQ_ALLOC_CACHE) bio_put_percpu_cache(bio); else bio_free(bio);}EXPORT_SYMBOL(bio_put);
static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp){ bio_set_flag(bio, BIO_CLONED); bio->bi_ioprio = bio_src->bi_ioprio; bio->bi_write_hint = bio_src->bi_write_hint; bio->bi_write_stream = bio_src->bi_write_stream; bio->bi_bvec_gap_bit = bio_src->bi_bvec_gap_bit; bio->bi_iter = bio_src->bi_iter;
if (op_is_dmabuf(bio->bi_opf)) bio->bi_dmabuf_map = bio_src->bi_dmabuf_map; else bio->bi_io_vec = bio_src->bi_io_vec;
if (bio->bi_bdev) { if (bio->bi_bdev == bio_src->bi_bdev && bio_flagged(bio_src, BIO_REMAPPED)) bio_set_flag(bio, BIO_REMAPPED); bio_clone_blkg_association(bio, bio_src); }
if (bio_crypt_clone(bio, bio_src, gfp) < 0) return -ENOMEM; if (bio_integrity(bio_src) && bio_integrity_clone(bio, bio_src, gfp) < 0) return -ENOMEM; return 0;}
/** * bio_alloc_clone - clone a bio that shares the original bio's biovec * @bdev: block_device to clone onto * @bio_src: bio to clone from * @gfp: allocation priority * @bs: bio_set to allocate from * * Allocate a new bio that is a clone of @bio_src. This reuses the bio_vecs * pointed to by @bio_src->bi_io_vec, and clones the iterator pointing to * the current position in it. The caller owns the returned bio, but not * the bio_vecs, and must ensure the bio is freed before the memory * pointed to by @bio_Src->bi_io_vecs. */struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src, gfp_t gfp, struct bio_set *bs){ struct bio *bio;
bio = bio_alloc_bioset(bdev, 0, bio_src->bi_opf, gfp, bs); if (!bio) return NULL;
if (__bio_clone(bio, bio_src, gfp) < 0) { bio_put(bio); return NULL; } return bio;}EXPORT_SYMBOL(bio_alloc_clone);
/** * bio_init_clone - clone a bio that shares the original bio's biovec * @bdev: block_device to clone onto * @bio: bio to clone into * @bio_src: bio to clone from * @gfp: allocation priority * * Initialize a new bio in caller provided memory that is a clone of @bio_src. * The same bio_vecs reuse and bio lifetime rules as bio_alloc_clone() apply. */int bio_init_clone(struct block_device *bdev, struct bio *bio, struct bio *bio_src, gfp_t gfp){ int ret;
bio_init(bio, bdev, NULL, 0, bio_src->bi_opf); ret = __bio_clone(bio, bio_src, gfp); if (ret) bio_uninit(bio); return ret;}EXPORT_SYMBOL(bio_init_clone);
/** * bio_full - check if the bio is full * @bio: bio to check * @len: length of one segment to be added * * Return true if @bio is full and one segment with @len bytes can't be * added to the bio, otherwise return false */static inline bool bio_full(struct bio *bio, unsigned len){ if (bio->bi_vcnt >= bio->bi_max_vecs) return true; if (bio->bi_iter.bi_size > BIO_MAX_SIZE - len) return true; return false;}
static bool bvec_try_merge_page(struct bio_vec *bv, struct page *page, unsigned int len, unsigned int off){ size_t bv_end = bv->bv_offset + bv->bv_len; phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + bv_end - 1; phys_addr_t page_addr = page_to_phys(page);
if (vec_end_addr + 1 != page_addr + off) return false; if (xen_domain() && !xen_biovec_phys_mergeable(bv, page)) return false;
if ((vec_end_addr & PAGE_MASK) != ((page_addr + off) & PAGE_MASK)) { if (IS_ENABLED(CONFIG_KMSAN)) return false; if (bv->bv_page + bv_end / PAGE_SIZE != page + off / PAGE_SIZE) return false; }
bv->bv_len += len; return true;}
/* * Try to merge a page into a segment, while obeying the hardware segment * size limit. * * This is kept around for the integrity metadata, which is still tries * to build the initial bio to the hardware limit and doesn't have proper * helpers to split. Hopefully this will go away soon. */bool bvec_try_merge_hw_page(struct request_queue *q, struct bio_vec *bv, struct page *page, unsigned len, unsigned offset){ unsigned long mask = queue_segment_boundary(q); phys_addr_t addr1 = bvec_phys(bv); phys_addr_t addr2 = page_to_phys(page) + offset + len - 1;
if ((addr1 | mask) != (addr2 | mask)) return false; if (len > queue_max_segment_size(q) - bv->bv_len) return false; return bvec_try_merge_page(bv, page, len, offset);}
/** * __bio_add_page - add page(s) to a bio in a new segment * @bio: destination bio * @page: start page to add * @len: length of the data to add, may cross pages * @off: offset of the data relative to @page, may cross pages * * Add the data at @page + @off to @bio as a new bvec. The caller must ensure * that @bio has space for another bvec. */void __bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int off){ WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); WARN_ON_ONCE(bio_full(bio, len));
if (is_pci_p2pdma_page(page)) bio->bi_opf |= REQ_NOMERGE;
bvec_set_page(&bio->bi_io_vec[bio->bi_vcnt], page, len, off); bio->bi_iter.bi_size += len; bio->bi_vcnt++;}EXPORT_SYMBOL_GPL(__bio_add_page);
/** * bio_add_virt_nofail - add data in the direct kernel mapping to a bio * @bio: destination bio * @vaddr: data to add * @len: length of the data to add, may cross pages * * Add the data at @vaddr to @bio. The caller must have ensure a segment * is available for the added data. No merging into an existing segment * will be performed. */void bio_add_virt_nofail(struct bio *bio, void *vaddr, unsigned len){ __bio_add_page(bio, virt_to_page(vaddr), len, offset_in_page(vaddr));}EXPORT_SYMBOL_GPL(bio_add_virt_nofail);
/** * bio_add_page - attempt to add page(s) to bio * @bio: destination bio * @page: start page to add * @len: vec entry length, may cross pages * @offset: vec entry offset relative to @page, may cross pages * * Attempt to add page(s) to the bio_vec maplist. This will only fail * if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio. */int bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int offset){ if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return 0; if (WARN_ON_ONCE(len == 0)) return 0; if (bio->bi_iter.bi_size > BIO_MAX_SIZE - len) return 0;
if (bio->bi_vcnt > 0) { struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
if (!zone_device_pages_compatible(bv->bv_page, page)) return 0; if (zone_device_pages_have_same_pgmap(bv->bv_page, page) && bvec_try_merge_page(bv, page, len, offset)) { bio->bi_iter.bi_size += len; return len; } }
if (bio->bi_vcnt >= bio->bi_max_vecs) return 0; __bio_add_page(bio, page, len, offset); return len;}EXPORT_SYMBOL(bio_add_page);
void bio_add_folio_nofail(struct bio *bio, struct folio *folio, size_t len, size_t off){ unsigned long nr = off / PAGE_SIZE;
WARN_ON_ONCE(len > BIO_MAX_SIZE); __bio_add_page(bio, folio_page(folio, nr), len, off % PAGE_SIZE);}EXPORT_SYMBOL_GPL(bio_add_folio_nofail);
/** * bio_add_folio - Attempt to add part of a folio to a bio. * @bio: BIO to add to. * @folio: Folio to add. * @len: How many bytes from the folio to add. * @off: First byte in this folio to add. * * Filesystems that use folios can call this function instead of calling * bio_add_page() for each page in the folio. If @off is bigger than * PAGE_SIZE, this function can create a bio_vec that starts in a page * after the bv_page. BIOs do not support folios that are 4GiB or larger. * * Return: Whether the addition was successful. */bool bio_add_folio(struct bio *bio, struct folio *folio, size_t len, size_t off){ unsigned long nr = off / PAGE_SIZE;
if (len > BIO_MAX_SIZE) return false; return bio_add_page(bio, folio_page(folio, nr), len, off % PAGE_SIZE) > 0;}EXPORT_SYMBOL(bio_add_folio);
/** * bio_add_vmalloc_chunk - add a vmalloc chunk to a bio * @bio: destination bio * @vaddr: vmalloc address to add * @len: total length in bytes of the data to add * * Add data starting at @vaddr to @bio and return how many bytes were added. * This may be less than the amount originally asked. Returns 0 if no data * could be added to @bio. * * This helper calls flush_kernel_vmap_range() for the range added. For reads * the caller still needs to manually call invalidate_kernel_vmap_range() in * the completion handler. */unsigned int bio_add_vmalloc_chunk(struct bio *bio, void *vaddr, unsigned len){ unsigned int offset = offset_in_page(vaddr);
len = min(len, PAGE_SIZE - offset); if (bio_add_page(bio, vmalloc_to_page(vaddr), len, offset) < len) return 0; if (op_is_write(bio_op(bio))) flush_kernel_vmap_range(vaddr, len); return len;}EXPORT_SYMBOL_GPL(bio_add_vmalloc_chunk);
/** * bio_add_vmalloc - add a vmalloc region to a bio * @bio: destination bio * @vaddr: vmalloc address to add * @len: total length in bytes of the data to add * * Add data starting at @vaddr to @bio. Return %true on success or %false if * @bio does not have enough space for the payload. * * This helper calls flush_kernel_vmap_range() for the range added. For reads * the caller still needs to manually call invalidate_kernel_vmap_range() in * the completion handler. */bool bio_add_vmalloc(struct bio *bio, void *vaddr, unsigned int len){ do { unsigned int added = bio_add_vmalloc_chunk(bio, vaddr, len);
if (!added) return false; vaddr += added; len -= added; } while (len);
return true;}EXPORT_SYMBOL_GPL(bio_add_vmalloc);
void __bio_release_pages(struct bio *bio, bool mark_dirty){ struct folio_iter fi;
bio_for_each_folio_all(fi, bio) { size_t nr_pages;
if (mark_dirty) { folio_lock(fi.folio); folio_mark_dirty(fi.folio); folio_unlock(fi.folio); } nr_pages = (fi.offset + fi.length - 1) / PAGE_SIZE - fi.offset / PAGE_SIZE + 1; unpin_user_folio(fi.folio, nr_pages); }}EXPORT_SYMBOL_GPL(__bio_release_pages);
bool bio_iov_iter_set(struct bio *bio, const struct iov_iter *iter){ if (!iov_iter_is_bvec(iter) && !iov_iter_is_dmabuf_map(iter)) return false;
WARN_ON_ONCE(bio->bi_max_vecs);
static_assert(offsetof(struct bio, bi_io_vec) == offsetof(struct bio, bi_dmabuf_map)); static_assert(offsetof(struct iov_iter, bvec) == offsetof(struct iov_iter, dmabuf_map));
bio->bi_io_vec = (struct bio_vec *)iter->bvec; bio->bi_iter.bi_idx = 0; bio->bi_iter.bi_offset = iter->iov_offset; bio->bi_iter.bi_size = iov_iter_count(iter); bio_set_flag(bio, BIO_CLONED); if (iov_iter_is_dmabuf_map(iter)) bio->bi_opf |= REQ_NOMERGE | REQ_DMABUF; return true;}
static unsigned int bvec_nr_pages(const struct bio_vec *bv){ return (bv->bv_offset + bv->bv_len - 1) / PAGE_SIZE - bv->bv_offset / PAGE_SIZE + 1;}
static void bvec_unpin(struct bio_vec *bv, bool mark_dirty){ struct folio *folio = bvec_folio(bv);
if (mark_dirty) folio_mark_dirty_lock(folio); unpin_user_folio(folio, bvec_nr_pages(bv));}
/* * Aligns the bio size to the len_align_mask, releasing excessive bio vecs that * __bio_iov_iter_get_pages may have inserted, and reverts the trimmed length * for the next iteration. */static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, unsigned len_align_mask){ struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; size_t nbytes = bio->bi_iter.bi_size & len_align_mask; unsigned int npages;
if (!nbytes) return 0;
iov_iter_revert(iter, nbytes); bio->bi_iter.bi_size -= nbytes; while (nbytes >= bv->bv_len) { if (bio_flagged(bio, BIO_PAGE_PINNED)) bvec_unpin(bv, false);
if (!--bio->bi_vcnt) return -EFAULT; nbytes -= bv->bv_len; bv--; }
/* * __bio_release_pages() only unpins the pages still covered by * the trimmed bv_len. Count the pages spanned before and after * the trim and unpin the difference. */ npages = bvec_nr_pages(bv); bv->bv_len -= nbytes; npages -= bvec_nr_pages(bv); if (npages && bio_flagged(bio, BIO_PAGE_PINNED)) unpin_user_folio(bvec_folio(bv), npages); return 0;}
#ifdef CONFIG_DEBUG_KERNELstatic inline bool bio_iov_bvec_aligned(const struct bio *bio, unsigned mem_align_mask){ struct bvec_iter iter; struct bio_vec bv;
/* * Correct callers never break the alignment requirements, so this * exhaustive check is only paid for in debug builds. */ for_each_mp_bvec(bv, bio->bi_io_vec, iter, bio->bi_iter) if ((bv.bv_offset | bv.bv_len) & mem_align_mask) return false; return true;}#elsestatic inline bool bio_iov_bvec_aligned(const struct bio *bio, unsigned mem_align_mask){ /* * We forward the bio_vec as-is, so ITER_BVEC callers must provide * segments already aligned to the device's DMA alignment. The only * unchecked user-controllable offset that reaches here is an io_uring * registered buffer where just the first segment can be unaligned * (the rest is virtually contiguous), so checking only that one is * sufficient to know if the entire vector is valid. */ return !(mp_bvec_iter_offset(bio->bi_io_vec, bio->bi_iter) & mem_align_mask);}#endif
/** * bio_iov_iter_get_pages - add user or kernel pages to a bio * @bio: bio to add pages to * @iter: iov iterator describing the region to be added * @maxlen: maximum size to consume from @iter * @mem_align_mask: the mask the source address and length must be aligned to, * 0 for no requirement * @len_align_mask: the mask to align the total size to, 0 for any length * * This takes either an iterator pointing to user memory, or one pointing to * kernel pages (BVEC iterator). If we're adding user pages, we pin them and * map them into the kernel. On IO completion, the caller should put those * pages. For bvec based iterators bio_iov_iter_get_pages() uses the provided * bvecs rather than copying them. Hence anyone issuing kiocb based IO needs * to ensure the bvecs and pages stay referenced until the submitted I/O is * completed by a call to ->ki_complete() or returns with an error other than * -EIOCBQUEUED. The caller needs to check if the bio is flagged BIO_NO_PAGE_REF * on IO completion. If it isn't, then pages should be released. * * The function tries, but does not guarantee, to pin as many pages as * fit into the bio, or are requested in @iter, whatever is smaller. If * MM encounters an error pinning the requested pages, it stops. Error * is returned only if 0 pages could be pinned. */int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, unsigned maxlen, unsigned mem_align_mask, unsigned len_align_mask){ iov_iter_extraction_t flags = 0; struct iov_iter saved_iter; int ret;
if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return -EIO;
if (bio_iov_iter_set(bio, iter)) { if (iov_iter_is_bvec(iter) && !bio_iov_bvec_aligned(bio, mem_align_mask)) return -EINVAL;
/* Truncate to the maximum size that the caller can handle */ bio->bi_iter.bi_size = min(bio->bi_iter.bi_size, maxlen); iov_iter_advance(iter, bio->bi_iter.bi_size); return 0; }
if (iov_iter_extract_will_pin(iter)) bio_set_flag(bio, BIO_PAGE_PINNED); if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue)) flags |= ITER_ALLOW_P2PDMA;
saved_iter = *iter; do { ssize_t size;
size = iov_iter_extract_bvecs(iter, bio->bi_io_vec, maxlen - bio->bi_iter.bi_size, &bio->bi_vcnt, bio->bi_max_vecs, mem_align_mask, flags); if (size <= 0) { /* A misaligned vector fails the whole I/O */ if (size == -EINVAL) goto out_release_pages; if (!bio->bi_vcnt) return size; break; } bio->bi_iter.bi_size += size; } while (iov_iter_count(iter) && !bio_full(bio, 0));
if (is_pci_p2pdma_page(bio->bi_io_vec->bv_page)) bio->bi_opf |= REQ_NOMERGE; ret = bio_iov_iter_align_down(bio, iter, len_align_mask); if (ret) return ret;
/* * An atomic write is submitted as a single bio, so it has to cover * the whole iterator or it would be torn. */ if ((bio->bi_opf & REQ_ATOMIC) && iov_iter_count(iter)) goto out_release_pages; return 0;
out_release_pages: /* * Release the pages pinned so far before failing, since this bio won't * be submitted to release them. */ bio_release_pages(bio, false); bio_clear_flag(bio, BIO_PAGE_PINNED); bio->bi_vcnt = 0; *iter = saved_iter; return -EINVAL;}
static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size, size_t minsize){ struct folio *folio;
while (*size > minsize) { folio = folio_alloc(gfp | __GFP_NORETRY | __GFP_NOWARN, get_order(*size)); if (folio) return folio; *size = rounddown_pow_of_two(*size - 1); }
return folio_alloc(gfp, get_order(*size));}
void bio_free_folios(struct bio *bio){ struct bio_vec *bv; int i;
bio_for_each_bvec_all(bv, bio, i) { struct folio *folio = bvec_folio(bv);
if (!is_zero_folio(folio) && !is_huge_zero_folio(folio)) folio_put(folio); }}
int bio_alloc_bounce_folios(struct bio *bio, size_t total_len, size_t minsize){ if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return -EINVAL; if (WARN_ON_ONCE(bio->bi_iter.bi_size)) return -EINVAL; if (WARN_ON_ONCE(bio->bi_vcnt >= bio->bi_max_vecs)) return -EINVAL;
do { size_t this_len = min(total_len, SZ_1M); struct folio *folio;
if (this_len > minsize * 2) this_len = rounddown_pow_of_two(this_len);
if (bio->bi_iter.bi_size > BIO_MAX_SIZE - this_len) break;
folio = folio_alloc_greedy(GFP_KERNEL, &this_len, minsize); if (!folio) break;
/* * Align down the size to the minimum alignment. In practice * this should not happen as minsize is expected to be a power * of two, as is the allocation size, but it offers us a cheap * extra safety belt. */ this_len &= ~(minsize - 1); bio_add_folio_nofail(bio, folio, this_len, 0); total_len -= this_len; } while (total_len && bio->bi_vcnt < bio->bi_max_vecs);
if (!bio->bi_iter.bi_size) return -ENOMEM; return 0;}
/** * bio_iov_iter_bounce_write - bounce buffer data from an iter into a bio * @bio: bio to send * @iter: iter to read from * @maxlen: maximum size to bounce * @minsize: minimum folio allocation size * * Helper for direct I/O write implementations that need to bounce buffer * because they need need to checksum the data or perform other operations that * require consistency. Allocates folios to back the bounce buffer, and copies * the data into it. Needs to be paired with bio_free_folios() called on * completion. */int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter, size_t maxlen, size_t minsize){ size_t total_len = min(maxlen, iov_iter_count(iter)); size_t total_copied = 0; struct bio_vec *bv; int i, error;
error = bio_alloc_bounce_folios(bio, total_len, minsize); if (error) return error;
bio_for_each_bvec_all(bv, bio, i) { struct folio *folio = page_folio(bv->bv_page); size_t copied;
if (iter->nofault) copied = copy_folio_from_iter_atomic(folio, 0, bv->bv_len, iter); else copied = copy_folio_from_iter(folio, 0, bv->bv_len, iter); total_copied += copied; if (copied < bv->bv_len) { iov_iter_revert(iter, total_copied); bio_free_folios(bio); return -EFAULT; } }
return 0;}EXPORT_SYMBOL_GPL(bio_iov_iter_bounce_write);
static void bio_wait_end_io(struct bio *bio){ complete(bio->bi_private);}
/** * bio_await - call a function on a bio, and wait until it completes * @bio: the bio which describes the I/O * @submit: function called to submit the bio * @priv: private data passed to @submit * * Wait for the bio as well as any bio chained off it after executing the * passed in callback @submit. The wait for the bio is set up before calling * @submit to ensure that the completion is captured. If @submit is %NULL, * submit_bio() is used instead to submit the bio. * * Note: this overrides the bi_private and bi_end_io fields in the bio. */void bio_await(struct bio *bio, void *priv, void (*submit)(struct bio *bio, void *priv)){ DECLARE_COMPLETION_ONSTACK_MAP(done, bio->bi_bdev->bd_disk->lockdep_map);
bio->bi_private = &done; bio->bi_end_io = bio_wait_end_io; bio->bi_opf |= REQ_SYNC; if (submit) submit(bio, priv); else submit_bio(bio); blk_wait_io(&done);}EXPORT_SYMBOL_GPL(bio_await);
/** * submit_bio_wait - submit a bio, and wait until it completes * @bio: The &struct bio which describes the I/O * * Simple wrapper around submit_bio(). Returns 0 on success, or the error from * bio_endio() on failure. * * WARNING: Unlike to how submit_bio() is usually used, this function does not * result in bio reference to be consumed. The caller must drop the reference * on his own. */int submit_bio_wait(struct bio *bio){ bio_await(bio, NULL, NULL); return blk_status_to_errno(bio->bi_status);}EXPORT_SYMBOL(submit_bio_wait);
static void bio_endio_cb(struct bio *bio, void *priv){ bio_endio(bio);}
/* * Submit @bio synchronously, or call bio_endio on it if the current process * is being killed. */int bio_submit_or_kill(struct bio *bio, unsigned int flags){ if ((flags & BLKDEV_ZERO_KILLABLE) && fatal_signal_pending(current)) { bio_await(bio, NULL, bio_endio_cb); return -EINTR; }
return submit_bio_wait(bio);}
/** * bdev_rw_virt - synchronously read into / write from kernel mapping * @bdev: block device to access * @sector: sector to access * @data: data to read/write * @len: length in byte to read/write * @op: operation (e.g. REQ_OP_READ/REQ_OP_WRITE) * * Performs synchronous I/O to @bdev for @data/@len. @data must be in * the kernel direct mapping and not a vmalloc address. */int bdev_rw_virt(struct block_device *bdev, sector_t sector, void *data, size_t len, enum req_op op){ struct bio_vec bv; struct bio bio; int error;
if (WARN_ON_ONCE(is_vmalloc_addr(data))) return -EIO;
bio_init(&bio, bdev, &bv, 1, op); bio.bi_iter.bi_sector = sector; bio_add_virt_nofail(&bio, data, len); error = submit_bio_wait(&bio); bio_uninit(&bio); return error;}EXPORT_SYMBOL_GPL(bdev_rw_virt);
void __bio_advance(struct bio *bio, unsigned bytes){ if (bio_integrity(bio)) bio_integrity_advance(bio, bytes);
bio_crypt_advance(bio, bytes); bio_advance_iter(bio, &bio->bi_iter, bytes);}EXPORT_SYMBOL(__bio_advance);
/** * bio_copy_data - copy contents of data buffers from one bio to another * @src: source bio * @dst: destination bio * * Stops when it reaches the end of either @src or @dst - that is, copies * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios). */void bio_copy_data(struct bio *dst, struct bio *src){ struct bvec_iter src_iter = src->bi_iter; struct bvec_iter dst_iter = dst->bi_iter;
while (src_iter.bi_size && dst_iter.bi_size) { struct bio_vec src_bv = bio_iter_iovec(src, src_iter); struct bio_vec dst_bv = bio_iter_iovec(dst, dst_iter); unsigned int bytes = min(src_bv.bv_len, dst_bv.bv_len); void *src_buf = bvec_kmap_local(&src_bv); void *dst_buf = bvec_kmap_local(&dst_bv);
memcpy(dst_buf, src_buf, bytes);
kunmap_local(dst_buf); kunmap_local(src_buf);
bio_advance_iter_single(src, &src_iter, bytes); bio_advance_iter_single(dst, &dst_iter, bytes); }}EXPORT_SYMBOL(bio_copy_data);
void bio_free_pages(struct bio *bio){ struct bio_vec *bvec; struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, iter_all) __free_page(bvec->bv_page);}EXPORT_SYMBOL(bio_free_pages);
/* * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions * for performing direct-IO in BIOs. * * The problem is that we cannot run folio_mark_dirty() from interrupt context * because the required locks are not interrupt-safe. So what we can do is to * mark the pages dirty _before_ performing IO. And in interrupt context, * check that the pages are still dirty. If so, fine. If not, redirty them * in process context. * * Note that this code is very hard to test under normal circumstances because * direct-io pins the pages with get_user_pages(). This makes * is_page_cache_freeable return false, and the VM will not clean the pages. * But other code (eg, flusher threads) could clean the pages if they are mapped * pagecache. * * Simply disabling the call to bio_set_pages_dirty() is a good way to test the * deferred bio dirtying paths. */
/* * bio_set_pages_dirty() will mark all the bio's pages as dirty. */void bio_set_pages_dirty(struct bio *bio){ struct folio_iter fi;
bio_for_each_folio_all(fi, bio) { folio_lock(fi.folio); folio_mark_dirty(fi.folio); folio_unlock(fi.folio); }}
/* * bio_check_pages_dirty() will check that all the BIO's pages are still dirty. * If they are, then fine. If, however, some pages are clean then they must * have been written out during the direct-IO read. So we take another ref on * the BIO and re-dirty the pages in process context. * * It is expected that bio_check_pages_dirty() will wholly own the BIO from * here on. It will unpin each page and will run one bio_put() against the * BIO. */
static void bio_dirty_fn(struct work_struct *work);
static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);static DEFINE_SPINLOCK(bio_dirty_lock);static struct bio *bio_dirty_list;
/* * This runs in process context */static void bio_dirty_fn(struct work_struct *work){ struct bio *bio, *next;
spin_lock_irq(&bio_dirty_lock); next = bio_dirty_list; bio_dirty_list = NULL; spin_unlock_irq(&bio_dirty_lock);
while ((bio = next) != NULL) { next = bio->bi_private;
bio_release_pages(bio, true); bio_put(bio); }}
void bio_check_pages_dirty(struct bio *bio){ struct folio_iter fi; unsigned long flags;
bio_for_each_folio_all(fi, bio) { if (!folio_test_dirty(fi.folio)) goto defer; }
bio_release_pages(bio, false); bio_put(bio); return;defer: spin_lock_irqsave(&bio_dirty_lock, flags); bio->bi_private = bio_dirty_list; bio_dirty_list = bio; spin_unlock_irqrestore(&bio_dirty_lock, flags); schedule_work(&bio_dirty_work);}
/* * Infrastructure for deferring bio completions to task-context via a per-CPU * workqueue. Triggered either by the BIO_COMPLETE_IN_TASK bio flag (static * decision at submit time) or by calling bio_complete_in_task() from * bi_end_io() (dynamic decision at completion time). */
struct bio_complete_batch { struct bio_list list; struct work_struct work; int cpu;};
static DEFINE_PER_CPU(struct bio_complete_batch, bio_complete_batch);static struct workqueue_struct *bio_complete_wq;
static void bio_complete_work_fn(struct work_struct *w){ struct bio_complete_batch *batch = container_of(w, struct bio_complete_batch, work);
while (1) { struct bio_list list; struct bio *bio;
local_irq_disable(); list = batch->list; bio_list_init(&batch->list); local_irq_enable();
if (bio_list_empty(&list)) break;
while ((bio = bio_list_pop(&list))) bio->bi_end_io(bio); }}
void __bio_complete_in_task(struct bio *bio){ struct bio_complete_batch *batch; unsigned long flags; bool was_empty;
local_irq_save(flags); batch = this_cpu_ptr(&bio_complete_batch); was_empty = bio_list_empty(&batch->list); bio_list_add(&batch->list, bio); local_irq_restore(flags);
if (was_empty) queue_work_on(batch->cpu, bio_complete_wq, &batch->work);}EXPORT_SYMBOL_GPL(__bio_complete_in_task);
static inline bool bio_remaining_done(struct bio *bio){ /* * If we're not chaining, then ->__bi_remaining is always 1 and * we always end io on the first invocation. */ if (!bio_flagged(bio, BIO_CHAIN)) return true;
BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
if (atomic_dec_and_test(&bio->__bi_remaining)) { bio_clear_flag(bio, BIO_CHAIN); return true; }
return false;}
/** * bio_endio - end I/O on a bio * @bio: bio * * Description: * bio_endio() will end I/O on the whole bio. bio_endio() is the preferred * way to end I/O on a bio. No one should call bi_end_io() directly on a * bio unless they own it and thus know that it has an end_io function. * * bio_endio() can be called several times on a bio that has been chained * using bio_chain(). The ->bi_end_io() function will only be called the * last time. **/void bio_endio(struct bio *bio){again: if (!bio_remaining_done(bio)) return; if (!bio_integrity_endio(bio)) return;
blk_zone_bio_endio(bio);
rq_qos_done_bio(bio);
if (bio->bi_bdev && bio_flagged(bio, BIO_TRACE_COMPLETION)) { trace_block_bio_complete(bdev_get_queue(bio->bi_bdev), bio); bio_clear_flag(bio, BIO_TRACE_COMPLETION); }
/* * Need to have a real endio function for chained bios, otherwise * various corner cases will break (like stacking block devices that * save/restore bi_end_io) - however, we want to avoid unbounded * recursion and blowing the stack. Tail call optimization would * handle this, but compiling with frame pointers also disables * gcc's sibling call optimization. */ if (bio->bi_end_io == bio_chain_endio) { bio = __bio_chain_endio(bio); goto again; }
#ifdef CONFIG_BLK_CGROUP /* * Release cgroup info. We shouldn't have to do this here, but quite * a few callers of bio_init fail to call bio_uninit, so we cover up * for that here at least for now. */ if (bio->bi_blkg) { blkg_put(bio->bi_blkg); bio->bi_blkg = NULL; }#endif
if (bio_flagged(bio, BIO_COMPLETE_IN_TASK) && bio_in_atomic()) __bio_complete_in_task(bio); else if (bio->bi_end_io) bio->bi_end_io(bio);}EXPORT_SYMBOL(bio_endio);
/** * bio_split - split a bio * @bio: bio to split * @sectors: number of sectors to split from the front of @bio * @gfp: gfp mask * @bs: bio set to allocate from * * Allocates and returns a new bio which represents @sectors from the start of * @bio, and updates @bio to represent the remaining sectors. * * Unless this is a discard request the newly allocated bio will point * to @bio's bi_io_vec. It is the caller's responsibility to ensure that * neither @bio nor @bs are freed before the split bio. */struct bio *bio_split(struct bio *bio, int sectors, gfp_t gfp, struct bio_set *bs){ struct bio *split;
if (WARN_ON_ONCE(sectors <= 0)) return ERR_PTR(-EINVAL); if (WARN_ON_ONCE(sectors >= bio_sectors(bio))) return ERR_PTR(-EINVAL);
/* Zone append commands cannot be split */ if (WARN_ON_ONCE(bio_op(bio) == REQ_OP_ZONE_APPEND)) return ERR_PTR(-EINVAL);
/* atomic writes cannot be split */ if (bio->bi_opf & REQ_ATOMIC) return ERR_PTR(-EINVAL);
split = bio_alloc_clone(bio->bi_bdev, bio, gfp, bs); if (!split) return ERR_PTR(-ENOMEM);
split->bi_iter.bi_size = sectors << 9;
if (bio_integrity(split)) bio_integrity_trim(split);
bio_advance(bio, split->bi_iter.bi_size);
/* * The gap bit is set when splitting to limits and only applies to the * front bio that was split off. The remaining bio will calcualte its * gap value when it is subsequently split to limits, so it is safe to * re-initialize the value back to 0. */ bio->bi_bvec_gap_bit = 0;
if (bio_flagged(bio, BIO_TRACE_COMPLETION)) bio_set_flag(split, BIO_TRACE_COMPLETION);
return split;}EXPORT_SYMBOL(bio_split);
/** * bio_trim - trim a bio * @bio: bio to trim * @offset: number of sectors to trim from the front of @bio * @size: size we want to trim @bio to, in sectors * * This function is typically used for bios that are cloned and submitted * to the underlying device in parts. */void bio_trim(struct bio *bio, sector_t offset, sector_t size){ /* We should never trim an atomic write */ if (WARN_ON_ONCE(bio->bi_opf & REQ_ATOMIC && size)) return;
if (WARN_ON_ONCE(offset > BIO_MAX_SECTORS || size > BIO_MAX_SECTORS || offset + size > bio_sectors(bio))) return;
size <<= 9; if (offset == 0 && size == bio->bi_iter.bi_size) return;
bio_advance(bio, offset << 9); bio->bi_iter.bi_size = size;
if (bio_integrity(bio)) bio_integrity_trim(bio);}EXPORT_SYMBOL_GPL(bio_trim);
/* * create memory pools for biovec's in a bio_set. * use the global biovec slabs created for general use. */static int biovec_init_pool(mempool_t *pool, int pool_entries){ struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1;
return mempool_init_slab_pool(pool, pool_entries, bp->slab);}
/* * bioset_exit - exit a bioset initialized with bioset_init() * * May be called on a zeroed but uninitialized bioset (i.e. allocated with * kzalloc()). */void bioset_exit(struct bio_set *bs){ bio_alloc_cache_destroy(bs); if (bs->rescue_workqueue) destroy_workqueue(bs->rescue_workqueue); bs->rescue_workqueue = NULL;
mempool_exit(&bs->bio_pool); mempool_exit(&bs->bvec_pool);
if (bs->bio_slab) bio_put_slab(bs); bs->bio_slab = NULL;}EXPORT_SYMBOL(bioset_exit);
/** * bioset_init - Initialize a bio_set * @bs: pool to initialize * @pool_size: Number of bio and bio_vecs to cache in the mempool * @front_pad: Number of bytes to allocate in front of the returned bio * @flags: Flags to modify behavior, currently %BIOSET_NEED_BVECS * and %BIOSET_NEED_RESCUER * * Description: * Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller * to ask for a number of bytes to be allocated in front of the bio. * Front pad allocation is useful for embedding the bio inside * another structure, to avoid allocating extra data to go with the bio. * Note that the bio must be embedded at the END of that structure always, * or things will break badly. * If %BIOSET_NEED_BVECS is set in @flags, a separate pool will be allocated * for allocating iovecs. This pool is not needed e.g. for bio_init_clone(). * If %BIOSET_NEED_RESCUER is set, a workqueue is created which can be used * to dispatch queued requests when the mempool runs out of space. * */int bioset_init(struct bio_set *bs, unsigned int pool_size, unsigned int front_pad, int flags){ bs->front_pad = front_pad; if (flags & BIOSET_NEED_BVECS) bs->back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec); else bs->back_pad = 0;
spin_lock_init(&bs->rescue_lock); bio_list_init(&bs->rescue_list); INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
bs->bio_slab = bio_find_or_create_slab(bs); if (!bs->bio_slab) return -ENOMEM;
if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab)) goto bad;
if ((flags & BIOSET_NEED_BVECS) && biovec_init_pool(&bs->bvec_pool, pool_size)) goto bad;
if (flags & BIOSET_NEED_RESCUER) { bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!bs->rescue_workqueue) goto bad; } if (flags & BIOSET_PERCPU_CACHE) { bs->cache = alloc_percpu(struct bio_alloc_cache); if (!bs->cache) goto bad; cpuhp_state_add_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead); }
return 0;bad: bioset_exit(bs); return -ENOMEM;}EXPORT_SYMBOL(bioset_init);
static int bio_complete_batch_cpu_online(unsigned int cpu){ struct bio_complete_batch *batch = &per_cpu(bio_complete_batch, cpu);
enable_work(&batch->work); if (!bio_list_empty(&batch->list)) queue_work_on(cpu, bio_complete_wq, &batch->work); return 0;}
/* * Disable this CPU's work item so that it cannot run on an unbound worker * after the CPU is offlined. */static int bio_complete_batch_cpu_down_prep(unsigned int cpu){ disable_work_sync(&per_cpu(bio_complete_batch, cpu).work); return 0;}
/* * Drain a dead CPU's deferred bio completions. The CPU is dead and the worker * is canceled so no locking is needed. */static int bio_complete_batch_cpu_dead(unsigned int cpu){ struct bio_complete_batch *batch = per_cpu_ptr(&bio_complete_batch, cpu); struct bio *bio;
while ((bio = bio_list_pop(&batch->list))) bio->bi_end_io(bio);
return 0;}
static void __init bio_complete_batch_init(int cpu){ struct bio_complete_batch *batch = per_cpu_ptr(&bio_complete_batch, cpu);
bio_list_init(&batch->list); INIT_WORK(&batch->work, bio_complete_work_fn); batch->cpu = cpu;
if (!cpu_online(cpu)) disable_work_sync(&batch->work);}
static int __init init_bio(void){ int i;
BUILD_BUG_ON(BIO_FLAG_LAST > 8 * sizeof_field(struct bio, bi_flags));
for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) { struct biovec_slab *bvs = bvec_slabs + i;
bvs->slab = kmem_cache_create(bvs->name, bvs->nr_vecs * sizeof(struct bio_vec), 0, SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL); }
for_each_possible_cpu(i) bio_complete_batch_init(i);
bio_complete_wq = alloc_workqueue("bio_complete", WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!bio_complete_wq) panic("bio: can't allocate bio_complete workqueue\n");
/* * bio task-context completion draining on hot-unplugged CPUs: * * 1. Stop the per-CPU work item while the CPU is still online, so * that it cannot run on an unbound worker later. * 2. Drain leftover bios added between worker disabling and CPU * offlining. */ cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "block/bio:complete:online", bio_complete_batch_cpu_online, bio_complete_batch_cpu_down_prep); cpuhp_setup_state_nocalls(CPUHP_BP_PREPARE_DYN, "block/bio:complete:dead", NULL, bio_complete_batch_cpu_dead);
cpuhp_setup_state_multi(CPUHP_BIO_DEAD, "block/bio:dead", NULL, bio_cpu_dead);
if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE)) panic("bio: can't allocate bios\n");
return 0;}subsys_initcall(init_bio);