Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422// SPDX-License-Identifier: GPL-2.0-or-later
#include <linux/blkdev.h>#include <linux/module.h>#include <linux/errno.h>#include <linux/slab.h>#include <linux/init.h>#include <linux/timer.h>#include <linux/sched.h>#include <linux/list.h>#include <linux/file.h>#include <linux/math64.h>#include <linux/seq_file.h>#include <trace/events/block.h>
#include "md.h"#include "md-bitmap.h"
/* * #### Background * * Redundant data is used to enhance data fault tolerance, and the storage * methods for redundant data vary depending on the RAID levels. And it's * important to maintain the consistency of redundant data. * * Bitmap is used to record which data blocks have been synchronized and which * ones need to be resynchronized or recovered. Each bit in the bitmap * represents a segment of data in the array. When a bit is set, it indicates * that the multiple redundant copies of that data segment may not be * consistent. Data synchronization can be performed based on the bitmap after * power failure or readding a disk. If there is no bitmap, a full disk * synchronization is required. * * #### Key Features * * - IO fastpath is lockless, if user issues lots of write IO to the same * bitmap bit in a short time, only the first write has additional overhead * to update bitmap bit, no additional overhead for the following writes; * - support only resync or recover written data, means in the case creating * new array or replacing with a new disk, there is no need to do a full disk * resync/recovery; * * #### Key Concept * * ##### State Machine * * Each bit is one byte, contain 6 different states, see llbitmap_state. And * there are total 8 different actions, see llbitmap_action, can change state: * * llbitmap state machine: transitions between states * * | | Startwrite | Startsync | Endsync | Abortsync| * | --------- | ---------- | --------- | ------- | ------- | * | Unwritten | Dirty | x | x | x | * | Clean | Dirty | x | x | x | * | Dirty | x | x | x | x | * | NeedSync | x | Syncing | x | x | * | Syncing | x | Syncing | Dirty | NeedSync | * * | | Reload | Daemon | Discard | Stale | * | --------- | -------- | ------ | --------- | --------- | * | Unwritten | x | x | x | x | * | Clean | x | x | Unwritten | NeedSync | * | Dirty | NeedSync | Clean | Unwritten | NeedSync | * | NeedSync | x | x | Unwritten | x | * | Syncing | NeedSync | x | Unwritten | NeedSync | * * Typical scenarios: * * 1) Create new array * All bits will be set to Unwritten by default, if --assume-clean is set, * all bits will be set to Clean instead. * * 2) write data, raid1/raid10 have full copy of data, while raid456 doesn't and * rely on xor data * * 2.1) write new data to raid1/raid10: * Unwritten --StartWrite--> Dirty * * 2.2) write new data to raid456: * Unwritten --StartWrite--> NeedSync * * Because the initial recover for raid456 is skipped, the xor data is not built * yet, the bit must be set to NeedSync first and after lazy initial recover is * finished, the bit will finally set to Dirty(see 5.1 and 5.4); * * 2.3) cover write * Clean --StartWrite--> Dirty * * 3) daemon, if the array is not degraded: * Dirty --Daemon--> Clean * * 4) discard * {Clean, Dirty, NeedSync, Syncing} --Discard--> Unwritten * * 5) resync and recover * * 5.1) common process * NeedSync --Startsync--> Syncing --Endsync--> Dirty --Daemon--> Clean * * 5.2) resync after power failure * Dirty --Reload--> NeedSync * * 5.3) recover while replacing with a new disk * By default, the old bitmap framework will recover all data, and llbitmap * implements this by a new helper, see llbitmap_skip_sync_blocks: * * skip recover for bits other than dirty or clean; * * 5.4) lazy initial recover for raid5: * By default, the old bitmap framework will only allow new recover when there * are spares(new disk), a new recovery flag MD_RECOVERY_LAZY_RECOVER is added * to perform raid456 lazy recover for set bits(from 2.2). * * 6. special handling for degraded array: * * - Dirty bits will never be cleared, daemon will just do nothing, so that if * a disk is readded, Clean bits can be skipped with recovery; * - Dirty bits will convert to Syncing from start write, to do data recovery * for new added disks; * - New write will convert bits to NeedSync directly; * * ##### Bitmap IO * * ##### Chunksize * * The default bitmap size is 128k, incluing 1k bitmap super block, and * the default size of segment of data in the array each bit(chunksize) is 64k, * and chunksize will adjust to twice the old size each time if the total number * bits is not less than 127k.(see llbitmap_init) * * ##### READ * * While creating bitmap, all pages will be allocated and read for llbitmap, * there won't be read afterwards * * ##### WRITE * * WRITE IO is divided into logical_block_size of the array, the dirty state * of each block is tracked independently, for example: * * each page is 4k, contain 8 blocks; each block is 512 bytes contain 512 bit; * * | page0 | page1 | ... | page 31 | * | | * | \-----------------------\ * | | * | block0 | block1 | ... | block 8| * | | * | \-----------------\ * | | * | bit0 | bit1 | ... | bit511 | * * From IO path, if one bit is changed to Dirty or NeedSync, the corresponding * subpage will be marked dirty, such block must write first before the IO is * issued. This behaviour will affect IO performance, to reduce the impact, if * multiple bits are changed in the same block in a short time, all bits in this * block will be changed to Dirty/NeedSync, so that there won't be any overhead * until daemon clears dirty bits. * * ##### Dirty Bits synchronization * * IO fast path will set bits to dirty, and those dirty bits will be cleared * by daemon after IO is done. llbitmap_page_ctl is used to synchronize between * IO path and daemon; * * IO path: * 1) try to grab a reference, if succeed, set expire time after 5s and return; * 2) if failed to grab a reference, wait for daemon to finish clearing dirty * bits; * * Daemon (Daemon will be woken up every daemon_sleep seconds): * For each page: * 1) check if page expired, if not skip this page; for expired page: * 2) suspend the page and wait for inflight write IO to be done; * 3) change dirty page to clean; * 4) resume the page; */
#define BITMAP_DATA_OFFSET 1024
/* 64k is the max IO size of sync IO for raid1/raid10 */#define MIN_CHUNK_SIZE (64 * 2)
/* By default, daemon will be woken up every 30s */#define DEFAULT_DAEMON_SLEEP 30
/* * Dirtied bits that have not been accessed for more than 5s will be cleared * by daemon. */#define DEFAULT_BARRIER_IDLE 5
enum llbitmap_state { /* No valid data, init state after assemble the array */ BitUnwritten = 0, /* data is consistent */ BitClean, /* data will be consistent after IO is done, set directly for writes */ BitDirty, /* * data need to be resynchronized: * 1) set directly for writes if array is degraded, prevent full disk * synchronization after readding a disk; * 2) reassemble the array after power failure, and dirty bits are * found after reloading the bitmap; * 3) set for first write for raid5, to build initial xor data lazily */ BitNeedSync, /* data is synchronizing */ BitSyncing, /* * Proactive sync requested for unwritten region (raid456 only). * Triggered via sysfs when user wants to pre-build XOR parity * for regions that have never been written. */ BitNeedSyncUnwritten, /* Proactive sync in progress for unwritten region */ BitSyncingUnwritten, /* * XOR parity has been pre-built for a region that has never had * user data written. When user writes to this region, it transitions * to BitDirty. */ BitCleanUnwritten, BitStateCount, BitNone = 0xff,};
enum llbitmap_action { /* User write new data, this is the only action from IO fast path */ BitmapActionStartwrite = 0, /* Start recovery */ BitmapActionStartsync, /* Finish recovery */ BitmapActionEndsync, /* Failed recovery */ BitmapActionAbortsync, /* Reassemble the array */ BitmapActionReload, /* Daemon thread is trying to clear dirty bits */ BitmapActionDaemon, /* Data is deleted */ BitmapActionDiscard, /* * Bitmap is stale, mark all bits in addition to BitUnwritten to * BitNeedSync. */ BitmapActionStale, /* * Proactive sync trigger for raid456 - builds XOR parity for * Unwritten regions without requiring user data write first. */ BitmapActionProactiveSync, BitmapActionClearUnwritten, BitmapActionCount, /* Init state is BitUnwritten */ BitmapActionInit,};
enum llbitmap_page_state { LLPageFlush = 0, LLPageDirty,};
struct llbitmap_page_ctl { char *state; struct page *page; unsigned long expire; unsigned long flags; wait_queue_head_t wait; struct percpu_ref active; /* Per block size dirty state, maximum 64k page / 1 sector = 128 */ unsigned long dirty[];};
struct llbitmap { struct mddev *mddev; struct llbitmap_page_ctl **pctl;
unsigned int nr_pages; unsigned int io_size; unsigned int blocks_per_page;
/* shift of one chunk */ unsigned long chunkshift; /* size of one chunk in sector */ unsigned long chunksize; /* total number of chunks */ unsigned long chunks; /* total number of sectors tracked by current bitmap geometry */ sector_t sync_size; unsigned long reshape_chunksize; unsigned long reshape_chunks; sector_t reshape_sync_size; unsigned long last_end_sync; /* * time in seconds that dirty bits will be cleared if the page is not * accessed. */ unsigned long barrier_idle; /* fires on first BitDirty state */ struct timer_list pending_timer; struct work_struct daemon_work; /* * Serialize reshape checkpoint remapping against normal I/O bitmap * updates without blocking concurrent I/O updates on each other. */ rwlock_t reshape_lock;
unsigned long flags; __u64 events_cleared;
/* for slow disks */ atomic_t behind_writes; wait_queue_head_t behind_wait;};
struct llbitmap_unplug_work { struct work_struct work; struct llbitmap *llbitmap; struct completion *done;};
static struct workqueue_struct *md_llbitmap_io_wq;static struct workqueue_struct *md_llbitmap_unplug_wq;
static char state_machine[BitStateCount][BitmapActionCount] = { [BitUnwritten] = { [BitmapActionStartwrite] = BitDirty, [BitmapActionStartsync] = BitNone, [BitmapActionEndsync] = BitNone, [BitmapActionAbortsync] = BitNone, [BitmapActionReload] = BitNone, [BitmapActionDaemon] = BitNone, [BitmapActionDiscard] = BitNone, [BitmapActionStale] = BitNone, [BitmapActionProactiveSync] = BitNeedSyncUnwritten, [BitmapActionClearUnwritten] = BitNone, }, [BitClean] = { [BitmapActionStartwrite] = BitDirty, [BitmapActionStartsync] = BitNone, [BitmapActionEndsync] = BitNone, [BitmapActionAbortsync] = BitNone, [BitmapActionReload] = BitNone, [BitmapActionDaemon] = BitNone, [BitmapActionDiscard] = BitUnwritten, [BitmapActionStale] = BitNeedSync, [BitmapActionProactiveSync] = BitNone, [BitmapActionClearUnwritten] = BitNone, }, [BitDirty] = { [BitmapActionStartwrite] = BitNone, [BitmapActionStartsync] = BitNone, [BitmapActionEndsync] = BitNone, [BitmapActionAbortsync] = BitNone, [BitmapActionReload] = BitNeedSync, [BitmapActionDaemon] = BitClean, [BitmapActionDiscard] = BitUnwritten, [BitmapActionStale] = BitNeedSync, [BitmapActionProactiveSync] = BitNone, [BitmapActionClearUnwritten] = BitNone, }, [BitNeedSync] = { [BitmapActionStartwrite] = BitNone, [BitmapActionStartsync] = BitSyncing, [BitmapActionEndsync] = BitNone, [BitmapActionAbortsync] = BitNone, [BitmapActionReload] = BitNone, [BitmapActionDaemon] = BitNone, [BitmapActionDiscard] = BitUnwritten, [BitmapActionStale] = BitNone, [BitmapActionProactiveSync] = BitNone, [BitmapActionClearUnwritten] = BitNone, }, [BitSyncing] = { [BitmapActionStartwrite] = BitNone, [BitmapActionStartsync] = BitSyncing, [BitmapActionEndsync] = BitDirty, [BitmapActionAbortsync] = BitNeedSync, [BitmapActionReload] = BitNeedSync, [BitmapActionDaemon] = BitNone, [BitmapActionDiscard] = BitUnwritten, [BitmapActionStale] = BitNeedSync, [BitmapActionProactiveSync] = BitNone, [BitmapActionClearUnwritten] = BitNone, }, [BitNeedSyncUnwritten] = { [BitmapActionStartwrite] = BitNeedSync, [BitmapActionStartsync] = BitSyncingUnwritten, [BitmapActionEndsync] = BitNone, [BitmapActionAbortsync] = BitUnwritten, [BitmapActionReload] = BitUnwritten, [BitmapActionDaemon] = BitNone, [BitmapActionDiscard] = BitUnwritten, [BitmapActionStale] = BitUnwritten, [BitmapActionProactiveSync] = BitNone, [BitmapActionClearUnwritten] = BitUnwritten, }, [BitSyncingUnwritten] = { [BitmapActionStartwrite] = BitSyncing, [BitmapActionStartsync] = BitSyncingUnwritten, [BitmapActionEndsync] = BitCleanUnwritten, [BitmapActionAbortsync] = BitUnwritten, [BitmapActionReload] = BitUnwritten, [BitmapActionDaemon] = BitNone, [BitmapActionDiscard] = BitUnwritten, [BitmapActionStale] = BitUnwritten, [BitmapActionProactiveSync] = BitNone, [BitmapActionClearUnwritten] = BitUnwritten, }, [BitCleanUnwritten] = { [BitmapActionStartwrite] = BitDirty, [BitmapActionStartsync] = BitNone, [BitmapActionEndsync] = BitNone, [BitmapActionAbortsync] = BitNone, [BitmapActionReload] = BitNone, [BitmapActionDaemon] = BitNone, [BitmapActionDiscard] = BitUnwritten, [BitmapActionStale] = BitUnwritten, [BitmapActionProactiveSync] = BitNone, [BitmapActionClearUnwritten] = BitUnwritten, },};
static void __llbitmap_flush(struct mddev *mddev);static void llbitmap_flush(struct mddev *mddev);static void llbitmap_update_sb(void *data);
static void llbitmap_calculate_chunks(struct mddev *mddev, sector_t blocks, unsigned long *chunksize, unsigned long *chunks){ *chunks = DIV_ROUND_UP_SECTOR_T(blocks, *chunksize); while (*chunks > mddev->bitmap_info.space << SECTOR_SHIFT) { *chunksize = *chunksize << 1; *chunks = DIV_ROUND_UP_SECTOR_T(blocks, *chunksize); }}
static sector_t llbitmap_personality_sync_size(struct llbitmap *llbitmap, bool previous){ struct mddev *mddev = llbitmap->mddev;
if (READ_ONCE(mddev->reshape_position) == MaxSector || !mddev->private || !mddev->pers || !mddev->pers->bitmap_sync_size) return llbitmap->sync_size; return mddev->pers->bitmap_sync_size(mddev, previous);}
static sector_t llbitmap_logical_size(struct llbitmap *llbitmap, bool previous){ struct mddev *mddev = llbitmap->mddev;
if (!mddev->private || !mddev->pers || !mddev->pers->bitmap_array_sectors) return llbitmap_personality_sync_size(llbitmap, previous); return mddev->pers->bitmap_array_sectors(mddev, previous);}
static void llbitmap_refresh_reshape(struct llbitmap *llbitmap){ unsigned long old_chunks = DIV_ROUND_UP_SECTOR_T(llbitmap->sync_size, llbitmap->chunksize); sector_t blocks = llbitmap_personality_sync_size(llbitmap, false); unsigned long chunksize = llbitmap->chunksize; unsigned long chunks = DIV_ROUND_UP_SECTOR_T(blocks, chunksize);
llbitmap->reshape_sync_size = blocks; llbitmap->reshape_chunksize = chunksize; llbitmap->reshape_chunks = chunks; llbitmap_calculate_chunks(llbitmap->mddev, blocks, &llbitmap->reshape_chunksize, &llbitmap->reshape_chunks); llbitmap->chunks = max(old_chunks, llbitmap->reshape_chunks);}
static void llbitmap_map_layout(struct llbitmap *llbitmap, sector_t *offset, unsigned long *sectors, bool previous){ sector_t limit = llbitmap_logical_size(llbitmap, previous); sector_t start = *offset; sector_t end = start + *sectors;
if (start >= limit) { *sectors = 0; return; } if (end > limit) end = limit;
*offset = start; *sectors = end - start; if (!*sectors) return;
if (llbitmap->mddev->pers->bitmap_sector_map) llbitmap->mddev->pers->bitmap_sector_map(llbitmap->mddev, offset, sectors, previous); else if (!previous && llbitmap->mddev->pers->bitmap_sector) llbitmap->mddev->pers->bitmap_sector(llbitmap->mddev, offset, sectors);
limit = llbitmap_personality_sync_size(llbitmap, previous); start = *offset; end = start + *sectors; if (start >= limit) *sectors = 0; else if (end > limit) *sectors = limit - start;}
static void llbitmap_encode_range(struct llbitmap *llbitmap, sector_t *offset, unsigned long *sectors, bool previous){ unsigned long chunksize = previous ? llbitmap->chunksize : llbitmap->reshape_chunksize; u64 start; u64 end;
if (!*sectors) { *offset = 0; return; }
start = div64_u64(*offset, chunksize); end = div64_u64(*offset + *sectors - 1, chunksize); *offset = (sector_t)start << llbitmap->chunkshift; *sectors = (end - start + 1) << llbitmap->chunkshift;}
static void llbitmap_encode_discard_range(struct llbitmap *llbitmap, sector_t *offset, unsigned long *sectors, bool previous){ unsigned long chunksize = previous ? llbitmap->chunksize : llbitmap->reshape_chunksize; sector_t end = *offset + *sectors; u64 start; u64 last;
if (!*sectors) { *offset = 0; return; }
start = DIV_ROUND_UP_SECTOR_T(*offset, chunksize); last = div64_u64(end, chunksize); if (start >= last) { *offset = 0; *sectors = 0; return; }
*offset = (sector_t)start << llbitmap->chunkshift; *sectors = (last - start) << llbitmap->chunkshift;}
static enum llbitmap_state llbitmap_read(struct llbitmap *llbitmap, loff_t pos){ unsigned int idx; unsigned int offset;
pos += BITMAP_DATA_OFFSET; idx = pos >> PAGE_SHIFT; offset = offset_in_page(pos);
return llbitmap->pctl[idx]->state[offset];}
/* set all the bits in the subpage as dirty */static void llbitmap_infect_dirty_bits(struct llbitmap *llbitmap, struct llbitmap_page_ctl *pctl, unsigned int block){ bool level_456 = raid_is_456(llbitmap->mddev); unsigned int io_size = llbitmap->io_size; int pos;
for (pos = block * io_size; pos < (block + 1) * io_size; pos++) { switch (pctl->state[pos]) { case BitUnwritten: pctl->state[pos] = level_456 ? BitNeedSync : BitDirty; break; case BitClean: case BitCleanUnwritten: pctl->state[pos] = BitDirty; break; } }}
static void llbitmap_set_page_dirty(struct llbitmap *llbitmap, int idx, int offset, bool infect){ struct llbitmap_page_ctl *pctl = llbitmap->pctl[idx]; unsigned int io_size = llbitmap->io_size; int block = offset / io_size; int pos;
if (!test_bit(LLPageDirty, &pctl->flags)) set_bit(LLPageDirty, &pctl->flags);
/* * For degraded array, dirty bits will never be cleared, and we must * resync all the dirty bits, hence skip infect new dirty bits to * prevent resync unnecessary data. */ if (llbitmap->mddev->degraded || !infect) { set_bit(block, pctl->dirty); return; }
/* * The subpage usually contains a total of 512 bits. If any single bit * within the subpage is marked as dirty, the entire sector will be * written. To avoid impacting write performance, when multiple bits * within the same sector are modified within llbitmap->barrier_idle, * all bits in the sector will be collectively marked as dirty at once. */ if (test_and_set_bit(block, pctl->dirty)) { llbitmap_infect_dirty_bits(llbitmap, pctl, block); return; }
for (pos = block * io_size; pos < (block + 1) * io_size; pos++) { if (pos == offset) continue; if (pctl->state[pos] == BitDirty || pctl->state[pos] == BitNeedSync) { llbitmap_infect_dirty_bits(llbitmap, pctl, block); return; } }}
static void llbitmap_write(struct llbitmap *llbitmap, enum llbitmap_state state, loff_t pos){ unsigned int idx; unsigned int bit;
pos += BITMAP_DATA_OFFSET; idx = pos >> PAGE_SHIFT; bit = offset_in_page(pos);
llbitmap->pctl[idx]->state[bit] = state; if (state == BitDirty || state == BitNeedSync) llbitmap_set_page_dirty(llbitmap, idx, bit, true); else if (state == BitNeedSyncUnwritten) llbitmap_set_page_dirty(llbitmap, idx, bit, false);}
static unsigned int llbitmap_used_pages(struct llbitmap *llbitmap, unsigned long chunks){ return DIV_ROUND_UP(chunks + BITMAP_DATA_OFFSET, PAGE_SIZE);}
static struct page *llbitmap_read_page(struct llbitmap *llbitmap, int idx){ struct mddev *mddev = llbitmap->mddev; struct page *page = NULL; struct md_rdev *rdev;
if (llbitmap->pctl && idx < llbitmap->nr_pages && llbitmap->pctl[idx]) page = llbitmap->pctl[idx]->page; if (page) return page;
page = alloc_page(GFP_NOIO | __GFP_ZERO); if (!page) return ERR_PTR(-ENOMEM); if (idx >= llbitmap_used_pages(llbitmap, llbitmap->chunks)) return page;
rdev_for_each(rdev, mddev) { sector_t sector;
if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags) || !test_bit(In_sync, &rdev->flags)) continue;
sector = mddev->bitmap_info.offset + (idx << PAGE_SECTORS_SHIFT);
if (sync_page_io(rdev, sector, PAGE_SIZE, page, REQ_OP_READ, true)) return page;
md_error(mddev, rdev); }
__free_page(page); return ERR_PTR(-EIO);}
static void llbitmap_write_page(struct llbitmap *llbitmap, int idx){ struct page *page = llbitmap->pctl[idx]->page; struct mddev *mddev = llbitmap->mddev; struct md_rdev *rdev; int block;
for (block = 0; block < llbitmap->blocks_per_page; block++) { struct llbitmap_page_ctl *pctl = llbitmap->pctl[idx];
if (!test_and_clear_bit(block, pctl->dirty)) continue;
rdev_for_each(rdev, mddev) { sector_t sector; sector_t bit_sector = llbitmap->io_size >> SECTOR_SHIFT;
if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags)) continue;
sector = mddev->bitmap_info.offset + rdev->sb_start + (idx << PAGE_SECTORS_SHIFT) + block * bit_sector; md_write_metadata(mddev, rdev, sector, llbitmap->io_size, page, block * llbitmap->io_size); } }}
static void active_release(struct percpu_ref *ref){ struct llbitmap_page_ctl *pctl = container_of(ref, struct llbitmap_page_ctl, active);
wake_up(&pctl->wait);}
static void llbitmap_free_pages(struct llbitmap *llbitmap){ int i;
if (!llbitmap->pctl) return;
for (i = 0; i < llbitmap->nr_pages; i++) { struct llbitmap_page_ctl *pctl = llbitmap->pctl[i];
if (!pctl) continue; if (pctl->page) __free_page(pctl->page); percpu_ref_exit(&pctl->active); kfree(pctl); }
kfree(llbitmap->pctl); llbitmap->pctl = NULL;}
static struct llbitmap_page_ctl *llbitmap_alloc_page_ctl(struct llbitmap *llbitmap, int idx){ struct llbitmap_page_ctl *pctl; struct page *page; unsigned int size = struct_size(pctl, dirty, BITS_TO_LONGS( llbitmap->blocks_per_page));
size = round_up(size, cache_line_size()); pctl = kzalloc(size, GFP_NOIO); if (!pctl) return ERR_PTR(-ENOMEM);
page = llbitmap_read_page(llbitmap, idx);
if (IS_ERR(page)) { kfree(pctl); return ERR_CAST(page); }
if (percpu_ref_init(&pctl->active, active_release, PERCPU_REF_ALLOW_REINIT, GFP_NOIO)) { __free_page(page); kfree(pctl); return ERR_PTR(-ENOMEM); }
pctl->page = page; pctl->state = page_address(page); init_waitqueue_head(&pctl->wait); return pctl;}
static unsigned int llbitmap_reserved_pages(struct llbitmap *llbitmap){ return DIV_ROUND_UP(llbitmap->mddev->bitmap_info.space << SECTOR_SHIFT, PAGE_SIZE);}
static int llbitmap_expand_pages(struct llbitmap *llbitmap, unsigned long chunks){ struct llbitmap_page_ctl **pctl; unsigned int old_nr_pages = llbitmap->nr_pages; unsigned int nr_pages = llbitmap_used_pages(llbitmap, chunks); unsigned int i; int ret;
if (nr_pages <= old_nr_pages) return 0;
pctl = kzalloc_objs(*pctl, nr_pages, GFP_NOIO); if (!pctl) return -ENOMEM;
if (llbitmap->pctl) memcpy(pctl, llbitmap->pctl, array_size(old_nr_pages, sizeof(*pctl)));
for (i = old_nr_pages; i < nr_pages; i++) { pctl[i] = llbitmap_alloc_page_ctl(llbitmap, i); if (IS_ERR(pctl[i])) goto err_alloc_ptr; }
kfree(llbitmap->pctl); llbitmap->pctl = pctl; llbitmap->nr_pages = nr_pages; return 0;
err_alloc_ptr: ret = PTR_ERR(pctl[i]); while (i-- > old_nr_pages) { __free_page(pctl[i]->page); percpu_ref_exit(&pctl[i]->active); kfree(pctl[i]); } kfree(pctl); return ret;}
static int llbitmap_alloc_pages(struct llbitmap *llbitmap){ unsigned int used_pages = llbitmap_used_pages(llbitmap, llbitmap->chunks); unsigned int nr_pages = max(used_pages, llbitmap_reserved_pages(llbitmap)); int i;
llbitmap->pctl = kzalloc_objs(*llbitmap->pctl, nr_pages, GFP_NOIO); if (!llbitmap->pctl) return -ENOMEM;
llbitmap->nr_pages = nr_pages;
for (i = 0; i < nr_pages; i++) { llbitmap->pctl[i] = llbitmap_alloc_page_ctl(llbitmap, i); if (IS_ERR(llbitmap->pctl[i])) { int ret = PTR_ERR(llbitmap->pctl[i]);
llbitmap->pctl[i] = NULL; llbitmap_free_pages(llbitmap); return ret; } }
return 0;}
/* * Check if all underlying disks support write_zeroes with unmap. */static bool llbitmap_all_disks_support_wzeroes_unmap(struct llbitmap *llbitmap){ struct mddev *mddev = llbitmap->mddev; struct md_rdev *rdev;
rdev_for_each(rdev, mddev) { if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags)) continue;
if (bdev_write_zeroes_unmap_sectors(rdev->bdev) == 0) return false; }
return true;}
/* * Issue write_zeroes to all underlying disks to zero their data regions. * This ensures parity consistency for RAID-456 (0 XOR 0 = 0). * Returns true if all disks were successfully zeroed. */static bool llbitmap_zero_all_disks(struct llbitmap *llbitmap){ struct mddev *mddev = llbitmap->mddev; struct md_rdev *rdev; sector_t dev_sectors = mddev->dev_sectors; int ret;
rdev_for_each(rdev, mddev) { if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags)) continue;
ret = blkdev_issue_zeroout(rdev->bdev, rdev->data_offset, dev_sectors, GFP_KERNEL, 0); if (ret) { pr_warn("md/llbitmap: failed to zero disk %pg: %d\n", rdev->bdev, ret); return false; } }
return true;}
static void llbitmap_mark_range(struct llbitmap *llbitmap, unsigned long start, unsigned long end, enum llbitmap_state state){ while (start <= end) { llbitmap_write(llbitmap, state, start); start++; }}
static int llbitmap_prepare_resize(struct llbitmap *llbitmap, unsigned long old_chunks, unsigned long new_chunks, unsigned long cache_chunks){ int ret;
llbitmap_flush(llbitmap->mddev); ret = llbitmap_expand_pages(llbitmap, cache_chunks); if (ret) return ret; if (new_chunks > old_chunks) llbitmap_mark_range(llbitmap, old_chunks, new_chunks - 1, BitUnwritten); return 0;}
static enum llbitmap_statellbitmap_rmerge_state(struct llbitmap *llbitmap, enum llbitmap_state dst, enum llbitmap_state src){ bool level_456 = raid_is_456(llbitmap->mddev);
if (dst == BitNeedSync || dst == BitSyncing || src == BitNeedSync || src == BitSyncing) return BitNeedSync;
if (dst == BitDirty || src == BitDirty) return BitDirty;
/* * Reshape generates valid target parity/data for both already-written * and not-yet-written regions in the checkpointed range, so a mix of * clean and unwritten still results in a clean destination bit. */ if (level_456 && ((dst == BitClean && src == BitUnwritten) || (src == BitClean && dst == BitUnwritten))) return BitClean; if (dst == BitClean || src == BitClean) return BitClean; return BitUnwritten;}
static void llbitmap_init_state(struct llbitmap *llbitmap){ struct mddev *mddev = llbitmap->mddev; enum llbitmap_state state = BitUnwritten; unsigned long i;
if (test_and_clear_bit(BITMAP_CLEAN, &llbitmap->flags)) { state = BitClean; } else if (raid_is_456(mddev) && llbitmap_all_disks_support_wzeroes_unmap(llbitmap)) { /* * All disks support write_zeroes with unmap. Zero all disks * to ensure parity consistency, then set BitCleanUnwritten * to skip initial sync. */ if (llbitmap_zero_all_disks(llbitmap)) state = BitCleanUnwritten; }
for (i = 0; i < llbitmap->chunks; i++) llbitmap_write(llbitmap, state, i);}
/* The return value is only used from resync, where @start == @end. */static enum llbitmap_state llbitmap_state_machine(struct llbitmap *llbitmap, unsigned long start, unsigned long end, enum llbitmap_action action){ struct mddev *mddev = llbitmap->mddev; enum llbitmap_state state = BitNone; bool level_456 = raid_is_456(llbitmap->mddev); bool need_resync = false; bool need_recovery = false;
if (test_bit(BITMAP_WRITE_ERROR, &llbitmap->flags)) return BitNone;
if (action == BitmapActionInit) { llbitmap_init_state(llbitmap); return BitNone; } if (start >= llbitmap->chunks) return BitNone; if (end >= llbitmap->chunks) end = llbitmap->chunks - 1; while (start <= end) { enum llbitmap_state c = llbitmap_read(llbitmap, start);
if (c < 0 || c >= BitStateCount) { pr_err("%s: invalid bit %lu state %d action %d, forcing resync\n", __func__, start, c, action); state = BitNeedSync; goto write_bitmap; }
if (c == BitNeedSync || c == BitNeedSyncUnwritten) need_resync = !mddev->degraded;
state = state_machine[c][action];write_bitmap: if (unlikely(mddev->degraded)) { /* For degraded array, mark new data as need sync. */ if (state == BitDirty && action == BitmapActionStartwrite) state = BitNeedSync; /* * For degraded array, resync dirty data as well, noted * if array is still degraded after resync is done, all * new data will still be dirty until array is clean. */ else if (c == BitDirty && action == BitmapActionStartsync) state = BitSyncing; } else if (c == BitUnwritten && state == BitDirty && action == BitmapActionStartwrite && level_456) { /* Delay raid456 initial recovery to first write. */ state = BitNeedSync; }
if (state == BitNone) { start++; continue; }
llbitmap_write(llbitmap, state, start); if (state == BitNeedSync || state == BitNeedSyncUnwritten) need_resync = !mddev->degraded; else if (state == BitDirty && !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags) && !timer_pending(&llbitmap->pending_timer)) mod_timer(&llbitmap->pending_timer, jiffies + mddev->bitmap_info.daemon_sleep * HZ);
start++; }
if (need_resync && level_456) need_recovery = true;
if (need_recovery) { set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); set_bit(MD_RECOVERY_LAZY_RECOVER, &mddev->recovery); md_wakeup_thread(mddev->thread); } else if (need_resync) { set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); set_bit(MD_RECOVERY_SYNC, &mddev->recovery); md_wakeup_thread(mddev->thread); }
return state;}
static void llbitmap_raise_barrier(struct llbitmap *llbitmap, int page_idx){ struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx];
retry: if (likely(percpu_ref_tryget_live(&pctl->active))) { WRITE_ONCE(pctl->expire, jiffies + llbitmap->barrier_idle * HZ); return; }
wait_event(pctl->wait, !percpu_ref_is_dying(&pctl->active)); goto retry;}
static void llbitmap_release_barrier(struct llbitmap *llbitmap, int page_idx){ struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx];
percpu_ref_put(&pctl->active);}
static int llbitmap_suspend_timeout(struct llbitmap *llbitmap, int page_idx){ struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx];
percpu_ref_kill(&pctl->active);
if (!wait_event_timeout(pctl->wait, percpu_ref_is_zero(&pctl->active), llbitmap->mddev->bitmap_info.daemon_sleep * HZ)) { percpu_ref_resurrect(&pctl->active); return -ETIMEDOUT; }
return 0;}
static void llbitmap_resume(struct llbitmap *llbitmap, int page_idx){ struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx];
pctl->expire = LONG_MAX; percpu_ref_resurrect(&pctl->active); wake_up(&pctl->wait);}
static int llbitmap_check_support(struct mddev *mddev){ if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) { pr_notice("md/llbitmap: %s: array with journal cannot have bitmap\n", mdname(mddev)); return -EBUSY; }
if (mddev->bitmap_info.space == 0) { if (mddev->bitmap_info.default_space == 0) { pr_notice("md/llbitmap: %s: no space for bitmap\n", mdname(mddev)); return -ENOSPC; } }
if (!mddev->persistent) { pr_notice("md/llbitmap: %s: array must be persistent\n", mdname(mddev)); return -EOPNOTSUPP; }
if (mddev->bitmap_info.file) { pr_notice("md/llbitmap: %s: doesn't support bitmap file\n", mdname(mddev)); return -EOPNOTSUPP; }
if (mddev->bitmap_info.external) { pr_notice("md/llbitmap: %s: doesn't support external metadata\n", mdname(mddev)); return -EOPNOTSUPP; }
if (mddev_is_dm(mddev)) { pr_notice("md/llbitmap: %s: doesn't support dm-raid\n", mdname(mddev)); return -EOPNOTSUPP; }
return 0;}
static int llbitmap_init(struct llbitmap *llbitmap){ struct mddev *mddev = llbitmap->mddev; sector_t blocks = mddev->resync_max_sectors; unsigned long chunksize = MIN_CHUNK_SIZE; unsigned long chunks = DIV_ROUND_UP(blocks, chunksize); unsigned long space = mddev->bitmap_info.space << SECTOR_SHIFT; int ret;
while (chunks > space) { chunksize = chunksize << 1; chunks = DIV_ROUND_UP_SECTOR_T(blocks, chunksize); }
llbitmap->barrier_idle = DEFAULT_BARRIER_IDLE; llbitmap->chunkshift = ffz(~chunksize); llbitmap->chunksize = chunksize; llbitmap->chunks = chunks; llbitmap->sync_size = blocks; llbitmap_refresh_reshape(llbitmap); mddev->bitmap_info.daemon_sleep = DEFAULT_DAEMON_SLEEP;
ret = llbitmap_alloc_pages(llbitmap); if (ret) return ret;
llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1, BitmapActionInit); /* flush initial llbitmap to disk */ __llbitmap_flush(mddev);
return 0;}
static int llbitmap_read_sb(struct llbitmap *llbitmap){ struct mddev *mddev = llbitmap->mddev; unsigned long daemon_sleep; unsigned long chunksize; unsigned long events; sector_t sync_size; struct page *sb_page; bitmap_super_t *sb; int ret = -EINVAL;
if (!mddev->bitmap_info.offset) { pr_err("md/llbitmap: %s: no super block found", mdname(mddev)); return -EINVAL; }
sb_page = llbitmap_read_page(llbitmap, 0); if (IS_ERR(sb_page)) { pr_err("md/llbitmap: %s: read super block failed", mdname(mddev)); return -EIO; }
sb = kmap_local_page(sb_page); if (sb->magic != cpu_to_le32(BITMAP_MAGIC)) { pr_err("md/llbitmap: %s: invalid super block magic number", mdname(mddev)); goto out_put_page; }
if (sb->version != cpu_to_le32(BITMAP_MAJOR_LOCKLESS)) { pr_err("md/llbitmap: %s: invalid super block version", mdname(mddev)); goto out_put_page; }
if (memcmp(sb->uuid, mddev->uuid, 16)) { pr_err("md/llbitmap: %s: bitmap superblock UUID mismatch\n", mdname(mddev)); goto out_put_page; }
if (mddev->bitmap_info.space == 0) { int room = le32_to_cpu(sb->sectors_reserved);
if (room) mddev->bitmap_info.space = room; else mddev->bitmap_info.space = mddev->bitmap_info.default_space; } llbitmap->flags = le32_to_cpu(sb->state) & ~BIT(BITMAP_SHUTDOWN); if (test_and_clear_bit(BITMAP_FIRST_USE, &llbitmap->flags)) { ret = llbitmap_init(llbitmap); goto out_put_page; }
sync_size = le64_to_cpu(sb->sync_size); if (!sync_size) sync_size = mddev->resync_max_sectors; if (sync_size > mddev->resync_max_sectors) { pr_err("md/llbitmap: %s: sync_size %llu exceeds array sync size %llu", mdname(mddev), sync_size, mddev->resync_max_sectors); goto out_put_page; } chunksize = le32_to_cpu(sb->chunksize); if (!is_power_of_2(chunksize)) { pr_err("md/llbitmap: %s: chunksize not a power of 2", mdname(mddev)); goto out_put_page; }
if (chunksize < DIV_ROUND_UP_SECTOR_T(sync_size, mddev->bitmap_info.space << SECTOR_SHIFT)) { pr_err("md/llbitmap: %s: chunksize too small %lu < %llu / %lu", mdname(mddev), chunksize, sync_size, mddev->bitmap_info.space); goto out_put_page; }
daemon_sleep = le32_to_cpu(sb->daemon_sleep); if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT / HZ) { pr_err("md/llbitmap: %s: daemon sleep %lu period out of range", mdname(mddev), daemon_sleep); goto out_put_page; }
events = le64_to_cpu(sb->events); if (events < mddev->events) { pr_warn("md/llbitmap :%s: bitmap file is out of date (%lu < %llu) -- forcing full recovery", mdname(mddev), events, mddev->events); set_bit(BITMAP_STALE, &llbitmap->flags); }
sb->sync_size = cpu_to_le64(mddev->resync_max_sectors); mddev->bitmap_info.chunksize = chunksize; mddev->bitmap_info.daemon_sleep = daemon_sleep;
llbitmap->barrier_idle = DEFAULT_BARRIER_IDLE; llbitmap->chunksize = chunksize; llbitmap->chunks = DIV_ROUND_UP_SECTOR_T(sync_size, chunksize); llbitmap->chunkshift = ffz(~chunksize); llbitmap->sync_size = sync_size; llbitmap_refresh_reshape(llbitmap); ret = llbitmap_alloc_pages(llbitmap);
out_put_page: __free_page(sb_page); kunmap_local(sb); return ret;}
static void llbitmap_pending_timer_fn(struct timer_list *pending_timer){ struct llbitmap *llbitmap = container_of(pending_timer, struct llbitmap, pending_timer);
if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) return;
if (work_busy(&llbitmap->daemon_work)) { pr_warn("md/llbitmap: %s daemon_work not finished in %lu seconds\n", mdname(llbitmap->mddev), llbitmap->mddev->bitmap_info.daemon_sleep); set_bit(BITMAP_DAEMON_BUSY, &llbitmap->flags); return; }
queue_work(md_llbitmap_io_wq, &llbitmap->daemon_work);}
static void md_llbitmap_daemon_fn(struct work_struct *work){ struct llbitmap *llbitmap = container_of(work, struct llbitmap, daemon_work); unsigned long start; unsigned long end; bool restart; int idx;
if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) return;
if (llbitmap->mddev->degraded) return;
retry: start = 0; end = min(llbitmap->chunks, PAGE_SIZE - BITMAP_DATA_OFFSET) - 1; restart = false;
for (idx = 0; idx < llbitmap->nr_pages; idx++) { struct llbitmap_page_ctl *pctl = llbitmap->pctl[idx]; bool flush = test_and_clear_bit(LLPageFlush, &pctl->flags);
if (idx > 0) { start = end + 1; end = min(end + PAGE_SIZE, llbitmap->chunks - 1); }
if (!flush && time_before(jiffies, pctl->expire)) { restart = true; continue; }
if (llbitmap_suspend_timeout(llbitmap, idx) < 0) { pr_warn("md/llbitmap: %s: %s waiting for page %d timeout\n", mdname(llbitmap->mddev), __func__, idx); continue; }
llbitmap_state_machine(llbitmap, start, end, BitmapActionDaemon); llbitmap_resume(llbitmap, idx); }
/* * If the daemon took a long time to finish, retry to prevent missing * clearing dirty bits. */ if (test_and_clear_bit(BITMAP_DAEMON_BUSY, &llbitmap->flags)) goto retry;
/* If some page is dirty but not expired, setup timer again */ if (restart && !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) mod_timer(&llbitmap->pending_timer, jiffies + llbitmap->mddev->bitmap_info.daemon_sleep * HZ);}
static int llbitmap_create(struct mddev *mddev){ struct llbitmap *llbitmap; int ret;
ret = llbitmap_check_support(mddev); if (ret) return ret;
llbitmap = kzalloc_obj(*llbitmap, GFP_NOIO); if (!llbitmap) return -ENOMEM;
llbitmap->mddev = mddev; llbitmap->io_size = bdev_logical_block_size(mddev->gendisk->part0); llbitmap->blocks_per_page = PAGE_SIZE / llbitmap->io_size;
timer_setup(&llbitmap->pending_timer, llbitmap_pending_timer_fn, 0); INIT_WORK(&llbitmap->daemon_work, md_llbitmap_daemon_fn); rwlock_init(&llbitmap->reshape_lock); atomic_set(&llbitmap->behind_writes, 0); init_waitqueue_head(&llbitmap->behind_wait);
mutex_lock(&mddev->bitmap_info.mutex); mddev->bitmap = llbitmap; ret = llbitmap_read_sb(llbitmap); if (ret) mddev->bitmap = NULL; mutex_unlock(&mddev->bitmap_info.mutex); if (ret) { kfree(llbitmap); }
return ret;}
static int llbitmap_resize(struct mddev *mddev, sector_t blocks, int chunksize){ struct llbitmap *llbitmap = mddev->bitmap; sector_t old_blocks = llbitmap->sync_size; unsigned long old_chunks = llbitmap->chunks; unsigned long chunks; unsigned long cache_chunks; int ret = 0; unsigned long bitmap_chunksize; bool reshape; bool quiesced = false;
if (chunksize == 0) chunksize = llbitmap->chunksize;
bitmap_chunksize = chunksize; llbitmap_calculate_chunks(mddev, blocks, &bitmap_chunksize, &chunks);
reshape = mddev->delta_disks || mddev->new_level != mddev->level || mddev->new_layout != mddev->layout || mddev->new_chunk_sectors != mddev->chunk_sectors; if (!reshape && bitmap_chunksize != llbitmap->chunksize) return -EOPNOTSUPP; if (blocks == old_blocks && chunks == llbitmap->chunks) return 0;
if (mddev->pers->quiesce) { mddev->pers->quiesce(mddev, 1); quiesced = true; }
mutex_lock(&mddev->bitmap_info.mutex); cache_chunks = reshape ? max(old_chunks, chunks) : chunks; ret = llbitmap_prepare_resize(llbitmap, old_chunks, chunks, cache_chunks); if (ret) goto out;
if (reshape) { llbitmap->reshape_sync_size = blocks; llbitmap->reshape_chunksize = bitmap_chunksize; llbitmap->reshape_chunks = chunks; llbitmap->chunks = max(old_chunks, chunks); } else { if (blocks < old_blocks && chunks < old_chunks) llbitmap_mark_range(llbitmap, chunks, old_chunks - 1, BitUnwritten); mddev->bitmap_info.chunksize = bitmap_chunksize; llbitmap->chunks = chunks; llbitmap->sync_size = blocks; llbitmap_refresh_reshape(llbitmap); llbitmap_update_sb(llbitmap); } __llbitmap_flush(mddev); mutex_unlock(&mddev->bitmap_info.mutex); if (quiesced) mddev->pers->quiesce(mddev, 0); return 0;
out: mutex_unlock(&mddev->bitmap_info.mutex); if (quiesced) mddev->pers->quiesce(mddev, 0); return ret;}
static int llbitmap_load(struct mddev *mddev){ enum llbitmap_action action = BitmapActionReload; struct llbitmap *llbitmap = mddev->bitmap; int ret;
if (test_and_clear_bit(BITMAP_STALE, &llbitmap->flags)) action = BitmapActionStale;
mutex_lock(&mddev->bitmap_info.mutex); llbitmap_refresh_reshape(llbitmap); ret = llbitmap_expand_pages(llbitmap, llbitmap->chunks); if (ret) { mutex_unlock(&mddev->bitmap_info.mutex); return ret; } llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1, action); mutex_unlock(&mddev->bitmap_info.mutex); return 0;}
static void llbitmap_destroy(struct mddev *mddev){ struct llbitmap *llbitmap = mddev->bitmap;
if (!llbitmap) return;
mutex_lock(&mddev->bitmap_info.mutex);
set_bit(BITMAP_SHUTDOWN, &llbitmap->flags); timer_shutdown_sync(&llbitmap->pending_timer); cancel_work_sync(&llbitmap->daemon_work); flush_workqueue(md_llbitmap_io_wq); flush_workqueue(md_llbitmap_unplug_wq);
mddev->bitmap = NULL; llbitmap_free_pages(llbitmap); kfree(llbitmap); mutex_unlock(&mddev->bitmap_info.mutex);}
static bool llbitmap_map_previous(struct llbitmap *llbitmap, sector_t offset, unsigned long sectors){ struct mddev *mddev = llbitmap->mddev; sector_t boundary = READ_ONCE(mddev->reshape_position);
if (boundary == MaxSector) return false;
WARN_ON_ONCE(sectors && offset < boundary && offset + sectors > boundary);
return mddev->reshape_backwards ? offset < boundary : offset >= boundary;}
static void llbitmap_prepare_range(struct mddev *mddev, sector_t *offset, unsigned long *sectors, bool discard){ struct llbitmap *llbitmap = mddev->bitmap; bool previous;
if (!llbitmap) return;
previous = llbitmap_map_previous(llbitmap, *offset, *sectors); llbitmap_map_layout(llbitmap, offset, sectors, previous); if (discard) llbitmap_encode_discard_range(llbitmap, offset, sectors, previous); else llbitmap_encode_range(llbitmap, offset, sectors, previous);}
static void llbitmap_start_write(struct mddev *mddev, sector_t offset, unsigned long sectors){ struct llbitmap *llbitmap = mddev->bitmap; unsigned long start = offset >> llbitmap->chunkshift; unsigned long end = (offset + sectors - 1) >> llbitmap->chunkshift; int page_start = (start + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; int page_end = (end + BITMAP_DATA_OFFSET) >> PAGE_SHIFT;
while (page_start <= page_end) { llbitmap_raise_barrier(llbitmap, page_start); page_start++; }
read_lock(&llbitmap->reshape_lock); llbitmap_state_machine(llbitmap, start, end, BitmapActionStartwrite); read_unlock(&llbitmap->reshape_lock);}
static void llbitmap_end_write(struct mddev *mddev, sector_t offset, unsigned long sectors){ struct llbitmap *llbitmap = mddev->bitmap; unsigned long start = offset >> llbitmap->chunkshift; unsigned long end = (offset + sectors - 1) >> llbitmap->chunkshift; int page_start = (start + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; int page_end = (end + BITMAP_DATA_OFFSET) >> PAGE_SHIFT;
while (page_start <= page_end) { llbitmap_release_barrier(llbitmap, page_start); page_start++; }}
static void llbitmap_start_discard(struct mddev *mddev, sector_t offset, unsigned long sectors){ struct llbitmap *llbitmap = mddev->bitmap; unsigned long start = DIV_ROUND_UP_SECTOR_T(offset, llbitmap->chunksize); unsigned long end = (offset + sectors - 1) >> llbitmap->chunkshift; int page_start = (start + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; int page_end = (end + BITMAP_DATA_OFFSET) >> PAGE_SHIFT;
while (page_start <= page_end) { llbitmap_raise_barrier(llbitmap, page_start); page_start++; }
read_lock(&llbitmap->reshape_lock); llbitmap_state_machine(llbitmap, start, end, BitmapActionDiscard); read_unlock(&llbitmap->reshape_lock);}
static void llbitmap_end_discard(struct mddev *mddev, sector_t offset, unsigned long sectors){ struct llbitmap *llbitmap = mddev->bitmap; unsigned long start = DIV_ROUND_UP_SECTOR_T(offset, llbitmap->chunksize); unsigned long end = (offset + sectors - 1) >> llbitmap->chunkshift; int page_start = (start + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; int page_end = (end + BITMAP_DATA_OFFSET) >> PAGE_SHIFT;
while (page_start <= page_end) { llbitmap_release_barrier(llbitmap, page_start); page_start++; }}
static void llbitmap_unplug_fn(struct work_struct *work){ struct llbitmap_unplug_work *unplug_work = container_of(work, struct llbitmap_unplug_work, work); struct llbitmap *llbitmap = unplug_work->llbitmap; struct blk_plug plug; int i;
blk_start_plug(&plug);
for (i = 0; i < llbitmap->nr_pages; i++) { if (!test_bit(LLPageDirty, &llbitmap->pctl[i]->flags) || !test_and_clear_bit(LLPageDirty, &llbitmap->pctl[i]->flags)) continue;
llbitmap_write_page(llbitmap, i); }
blk_finish_plug(&plug); md_super_wait(llbitmap->mddev); complete(unplug_work->done);}
static bool llbitmap_dirty(struct llbitmap *llbitmap){ int i;
for (i = 0; i < llbitmap->nr_pages; i++) if (test_bit(LLPageDirty, &llbitmap->pctl[i]->flags)) return true;
return false;}
static void llbitmap_unplug(struct mddev *mddev, bool sync){ DECLARE_COMPLETION_ONSTACK(done); struct llbitmap *llbitmap = mddev->bitmap; struct llbitmap_unplug_work unplug_work = { .llbitmap = llbitmap, .done = &done, };
if (!llbitmap_dirty(llbitmap)) return;
/* * Issue new bitmap IO under submit_bio() context will deadlock: * - the bio will wait for bitmap bio to be done, before it can be * issued; * - bitmap bio will be added to current->bio_list and wait for this * bio to be issued; */ INIT_WORK_ONSTACK(&unplug_work.work, llbitmap_unplug_fn); queue_work(md_llbitmap_unplug_wq, &unplug_work.work); wait_for_completion(&done); destroy_work_on_stack(&unplug_work.work);}
/* * Force to write all bitmap pages to disk, called when stopping the array, or * every daemon_sleep seconds when sync_thread is running. */static void __llbitmap_flush(struct mddev *mddev){ struct llbitmap *llbitmap = mddev->bitmap; struct blk_plug plug; int i;
blk_start_plug(&plug); for (i = 0; i < llbitmap->nr_pages; i++) { struct llbitmap_page_ctl *pctl = llbitmap->pctl[i];
/* mark all blocks as dirty */ set_bit(LLPageDirty, &pctl->flags); bitmap_fill(pctl->dirty, llbitmap->blocks_per_page); llbitmap_write_page(llbitmap, i); } blk_finish_plug(&plug); md_super_wait(llbitmap->mddev);}
static void llbitmap_flush(struct mddev *mddev){ struct llbitmap *llbitmap = mddev->bitmap; int i;
for (i = 0; i < llbitmap->nr_pages; i++) set_bit(LLPageFlush, &llbitmap->pctl[i]->flags);
timer_delete_sync(&llbitmap->pending_timer); queue_work(md_llbitmap_io_wq, &llbitmap->daemon_work); flush_work(&llbitmap->daemon_work);
__llbitmap_flush(mddev);}
/* This is used for raid5 lazy initial recovery */static bool llbitmap_blocks_synced(struct mddev *mddev, sector_t offset){ struct llbitmap *llbitmap = mddev->bitmap; unsigned long p = offset >> llbitmap->chunkshift; enum llbitmap_state c;
if (p >= llbitmap->chunks) return false; c = llbitmap_read(llbitmap, p);
return c == BitClean || c == BitDirty || c == BitCleanUnwritten;}
static sector_t llbitmap_skip_sync_blocks(struct mddev *mddev, sector_t offset){ struct llbitmap *llbitmap = mddev->bitmap; unsigned long p = offset >> llbitmap->chunkshift; int blocks = llbitmap->chunksize - (offset & (llbitmap->chunksize - 1)); enum llbitmap_state c;
if (p >= llbitmap->chunks) return 0; c = llbitmap_read(llbitmap, p);
/* * Reshape progress is tracked by array metadata rather than llbitmap. * Skipping reshape ranges from stale bitmap state can lose data after a * restart before the corresponding bits are checkpointed to disk. */ if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) return 0;
/* always skip unwritten blocks */ if (c == BitUnwritten) return blocks;
/* Skip CleanUnwritten - no user data, will be reset after recovery */ if (c == BitCleanUnwritten) return blocks;
/* For degraded array, don't skip */ if (mddev->degraded) return 0;
/* For resync also skip clean/dirty blocks */ if ((c == BitClean || c == BitDirty) && test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) return blocks;
return 0;}
static bool llbitmap_start_sync(struct mddev *mddev, sector_t offset, sector_t *blocks, bool degraded){ struct llbitmap *llbitmap = mddev->bitmap; unsigned long p = offset >> llbitmap->chunkshift; enum llbitmap_state state;
/* * Before recovery starts, convert CleanUnwritten to Unwritten. * This ensures the new disk won't have stale parity data. */ if (offset == 0 && test_bit(MD_RECOVERY_RECOVER, &mddev->recovery) && !test_bit(MD_RECOVERY_LAZY_RECOVER, &mddev->recovery)) llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1, BitmapActionClearUnwritten);
/* * Handle one bit at a time, this is much simpler. And it doesn't matter * if md_do_sync() loop more times. */ *blocks = llbitmap->chunksize - (offset & (llbitmap->chunksize - 1)); if (p >= llbitmap->chunks) return false; state = llbitmap_state_machine(llbitmap, p, p, BitmapActionStartsync); return state == BitSyncing || state == BitSyncingUnwritten;}
/* Something is wrong, sync_thread stop at @offset */static void llbitmap_end_sync(struct mddev *mddev, sector_t offset, sector_t *blocks){ struct llbitmap *llbitmap = mddev->bitmap; unsigned long p = offset >> llbitmap->chunkshift;
*blocks = llbitmap->chunksize - (offset & (llbitmap->chunksize - 1)); llbitmap_state_machine(llbitmap, p, llbitmap->chunks - 1, BitmapActionAbortsync);}
/* A full sync_thread is finished */static void llbitmap_close_sync(struct mddev *mddev){ struct llbitmap *llbitmap = mddev->bitmap; int i;
for (i = 0; i < llbitmap->nr_pages; i++) { struct llbitmap_page_ctl *pctl = llbitmap->pctl[i];
/* let daemon_fn clear dirty bits immediately */ WRITE_ONCE(pctl->expire, jiffies); }
llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1, BitmapActionEndsync);}
/* * sync_thread have reached @sector, update metadata every daemon_sleep seconds, * just in case sync_thread have to restart after power failure. */static void llbitmap_cond_end_sync(struct mddev *mddev, sector_t sector, bool force){ struct llbitmap *llbitmap = mddev->bitmap; sector_t complete;
if (sector == 0) { llbitmap->last_end_sync = jiffies; return; }
if (!force && time_before(jiffies, llbitmap->last_end_sync + HZ * mddev->bitmap_info.daemon_sleep)) return;
wait_event(mddev->recovery_wait, !atomic_read(&mddev->recovery_active));
mddev->curr_resync_completed = sector; set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
complete = round_down(sector, llbitmap->chunksize); if (complete) llbitmap_state_machine(llbitmap, 0, (complete >> llbitmap->chunkshift) - 1, BitmapActionEndsync); __llbitmap_flush(mddev);
llbitmap->last_end_sync = jiffies; sysfs_notify_dirent_safe(mddev->sysfs_completed);}
static bool llbitmap_enabled(void *data, bool flush){ struct llbitmap *llbitmap = data;
return llbitmap && !test_bit(BITMAP_WRITE_ERROR, &llbitmap->flags);}
static void llbitmap_dirty_bits(struct mddev *mddev, unsigned long s, unsigned long e){ llbitmap_state_machine(mddev->bitmap, s, e, BitmapActionStartwrite);}
static int llbitmap_reshape_can_start(struct mddev *mddev){ struct llbitmap *llbitmap = mddev->bitmap; unsigned long chunk; int ret = 0;
if (!llbitmap) return 0;
mutex_lock(&mddev->bitmap_info.mutex); for (chunk = 0; chunk < llbitmap->chunks; chunk++) { enum llbitmap_state state = llbitmap_read(llbitmap, chunk);
if (state == BitNeedSync || state == BitSyncing) { ret = -EBUSY; break; } } mutex_unlock(&mddev->bitmap_info.mutex);
return ret;}
struct llbitmap_reshape_range { sector_t offset; unsigned long sectors; sector_t start; sector_t end;};
static enum llbitmap_statellbitmap_reshape_init_dst(struct llbitmap *llbitmap, unsigned long dst, const struct llbitmap_reshape_range *new){ u64 bit_start = (u64)dst * llbitmap->reshape_chunksize; u64 bit_end = bit_start + llbitmap->reshape_chunksize;
if (!llbitmap->mddev->reshape_backwards) return bit_start < new->offset ? llbitmap_read(llbitmap, dst) : BitUnwritten; return bit_end > new->end ? llbitmap_read(llbitmap, dst) : BitUnwritten;}
static void llbitmap_reshape_dst_range(struct llbitmap *llbitmap, unsigned long dst, const struct llbitmap_reshape_range *new, struct llbitmap_reshape_range *dst_range){ sector_t dst_bit_start = (sector_t)dst * llbitmap->reshape_chunksize;
dst_range->start = max(dst_bit_start, new->offset); dst_range->end = min(dst_bit_start + llbitmap->reshape_chunksize, new->end); dst_range->offset = dst_range->start; dst_range->sectors = dst_range->end - dst_range->start;}
static void llbitmap_reshape_map_range(struct llbitmap *llbitmap, sector_t lo, sector_t hi, bool previous, struct llbitmap_reshape_range *range){ range->offset = lo; range->sectors = hi - lo; llbitmap_map_layout(llbitmap, &range->offset, &range->sectors, previous); range->start = range->offset; range->end = range->offset + range->sectors;}
static bool llbitmap_reshape_src_range(const struct llbitmap_reshape_range *old, const struct llbitmap_reshape_range *new, const struct llbitmap_reshape_range *dst, struct llbitmap_reshape_range *src){ if (!old->sectors) return false;
src->start = old->offset + mul_u64_u64_div_u64(dst->start - new->offset, old->sectors, new->sectors); src->end = old->offset + mul_u64_u64_div_u64_roundup(dst->end - new->offset, old->sectors, new->sectors); if (src->end > old->end) src->end = old->end; src->offset = src->start; src->sectors = src->end - src->start;
return src->sectors;}
static enum llbitmap_state llbitmap_rmerge_src(struct llbitmap *llbitmap, enum llbitmap_state state, const struct llbitmap_reshape_range *src){ unsigned long bit = div64_u64(src->start, llbitmap->chunksize); unsigned long end = div64_u64(src->end - 1, llbitmap->chunksize);
while (bit <= end) { enum llbitmap_state src_state = llbitmap_read(llbitmap, bit);
state = llbitmap_rmerge_state(llbitmap, state, src_state); bit++; }
return state;}
static void llbitmap_reshape_merge(struct llbitmap *llbitmap, const struct llbitmap_reshape_range *old, const struct llbitmap_reshape_range *new){ unsigned long dst_start; unsigned long dst_end; unsigned long dst; bool backwards = false;
if (!new->sectors) return;
dst_start = div64_u64(new->offset, llbitmap->reshape_chunksize); dst_end = div64_u64(new->end - 1, llbitmap->reshape_chunksize); if (old->sectors) { unsigned long src_start = div64_u64(old->offset, llbitmap->chunksize); unsigned long src_end = div64_u64(old->end - 1, llbitmap->chunksize);
backwards = src_start < dst_start && src_end >= dst_start; }
dst = backwards ? dst_end : dst_start; while (true) { struct llbitmap_reshape_range dst_range; struct llbitmap_reshape_range src; enum llbitmap_state state;
llbitmap_reshape_dst_range(llbitmap, dst, new, &dst_range); state = llbitmap_reshape_init_dst(llbitmap, dst, new); if (llbitmap_reshape_src_range(old, new, &dst_range, &src)) state = llbitmap_rmerge_src(llbitmap, state, &src); else state = llbitmap_rmerge_state(llbitmap, state, BitUnwritten); llbitmap_write(llbitmap, state, dst); if (dst == (backwards ? dst_start : dst_end)) break; if (backwards) dst--; else dst++; }}
static void llbitmap_reshape_finish(struct mddev *mddev){ struct llbitmap *llbitmap = mddev->bitmap;
if (mddev->pers->quiesce) mddev->pers->quiesce(mddev, 1);
mutex_lock(&mddev->bitmap_info.mutex); llbitmap_flush(mddev);
llbitmap->chunksize = llbitmap->reshape_chunksize; llbitmap->chunkshift = ffz(~llbitmap->chunksize); llbitmap->chunks = llbitmap->reshape_chunks; llbitmap->sync_size = llbitmap->reshape_sync_size; llbitmap_refresh_reshape(llbitmap); mddev->bitmap_info.chunksize = llbitmap->chunksize; llbitmap_update_sb(llbitmap); __llbitmap_flush(mddev); mutex_unlock(&mddev->bitmap_info.mutex);
if (mddev->pers->quiesce) mddev->pers->quiesce(mddev, 0);}
static void llbitmap_reshape_mark(struct mddev *mddev, sector_t old_pos, sector_t new_pos){ struct llbitmap *llbitmap = mddev->bitmap; sector_t lo; sector_t hi; struct llbitmap_reshape_range old; struct llbitmap_reshape_range new;
if (!llbitmap || old_pos == new_pos) return;
lo = min(old_pos, new_pos); hi = max(old_pos, new_pos); if (!hi) return;
llbitmap_reshape_map_range(llbitmap, lo, hi, true, &old); llbitmap_reshape_map_range(llbitmap, lo, hi, false, &new); if (!new.sectors) return;
write_lock(&llbitmap->reshape_lock); llbitmap_reshape_merge(llbitmap, &old, &new); write_unlock(&llbitmap->reshape_lock);}
static void llbitmap_write_sb(struct llbitmap *llbitmap){ int nr_blocks = DIV_ROUND_UP(BITMAP_DATA_OFFSET, llbitmap->io_size);
bitmap_fill(llbitmap->pctl[0]->dirty, nr_blocks); llbitmap_write_page(llbitmap, 0); md_super_wait(llbitmap->mddev);}
static void llbitmap_update_sb(void *data){ struct llbitmap *llbitmap = data; struct mddev *mddev = llbitmap->mddev; struct page *sb_page; bitmap_super_t *sb;
if (test_bit(BITMAP_WRITE_ERROR, &llbitmap->flags)) return;
sb_page = llbitmap_read_page(llbitmap, 0); if (IS_ERR(sb_page)) { pr_err("%s: %s: read super block failed", __func__, mdname(mddev)); set_bit(BITMAP_WRITE_ERROR, &llbitmap->flags); return; }
if (mddev->events < llbitmap->events_cleared) llbitmap->events_cleared = mddev->events;
sb = kmap_local_page(sb_page); sb->events = cpu_to_le64(mddev->events); sb->state = cpu_to_le32(llbitmap->flags & ~BIT(BITMAP_SHUTDOWN)); sb->chunksize = cpu_to_le32(llbitmap->chunksize); sb->sync_size = cpu_to_le64(llbitmap->sync_size); sb->events_cleared = cpu_to_le64(llbitmap->events_cleared); sb->sectors_reserved = cpu_to_le32(mddev->bitmap_info.space); sb->daemon_sleep = cpu_to_le32(mddev->bitmap_info.daemon_sleep);
kunmap_local(sb); llbitmap_write_sb(llbitmap);}
static int llbitmap_get_stats(void *data, struct md_bitmap_stats *stats){ struct llbitmap *llbitmap = data;
memset(stats, 0, sizeof(*stats));
stats->missing_pages = 0; stats->pages = llbitmap->nr_pages; stats->file_pages = llbitmap->nr_pages; stats->sync_size = llbitmap->sync_size;
stats->behind_writes = atomic_read(&llbitmap->behind_writes); stats->behind_wait = wq_has_sleeper(&llbitmap->behind_wait); stats->events_cleared = llbitmap->events_cleared;
return 0;}
/* just flag all pages as needing to be written */static void llbitmap_write_all(struct mddev *mddev){ int i; struct llbitmap *llbitmap = mddev->bitmap;
for (i = 0; i < llbitmap->nr_pages; i++) { struct llbitmap_page_ctl *pctl = llbitmap->pctl[i];
set_bit(LLPageDirty, &pctl->flags); bitmap_fill(pctl->dirty, llbitmap->blocks_per_page); }}
static void llbitmap_start_behind_write(struct mddev *mddev){ struct llbitmap *llbitmap = mddev->bitmap;
atomic_inc(&llbitmap->behind_writes);}
static void llbitmap_end_behind_write(struct mddev *mddev){ struct llbitmap *llbitmap = mddev->bitmap;
if (atomic_dec_and_test(&llbitmap->behind_writes)) wake_up(&llbitmap->behind_wait);}
static void llbitmap_wait_behind_writes(struct mddev *mddev){ struct llbitmap *llbitmap = mddev->bitmap;
if (llbitmap && atomic_read(&llbitmap->behind_writes) > 0) wait_event(llbitmap->behind_wait, atomic_read(&llbitmap->behind_writes) == 0);}
static ssize_t bits_show(struct mddev *mddev, char *page){ struct llbitmap *llbitmap; int bits[BitStateCount] = {0}; loff_t start = 0;
mutex_lock(&mddev->bitmap_info.mutex); llbitmap = mddev->bitmap; if (!llbitmap || !llbitmap->pctl) { mutex_unlock(&mddev->bitmap_info.mutex); return sprintf(page, "no bitmap\n"); }
if (test_bit(BITMAP_WRITE_ERROR, &llbitmap->flags)) { mutex_unlock(&mddev->bitmap_info.mutex); return sprintf(page, "bitmap io error\n"); }
while (start < llbitmap->chunks) { enum llbitmap_state c = llbitmap_read(llbitmap, start);
if (c < 0 || c >= BitStateCount) pr_err("%s: invalid bit %llu state %d\n", __func__, start, c); else bits[c]++; start++; }
mutex_unlock(&mddev->bitmap_info.mutex); return sprintf(page, "unwritten %d\nclean %d\ndirty %d\n" "need sync %d\nsyncing %d\n" "need sync unwritten %d\nsyncing unwritten %d\n" "clean unwritten %d\n", bits[BitUnwritten], bits[BitClean], bits[BitDirty], bits[BitNeedSync], bits[BitSyncing], bits[BitNeedSyncUnwritten], bits[BitSyncingUnwritten], bits[BitCleanUnwritten]);}
static struct md_sysfs_entry llbitmap_bits = __ATTR_RO(bits);
static ssize_t metadata_show(struct mddev *mddev, char *page){ struct llbitmap *llbitmap; ssize_t ret;
mutex_lock(&mddev->bitmap_info.mutex); llbitmap = mddev->bitmap; if (!llbitmap) { mutex_unlock(&mddev->bitmap_info.mutex); return sprintf(page, "no bitmap\n"); }
ret = sprintf(page, "chunksize %lu\nchunkshift %lu\nchunks %lu\noffset %llu\ndaemon_sleep %lu\n", llbitmap->chunksize, llbitmap->chunkshift, llbitmap->chunks, mddev->bitmap_info.offset, llbitmap->mddev->bitmap_info.daemon_sleep); mutex_unlock(&mddev->bitmap_info.mutex);
return ret;}
static struct md_sysfs_entry llbitmap_metadata = __ATTR_RO(metadata);
static ssize_tdaemon_sleep_show(struct mddev *mddev, char *page){ return sprintf(page, "%lu\n", mddev->bitmap_info.daemon_sleep);}
static ssize_tdaemon_sleep_store(struct mddev *mddev, const char *buf, size_t len){ unsigned long timeout; int rv = kstrtoul(buf, 10, &timeout);
if (rv) return rv;
mddev->bitmap_info.daemon_sleep = timeout; return len;}
static struct md_sysfs_entry llbitmap_daemon_sleep = __ATTR_RW(daemon_sleep);
static ssize_tbarrier_idle_show(struct mddev *mddev, char *page){ struct llbitmap *llbitmap = mddev->bitmap;
return sprintf(page, "%lu\n", llbitmap->barrier_idle);}
static ssize_tbarrier_idle_store(struct mddev *mddev, const char *buf, size_t len){ struct llbitmap *llbitmap = mddev->bitmap; unsigned long timeout; int rv = kstrtoul(buf, 10, &timeout);
if (rv) return rv;
llbitmap->barrier_idle = timeout; return len;}
static struct md_sysfs_entry llbitmap_barrier_idle = __ATTR_RW(barrier_idle);
static ssize_tproactive_sync_store(struct mddev *mddev, const char *buf, size_t len){ struct llbitmap *llbitmap;
/* Only for RAID-456 */ if (!raid_is_456(mddev)) return -EINVAL;
mutex_lock(&mddev->bitmap_info.mutex); llbitmap = mddev->bitmap; if (!llbitmap || !llbitmap->pctl) { mutex_unlock(&mddev->bitmap_info.mutex); return -ENODEV; }
/* Trigger proactive sync on all Unwritten regions */ llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1, BitmapActionProactiveSync);
mutex_unlock(&mddev->bitmap_info.mutex); return len;}
static struct md_sysfs_entry llbitmap_proactive_sync = __ATTR(proactive_sync, 0200, NULL, proactive_sync_store);
static struct attribute *md_llbitmap_attrs[] = { &llbitmap_bits.attr, &llbitmap_metadata.attr, &llbitmap_daemon_sleep.attr, &llbitmap_barrier_idle.attr, &llbitmap_proactive_sync.attr, NULL};
static struct attribute_group md_llbitmap_group = { .name = "llbitmap", .attrs = md_llbitmap_attrs,};
static const struct attribute_group *md_llbitmap_groups[] = { &md_llbitmap_group, NULL,};
static struct bitmap_operations llbitmap_ops = { .head = { .type = MD_BITMAP, .id = ID_LLBITMAP, .name = "llbitmap", },
.enabled = llbitmap_enabled, .create = llbitmap_create, .resize = llbitmap_resize, .load = llbitmap_load, .destroy = llbitmap_destroy,
.start_write = llbitmap_start_write, .end_write = llbitmap_end_write, .start_discard = llbitmap_start_discard, .end_discard = llbitmap_end_discard, .unplug = llbitmap_unplug, .flush = llbitmap_flush,
.start_behind_write = llbitmap_start_behind_write, .end_behind_write = llbitmap_end_behind_write, .wait_behind_writes = llbitmap_wait_behind_writes,
.blocks_synced = llbitmap_blocks_synced, .skip_sync_blocks = llbitmap_skip_sync_blocks, .start_sync = llbitmap_start_sync, .end_sync = llbitmap_end_sync, .close_sync = llbitmap_close_sync, .cond_end_sync = llbitmap_cond_end_sync,
.update_sb = llbitmap_update_sb, .get_stats = llbitmap_get_stats, .dirty_bits = llbitmap_dirty_bits, .prepare_range = llbitmap_prepare_range, .reshape_finish = llbitmap_reshape_finish, .reshape_can_start = llbitmap_reshape_can_start, .reshape_mark = llbitmap_reshape_mark, .write_all = llbitmap_write_all,
.groups = md_llbitmap_groups,};
int md_llbitmap_init(void){ md_llbitmap_io_wq = alloc_workqueue("md_llbitmap_io", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); if (!md_llbitmap_io_wq) return -ENOMEM;
md_llbitmap_unplug_wq = alloc_workqueue("md_llbitmap_unplug", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); if (!md_llbitmap_unplug_wq) { destroy_workqueue(md_llbitmap_io_wq); md_llbitmap_io_wq = NULL; return -ENOMEM; }
return register_md_submodule(&llbitmap_ops.head);}
void md_llbitmap_exit(void){ destroy_workqueue(md_llbitmap_io_wq); md_llbitmap_io_wq = NULL; destroy_workqueue(md_llbitmap_unplug_wq); md_llbitmap_unplug_wq = NULL; unregister_md_submodule(&llbitmap_ops.head);}