Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616// SPDX-License-Identifier: GPL-2.0/* * fs/f2fs/node.c * * Copyright (c) 2012 Samsung Electronics Co., Ltd. * http://www.samsung.com/ */#include <linux/fs.h>#include <linux/f2fs_fs.h>#include <linux/sched/mm.h>#include <linux/blkdev.h>#include <linux/folio_batch.h>#include <linux/swap.h>#include <linux/fserror.h>#include <linux/freezer.h>
#include "f2fs.h"#include "node.h"#include "segment.h"#include "xattr.h"#include "iostat.h"#include <trace/events/f2fs.h>
#define on_f2fs_build_free_nids(nm_i) mutex_is_locked(&(nm_i)->build_lock)
static struct kmem_cache *nat_entry_slab;static struct kmem_cache *free_nid_slab;static struct kmem_cache *nat_entry_set_slab;static struct kmem_cache *fsync_node_entry_slab;
static inline bool is_invalid_nid(struct f2fs_sb_info *sbi, nid_t nid){ return nid < F2FS_ROOT_INO(sbi) || nid >= NM_I(sbi)->max_nid;}
/* * Check whether the given nid is within node id range. */int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid){ if (unlikely(is_invalid_nid(sbi, nid))) { set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_warn(sbi, "%s: out-of-range nid=%x, run fsck to fix.", __func__, nid); f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); return -EFSCORRUPTED; } return 0;}
bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; struct sysinfo val; unsigned long avail_ram; unsigned long mem_size = 0; bool res = false;
if (!nm_i) return true;
si_meminfo(&val);
/* only uses low memory */ avail_ram = val.totalram - val.totalhigh;
/* * give 25%, 25%, 50%, 50%, 25%, 25% memory for each components respectively */ if (type == FREE_NIDS) { mem_size = (nm_i->nid_cnt[FREE_NID] * sizeof(struct free_nid)) >> PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); } else if (type == NAT_ENTRIES) { /* * nat_cnt[] is heuristic accounting. Sample it locklessly here * to avoid taking nat_tree_lock in the balance path. */ mem_size = (data_race(READ_ONCE(nm_i->nat_cnt[TOTAL_NAT])) * sizeof(struct nat_entry)) >> PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); if (excess_cached_nats(sbi)) res = false; } else if (type == DIRTY_DENTS) { if (bdi_wb_dirty_exceeded(sbi->sb->s_bdi)) return false; mem_size = get_nr_caches(sbi, F2FS_DIRTY_DENTS); res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); } else if (type == INO_ENTRIES) { int i;
for (i = 0; i < MAX_INO_ENTRY; i++) mem_size += sbi->im[i].ino_num * sizeof(struct ino_entry); mem_size >>= PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); } else if (type == READ_EXTENT_CACHE || type == AGE_EXTENT_CACHE) { enum extent_type etype = type == READ_EXTENT_CACHE ? EX_READ : EX_BLOCK_AGE; struct extent_tree_info *eti = &sbi->extent_tree[etype];
mem_size = (atomic_read(&eti->total_ext_tree) * sizeof(struct extent_tree) + atomic_read(&eti->total_ext_node) * sizeof(struct extent_node)) >> PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); } else if (type == DISCARD_CACHE) { mem_size = (atomic_read(&dcc->discard_cmd_cnt) * sizeof(struct discard_cmd)) >> PAGE_SHIFT; res = mem_size < (avail_ram * nm_i->ram_thresh / 100); } else if (type == COMPRESS_BLOCK) {#ifdef CONFIG_F2FS_FS_COMPRESSION unsigned long free_ram = val.freeram;
/* * free memory is lower than watermark or cached page count * exceed threshold, deny caching compress page. */ res = (free_ram > avail_ram * sbi->compress_watermark / 100) && (COMPRESS_CACHE(sbi)->num_entries < free_ram * sbi->compress_percent / 100);#else res = false;#endif } else if (type == META_BLOCK) { mem_size = (META_CACHE(sbi)->num_entries * (sizeof(struct f2fs_cached_block) + F2FS_BLKSIZE(sbi))) >> PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); } else if (type == NODE_BLOCK) { mem_size = (NODE_CACHE(sbi)->num_entries * (sizeof(struct f2fs_cached_block) + F2FS_BLKSIZE(sbi))) >> PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); } else { if (!bdi_wb_dirty_exceeded(sbi->sb->s_bdi)) return true; } return res;}
static struct f2fs_cached_block *get_current_nat_cache(struct f2fs_sb_info *sbi, nid_t nid){ return f2fs_get_meta_cache_retry(sbi, current_nat_addr(sbi, nid));}
static struct f2fs_cached_block *get_next_nat_cache(struct f2fs_sb_info *sbi, nid_t nid){ struct f2fs_cached_block *src_entry; struct f2fs_cached_block *dst_entry; pgoff_t dst_off; struct f2fs_nm_info *nm_i = NM_I(sbi);
dst_off = next_nat_addr(sbi, current_nat_addr(sbi, nid));
/* get current nat cached block with lock */ src_entry = get_current_nat_cache(sbi, nid); if (IS_ERR(src_entry)) return src_entry; dst_entry = f2fs_grab_meta_cache(sbi, dst_off); f2fs_bug_on(sbi, f2fs_cache_test_dirty(src_entry));
memcpy(cache_address(dst_entry), cache_address(src_entry), F2FS_BLKSIZE(sbi)); f2fs_mark_cache_dirty(dst_entry); f2fs_put_cache(src_entry, true);
set_to_next_nat(sbi, nm_i, nid);
return dst_entry;}
static struct nat_entry *__alloc_nat_entry(struct f2fs_sb_info *sbi, nid_t nid, bool no_fail){ struct nat_entry *new;
new = f2fs_kmem_cache_alloc(nat_entry_slab, GFP_F2FS_ZERO, no_fail, sbi); if (new) { nat_set_nid(new, nid); nat_reset_flag(new); } return new;}
static void __free_nat_entry(struct nat_entry *e){ kmem_cache_free(nat_entry_slab, e);}
/* must be locked by nat_tree_lock */static struct nat_entry *__init_nat_entry(struct f2fs_nm_info *nm_i, struct nat_entry *ne, struct f2fs_nat_entry *raw_ne, bool no_fail, bool init_dirty){ if (no_fail) f2fs_radix_tree_insert(&nm_i->nat_root, nat_get_nid(ne), ne); else if (radix_tree_insert(&nm_i->nat_root, nat_get_nid(ne), ne)) return NULL;
if (raw_ne) node_info_from_raw_nat(&ne->ni, raw_ne);
if (init_dirty) { INIT_LIST_HEAD(&ne->list); nm_i->nat_cnt[TOTAL_NAT]++; return ne; }
spin_lock(&nm_i->nat_list_lock); list_add_tail(&ne->list, &nm_i->nat_entries); spin_unlock(&nm_i->nat_list_lock);
nm_i->nat_cnt[TOTAL_NAT]++; nm_i->nat_cnt[RECLAIMABLE_NAT]++; return ne;}
static struct nat_entry *__lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t n, bool for_dirty){ struct nat_entry *ne;
ne = radix_tree_lookup(&nm_i->nat_root, n);
/* * for recent accessed nat entry which will not be dirtied soon * later, move it to tail of lru list. */ if (ne && !get_nat_flag(ne, IS_DIRTY) && !for_dirty) { spin_lock(&nm_i->nat_list_lock); if (!list_empty(&ne->list)) list_move_tail(&ne->list, &nm_i->nat_entries); spin_unlock(&nm_i->nat_list_lock); }
return ne;}
static unsigned int __gang_lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t start, unsigned int nr, struct nat_entry **ep){ return radix_tree_gang_lookup(&nm_i->nat_root, (void **)ep, start, nr);}
static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e){ radix_tree_delete(&nm_i->nat_root, nat_get_nid(e)); nm_i->nat_cnt[TOTAL_NAT]--; nm_i->nat_cnt[RECLAIMABLE_NAT]--; __free_nat_entry(e);}
static struct nat_entry_set *__grab_nat_entry_set(struct f2fs_sb_info *sbi, struct f2fs_nm_info *nm_i, struct nat_entry *ne){ nid_t set = NAT_BLOCK_OFFSET(sbi, ne->ni.nid); struct nat_entry_set *head;
head = radix_tree_lookup(&nm_i->nat_set_root, set); if (!head) { head = f2fs_kmem_cache_alloc(nat_entry_set_slab, GFP_NOFS, true, NULL);
INIT_LIST_HEAD(&head->entry_list); INIT_LIST_HEAD(&head->set_list); head->set = set; head->entry_cnt = 0; f2fs_radix_tree_insert(&nm_i->nat_set_root, set, head); } return head;}
static void __set_nat_cache_dirty(struct f2fs_sb_info *sbi, struct f2fs_nm_info *nm_i, struct nat_entry *ne, bool init_dirty){ struct nat_entry_set *head; bool new_ne = nat_get_blkaddr(ne) == NEW_ADDR;
if (!new_ne) head = __grab_nat_entry_set(sbi, nm_i, ne);
/* * update entry_cnt in below condition: * 1. update NEW_ADDR to valid block address; * 2. update old block address to new one; */ if (!new_ne && (get_nat_flag(ne, IS_PREALLOC) || !get_nat_flag(ne, IS_DIRTY))) head->entry_cnt++;
set_nat_flag(ne, IS_PREALLOC, new_ne);
if (get_nat_flag(ne, IS_DIRTY)) goto refresh_list;
nm_i->nat_cnt[DIRTY_NAT]++; if (!init_dirty) nm_i->nat_cnt[RECLAIMABLE_NAT]--; set_nat_flag(ne, IS_DIRTY, true);refresh_list: spin_lock(&nm_i->nat_list_lock); if (new_ne) list_del_init(&ne->list); else list_move_tail(&ne->list, &head->entry_list); spin_unlock(&nm_i->nat_list_lock);}
static void __clear_nat_cache_dirty(struct f2fs_nm_info *nm_i, struct nat_entry_set *set, struct nat_entry *ne){ spin_lock(&nm_i->nat_list_lock); list_move_tail(&ne->list, &nm_i->nat_entries); spin_unlock(&nm_i->nat_list_lock);
set_nat_flag(ne, IS_DIRTY, false); set->entry_cnt--; nm_i->nat_cnt[DIRTY_NAT]--; nm_i->nat_cnt[RECLAIMABLE_NAT]++;}
static unsigned int __gang_lookup_nat_set(struct f2fs_nm_info *nm_i, nid_t start, unsigned int nr, struct nat_entry_set **ep){ return radix_tree_gang_lookup(&nm_i->nat_set_root, (void **)ep, start, nr);}
bool f2fs_in_warm_node_list(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry){ return f2fs_is_node_cache(entry) && IS_DNODE(sbi, entry) && is_cold_node(sbi, entry);}
void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi){ spin_lock_init(&sbi->fsync_node_lock); INIT_LIST_HEAD(&sbi->fsync_node_list); sbi->fsync_seg_id = 0; sbi->fsync_node_num = 0;}
static unsigned int f2fs_add_fsync_node_entry(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry){ struct fsync_node_entry *fn; unsigned long flags; unsigned int seq_id;
fn = f2fs_kmem_cache_alloc(fsync_node_entry_slab, GFP_NOFS, true, NULL);
f2fs_cache_get(entry); fn->entry = entry; INIT_LIST_HEAD(&fn->list);
spin_lock_irqsave(&sbi->fsync_node_lock, flags); list_add_tail(&fn->list, &sbi->fsync_node_list); fn->seq_id = sbi->fsync_seg_id++; seq_id = fn->seq_id; sbi->fsync_node_num++; spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);
return seq_id;}
void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry){ struct fsync_node_entry *fn; unsigned long flags;
spin_lock_irqsave(&sbi->fsync_node_lock, flags); list_for_each_entry(fn, &sbi->fsync_node_list, list) { if (fn->entry == entry) { list_del(&fn->list); sbi->fsync_node_num--; spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); f2fs_put_cache(fn->entry, false); kmem_cache_free(fsync_node_entry_slab, fn); return; } } spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); f2fs_bug_on(sbi, 1);}
void f2fs_reset_fsync_node_info(struct f2fs_sb_info *sbi){ unsigned long flags;
spin_lock_irqsave(&sbi->fsync_node_lock, flags); sbi->fsync_seg_id = 0; spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);}
bool f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; bool need = false; struct f2fs_lock_context lc;
f2fs_down_read_trace(&nm_i->nat_tree_lock, &lc); e = __lookup_nat_cache(nm_i, nid, false); if (e) { if (!get_nat_flag(e, IS_CHECKPOINTED) && !get_nat_flag(e, HAS_FSYNCED_INODE)) need = true; } f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); return need;}
bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; bool is_cp = true; struct f2fs_lock_context lc;
f2fs_down_read_trace(&nm_i->nat_tree_lock, &lc); e = __lookup_nat_cache(nm_i, nid, false); if (e && !get_nat_flag(e, IS_CHECKPOINTED)) is_cp = false; f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); return is_cp;}
bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; bool need_update = true; struct f2fs_lock_context lc, nlc;
f2fs_down_read_trace(&sbi->node_write, &lc); f2fs_down_read_trace(&nm_i->nat_tree_lock, &nlc); e = __lookup_nat_cache(nm_i, ino, false); if (e && get_nat_flag(e, HAS_LAST_FSYNC) && (get_nat_flag(e, IS_CHECKPOINTED) || get_nat_flag(e, HAS_FSYNCED_INODE))) need_update = false; f2fs_up_read_trace(&nm_i->nat_tree_lock, &nlc); f2fs_up_read_trace(&sbi->node_write, &lc); return need_update;}
/* must be locked by nat_tree_lock */static void cache_nat_entry(struct f2fs_sb_info *sbi, nid_t nid, struct f2fs_nat_entry *ne){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *new, *e; struct f2fs_lock_context lc;
/* Let's mitigate lock contention of nat_tree_lock during checkpoint */ if (f2fs_rwsem_is_locked(&sbi->cp_global_sem)) return;
new = __alloc_nat_entry(sbi, nid, false); if (!new) return;
f2fs_down_write_trace(&nm_i->nat_tree_lock, &lc); e = __lookup_nat_cache(nm_i, nid, false); if (!e) e = __init_nat_entry(nm_i, new, ne, false, false); else f2fs_bug_on(sbi, nat_get_ino(e) != le32_to_cpu(ne->ino) || nat_get_blkaddr(e) != le32_to_cpu(ne->block_addr) || nat_get_version(e) != ne->version); f2fs_up_write_trace(&nm_i->nat_tree_lock, &lc); if (e != new) __free_nat_entry(new);}
static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni, block_t new_blkaddr, bool fsync_done){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; struct nat_entry *new = __alloc_nat_entry(sbi, ni->nid, true); bool init_dirty = false; struct f2fs_lock_context lc;
f2fs_down_write_trace(&nm_i->nat_tree_lock, &lc); e = __lookup_nat_cache(nm_i, ni->nid, true); if (!e) { init_dirty = true; e = __init_nat_entry(nm_i, new, NULL, true, true); copy_node_info(&e->ni, ni); f2fs_bug_on(sbi, ni->blk_addr == NEW_ADDR); } else if (new_blkaddr == NEW_ADDR) { /* * when nid is reallocated, * previous nat entry can be remained in nat cache. * So, reinitialize it with new information. */ copy_node_info(&e->ni, ni); f2fs_bug_on(sbi, ni->blk_addr != NULL_ADDR); } /* let's free early to reduce memory consumption */ if (e != new) __free_nat_entry(new);
/* sanity check */ f2fs_bug_on(sbi, nat_get_blkaddr(e) != ni->blk_addr); f2fs_bug_on(sbi, nat_get_blkaddr(e) == NULL_ADDR && new_blkaddr == NULL_ADDR); f2fs_bug_on(sbi, nat_get_blkaddr(e) == NEW_ADDR && new_blkaddr == NEW_ADDR); f2fs_bug_on(sbi, __is_valid_data_blkaddr(nat_get_blkaddr(e)) && new_blkaddr == NEW_ADDR);
/* increment version no as node is removed */ if (nat_get_blkaddr(e) != NEW_ADDR && new_blkaddr == NULL_ADDR) { unsigned char version = nat_get_version(e);
nat_set_version(e, inc_node_version(version)); }
/* change address */ nat_set_blkaddr(e, new_blkaddr); if (!__is_valid_data_blkaddr(new_blkaddr)) set_nat_flag(e, IS_CHECKPOINTED, false); __set_nat_cache_dirty(sbi, nm_i, e, init_dirty);
/* update fsync_mark if its inode nat entry is still alive */ if (ni->nid != ni->ino) e = __lookup_nat_cache(nm_i, ni->ino, false); if (e) { if (fsync_done && ni->nid == ni->ino) set_nat_flag(e, HAS_FSYNCED_INODE, true); set_nat_flag(e, HAS_LAST_FSYNC, fsync_done); } f2fs_up_write_trace(&nm_i->nat_tree_lock, &lc);}
int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink){ struct f2fs_nm_info *nm_i = NM_I(sbi); int nr = nr_shrink; struct f2fs_lock_context lc;
if (!f2fs_down_write_trylock_trace(&nm_i->nat_tree_lock, &lc)) return 0;
spin_lock(&nm_i->nat_list_lock); while (nr_shrink) { struct nat_entry *ne;
if (list_empty(&nm_i->nat_entries)) break;
ne = list_first_entry(&nm_i->nat_entries, struct nat_entry, list); list_del(&ne->list); spin_unlock(&nm_i->nat_list_lock);
__del_from_nat_cache(nm_i, ne); nr_shrink--;
spin_lock(&nm_i->nat_list_lock); } spin_unlock(&nm_i->nat_list_lock);
f2fs_up_write_trace(&nm_i->nat_tree_lock, &lc); return nr - nr_shrink;}
int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid, struct node_info *ni, bool checkpoint_context){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); struct f2fs_journal *journal = curseg->journal; nid_t start_nid = f2fs_start_nid(sbi, nid); struct f2fs_nat_block *nat_blk; struct f2fs_cached_block *entry = NULL; struct f2fs_nat_entry ne; struct nat_entry *e; pgoff_t index; int i; bool need_cache = true; struct f2fs_lock_context lc;
ni->flag = 0; ni->nid = nid;retry: /* Check nat cache */ f2fs_down_read_trace(&nm_i->nat_tree_lock, &lc); e = __lookup_nat_cache(nm_i, nid, false); if (e) { ni->ino = nat_get_ino(e); ni->blk_addr = nat_get_blkaddr(e); ni->version = nat_get_version(e); f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); if (IS_ENABLED(CONFIG_F2FS_CHECK_FS)) { need_cache = false; goto sanity_check; } return 0; }
/* * Check current segment summary by trying to grab journal_rwsem first. * This sem is on the critical path on the checkpoint requiring the above * nat_tree_lock. Therefore, we should retry, if we failed to grab here * while not bothering checkpoint. */ if (!f2fs_rwsem_is_locked(&sbi->cp_global_sem) || checkpoint_context) { down_read(&curseg->journal_rwsem); } else if (f2fs_rwsem_is_contended(&nm_i->nat_tree_lock) || !down_read_trylock(&curseg->journal_rwsem)) { f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); goto retry; }
i = f2fs_lookup_journal_in_cursum(sbi, journal, NAT_JOURNAL, nid, 0); if (i >= 0) { ne = nat_in_journal(journal, i); node_info_from_raw_nat(ni, &ne); } up_read(&curseg->journal_rwsem); if (i >= 0) { f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); goto sanity_check; }
/* Fill node_info from nat block */ index = current_nat_addr(sbi, nid); f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc);
entry = f2fs_get_meta_cache(sbi, index); if (IS_ERR(entry)) return PTR_ERR(entry);
nat_blk = cache_address(entry); ne = nat_blk->entries[nid - start_nid]; node_info_from_raw_nat(ni, &ne); f2fs_put_cache(entry, true);sanity_check: if (__is_valid_data_blkaddr(ni->blk_addr) && !f2fs_is_valid_blkaddr(sbi, ni->blk_addr, DATA_GENERIC_ENHANCE)) { set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_err_ratelimited(sbi, "f2fs_get_node_info of %pS: inconsistent nat entry, " "ino:%u, nid:%u, blkaddr:%u, ver:%u, flag:%u", __builtin_return_address(0), ni->ino, ni->nid, ni->blk_addr, ni->version, ni->flag); f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT); return -EFSCORRUPTED; }
if (unlikely(f2fs_quota_file(sbi, ni->nid) && !__is_valid_data_blkaddr(ni->blk_addr))) { set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_err_ratelimited(sbi, "f2fs_get_node_info of %pS: inconsistent nat entry from qf_ino, " "ino:%u, nid:%u, blkaddr:%u, ver:%u, flag:%u", __builtin_return_address(0), ni->ino, ni->nid, ni->blk_addr, ni->version, ni->flag); f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT); return -EFSCORRUPTED; }
/* cache nat entry */ if (need_cache) cache_nat_entry(sbi, nid, &ne); return 0;}
/* * readahead MAX_RA_NODE number of node caches. */static void f2fs_ra_node_caches(struct f2fs_cached_block *parent, int start, int n){ struct f2fs_sb_info *sbi = parent->cache->sbi; struct blk_plug plug; int i, end; nid_t nid;
blk_start_plug(&plug);
/* Then, try readahead for siblings of the desired node */ end = start + n; end = min_t(int, end, NIDS_PER_BLOCK(sbi)); for (i = start; i < end; i++) { nid = get_nid(sbi, parent, i, false); f2fs_ra_node_cache(sbi, nid); }
blk_finish_plug(&plug);}
pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs){ struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); const long direct_index = ADDRS_PER_INODE(dn->inode); const long direct_blks = ADDRS_PER_BLOCK(dn->inode); const long indirect_blks = ADDRS_PER_BLOCK(dn->inode) * NIDS_PER_BLOCK(sbi); unsigned int skipped_unit = ADDRS_PER_BLOCK(dn->inode); int cur_level = dn->cur_level; int max_level = dn->max_level; pgoff_t base = 0;
if (!dn->max_level) return pgofs + 1;
while (max_level-- > cur_level) skipped_unit *= NIDS_PER_BLOCK(sbi);
switch (dn->max_level) { case 3: base += 2 * indirect_blks; fallthrough; case 2: base += 2 * direct_blks; fallthrough; case 1: base += direct_index; break; default: f2fs_bug_on(F2FS_I_SB(dn->inode), 1); }
return ((pgofs - base) / skipped_unit + 1) * skipped_unit + base;}
/* * The maximum depth is four. * Offset[0] will have raw inode offset. */static int get_node_path(struct inode *inode, long block, int offset[4], unsigned int noffset[4]){ struct f2fs_sb_info *sbi = F2FS_I_SB(inode); const long direct_index = ADDRS_PER_INODE(inode); const long direct_blks = ADDRS_PER_BLOCK(inode); const long dptrs_per_blk = NIDS_PER_BLOCK(sbi); const long indirect_blks = ADDRS_PER_BLOCK(inode) * NIDS_PER_BLOCK(sbi); const long dindirect_blks = indirect_blks * NIDS_PER_BLOCK(sbi); int n = 0; int level = 0;
noffset[0] = 0;
if (block < direct_index) { offset[n] = block; goto got; } block -= direct_index; if (block < direct_blks) { offset[n++] = NODE_DIR1_BLOCK(sbi); noffset[n] = 1; offset[n] = block; level = 1; goto got; } block -= direct_blks; if (block < direct_blks) { offset[n++] = NODE_DIR2_BLOCK(sbi); noffset[n] = 2; offset[n] = block; level = 1; goto got; } block -= direct_blks; if (block < indirect_blks) { offset[n++] = NODE_IND1_BLOCK(sbi); noffset[n] = 3; offset[n++] = block / direct_blks; noffset[n] = 4 + offset[n - 1]; offset[n] = block % direct_blks; level = 2; goto got; } block -= indirect_blks; if (block < indirect_blks) { offset[n++] = NODE_IND2_BLOCK(sbi); noffset[n] = 4 + dptrs_per_blk; offset[n++] = block / direct_blks; noffset[n] = 5 + dptrs_per_blk + offset[n - 1]; offset[n] = block % direct_blks; level = 2; goto got; } block -= indirect_blks; if (block < dindirect_blks) { offset[n++] = NODE_DIND_BLOCK(sbi); noffset[n] = 5 + (dptrs_per_blk * 2); offset[n++] = block / indirect_blks; noffset[n] = 6 + (dptrs_per_blk * 2) + offset[n - 1] * (dptrs_per_blk + 1); offset[n++] = (block / direct_blks) % dptrs_per_blk; noffset[n] = 7 + (dptrs_per_blk * 2) + offset[n - 2] * (dptrs_per_blk + 1) + offset[n - 1]; offset[n] = block % direct_blks; level = 3; goto got; } else { return -E2BIG; }got: return level;}
static struct f2fs_cached_block *f2fs_get_node_cache_ra( struct f2fs_cached_block *parent, int start);
/* * Caller should call f2fs_put_dnode(dn). * Also, it should grab and release a rwsem by calling f2fs_lock_op() and * f2fs_unlock_op() only if mode is set with ALLOC_NODE. */int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode){ struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); struct f2fs_cached_block *nentry[4]; struct f2fs_cached_block *parent = NULL; int offset[4]; unsigned int noffset[4]; nid_t nids[4]; int level, i = 0; int err = 0;
level = get_node_path(dn->inode, index, offset, noffset); if (level < 0) return level;
nids[0] = dn->inode->i_ino;
if (!dn->inode_entry) { nentry[0] = f2fs_get_inode_cache(sbi, nids[0]); if (IS_ERR(nentry[0])) { err = PTR_ERR(nentry[0]); /* * cp_error drops dirty node caches, so the inode block * of an inode that was never written reads as NEW_ADDR * and returns -ENOENT. It is not a hole; the inode is * gone. Callers treat -ENOENT as a hole and keep * looking block by block (e.g. fiemap up to maxbytes). */ if (err == -ENOENT && (f2fs_cp_error(sbi) || is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN))) err = -EIO; return err; } } else { nentry[0] = dn->inode_entry; }
/* if inline_data is set, should not report any block indices */ if (f2fs_has_inline_data(dn->inode) && index) { err = -ENOENT; f2fs_put_cache(nentry[0], true); goto release_out; }
parent = nentry[0]; if (level != 0) nids[1] = get_nid(sbi, parent, offset[0], true); dn->inode_entry = nentry[0]; dn->inode_entry_locked = true;
/* get indirect or direct nodes */ for (i = 1; i <= level; i++) { bool done = false;
if (nids[i] && nids[i] == dn->inode->i_ino) { err = -EFSCORRUPTED; f2fs_err_ratelimited(sbi, "inode mapping table is corrupted, run fsck to fix it, " "ino:%llu, nid:%u, level:%d, offset:%d", dn->inode->i_ino, nids[i], level, offset[level]); set_sbi_flag(sbi, SBI_NEED_FSCK); goto release_caches; }
if (!nids[i] && mode == ALLOC_NODE) { /* alloc new node */ if (!f2fs_alloc_nid(sbi, &(nids[i]))) { err = -ENOSPC; goto release_caches; }
dn->nid = nids[i]; nentry[i] = f2fs_new_node_cache(dn, noffset[i]); if (IS_ERR(nentry[i])) { f2fs_alloc_nid_failed(sbi, nids[i]); err = PTR_ERR(nentry[i]); goto release_caches; }
set_nid(sbi, parent, offset[i - 1], nids[i], i == 1); f2fs_alloc_nid_done(sbi, nids[i]); done = true; } else if (mode == LOOKUP_NODE_RA && i == level && level > 1) { nentry[i] = f2fs_get_node_cache_ra(parent, offset[i - 1]); if (IS_ERR(nentry[i])) { err = PTR_ERR(nentry[i]); goto release_caches; } done = true; } if (i == 1) { dn->inode_entry_locked = false; f2fs_unlock_cache(parent); } else { f2fs_put_cache(parent, true); }
if (!done) { nentry[i] = f2fs_get_node_cache(sbi, nids[i], NODE_TYPE_NON_INODE); if (IS_ERR(nentry[i])) { err = PTR_ERR(nentry[i]); f2fs_put_cache(nentry[0], false); goto release_out; } } if (i < level) { parent = nentry[i]; nids[i + 1] = get_nid(sbi, parent, offset[i], false); } } dn->nid = nids[level]; dn->ofs_in_node = offset[level]; dn->node_entry = nentry[level]; dn->data_blkaddr = f2fs_data_blkaddr(dn);
if (is_inode_flag_set(dn->inode, FI_COMPRESSED_FILE) && f2fs_sb_has_readonly(sbi)) { unsigned int cluster_size = F2FS_I(dn->inode)->i_cluster_size; unsigned int ofs_in_node = dn->ofs_in_node; pgoff_t fofs = index; unsigned int c_len; block_t blkaddr;
/* should align fofs and ofs_in_node to cluster_size */ if (fofs % cluster_size) { fofs = round_down(fofs, cluster_size); ofs_in_node = round_down(ofs_in_node, cluster_size); }
c_len = f2fs_cluster_blocks_are_contiguous(dn, ofs_in_node); if (!c_len) goto out;
blkaddr = data_blkaddr(dn->inode, dn->node_entry, ofs_in_node); if (blkaddr == COMPRESS_ADDR) blkaddr = data_blkaddr(dn->inode, dn->node_entry, ofs_in_node + 1);
f2fs_update_read_extent_tree_range_compressed(dn->inode, fofs, blkaddr, cluster_size, c_len); }out: return 0;
release_caches: f2fs_put_cache(parent, true); if (i > 1) f2fs_put_cache(nentry[0], false);release_out: dn->inode_entry = NULL; dn->node_entry = NULL; if (err == -ENOENT) { dn->cur_level = i; dn->max_level = level; dn->ofs_in_node = offset[level]; } return err;}
static int truncate_node(struct dnode_of_data *dn){ struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); struct node_info ni; int err; pgoff_t index;
err = f2fs_get_node_info(sbi, dn->nid, &ni, false); if (err) return err;
if (ni.blk_addr != NEW_ADDR && !f2fs_is_valid_blkaddr(sbi, ni.blk_addr, DATA_GENERIC_ENHANCE)) { f2fs_err_ratelimited(sbi, "nat entry is corrupted, run fsck to fix it, ino:%u, " "nid:%u, blkaddr:%u", ni.ino, ni.nid, ni.blk_addr); set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT); return -EFSCORRUPTED; }
/* Deallocate node address */ f2fs_invalidate_blocks(sbi, ni.blk_addr, 1); dec_valid_node_count(sbi, dn->inode, dn->nid == dn->inode->i_ino); set_node_addr(sbi, &ni, NULL_ADDR, false);
if (dn->nid == dn->inode->i_ino) { f2fs_remove_orphan_inode(sbi, dn->nid); dec_valid_inode_count(sbi); f2fs_inode_synced(dn->inode); }
f2fs_drop_cache_dirty(dn->node_entry); set_sbi_flag(sbi, SBI_IS_DIRTY);
index = dn->node_entry->index; f2fs_put_cache(dn->node_entry, true);
f2fs_invalidate_node_cache(sbi, index);
dn->node_entry = NULL; trace_f2fs_truncate_node(dn->inode, dn->nid, ni.blk_addr);
return 0;}
static int truncate_dnode(struct dnode_of_data *dn){ struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); struct f2fs_cached_block *entry; int err;
if (dn->nid == 0) return 1;
/* get direct node */ entry = f2fs_get_node_cache(sbi, dn->nid, NODE_TYPE_NON_INODE); if (PTR_ERR(entry) == -ENOENT) return 1; else if (IS_ERR(entry)) return PTR_ERR(entry);
if (IS_INODE(sbi, entry) || ino_of_node(sbi, entry) != dn->inode->i_ino) { f2fs_err(sbi, "incorrect node reference, ino: %llu, nid: %u, ino_of_node: %u", dn->inode->i_ino, dn->nid, ino_of_node(sbi, entry)); set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_handle_error(sbi, ERROR_INVALID_NODE_REFERENCE); f2fs_put_cache(entry, true); return -EFSCORRUPTED; }
/* Make dnode_of_data for parameter */ dn->node_entry = entry; dn->ofs_in_node = 0; f2fs_truncate_data_blocks_range(dn, ADDRS_PER_BLOCK(dn->inode)); err = truncate_node(dn); if (err) { f2fs_put_cache(entry, true); return err; }
return 1;}
static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs, int ofs, int depth){ struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); struct dnode_of_data rdn = *dn; struct f2fs_cached_block *entry; struct f2fs_node *rn; nid_t child_nid; unsigned int child_nofs; int freed = 0; int i, ret;
if (dn->nid == 0) return NIDS_PER_BLOCK(sbi) + 1;
trace_f2fs_truncate_nodes_enter(dn->inode, dn->nid, dn->data_blkaddr);
entry = f2fs_get_node_cache(F2FS_I_SB(dn->inode), dn->nid, NODE_TYPE_NON_INODE); if (IS_ERR(entry)) { trace_f2fs_truncate_nodes_exit(dn->inode, PTR_ERR(entry)); return PTR_ERR(entry); }
f2fs_ra_node_caches(entry, ofs, NIDS_PER_BLOCK(sbi));
rn = CACHED_NODE(entry); if (depth < 3) { for (i = ofs; i < NIDS_PER_BLOCK(sbi); i++, freed++) { child_nid = le32_to_cpu(rn->in.nid[i]); if (child_nid == 0) continue; rdn.nid = child_nid; ret = truncate_dnode(&rdn); if (ret < 0) goto out_err; if (set_nid(sbi, entry, i, 0, false)) dn->node_changed = true; } } else { child_nofs = nofs + ofs * (NIDS_PER_BLOCK(sbi) + 1) + 1; for (i = ofs; i < NIDS_PER_BLOCK(sbi); i++) { child_nid = le32_to_cpu(rn->in.nid[i]); if (child_nid == 0) { child_nofs += NIDS_PER_BLOCK(sbi) + 1; continue; } rdn.nid = child_nid; ret = truncate_nodes(&rdn, child_nofs, 0, depth - 1); if (ret == (NIDS_PER_BLOCK(sbi) + 1)) { if (set_nid(sbi, entry, i, 0, false)) dn->node_changed = true; child_nofs += ret; } else if (ret < 0 && ret != -ENOENT) { goto out_err; } } freed = child_nofs; }
if (!ofs) { /* remove current indirect node */ dn->node_entry = entry; ret = truncate_node(dn); if (ret) goto out_err; freed++; } else { f2fs_put_cache(entry, true); } trace_f2fs_truncate_nodes_exit(dn->inode, freed); return freed;
out_err: f2fs_put_cache(entry, true); trace_f2fs_truncate_nodes_exit(dn->inode, ret); return ret;}
static int truncate_partial_nodes(struct dnode_of_data *dn, int *offset, int depth){ struct f2fs_cached_block *entries[2]; nid_t nid[3]; nid_t child_nid; int err = 0; int i; int idx = depth - 2;
nid[0] = get_nid(F2FS_I_SB(dn->inode), dn->inode_entry, offset[0], true); if (!nid[0]) return 0;
/* get indirect nodes in the path */ for (i = 0; i < idx + 1; i++) { /* reference count'll be increased */ entries[i] = f2fs_get_node_cache(F2FS_I_SB(dn->inode), nid[i], NODE_TYPE_NON_INODE); if (IS_ERR(entries[i])) { err = PTR_ERR(entries[i]); idx = i - 1; goto fail; } nid[i + 1] = get_nid(F2FS_I_SB(dn->inode), entries[i], offset[i + 1], false); }
f2fs_ra_node_caches(entries[idx], offset[idx + 1], NIDS_PER_BLOCK(F2FS_I_SB(dn->inode)));
/* free direct nodes linked to a partial indirect node */ for (i = offset[idx + 1]; i < NIDS_PER_BLOCK(F2FS_I_SB(dn->inode)); i++) { child_nid = get_nid(F2FS_I_SB(dn->inode), entries[idx], i, false); if (!child_nid) continue; dn->nid = child_nid; err = truncate_dnode(dn); if (err < 0) goto fail; if (set_nid(F2FS_I_SB(dn->inode), entries[idx], i, 0, false)) dn->node_changed = true; }
if (offset[idx + 1] == 0) { dn->node_entry = entries[idx]; dn->nid = nid[idx]; err = truncate_node(dn); if (err) goto fail; } else { f2fs_put_cache(entries[idx], true); } offset[idx]++; offset[idx + 1] = 0; idx--;fail: for (i = idx; i >= 0; i--) f2fs_put_cache(entries[i], true);
trace_f2fs_truncate_partial_nodes(dn->inode, nid, depth, err);
return err;}
/* * All the node blocks actually belong to the inode will be released. * If the level is 0, we will simply truncate the dnode, * or else we should do dynamic truncate for the node pointers with the depth. */int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from){ struct f2fs_sb_info *sbi = F2FS_I_SB(inode); int err = 0, cont = 1; int level, offset[4], noffset[4]; unsigned int nofs = 0; struct dnode_of_data dn; struct f2fs_cached_block *entry;
trace_f2fs_truncate_inode_blocks_enter(inode, from);
level = get_node_path(inode, from, offset, noffset); if (level <= 0) { if (!level) { level = -EFSCORRUPTED; f2fs_err(sbi, "%s: inode ino=%llx has corrupted node block, from:%lu addrs:%u", __func__, inode->i_ino, from, ADDRS_PER_INODE(inode)); set_sbi_flag(sbi, SBI_NEED_FSCK); } trace_f2fs_truncate_inode_blocks_exit(inode, level); return level; }
entry = f2fs_get_inode_cache(sbi, inode->i_ino); if (IS_ERR(entry)) { trace_f2fs_truncate_inode_blocks_exit(inode, PTR_ERR(entry)); return PTR_ERR(entry); }
set_new_dnode(&dn, inode, entry, NULL, 0); f2fs_unlock_cache(entry);
switch (level) { case 0: case 1: nofs = noffset[1]; break; case 2: nofs = noffset[1]; if (!offset[level - 1]) goto skip_partial; err = truncate_partial_nodes(&dn, offset, level); if (err < 0 && err != -ENOENT) goto fail; nofs += 1 + NIDS_PER_BLOCK(sbi); break; case 3: nofs = 5 + 2 * NIDS_PER_BLOCK(sbi); if (!offset[level - 1]) goto skip_partial; err = truncate_partial_nodes(&dn, offset, level); if (err < 0 && err != -ENOENT) goto fail; break; default: BUG(); }
skip_partial: while (cont) { dn.nid = get_nid(sbi, entry, offset[0], true); if (offset[0] == NODE_DIR1_BLOCK(sbi) || offset[0] == NODE_DIR2_BLOCK(sbi)) { err = truncate_dnode(&dn); } else if (offset[0] == NODE_IND1_BLOCK(sbi) || offset[0] == NODE_IND2_BLOCK(sbi)) { err = truncate_nodes(&dn, nofs, offset[1], 2); } else if (offset[0] == NODE_DIND_BLOCK(sbi)) { err = truncate_nodes(&dn, nofs, offset[1], 3); cont = 0; } else { BUG(); } if (err == -ENOENT) { set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); fserror_report_file_metadata(dn.inode, -EFSCORRUPTED, GFP_NOFS); f2fs_err_ratelimited(sbi, "truncate node fail, ino:%llu, nid:%u, " "offset[0]:%d, offset[1]:%d, nofs:%d", inode->i_ino, dn.nid, offset[0], offset[1], nofs); err = 0; } if (err < 0) goto fail; if (offset[1] == 0 && get_nid(sbi, entry, offset[0], true)) { f2fs_lock_cache(entry); f2fs_bug_on(sbi, !f2fs_is_node_cache(entry)); set_nid(sbi, entry, offset[0], 0, true); f2fs_unlock_cache(entry); } offset[1] = 0; offset[0]++; nofs += err; }fail: f2fs_put_cache(entry, false); trace_f2fs_truncate_inode_blocks_exit(inode, err); return err > 0 ? 0 : err;}
/* caller must lock inode cached block */int f2fs_truncate_xattr_node(struct inode *inode){ struct f2fs_sb_info *sbi = F2FS_I_SB(inode); nid_t nid = F2FS_I(inode)->i_xattr_nid; struct dnode_of_data dn; struct f2fs_cached_block *nentry; int err;
if (!nid) return 0;
nentry = f2fs_get_xnode_cache(sbi, nid); if (IS_ERR(nentry)) return PTR_ERR(nentry);
set_new_dnode(&dn, inode, NULL, nentry, nid); err = truncate_node(&dn); if (err) { f2fs_put_cache(nentry, true); return err; }
f2fs_i_xnid_write(inode, 0);
return 0;}
/* * Caller should grab and release a rwsem by calling f2fs_lock_op() and * f2fs_unlock_op(). */int f2fs_remove_inode_cache(struct inode *inode){ struct dnode_of_data dn; int err;
set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino); err = f2fs_get_dnode_of_data(&dn, 0, LOOKUP_NODE); if (err) return err;
err = f2fs_truncate_xattr_node(inode); if (err) { f2fs_put_dnode(&dn); return err; }
/* remove potential inline_data blocks */ if (!IS_DEVICE_ALIASING(inode) && (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))) f2fs_truncate_data_blocks_range(&dn, 1);
/* 0 is possible, after f2fs_new_inode() has failed */ if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) { f2fs_put_dnode(&dn); return -EIO; }
if (unlikely(inode->i_blocks != 0 && inode->i_blocks != 8)) { f2fs_warn(F2FS_I_SB(inode), "f2fs_remove_inode_cache: inconsistent i_blocks, ino:%llu, iblocks:%llu", inode->i_ino, (unsigned long long)inode->i_blocks); set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK); }
/* will put inode & node cached blocks */ err = truncate_node(&dn); if (err) { f2fs_put_dnode(&dn); return err; } return 0;}
struct f2fs_cached_block *f2fs_new_inode_cache(struct inode *inode){ struct dnode_of_data dn;
/* allocate inode block for new inode */ set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino);
/* caller should f2fs_put_cache(entry, true); */ return f2fs_new_node_cache(&dn, 0);}
struct f2fs_cached_block *f2fs_new_node_cache(struct dnode_of_data *dn, unsigned int ofs){ struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); struct node_info new_ni; struct f2fs_cached_block *entry; int err;
if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) return ERR_PTR(-EPERM);
entry = f2fs_grab_node_cache(sbi, dn->nid); if (IS_ERR(entry)) return entry;
if (unlikely((err = inc_valid_node_count(sbi, dn->inode, !ofs)))) goto fail;
#ifdef CONFIG_F2FS_CHECK_FS err = f2fs_get_node_info(sbi, dn->nid, &new_ni, false); if (err) { dec_valid_node_count(sbi, dn->inode, !ofs); goto fail; } if (unlikely(new_ni.blk_addr != NULL_ADDR)) { err = -EFSCORRUPTED; dec_valid_node_count(sbi, dn->inode, !ofs); set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_warn_ratelimited(sbi, "f2fs_new_node_cache: inconsistent nat entry, " "ino:%u, nid:%u, blkaddr:%u, ver:%u, flag:%u", new_ni.ino, new_ni.nid, new_ni.blk_addr, new_ni.version, new_ni.flag); f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT); goto fail; }#endif new_ni.nid = dn->nid; new_ni.ino = dn->inode->i_ino; new_ni.blk_addr = NULL_ADDR; new_ni.flag = 0; new_ni.version = 0; set_node_addr(sbi, &new_ni, NEW_ADDR, false);
f2fs_cache_wait_writeback(entry); fill_node_footer(sbi, entry, dn->nid, dn->inode->i_ino, ofs, true); set_cold_node(sbi, entry, S_ISDIR(dn->inode->i_mode)); f2fs_cache_set_uptodate(entry); if (f2fs_mark_cache_dirty(entry)) dn->node_changed = true;
if (f2fs_has_xattr_block(ofs)) f2fs_i_xnid_write(dn->inode, dn->nid);
if (ofs == 0) inc_valid_inode_count(sbi); return entry;fail: f2fs_drop_cache_dirty(entry); f2fs_put_cache(entry, true); return ERR_PTR(err);}
/* * Caller should do after getting the following values. * 0: f2fs_put_cache(cache, false) * LOCKED_CACHE or error: f2fs_put_cache(entry, true) */static int read_node_cache(struct f2fs_cached_block *entry, blk_opf_t op_flags){ struct f2fs_sb_info *sbi = entry->cache->sbi; struct node_info ni; struct f2fs_io_info fio = { .sbi = sbi, .type = NODE, .op = REQ_OP_READ, .op_flags = op_flags, .encrypted_page = NULL, .cache_entry = entry, .is_cache = 1, }; int err;
if (f2fs_cache_test_uptodate(entry)) { if (!f2fs_inode_chksum_verify(sbi, entry)) { f2fs_cache_clear_uptodate(entry); return -EFSBADCRC; } return LOCKED_CACHE; }
err = f2fs_get_node_info(sbi, entry->index, &ni, false); if (err) return err;
/* NEW_ADDR can be seen, after cp_error drops some dirty node caches */ if (unlikely(ni.blk_addr == NULL_ADDR || ni.blk_addr == NEW_ADDR)) { f2fs_cache_clear_uptodate(entry); return -ENOENT; }
fio.new_blkaddr = fio.old_blkaddr = ni.blk_addr;
err = f2fs_submit_cache_read(&fio); if (!err) f2fs_update_iostat(sbi, NULL, FS_NODE_READ_IO, F2FS_BLKSIZE(sbi));
return err;}
/* * Readahead a node cache */void f2fs_ra_node_cache(struct f2fs_sb_info *sbi, nid_t nid){ struct f2fs_cached_block *entry; int err;
if (!nid) return; if (f2fs_check_nid_range(sbi, nid)) return;
entry = xa_load(&NODE_CACHE(sbi)->root, nid); if (entry) return;
entry = f2fs_grab_node_cache(sbi, nid); if (IS_ERR(entry)) return;
err = read_node_cache(entry, REQ_RAHEAD); f2fs_put_cache(entry, err ? true : false);}
int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry, pgoff_t nid, enum node_type ntype, bool in_irq){ bool is_inode, is_xnode;
if (unlikely(nid != nid_of_node(sbi, entry))) goto out_err;
is_inode = IS_INODE(sbi, entry); is_xnode = f2fs_has_xattr_block(ofs_of_node(sbi, entry));
switch (ntype) { case NODE_TYPE_REGULAR: if (is_inode && is_xnode) goto out_err; break; case NODE_TYPE_INODE: if (!is_inode || is_xnode) goto out_err; break; case NODE_TYPE_XATTR: if (is_inode || !is_xnode) goto out_err; break; case NODE_TYPE_NON_INODE: if (is_inode) goto out_err; break; case NODE_TYPE_NON_IXNODE: if (is_inode || is_xnode) goto out_err; break; default: break; } if (time_to_inject(sbi, FAULT_INCONSISTENT_FOOTER)) goto out_err; return 0;out_err: set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_warn_ratelimited(sbi, "inconsistent node block, node_type:%d, nid:%lu, " "node_footer[nid:%u,ino:%u,ofs:%u,cpver:%llu,blkaddr:%u]", ntype, nid, nid_of_node(sbi, entry), ino_of_node(sbi, entry), ofs_of_node(sbi, entry), cpver_of_node(sbi, entry), next_blkaddr_of_node(sbi, entry));
f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER); fserror_report_metadata(sbi->sb, -EFSCORRUPTED, in_irq ? GFP_NOWAIT : GFP_NOFS); return -EFSCORRUPTED;}
static struct f2fs_cached_block *__get_node_cache(struct f2fs_sb_info *sbi, pgoff_t nid, struct f2fs_cached_block *parent, int start, enum node_type ntype){ struct f2fs_cached_block *entry; int err;
if (!nid) return ERR_PTR(-ENOENT); if (f2fs_check_nid_range(sbi, nid)) return ERR_PTR(-EINVAL);repeat: entry = f2fs_grab_node_cache(sbi, nid); if (IS_ERR(entry)) return entry;
err = read_node_cache(entry, 0); if (err < 0) goto out_put_err; if (err == LOCKED_CACHE) goto entry_hit;
if (parent) f2fs_ra_node_caches(parent, start + 1, MAX_RA_NODE);
f2fs_lock_cache(entry);
if (unlikely(!f2fs_is_node_cache(entry))) { f2fs_put_cache(entry, true); goto repeat; }
if (unlikely(!f2fs_cache_test_uptodate(entry))) { err = -EIO; goto out_put_err; }
if (!f2fs_inode_chksum_verify(sbi, entry)) { err = -EFSBADCRC; goto out_err; }entry_hit: err = f2fs_sanity_check_node_footer(sbi, entry, nid, ntype, false); if (!err) return entry;out_err: f2fs_drop_cache_dirty(entry);out_put_err: /* ENOENT comes from read_node_folio which is not an error. */ if (err != -ENOENT) f2fs_handle_page_eio(sbi, entry->index, NODE); f2fs_put_cache(entry, true); return ERR_PTR(err);}
struct f2fs_cached_block *f2fs_get_node_cache(struct f2fs_sb_info *sbi, pgoff_t nid, enum node_type node_type){ return __get_node_cache(sbi, nid, NULL, 0, node_type);}
struct f2fs_cached_block *f2fs_get_inode_cache(struct f2fs_sb_info *sbi, pgoff_t ino){ return __get_node_cache(sbi, ino, NULL, 0, NODE_TYPE_INODE);}
struct f2fs_cached_block *f2fs_get_xnode_cache(struct f2fs_sb_info *sbi, pgoff_t xnid){ return __get_node_cache(sbi, xnid, NULL, 0, NODE_TYPE_XATTR);}
static struct f2fs_cached_block *f2fs_get_node_cache_ra(struct f2fs_cached_block *parent, int start){ struct f2fs_sb_info *sbi = parent->cache->sbi; nid_t nid = get_nid(sbi, parent, start, false);
return __get_node_cache(sbi, nid, parent, start, NODE_TYPE_NON_IXNODE);}
static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino){ struct inode *inode; struct folio *folio; int ret;
/* should flush inline_data before evict_inode */ inode = ilookup(sbi->sb, ino); if (!inode) return;
folio = f2fs_filemap_get_folio(inode->i_mapping, 0, FGP_LOCK|FGP_NOWAIT, 0); if (IS_ERR(folio)) goto iput_out;
if (!folio_test_uptodate(folio)) goto folio_out;
if (!folio_test_dirty(folio)) goto folio_out;
if (!folio_clear_dirty_for_io(folio)) goto folio_out;
ret = f2fs_write_inline_data(inode, folio); inode_dec_dirty_pages(inode); f2fs_remove_dirty_inode(inode); if (ret) folio_mark_dirty(folio);folio_out: f2fs_folio_put(folio, true);iput_out: iput(inode);}
static struct f2fs_cached_block *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino){ pgoff_t index = 0; struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES]; struct f2fs_cached_block *last_entry = NULL; unsigned int nr;
while ((nr = f2fs_cache_gang_lookup_tag(NODE_CACHE(sbi), entries, &index, F2FS_ONSTACK_CACHES, F2FS_CACHE_TAG_DIRTY))) { int i;
for (i = 0; i < nr; i++) { struct f2fs_cached_block *entry = entries[i];
if (unlikely(f2fs_cp_error(sbi))) { f2fs_put_cache(last_entry, false); f2fs_cache_gang_release(entries, nr); return ERR_PTR(-EIO); }
if (!IS_DNODE(sbi, entry) || !is_cold_node(sbi, entry)) continue; if (ino_of_node(sbi, entry) != ino) continue;
f2fs_lock_cache(entry);
if (unlikely(!f2fs_is_node_cache(entry))) {continue_unlock: f2fs_unlock_cache(entry); continue; } if (ino_of_node(sbi, entry) != ino) goto continue_unlock;
if (!f2fs_cache_test_dirty(entry)) { /* someone wrote it for us */ goto continue_unlock; }
if (last_entry) f2fs_put_cache(last_entry, false);
f2fs_cache_get(entry); last_entry = entry; f2fs_unlock_cache(entry); } f2fs_cache_gang_release(entries, nr); cond_resched(); } return last_entry;}
static bool __write_node_cache(struct f2fs_cached_block *entry, bool atomic, bool do_fsync, bool *submitted, bool sync, bool do_balance, enum iostat_type io_type, unsigned int *seq_id){ struct f2fs_sb_info *sbi = entry->cache->sbi; nid_t nid; struct node_info ni; struct f2fs_io_info fio = { .sbi = sbi, .ino = ino_of_node(sbi, entry), .type = NODE, .op = REQ_OP_WRITE, .op_flags = sync ? REQ_SYNC : REQ_BACKGROUND, .cache_entry = entry, .encrypted_page = NULL, .submitted = 0, .io_type = io_type, .is_cache = true, }; struct f2fs_lock_context lc; unsigned int seq;
if (unlikely(f2fs_cp_error(sbi))) { /* keep node caches in remount-ro mode */ if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_READONLY) goto redirty_out; f2fs_cache_clear_uptodate(entry); f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_DIRTY, F2FS_CACHE_TAG_NONE); dec_cache_count(entry->cache->sbi, F2FS_DIRTY_NODES); f2fs_unlock_cache(entry); return true; }
if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) goto redirty_out;
if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) && !sync && IS_DNODE(sbi, entry) && is_cold_node(sbi, entry)) goto redirty_out;
/* get old block addr of this node cache */ nid = nid_of_node(sbi, entry);
if (f2fs_sanity_check_node_footer(sbi, entry, entry->index, NODE_TYPE_REGULAR, false)) { fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS); f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_NID); goto redirty_out; }
if (f2fs_get_node_info(sbi, nid, &ni, !do_balance)) goto redirty_out;
f2fs_down_read_trace(&sbi->node_write, &lc);
/* This cache is already truncated */ if (unlikely(ni.blk_addr == NULL_ADDR)) { f2fs_cache_clear_uptodate(entry); f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_DIRTY, F2FS_CACHE_TAG_NONE); dec_cache_count(entry->cache->sbi, F2FS_DIRTY_NODES); f2fs_up_read_trace(&sbi->node_write, &lc); f2fs_unlock_cache(entry); return true; }
if (__is_valid_data_blkaddr(ni.blk_addr) && !f2fs_is_valid_blkaddr(sbi, ni.blk_addr, DATA_GENERIC_ENHANCE)) { f2fs_up_read_trace(&sbi->node_write, &lc); goto redirty_out; }
if (atomic && !test_opt(sbi, NOBARRIER)) fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
set_dentry_mark(sbi, entry, false); set_fsync_mark(sbi, entry, do_fsync); if (IS_INODE(sbi, entry) && (atomic || is_fsync_dnode(sbi, entry))) set_dentry_mark(sbi, entry, f2fs_need_dentry_mark(sbi, ino_of_node(sbi, entry)));
/* should add to global list before clearing CACHE status */ if (f2fs_in_warm_node_list(sbi, entry)) { seq = f2fs_add_fsync_node_entry(sbi, entry); if (seq_id) *seq_id = seq; }
f2fs_start_cache_writeback(entry);
fio.old_blkaddr = ni.blk_addr; f2fs_do_write_node_cache(nid, &fio); set_node_addr(sbi, &ni, fio.new_blkaddr, is_fsync_dnode(sbi, entry)); dec_cache_count(sbi, F2FS_DIRTY_NODES); f2fs_up_read_trace(&sbi->node_write, &lc);
f2fs_unlock_cache(entry);
if (unlikely(f2fs_cp_error(sbi))) { f2fs_submit_merged_write(sbi, NODE); submitted = NULL; } if (submitted) *submitted = fio.submitted;
if (do_balance) f2fs_balance_fs(sbi, false); return true;
redirty_out: f2fs_cache_set_dirty(entry); f2fs_unlock_cache(entry); return false;}
int f2fs_write_node_cache(struct f2fs_cached_block *node_entry, int sync_mode, bool mark_dirty, enum iostat_type io_type){ int err = 0;
if (!sync_mode) { /* set cache dirty and write it */ if (!f2fs_cache_test_writeback(node_entry)) f2fs_mark_cache_dirty(node_entry); goto out_entry; }
f2fs_cache_wait_writeback(node_entry);
if (mark_dirty) f2fs_mark_cache_dirty(node_entry); else if (!f2fs_cache_test_dirty(node_entry)) goto out_entry;
if (!f2fs_cache_test_and_clear_dirty(node_entry)) { err = -EAGAIN; goto out_entry; }
if (!__write_node_cache(node_entry, false, false, NULL, true, false, io_type, NULL)) err = -EAGAIN; goto release_entry;out_entry: f2fs_unlock_cache(node_entry);release_entry: f2fs_put_cache(node_entry, false); return err;}
int f2fs_move_node_cache(struct f2fs_cached_block *entry, int gc_type){ return f2fs_write_node_cache(entry, gc_type == FG_GC, true, FS_GC_NODE_IO);}
int f2fs_fsync_node_caches(struct f2fs_sb_info *sbi, struct inode *inode, bool atomic, unsigned int *seq_id){ pgoff_t index; struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES]; int ret = 0; struct f2fs_cached_block *last_entry = NULL; bool marked = false; nid_t ino = inode->i_ino; int nr; int nwritten = 0;
if (atomic) { last_entry = last_fsync_dnode(sbi, ino); if (IS_ERR_OR_NULL(last_entry)) return PTR_ERR_OR_ZERO(last_entry); }retry: index = 0;
while ((nr = f2fs_cache_gang_lookup_tag(NODE_CACHE(sbi), entries, &index, F2FS_ONSTACK_CACHES, F2FS_CACHE_TAG_DIRTY))) { int i;
for (i = 0; i < nr; i++) { struct f2fs_cached_block *entry = entries[i]; bool submitted = false; bool do_fsync = false;
if (unlikely(f2fs_cp_error(sbi))) { f2fs_put_cache(last_entry, false); f2fs_cache_gang_release(entries, nr); ret = -EIO; goto out; }
if (!IS_DNODE(sbi, entry) || !is_cold_node(sbi, entry)) continue; if (ino_of_node(sbi, entry) != ino) continue;
f2fs_lock_cache(entry);
if (unlikely(!f2fs_is_node_cache(entry))) {continue_unlock: f2fs_unlock_cache(entry); continue; } if (ino_of_node(sbi, entry) != ino) goto continue_unlock;
if (!f2fs_cache_test_dirty(entry) && entry != last_entry) { /* someone wrote it for us */ goto continue_unlock; }
f2fs_cache_wait_writeback(entry);
if (!atomic || entry == last_entry) { do_fsync = true; percpu_counter_inc(&sbi->rf_node_block_count); if (IS_INODE(sbi, entry)) { if (is_inode_flag_set(inode, FI_DIRTY_INODE)) f2fs_update_inode(inode, entry); } /* may be written by other thread */ if (!f2fs_cache_test_dirty(entry)) f2fs_mark_cache_dirty(entry); }
if (!f2fs_cache_test_and_clear_dirty(entry)) goto continue_unlock;
if (!__write_node_cache(entry, atomic && entry == last_entry, do_fsync, &submitted, true, true, FS_NODE_IO, seq_id)) { f2fs_put_cache(last_entry, false); f2fs_cache_gang_release(entries, nr); ret = -EIO; goto out; } if (submitted) nwritten++;
if (entry == last_entry) { f2fs_put_cache(entry, false); f2fs_cache_gang_release(entries, nr); marked = true; goto out; } } f2fs_cache_gang_release(entries, nr); cond_resched(); } if (atomic && !marked) { f2fs_debug(sbi, "Retry to write fsync mark: ino=%u, idx=%lx", ino, last_entry->index); f2fs_lock_cache(last_entry); if (unlikely(!f2fs_is_node_cache(last_entry))) { f2fs_put_cache(last_entry, true); ret = -EAGAIN; goto out; } f2fs_cache_wait_writeback(last_entry); f2fs_mark_cache_dirty(last_entry); f2fs_unlock_cache(last_entry); goto retry; }out: if (nwritten) f2fs_submit_merged_write_cache(sbi, NULL, ino, NODE); return ret;}
static int f2fs_match_ino(struct inode *inode, u64 ino, void *data){ struct f2fs_sb_info *sbi = F2FS_I_SB(inode); bool clean;
if (inode->i_ino != ino) return 0;
if (!is_inode_flag_set(inode, FI_DIRTY_INODE)) return 0;
spin_lock(&sbi->inode_lock[DIRTY_META]); clean = list_empty(&F2FS_I(inode)->gdirty_list); spin_unlock(&sbi->inode_lock[DIRTY_META]);
if (clean) return 0;
inode = igrab(inode); if (!inode) return 0; return 1;}
static bool flush_dirty_cache_inode(struct f2fs_cached_block *entry){ struct f2fs_sb_info *sbi = entry->cache->sbi; struct inode *inode; nid_t ino = ino_of_node(sbi, entry);
inode = find_inode_nowait(sbi->sb, ino, f2fs_match_ino, NULL); if (!inode) return false;
f2fs_update_inode(inode, entry); f2fs_unlock_cache(entry);
iput(inode); return true;}
void f2fs_flush_inline_data(struct f2fs_sb_info *sbi){ pgoff_t index = 0; struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES]; unsigned int nr;
while ((nr = f2fs_cache_gang_lookup_tag(NODE_CACHE(sbi), entries, &index, F2FS_ONSTACK_CACHES, F2FS_CACHE_TAG_DIRTY))) { int i;
for (i = 0; i < nr; i++) { struct f2fs_cached_block *entry = entries[i];
f2fs_lock_cache(entry);
if (unlikely(!f2fs_is_node_cache(entry))) goto unlock; if (!IS_INODE(sbi, entry)) goto unlock; if (!f2fs_cache_test_dirty(entry)) goto unlock;
/* flush inline_data, if it's async context. */ if (f2fs_cache_test_inline(entry)) { nid_t ino = ino_of_node(sbi, entry);
f2fs_cache_clear_inline(entry); f2fs_unlock_cache(entry); flush_inline_data(sbi, ino); continue; }unlock: f2fs_unlock_cache(entry); } f2fs_cache_gang_release(entries, nr); cond_resched(); }}
int f2fs_writeback_node_caches(struct f2fs_sb_info *sbi, long nr_to_write, bool sync, bool do_balance, enum iostat_type io_type, long *nr_written){ pgoff_t index; struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES]; int step = 0; int nwritten = 0; int ret = 0; int nr, done = 0;
trace_f2fs_write_caches(sbi, nr_to_write, 0, NODE);
next_step: index = 0;
while (!done && (nr = f2fs_cache_gang_lookup_tag(NODE_CACHE(sbi), entries, &index, F2FS_ONSTACK_CACHES, F2FS_CACHE_TAG_DIRTY))) { int i;
for (i = 0; i < nr; i++) { struct f2fs_cached_block *entry = entries[i]; bool submitted = false;
if (!sync && unlikely(freezing(current))) { done = 1; break; }
/* give a priority to WB_SYNC threads */ if (atomic_read(&sbi->wb_sync_req[NODE]) && !sync) { done = 1; break; }
/* * flushing sequence with step: * 0. indirect nodes * 1. dentry dnodes * 2. file dnodes */ if (step == 0 && IS_DNODE(sbi, entry)) continue; if (step == 1 && (!IS_DNODE(sbi, entry) || is_cold_node(sbi, entry))) continue; if (step == 2 && (!IS_DNODE(sbi, entry) || !is_cold_node(sbi, entry))) continue;lock_node: if (sync) f2fs_lock_cache(entry); else if (!f2fs_trylock_cache(entry)) continue;
if (unlikely(!f2fs_is_node_cache(entry))) {continue_unlock: f2fs_unlock_cache(entry); continue; }
if (!f2fs_cache_test_dirty(entry)) { /* someone wrote it for us */ goto continue_unlock; }
/* flush inline_data/inode, if it's async context. */ if (!do_balance) goto write_node;
/* flush inline_data */ if (f2fs_cache_test_inline(entry)) { nid_t ino = ino_of_node(sbi, entry);
f2fs_cache_clear_inline(entry); f2fs_unlock_cache(entry); flush_inline_data(sbi, ino); goto lock_node; }
/* flush dirty inode */ if (IS_INODE(sbi, entry) && flush_dirty_cache_inode(entry)) goto lock_node;write_node: f2fs_cache_wait_writeback(entry);
if (!f2fs_cache_test_and_clear_dirty(entry)) goto continue_unlock;
if (!__write_node_cache(entry, false, false, &submitted, sync, do_balance, io_type, NULL)) { f2fs_cache_gang_release(entries, nr); ret = -EIO; goto out; } if (submitted) nwritten++;
if (nwritten >= nr_to_write) break; } f2fs_cache_gang_release(entries, nr); cond_resched();
if (nwritten >= nr_to_write) { step = 2; break; } }
if (step < 2) { if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) && !sync && step == 1) goto out; step++; goto next_step; }out: if (nwritten) f2fs_submit_merged_write(sbi, NODE);
trace_f2fs_write_caches(sbi, nr_to_write, nwritten, NODE);
if (nr_written) *nr_written = nwritten;
if (unlikely(f2fs_cp_error(sbi))) return -EIO; return ret;}
int f2fs_wait_on_node_caches_writeback(struct f2fs_sb_info *sbi, unsigned int seq_id){ struct fsync_node_entry *fn; struct list_head *head = &sbi->fsync_node_list; unsigned long flags; unsigned int cur_seq_id = 0;
while (seq_id && cur_seq_id < seq_id) { struct f2fs_cached_block *entry;
spin_lock_irqsave(&sbi->fsync_node_lock, flags); if (list_empty(head)) { spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); break; } fn = list_first_entry(head, struct fsync_node_entry, list); if (fn->seq_id > seq_id) { spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); break; } cur_seq_id = fn->seq_id; entry = fn->entry; f2fs_cache_get(entry); spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);
f2fs_lock_cache(entry); f2fs_cache_wait_writeback(entry);
f2fs_put_cache(entry, true); }
return f2fs_cp_error(sbi) ? -EIO : 0;}
int f2fs_write_node_caches(struct f2fs_sb_info *sbi, long *nr_written){ struct blk_plug plug; long nr_to_write = LONG_MAX;
if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) return -EAGAIN;
/* balancing f2fs's metadata in background */ f2fs_balance_fs_bg(sbi, true);
/* collect a number of dirty node caches and write together */ if (get_nr_caches(sbi, F2FS_DIRTY_NODES) < nr_caches_to_skip(sbi, NODE)) return -EAGAIN;
if (atomic_read(&sbi->wb_sync_req[NODE])) { /* to avoid potential deadlock */ if (current->plug) blk_finish_plug(current->plug); return -EAGAIN; }
nr_to_write = adjust_flush_cache_number(sbi, NODE); blk_start_plug(&plug); f2fs_writeback_node_caches(sbi, nr_to_write, false, true, FS_NODE_IO, nr_written); blk_finish_plug(&plug); return 0;}
static struct free_nid *__lookup_free_nid_list(struct f2fs_nm_info *nm_i, nid_t n){ return radix_tree_lookup(&nm_i->free_nid_root, n);}
static int __insert_free_nid(struct f2fs_sb_info *sbi, struct free_nid *i){ struct f2fs_nm_info *nm_i = NM_I(sbi); int err = radix_tree_insert(&nm_i->free_nid_root, i->nid, i);
if (err) return err;
nm_i->nid_cnt[FREE_NID]++; list_add_tail(&i->list, &nm_i->free_nid_list); return 0;}
static void __remove_free_nid(struct f2fs_sb_info *sbi, struct free_nid *i, enum nid_state state){ struct f2fs_nm_info *nm_i = NM_I(sbi);
f2fs_bug_on(sbi, state != i->state); nm_i->nid_cnt[state]--; if (state == FREE_NID) list_del(&i->list); radix_tree_delete(&nm_i->free_nid_root, i->nid);}
static void __move_free_nid(struct f2fs_sb_info *sbi, struct free_nid *i, enum nid_state org_state, enum nid_state dst_state){ struct f2fs_nm_info *nm_i = NM_I(sbi);
f2fs_bug_on(sbi, org_state != i->state); i->state = dst_state; nm_i->nid_cnt[org_state]--; nm_i->nid_cnt[dst_state]++;
switch (dst_state) { case PREALLOC_NID: list_del(&i->list); break; case FREE_NID: list_add_tail(&i->list, &nm_i->free_nid_list); break; default: BUG_ON(1); }}
static void update_free_nid_bitmap(struct f2fs_sb_info *sbi, nid_t nid, bool set, bool build){ struct f2fs_nm_info *nm_i = NM_I(sbi); unsigned int nat_ofs = NAT_BLOCK_OFFSET(sbi, nid); unsigned int nid_ofs = nid - f2fs_start_nid(sbi, nid);
if (!test_bit_le(nat_ofs, nm_i->nat_block_bitmap)) return;
if (set) { if (test_bit_le(nid_ofs, nm_i->free_nid_bitmap[nat_ofs])) return; __set_bit_le(nid_ofs, nm_i->free_nid_bitmap[nat_ofs]); nm_i->free_nid_count[nat_ofs]++; } else { if (!test_bit_le(nid_ofs, nm_i->free_nid_bitmap[nat_ofs])) return; __clear_bit_le(nid_ofs, nm_i->free_nid_bitmap[nat_ofs]); if (!build) nm_i->free_nid_count[nat_ofs]--; }}
/* return if the nid is recognized as free */static bool add_free_nid(struct f2fs_sb_info *sbi, nid_t nid, bool build, bool update){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct free_nid *i, *e; struct nat_entry *ne; int err; bool ret = false;
/* 0 nid should not be used */ if (unlikely(nid == 0)) return false;
if (unlikely(f2fs_check_nid_range(sbi, nid))) return false;
i = f2fs_kmem_cache_alloc(free_nid_slab, GFP_NOFS, true, NULL); i->nid = nid; i->state = FREE_NID;
err = radix_tree_preload(GFP_NOFS | __GFP_NOFAIL); f2fs_bug_on(sbi, err);
err = -EINVAL;
spin_lock(&nm_i->nid_list_lock);
if (build) { /* * Thread A Thread B * - f2fs_create * - f2fs_new_inode * - f2fs_alloc_nid * - __insert_nid_to_list(PREALLOC_NID) * - f2fs_balance_fs_bg * - f2fs_build_free_nids * - __f2fs_build_free_nids * - scan_nat_blocks * - add_free_nid * - __lookup_nat_cache * - f2fs_add_link * - f2fs_init_inode_metadata * - f2fs_new_inode_folio * - f2fs_new_node_folio * - set_node_addr * - f2fs_alloc_nid_done * - __remove_nid_from_list(PREALLOC_NID) * - __insert_nid_to_list(FREE_NID) */ ne = __lookup_nat_cache(nm_i, nid, false); if (ne && (!get_nat_flag(ne, IS_CHECKPOINTED) || nat_get_blkaddr(ne) != NULL_ADDR)) goto err_out;
e = __lookup_free_nid_list(nm_i, nid); if (e) { if (e->state == FREE_NID) ret = true; goto err_out; } } ret = true; err = __insert_free_nid(sbi, i);err_out: if (update) { update_free_nid_bitmap(sbi, nid, ret, build); if (!build) nm_i->available_nids++; } spin_unlock(&nm_i->nid_list_lock); radix_tree_preload_end();
if (err) kmem_cache_free(free_nid_slab, i); return ret;}
static void remove_free_nid(struct f2fs_sb_info *sbi, nid_t nid){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct free_nid *i; bool need_free = false;
spin_lock(&nm_i->nid_list_lock); i = __lookup_free_nid_list(nm_i, nid); if (i && i->state == FREE_NID) { __remove_free_nid(sbi, i, FREE_NID); need_free = true; } spin_unlock(&nm_i->nid_list_lock);
if (need_free) kmem_cache_free(free_nid_slab, i);}
static int scan_nat_block(struct f2fs_sb_info *sbi, struct f2fs_nat_block *nat_blk, nid_t start_nid){ struct f2fs_nm_info *nm_i = NM_I(sbi); block_t blk_addr; unsigned int nat_ofs = NAT_BLOCK_OFFSET(sbi, start_nid); int i;
__set_bit_le(nat_ofs, nm_i->nat_block_bitmap);
i = start_nid % NAT_ENTRY_PER_BLOCK(sbi);
for (; i < NAT_ENTRY_PER_BLOCK(sbi); i++, start_nid++) { if (unlikely(start_nid >= nm_i->max_nid)) break;
blk_addr = le32_to_cpu(nat_blk->entries[i].block_addr);
if (blk_addr == NEW_ADDR) return -EFSCORRUPTED;
if (blk_addr == NULL_ADDR) { add_free_nid(sbi, start_nid, true, true); } else { spin_lock(&NM_I(sbi)->nid_list_lock); update_free_nid_bitmap(sbi, start_nid, false, true); spin_unlock(&NM_I(sbi)->nid_list_lock); } }
return 0;}
static void scan_curseg_cache(struct f2fs_sb_info *sbi){ struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); struct f2fs_journal *journal = curseg->journal; int i;
down_read(&curseg->journal_rwsem); for (i = 0; i < nats_in_cursum(journal); i++) { block_t addr; nid_t nid;
addr = le32_to_cpu(nat_in_journal(journal, i).block_addr); nid = le32_to_cpu(nid_in_journal(journal, i)); if (addr == NULL_ADDR) add_free_nid(sbi, nid, true, false); else remove_free_nid(sbi, nid); } up_read(&curseg->journal_rwsem);}
static void scan_free_nid_bits(struct f2fs_sb_info *sbi){ struct f2fs_nm_info *nm_i = NM_I(sbi); unsigned int i, idx; nid_t nid; struct f2fs_lock_context lc;
f2fs_down_read_trace(&nm_i->nat_tree_lock, &lc);
for (i = 0; i < nm_i->nat_blocks; i++) { if (!test_bit_le(i, nm_i->nat_block_bitmap)) continue; if (!nm_i->free_nid_count[i]) continue; for (idx = 0; idx < NAT_ENTRY_PER_BLOCK(sbi); idx++) { idx = find_next_bit_le(nm_i->free_nid_bitmap[i], NAT_ENTRY_PER_BLOCK(sbi), idx); if (idx >= NAT_ENTRY_PER_BLOCK(sbi)) break;
nid = i * NAT_ENTRY_PER_BLOCK(sbi) + idx; add_free_nid(sbi, nid, true, false);
if (nm_i->nid_cnt[FREE_NID] >= MAX_FREE_NIDS(sbi)) goto out; } }out: scan_curseg_cache(sbi);
f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc);}
static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct f2fs_cached_block *entry = NULL; int i = 0, ret; nid_t nid = nm_i->next_scan_nid; struct f2fs_lock_context lc;
if (unlikely(nid >= nm_i->max_nid)) nid = 0;
if (unlikely(nid % NAT_ENTRY_PER_BLOCK(sbi))) nid = NAT_BLOCK_OFFSET(sbi, nid) * NAT_ENTRY_PER_BLOCK(sbi);
/* Enough entries */ if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK(sbi)) return 0;
if (!sync && !f2fs_available_free_memory(sbi, FREE_NIDS)) return 0;
if (!mount) { /* try to find free nids in free_nid_bitmap */ scan_free_nid_bits(sbi);
if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK(sbi)) return 0; }
/* readahead nat blocks to be scanned */ f2fs_ra_meta_caches(sbi, NAT_BLOCK_OFFSET(sbi, nid), FREE_NID_BLOCKS, META_NAT, true);
f2fs_down_read_trace(&nm_i->nat_tree_lock, &lc);
while (1) { if (!test_bit_le(NAT_BLOCK_OFFSET(sbi, nid), nm_i->nat_block_bitmap)) { entry = get_current_nat_cache(sbi, nid);
if (IS_ERR(entry)) { ret = PTR_ERR(entry); } else { ret = scan_nat_block(sbi, cache_address(entry), nid); f2fs_put_cache(entry, true); }
if (ret) { f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc);
if (ret == -EFSCORRUPTED) { f2fs_err(sbi, "NAT is corrupt, run fsck to fix it"); set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT); }
return ret; } }
nid += NAT_ENTRY_PER_BLOCK(sbi) - (nid % NAT_ENTRY_PER_BLOCK(sbi)); if (unlikely(nid >= nm_i->max_nid)) nid = 0;
if (++i >= FREE_NID_BLOCKS) break; }
/* go to the next free nat blocks to find free nids abundantly */ nm_i->next_scan_nid = nid;
/* find free nids from current sum_blocks */ scan_curseg_cache(sbi);
f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc);
f2fs_ra_meta_caches(sbi, NAT_BLOCK_OFFSET(sbi, nm_i->next_scan_nid), nm_i->ra_nid_blocks, META_NAT, false);
return 0;}
int f2fs_build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount){ int ret;
mutex_lock(&NM_I(sbi)->build_lock); ret = __f2fs_build_free_nids(sbi, sync, mount); mutex_unlock(&NM_I(sbi)->build_lock);
return ret;}
/* * If this function returns success, caller can obtain a new nid * from second parameter of this function. * The returned nid could be used ino as well as nid when inode is created. */bool f2fs_alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct free_nid *i = NULL;retry: if (time_to_inject(sbi, FAULT_ALLOC_NID)) return false;
spin_lock(&nm_i->nid_list_lock);
if (unlikely(nm_i->available_nids == 0)) { spin_unlock(&nm_i->nid_list_lock); return false; }
/* We should not use stale free nids created by f2fs_build_free_nids */ if (nm_i->nid_cnt[FREE_NID] && !on_f2fs_build_free_nids(nm_i)) { f2fs_bug_on(sbi, list_empty(&nm_i->free_nid_list)); i = list_first_entry(&nm_i->free_nid_list, struct free_nid, list);
if (unlikely(is_invalid_nid(sbi, i->nid))) { spin_unlock(&nm_i->nid_list_lock); f2fs_err(sbi, "Corrupted nid %u in free_nid_list", i->nid); fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS); f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_NID); return false; }
*nid = i->nid;
__move_free_nid(sbi, i, FREE_NID, PREALLOC_NID); nm_i->available_nids--;
update_free_nid_bitmap(sbi, *nid, false, false);
spin_unlock(&nm_i->nid_list_lock); return true; } spin_unlock(&nm_i->nid_list_lock);
/* Let's scan nat blocks and its caches to get free nids */ if (!f2fs_build_free_nids(sbi, true, false)) goto retry; return false;}
/* * f2fs_alloc_nid() should be called prior to this function. */void f2fs_alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct free_nid *i;
spin_lock(&nm_i->nid_list_lock); i = __lookup_free_nid_list(nm_i, nid); f2fs_bug_on(sbi, !i); __remove_free_nid(sbi, i, PREALLOC_NID); spin_unlock(&nm_i->nid_list_lock);
kmem_cache_free(free_nid_slab, i);}
/* * f2fs_alloc_nid() should be called prior to this function. */void f2fs_alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct free_nid *i; bool need_free = false;
if (!nid) return;
spin_lock(&nm_i->nid_list_lock); i = __lookup_free_nid_list(nm_i, nid); f2fs_bug_on(sbi, !i);
if (!f2fs_available_free_memory(sbi, FREE_NIDS)) { __remove_free_nid(sbi, i, PREALLOC_NID); need_free = true; } else { __move_free_nid(sbi, i, PREALLOC_NID, FREE_NID); }
nm_i->available_nids++;
update_free_nid_bitmap(sbi, nid, true, false);
spin_unlock(&nm_i->nid_list_lock);
if (need_free) kmem_cache_free(free_nid_slab, i);}
int f2fs_try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink){ struct f2fs_nm_info *nm_i = NM_I(sbi); int nr = nr_shrink;
if (nm_i->nid_cnt[FREE_NID] <= MAX_FREE_NIDS(sbi)) return 0;
if (!mutex_trylock(&nm_i->build_lock)) return 0;
while (nr_shrink && nm_i->nid_cnt[FREE_NID] > MAX_FREE_NIDS(sbi)) { struct free_nid *i, *next; unsigned int batch = SHRINK_NID_BATCH_SIZE;
spin_lock(&nm_i->nid_list_lock); list_for_each_entry_safe(i, next, &nm_i->free_nid_list, list) { if (!nr_shrink || !batch || nm_i->nid_cnt[FREE_NID] <= MAX_FREE_NIDS(sbi)) break; __remove_free_nid(sbi, i, FREE_NID); kmem_cache_free(free_nid_slab, i); nr_shrink--; batch--; } spin_unlock(&nm_i->nid_list_lock); }
mutex_unlock(&nm_i->build_lock);
return nr - nr_shrink;}
int f2fs_recover_inline_xattr(struct inode *inode, struct f2fs_cached_block *entry){ void *src_addr, *dst_addr; size_t inline_size; struct f2fs_cached_block *ientry; struct f2fs_inode *ri;
ientry = f2fs_get_inode_cache(F2FS_I_SB(inode), inode->i_ino); if (IS_ERR(ientry)) return PTR_ERR(ientry);
ri = &CACHED_NODE(entry)->i; if (ri->i_inline & F2FS_INLINE_XATTR) { if (!f2fs_has_inline_xattr(inode)) { set_inode_flag(inode, FI_INLINE_XATTR); stat_inc_inline_xattr(inode); } } else { if (f2fs_has_inline_xattr(inode)) { stat_dec_inline_xattr(inode); clear_inode_flag(inode, FI_INLINE_XATTR); } goto update_inode; }
dst_addr = inline_xattr_addr(inode, ientry); src_addr = inline_xattr_addr(inode, entry); inline_size = inline_xattr_size(inode);
f2fs_cache_wait_writeback(ientry); memcpy(dst_addr, src_addr, inline_size);update_inode: /* * The xattr node of the checkpoint was removed before fsync, e.g. the * xattrs became small enough to fit inline. Release it as well, or * the inode keeps pointing to the stale xattrs. */ if (!ri->i_xattr_nid && F2FS_I(inode)->i_xattr_nid) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct node_info ni; int err;
err = f2fs_get_node_info(sbi, F2FS_I(inode)->i_xattr_nid, &ni, false); if (err) { f2fs_put_cache(ientry, true); return err; }
if (ni.blk_addr != NEW_ADDR && !f2fs_is_valid_blkaddr(sbi, ni.blk_addr, DATA_GENERIC_ENHANCE)) { f2fs_err_ratelimited(sbi, "nat entry is corrupted, run fsck to fix it, ino:%u, nid:%u, blkaddr:%u", ni.ino, ni.nid, ni.blk_addr); set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT); f2fs_put_cache(ientry, true); return -EFSCORRUPTED; }
f2fs_invalidate_blocks(sbi, ni.blk_addr, 1); dec_valid_node_count(sbi, inode, false); set_node_addr(sbi, &ni, NULL_ADDR, false); /* drop the cached xattr node, which may be dirty */ f2fs_truncate_node_caches(sbi, ni.nid, 1); f2fs_i_xnid_write(inode, 0); }
f2fs_update_inode(inode, ientry); f2fs_put_cache(ientry, true); return 0;}
int f2fs_recover_xattr_data(struct inode *inode, struct f2fs_cached_block *entry){ struct f2fs_sb_info *sbi = F2FS_I_SB(inode); nid_t prev_xnid = F2FS_I(inode)->i_xattr_nid; nid_t new_xnid; struct dnode_of_data dn; struct node_info ni; struct f2fs_cached_block *xentry; int err;
if (!prev_xnid) goto recover_xnid;
/* 1: invalidate the previous xattr nid */ err = f2fs_get_node_info(sbi, prev_xnid, &ni, false); if (err) return err;
f2fs_invalidate_blocks(sbi, ni.blk_addr, 1); dec_valid_node_count(sbi, inode, false); set_node_addr(sbi, &ni, NULL_ADDR, false);
recover_xnid: /* 2: update xattr nid in inode */ if (!f2fs_alloc_nid(sbi, &new_xnid)) return -ENOSPC;
set_new_dnode(&dn, inode, NULL, NULL, new_xnid); xentry = f2fs_new_node_cache(&dn, XATTR_NODE_OFFSET); if (IS_ERR(xentry)) { f2fs_alloc_nid_failed(sbi, new_xnid); return PTR_ERR(xentry); }
f2fs_alloc_nid_done(sbi, new_xnid); f2fs_update_inode_cache(inode);
/* 3: update and set xattr node cache dirty */ if (entry) { memcpy(CACHED_NODE(xentry), CACHED_NODE(entry), VALID_XATTR_BLOCK_SIZE(inode)); f2fs_mark_cache_dirty(xentry); } f2fs_put_cache(xentry, true);
return 0;}
int f2fs_recover_inode_cache(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry){ struct f2fs_inode *src, *dst; nid_t ino = ino_of_node(sbi, entry); struct node_info old_ni, new_ni; struct f2fs_cached_block *ientry; int err;
err = f2fs_get_node_info(sbi, ino, &old_ni, false); if (err) return err;
if (unlikely(old_ni.blk_addr != NULL_ADDR)) return -EINVAL;retry: ientry = f2fs_grab_node_cache(sbi, ino); if (IS_ERR(ientry)) { memalloc_retry_wait(GFP_NOFS); goto retry; }
/* Should not use this inode from free nid list */ remove_free_nid(sbi, ino);
f2fs_cache_set_uptodate(ientry); fill_node_footer(sbi, ientry, ino, ino, 0, true); set_cold_node(sbi, ientry, false);
src = &CACHED_NODE(entry)->i; dst = F2FS_INODE(ientry);
memcpy(dst, src, offsetof(struct f2fs_inode, i_ext)); dst->i_size = 0; dst->i_blocks = cpu_to_le64(1); dst->i_links = cpu_to_le32(1); dst->i_xattr_nid = 0; dst->i_inline = src->i_inline & (F2FS_INLINE_XATTR | F2FS_EXTRA_ATTR); if (dst->i_inline & F2FS_EXTRA_ATTR) { dst->i_extra_isize = src->i_extra_isize;
if (f2fs_sb_has_flexible_inline_xattr(sbi) && F2FS_FITS_IN_INODE(src, le16_to_cpu(src->i_extra_isize), i_inline_xattr_size)) dst->i_inline_xattr_size = src->i_inline_xattr_size;
if (f2fs_sb_has_project_quota(sbi) && F2FS_FITS_IN_INODE(src, le16_to_cpu(src->i_extra_isize), i_projid)) dst->i_projid = src->i_projid;
if (f2fs_sb_has_inode_crtime(sbi) && F2FS_FITS_IN_INODE(src, le16_to_cpu(src->i_extra_isize), i_crtime_nsec)) { dst->i_crtime = src->i_crtime; dst->i_crtime_nsec = src->i_crtime_nsec; } }
new_ni = old_ni; new_ni.ino = ino;
if (unlikely(inc_valid_node_count(sbi, NULL, true))) WARN_ON(1); set_node_addr(sbi, &new_ni, NEW_ADDR, false); inc_valid_inode_count(sbi); f2fs_mark_cache_dirty(ientry); f2fs_put_cache(ientry, true); return 0;}
int f2fs_restore_node_summary(struct f2fs_sb_info *sbi, unsigned int segno, struct f2fs_summary_block *sum){ struct f2fs_summary *sum_entry; block_t addr; int i, idx, last_offset, nrblocks;
/* scan the node segment */ last_offset = BLKS_PER_SEG(sbi); addr = START_BLOCK(sbi, segno); sum_entry = sum_entries(sum);
for (i = 0; i < last_offset; i += nrblocks, addr += nrblocks) { nrblocks = bio_max_segs(last_offset - i);
/* readahead node blocks */ f2fs_ra_meta_caches(sbi, addr, nrblocks, META_POR, true);
for (idx = addr; idx < addr + nrblocks; idx++) { struct f2fs_cached_block *entry = f2fs_get_tmp_cache(sbi, idx);
if (IS_ERR(entry)) return PTR_ERR(entry);
struct node_footer *footer = (struct node_footer *)(cache_address(entry) + F2FS_BLKSIZE(sbi) - sizeof(struct node_footer)); sum_entry->nid = footer->nid; sum_entry->version = 0; sum_entry->ofs_in_node = 0; sum_entry++; f2fs_put_cache(entry, true); }
f2fs_truncate_meta_caches(sbi, addr, nrblocks); } return 0;}
static void remove_nats_in_journal(struct f2fs_sb_info *sbi){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); struct f2fs_journal *journal = curseg->journal; int i; bool init_dirty;
down_write(&curseg->journal_rwsem); for (i = 0; i < nats_in_cursum(journal); i++) { struct nat_entry *ne; struct f2fs_nat_entry raw_ne; nid_t nid = le32_to_cpu(nid_in_journal(journal, i));
if (f2fs_check_nid_range(sbi, nid)) continue;
init_dirty = false;
raw_ne = nat_in_journal(journal, i);
ne = __lookup_nat_cache(nm_i, nid, true); if (!ne) { init_dirty = true; ne = __alloc_nat_entry(sbi, nid, true); __init_nat_entry(nm_i, ne, &raw_ne, true, true); }
/* * if a free nat in journal has not been used after last * checkpoint, we should remove it from available nids, * since later we will add it again. */ if (!get_nat_flag(ne, IS_DIRTY) && le32_to_cpu(raw_ne.block_addr) == NULL_ADDR) { spin_lock(&nm_i->nid_list_lock); nm_i->available_nids--; spin_unlock(&nm_i->nid_list_lock); }
__set_nat_cache_dirty(sbi, nm_i, ne, init_dirty); } update_nats_in_cursum(journal, -i); up_write(&curseg->journal_rwsem);}
static void __adjust_nat_entry_set(struct nat_entry_set *nes, struct list_head *head, int max){ struct nat_entry_set *cur;
if (nes->entry_cnt >= max) goto add_out;
list_for_each_entry(cur, head, set_list) { if (cur->entry_cnt >= nes->entry_cnt) { list_add(&nes->set_list, cur->set_list.prev); return; } }add_out: list_add_tail(&nes->set_list, head);}
static void __update_nat_bits(struct f2fs_sb_info *sbi, nid_t start_nid, const struct f2fs_nat_block *nat_blk){ struct f2fs_nm_info *nm_i = NM_I(sbi); unsigned int nat_index = start_nid / NAT_ENTRY_PER_BLOCK(sbi); int valid = 0; int i = 0;
if (!enabled_nat_bits(sbi, NULL)) return;
if (nat_index == 0) { valid = 1; i = 1; } for (; i < NAT_ENTRY_PER_BLOCK(sbi); i++) { if (le32_to_cpu(nat_blk->entries[i].block_addr) != NULL_ADDR) valid++; } if (valid == 0) { __set_bit_le(nat_index, nm_i->empty_nat_bits); __clear_bit_le(nat_index, nm_i->full_nat_bits); return; }
__clear_bit_le(nat_index, nm_i->empty_nat_bits); if (valid == NAT_ENTRY_PER_BLOCK(sbi)) __set_bit_le(nat_index, nm_i->full_nat_bits); else __clear_bit_le(nat_index, nm_i->full_nat_bits);}
static int __flush_nat_entry_set(struct f2fs_sb_info *sbi, struct nat_entry_set *set, struct cp_control *cpc){ struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); struct f2fs_journal *journal = curseg->journal; nid_t start_nid = set->set * NAT_ENTRY_PER_BLOCK(sbi); bool to_journal = true; struct f2fs_nat_block *nat_blk; struct nat_entry *ne, *cur; struct f2fs_cached_block *entry = NULL;
/* * there are two steps to flush nat entries: * #1, flush nat entries to journal in current hot data summary block. * #2, flush nat entries to nat block. */ if (enabled_nat_bits(sbi, cpc) || !__has_cursum_space(sbi, journal, set->entry_cnt, NAT_JOURNAL)) to_journal = false;
if (to_journal) { down_write(&curseg->journal_rwsem); } else { entry = get_next_nat_cache(sbi, start_nid); if (IS_ERR(entry)) return PTR_ERR(entry);
nat_blk = cache_address(entry); f2fs_bug_on(sbi, !nat_blk); }
/* flush dirty nats in nat entry set */ list_for_each_entry_safe(ne, cur, &set->entry_list, list) { struct f2fs_nat_entry *raw_ne; nid_t nid = nat_get_nid(ne); int offset;
f2fs_bug_on(sbi, nat_get_blkaddr(ne) == NEW_ADDR);
if (to_journal) { offset = f2fs_lookup_journal_in_cursum(sbi, journal, NAT_JOURNAL, nid, 1); f2fs_bug_on(sbi, offset < 0); raw_ne = &nat_in_journal(journal, offset); nid_in_journal(journal, offset) = cpu_to_le32(nid); } else { raw_ne = &nat_blk->entries[nid - start_nid]; } raw_nat_from_node_info(raw_ne, &ne->ni); nat_reset_flag(ne); __clear_nat_cache_dirty(NM_I(sbi), set, ne); if (nat_get_blkaddr(ne) == NULL_ADDR) { add_free_nid(sbi, nid, false, true); } else { spin_lock(&NM_I(sbi)->nid_list_lock); update_free_nid_bitmap(sbi, nid, false, false); spin_unlock(&NM_I(sbi)->nid_list_lock); } }
if (to_journal) { up_write(&curseg->journal_rwsem); } else { __update_nat_bits(sbi, start_nid, nat_blk); f2fs_put_cache(entry, true); }
/* Allow dirty nats by node block allocation in write_begin */ if (!set->entry_cnt) { radix_tree_delete(&NM_I(sbi)->nat_set_root, set->set); kmem_cache_free(nat_entry_set_slab, set); } return 0;}
/* * This function is called during the checkpointing process. */int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); struct f2fs_journal *journal = curseg->journal; struct nat_entry_set *setvec[NAT_VEC_SIZE]; struct nat_entry_set *set, *tmp; unsigned int found, entry_count = 0; nid_t set_idx = 0; LIST_HEAD(sets); int err = 0; struct f2fs_lock_context lc;
/* * during unmount, let's flush nat_bits before checking * nat_cnt[DIRTY_NAT]. */ if (enabled_nat_bits(sbi, cpc)) { f2fs_down_write_trace(&nm_i->nat_tree_lock, &lc); remove_nats_in_journal(sbi); f2fs_up_write_trace(&nm_i->nat_tree_lock, &lc); }
if (!nm_i->nat_cnt[DIRTY_NAT]) return 0;
f2fs_down_write_trace(&nm_i->nat_tree_lock, &lc);
/* * if there are no enough space in journal to store dirty nat * entries, remove all entries from journal and merge them * into nat entry set. */ if (enabled_nat_bits(sbi, cpc) || !__has_cursum_space(sbi, journal, nm_i->nat_cnt[DIRTY_NAT], NAT_JOURNAL)) remove_nats_in_journal(sbi);
while ((found = __gang_lookup_nat_set(nm_i, set_idx, NAT_VEC_SIZE, setvec))) { unsigned idx;
set_idx = setvec[found - 1]->set + 1; for (idx = 0; idx < found; idx++) __adjust_nat_entry_set(setvec[idx], &sets, MAX_NAT_JENTRIES(sbi, journal)); }
/* * Readahead the current NAT block to prevent read requests from * being issued and waited on one by one. */ list_for_each_entry(set, &sets, set_list) { entry_count += set->entry_cnt; if (!enabled_nat_bits(sbi, cpc) && __has_cursum_space(sbi, journal, entry_count, NAT_JOURNAL)) continue; f2fs_ra_meta_caches(sbi, set->set, 1, META_NAT, true); } /* flush dirty nats in nat entry set */ list_for_each_entry_safe(set, tmp, &sets, set_list) { err = __flush_nat_entry_set(sbi, set, cpc); if (err) break; }
f2fs_up_write_trace(&nm_i->nat_tree_lock, &lc); /* Allow dirty nats by node block allocation in write_begin */
return err;}
static int __get_nat_bitmaps(struct f2fs_sb_info *sbi){ struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); struct f2fs_nm_info *nm_i = NM_I(sbi); unsigned int nat_bits_bytes = nm_i->nat_blocks / BITS_PER_BYTE; unsigned int i; __u64 cp_ver = cur_cp_version(ckpt); block_t nat_bits_addr;
if (!enabled_nat_bits(sbi, NULL)) return 0;
nm_i->nat_bits_blocks = F2FS_BLK_ALIGN(sbi, (nat_bits_bytes << 1) + 8); nm_i->nat_bits = f2fs_kvzalloc(sbi, F2FS_BLK_TO_BYTES(sbi, nm_i->nat_bits_blocks), GFP_KERNEL); if (!nm_i->nat_bits) return -ENOMEM;
nat_bits_addr = __start_cp_addr(sbi) + BLKS_PER_SEG(sbi) - nm_i->nat_bits_blocks; for (i = 0; i < nm_i->nat_bits_blocks; i++) { struct f2fs_cached_block *entry;
entry = f2fs_get_meta_cache(sbi, nat_bits_addr++); if (IS_ERR(entry)) return PTR_ERR(entry);
memcpy(nm_i->nat_bits + F2FS_BLK_TO_BYTES(sbi, i), cache_address(entry), F2FS_BLKSIZE(sbi)); f2fs_put_cache(entry, true); }
cp_ver |= (cur_cp_crc(ckpt) << 32); if (cpu_to_le64(cp_ver) != *(__le64 *)nm_i->nat_bits) { disable_nat_bits(sbi, true); return 0; }
nm_i->full_nat_bits = nm_i->nat_bits + 8; nm_i->empty_nat_bits = nm_i->full_nat_bits + nat_bits_bytes;
f2fs_notice(sbi, "Found nat_bits in checkpoint"); return 0;}
static inline void load_free_nid_bitmap(struct f2fs_sb_info *sbi){ struct f2fs_nm_info *nm_i = NM_I(sbi); unsigned int i = 0; nid_t nid, last_nid;
if (!enabled_nat_bits(sbi, NULL)) return;
for (i = 0; i < nm_i->nat_blocks; i++) { i = find_next_bit_le(nm_i->empty_nat_bits, nm_i->nat_blocks, i); if (i >= nm_i->nat_blocks) break;
__set_bit_le(i, nm_i->nat_block_bitmap);
nid = i * NAT_ENTRY_PER_BLOCK(sbi); last_nid = nid + NAT_ENTRY_PER_BLOCK(sbi);
spin_lock(&NM_I(sbi)->nid_list_lock); for (; nid < last_nid; nid++) update_free_nid_bitmap(sbi, nid, true, true); spin_unlock(&NM_I(sbi)->nid_list_lock); }
for (i = 0; i < nm_i->nat_blocks; i++) { i = find_next_bit_le(nm_i->full_nat_bits, nm_i->nat_blocks, i); if (i >= nm_i->nat_blocks) break;
__set_bit_le(i, nm_i->nat_block_bitmap); }}
static int init_node_manager(struct f2fs_sb_info *sbi){ struct f2fs_super_block *sb_raw = F2FS_RAW_SUPER(sbi); struct f2fs_nm_info *nm_i = NM_I(sbi); unsigned char *version_bitmap; unsigned int nat_segs; int err;
nm_i->nat_blkaddr = le32_to_cpu(sb_raw->nat_blkaddr);
/* segment_count_nat includes pair segment so divide to 2. */ nat_segs = le32_to_cpu(sb_raw->segment_count_nat) >> 1; nm_i->nat_blocks = nat_segs << le32_to_cpu(sb_raw->log_blocks_per_seg); nm_i->max_nid = NAT_ENTRY_PER_BLOCK(sbi) * nm_i->nat_blocks;
/* not used nids: 0, node, meta, (and root counted as valid node) */ nm_i->available_nids = nm_i->max_nid - sbi->total_valid_node_count - F2FS_RESERVED_NODE_NUM; nm_i->nid_cnt[FREE_NID] = 0; nm_i->nid_cnt[PREALLOC_NID] = 0; nm_i->ram_thresh = DEF_RAM_THRESHOLD; nm_i->ra_nid_blocks = DEF_RA_NID_BLOCKS; nm_i->dirty_nats_ratio = DEF_DIRTY_NAT_RATIO_THRESHOLD; nm_i->max_rf_node_blocks = DEF_RF_NODE_BLOCKS;
INIT_RADIX_TREE(&nm_i->free_nid_root, GFP_ATOMIC); INIT_LIST_HEAD(&nm_i->free_nid_list); INIT_RADIX_TREE(&nm_i->nat_root, GFP_NOIO); INIT_RADIX_TREE(&nm_i->nat_set_root, GFP_NOIO); INIT_LIST_HEAD(&nm_i->nat_entries); spin_lock_init(&nm_i->nat_list_lock);
mutex_init(&nm_i->build_lock); spin_lock_init(&nm_i->nid_list_lock); init_f2fs_rwsem_trace(&nm_i->nat_tree_lock, sbi, LOCK_NAME_NAT_TREE_LOCK);
nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid); nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP); version_bitmap = __bitmap_ptr(sbi, NAT_BITMAP); nm_i->nat_bitmap = kmemdup(version_bitmap, nm_i->bitmap_size, GFP_KERNEL); if (!nm_i->nat_bitmap) return -ENOMEM;
if (!test_opt(sbi, NAT_BITS)) disable_nat_bits(sbi, true);
err = __get_nat_bitmaps(sbi); if (err) return err;
#ifdef CONFIG_F2FS_CHECK_FS nm_i->nat_bitmap_mir = kmemdup(version_bitmap, nm_i->bitmap_size, GFP_KERNEL); if (!nm_i->nat_bitmap_mir) return -ENOMEM;#endif
return 0;}
static int init_free_nid_cache(struct f2fs_sb_info *sbi){ struct f2fs_nm_info *nm_i = NM_I(sbi); int i;
nm_i->free_nid_bitmap = f2fs_kvzalloc(sbi, array_size(sizeof(unsigned char *), nm_i->nat_blocks), GFP_KERNEL); if (!nm_i->free_nid_bitmap) return -ENOMEM;
for (i = 0; i < nm_i->nat_blocks; i++) { nm_i->free_nid_bitmap[i] = f2fs_kvzalloc(sbi, f2fs_bitmap_size(NAT_ENTRY_PER_BLOCK(sbi)), GFP_KERNEL); if (!nm_i->free_nid_bitmap[i]) return -ENOMEM; }
nm_i->nat_block_bitmap = f2fs_kvzalloc(sbi, nm_i->nat_blocks / 8, GFP_KERNEL); if (!nm_i->nat_block_bitmap) return -ENOMEM;
nm_i->free_nid_count = f2fs_kvzalloc(sbi, array_size(sizeof(unsigned short), nm_i->nat_blocks), GFP_KERNEL); if (!nm_i->free_nid_count) return -ENOMEM; return 0;}
int f2fs_build_node_manager(struct f2fs_sb_info *sbi){ int err;
sbi->nm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_nm_info), GFP_KERNEL); if (!sbi->nm_info) return -ENOMEM;
err = init_node_manager(sbi); if (err) return err;
err = init_free_nid_cache(sbi); if (err) return err;
/* load free nid status from nat_bits table */ load_free_nid_bitmap(sbi);
return f2fs_build_free_nids(sbi, true, true);}
void f2fs_destroy_node_manager(struct f2fs_sb_info *sbi){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct free_nid *i, *next_i; void *vec[NAT_VEC_SIZE]; struct nat_entry **natvec = (struct nat_entry **)vec; struct nat_entry_set **setvec = (struct nat_entry_set **)vec; nid_t nid = 0; unsigned int found; struct f2fs_lock_context lc;
if (!nm_i) return;
/* destroy free nid list */ spin_lock(&nm_i->nid_list_lock); list_for_each_entry_safe(i, next_i, &nm_i->free_nid_list, list) { __remove_free_nid(sbi, i, FREE_NID); spin_unlock(&nm_i->nid_list_lock); kmem_cache_free(free_nid_slab, i); spin_lock(&nm_i->nid_list_lock); } f2fs_bug_on(sbi, nm_i->nid_cnt[FREE_NID]); f2fs_bug_on(sbi, nm_i->nid_cnt[PREALLOC_NID]); f2fs_bug_on(sbi, !list_empty(&nm_i->free_nid_list)); spin_unlock(&nm_i->nid_list_lock);
/* destroy nat cache */ f2fs_down_write_trace(&nm_i->nat_tree_lock, &lc); while ((found = __gang_lookup_nat_cache(nm_i, nid, NAT_VEC_SIZE, natvec))) { unsigned idx;
nid = nat_get_nid(natvec[found - 1]) + 1; for (idx = 0; idx < found; idx++) { spin_lock(&nm_i->nat_list_lock); list_del(&natvec[idx]->list); spin_unlock(&nm_i->nat_list_lock);
__del_from_nat_cache(nm_i, natvec[idx]); } } f2fs_bug_on(sbi, nm_i->nat_cnt[TOTAL_NAT]);
/* destroy nat set cache */ nid = 0; memset(vec, 0, sizeof(void *) * NAT_VEC_SIZE); while ((found = __gang_lookup_nat_set(nm_i, nid, NAT_VEC_SIZE, setvec))) { unsigned idx;
nid = setvec[found - 1]->set + 1; for (idx = 0; idx < found; idx++) { /* entry_cnt is not zero, when cp_error was occurred */ f2fs_bug_on(sbi, !list_empty(&setvec[idx]->entry_list)); radix_tree_delete(&nm_i->nat_set_root, setvec[idx]->set); kmem_cache_free(nat_entry_set_slab, setvec[idx]); } } f2fs_up_write_trace(&nm_i->nat_tree_lock, &lc);
kvfree(nm_i->nat_block_bitmap); if (nm_i->free_nid_bitmap) { int i;
for (i = 0; i < nm_i->nat_blocks; i++) kvfree(nm_i->free_nid_bitmap[i]); kvfree(nm_i->free_nid_bitmap); } kvfree(nm_i->free_nid_count);
kfree(nm_i->nat_bitmap); kvfree(nm_i->nat_bits);#ifdef CONFIG_F2FS_CHECK_FS kfree(nm_i->nat_bitmap_mir);#endif sbi->nm_info = NULL; kfree(nm_i);}
int __init f2fs_create_node_manager_caches(void){ nat_entry_slab = f2fs_kmem_cache_create("f2fs_nat_entry", sizeof(struct nat_entry)); if (!nat_entry_slab) goto fail;
free_nid_slab = f2fs_kmem_cache_create("f2fs_free_nid", sizeof(struct free_nid)); if (!free_nid_slab) goto destroy_nat_entry;
nat_entry_set_slab = f2fs_kmem_cache_create("f2fs_nat_entry_set", sizeof(struct nat_entry_set)); if (!nat_entry_set_slab) goto destroy_free_nid;
fsync_node_entry_slab = f2fs_kmem_cache_create("f2fs_fsync_node_entry", sizeof(struct fsync_node_entry)); if (!fsync_node_entry_slab) goto destroy_nat_entry_set; return 0;
destroy_nat_entry_set: kmem_cache_destroy(nat_entry_set_slab);destroy_free_nid: kmem_cache_destroy(free_nid_slab);destroy_nat_entry: kmem_cache_destroy(nat_entry_slab);fail: return -ENOMEM;}
void f2fs_destroy_node_manager_caches(void){ kmem_cache_destroy(fsync_node_entry_slab); kmem_cache_destroy(nat_entry_set_slab); kmem_cache_destroy(free_nid_slab); kmem_cache_destroy(nat_entry_slab);}