Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767// SPDX-License-Identifier: GPL-2.0
/* * fs/ext4/fast_commit.c * * Written by Harshad Shirwadkar <harshadshirwadkar@gmail.com> * * Ext4 fast commits routines. */#include "ext4.h"#include "ext4_jbd2.h"#include "ext4_extents.h"#include "mballoc.h"
#include <linux/lockdep.h>#include <linux/wait_bit.h>/* * Ext4 Fast Commits * ----------------- * * Ext4 fast commits implement fine grained journalling for Ext4. * * Fast commits are organized as a log of tag-length-value (TLV) structs. (See * struct ext4_fc_tl). Each TLV contains some delta that is replayed TLV by * TLV during the recovery phase. For the scenarios for which we currently * don't have replay code, fast commit falls back to full commits. * Fast commits record delta in one of the following three categories. * * (A) Directory entry updates: * * - EXT4_FC_TAG_UNLINK - records directory entry unlink * - EXT4_FC_TAG_LINK - records directory entry link * - EXT4_FC_TAG_CREAT - records inode and directory entry creation * * (B) File specific data range updates: * * - EXT4_FC_TAG_ADD_RANGE - records addition of new blocks to an inode * - EXT4_FC_TAG_DEL_RANGE - records deletion of blocks from an inode * * (C) Inode metadata (mtime / ctime etc): * * - EXT4_FC_TAG_INODE - record the inode that should be replayed * during recovery. Note that iblocks field is * not replayed and instead derived during * replay. * Commit Operation * ---------------- * With fast commits, we maintain all the directory entry operations in the * order in which they are issued in an in-memory queue. This queue is flushed * to disk during the commit operation. We also maintain a list of inodes * that need to be committed during a fast commit in another in memory queue of * inodes. During the commit operation, we commit in the following order: * * [1] Prepare all the inodes to write out their data by setting * "EXT4_STATE_FC_FLUSHING_DATA". This ensures that inode cannot be * deleted while it is being flushed. * [2] Flush data buffers to disk and clear "EXT4_STATE_FC_FLUSHING_DATA" * state. * [3] Lock the journal by calling jbd2_journal_lock_updates(). This ensures * that all the existing handles finish and no new handles can start. * [4] Mark all the fast commit eligible inodes as undergoing fast commit by * setting "EXT4_STATE_FC_COMMITTING" state, and snapshot the inode state * needed for log writing. * [5] Unlock the journal by calling jbd2_journal_unlock_updates(). This allows * starting of new handles. Updates to inodes being fast committed are * tracked for requeue rather than blocking. * [6] Commit all the directory entry updates in the fast commit space. * [7] Commit all the changed inodes in the fast commit space. * [8] Write tail tag (this tag ensures the atomicity, please read the following * section for more details). * [9] Clear "EXT4_STATE_FC_COMMITTING" and wake up waiters in * ext4_fc_cleanup(). * * All the inode updates must be enclosed within jbd2_journal_start() * and jbd2_journal_stop() similar to JBD2 journaling. * * Fast Commit Ineligibility * ------------------------- * * Not all operations are supported by fast commits today (e.g extended * attributes). Fast commit ineligibility is marked by calling * ext4_fc_mark_ineligible(): This makes next fast commit operation to fall back * to full commit. * * Atomicity of commits * -------------------- * In order to guarantee atomicity during the commit operation, fast commit * uses "EXT4_FC_TAG_TAIL" tag that marks a fast commit as complete. Tail * tag contains CRC of the contents and TID of the transaction after which * this fast commit should be applied. Recovery code replays fast commit * logs only if there's at least 1 valid tail present. For every fast commit * operation, there is 1 tail. This means, we may end up with multiple tails * in the fast commit space. Here's an example: * * - Create a new file A and remove existing file B * - fsync() * - Append contents to file A * - Truncate file A * - fsync() * * The fast commit space at the end of above operations would look like this: * [HEAD] [CREAT A] [UNLINK B] [TAIL] [ADD_RANGE A] [DEL_RANGE A] [TAIL] * |<--- Fast Commit 1 --->|<--- Fast Commit 2 ---->| * * Replay code should thus check for all the valid tails in the FC area. * * Fast Commit Replay Idempotence * ------------------------------ * * Fast commits tags are idempotent in nature provided the recovery code follows * certain rules. The guiding principle that the commit path follows while * committing is that it stores the result of a particular operation instead of * storing the procedure. * * Let's consider this rename operation: 'mv /a /b'. Let's assume dirent '/a' * was associated with inode 10. During fast commit, instead of storing this * operation as a procedure "rename a to b", we store the resulting file system * state as a "series" of outcomes: * * - Link dirent b to inode 10 * - Unlink dirent a * - Inode <10> with valid refcount * * Now when recovery code runs, it needs "enforce" this state on the file * system. This is what guarantees idempotence of fast commit replay. * * Let's take an example of a procedure that is not idempotent and see how fast * commits make it idempotent. Consider following sequence of operations: * * rm A; mv B A; read A * (x) (y) (z) * * (x), (y) and (z) are the points at which we can crash. If we store this * sequence of operations as is then the replay is not idempotent. Let's say * while in replay, we crash at (z). During the second replay, file A (which was * actually created as a result of "mv B A" operation) would get deleted. Thus, * file named A would be absent when we try to read A. So, this sequence of * operations is not idempotent. However, as mentioned above, instead of storing * the procedure fast commits store the outcome of each procedure. Thus the fast * commit log for above procedure would be as follows: * * (Let's assume dirent A was linked to inode 10 and dirent B was linked to * inode 11 before the replay) * * [Unlink A] [Link A to inode 11] [Unlink B] [Inode 11] * (w) (x) (y) (z) * * If we crash at (z), we will have file A linked to inode 11. During the second * replay, we will remove file A (inode 11). But we will create it back and make * it point to inode 11. We won't find B, so we'll just skip that step. At this * point, the refcount for inode 11 is not reliable, but that gets fixed by the * replay of last inode 11 tag. Crashes at points (w), (x) and (y) get handled * similarly. Thus, by converting a non-idempotent procedure into a series of * idempotent outcomes, fast commits ensured idempotence during the replay. * * Locking * ------- * sbi->s_fc_lock protects the fast commit inodes queue and the fast commit * dentry queue. ei->i_fc_lock protects the fast commit related info in a given * inode. Most of the code avoids acquiring both the locks, but if one must do * that then sbi->s_fc_lock must be acquired before ei->i_fc_lock. * * TODOs * ----- * * 0) Fast commit replay path hardening: Fast commit replay code should use * journal handles to make sure all the updates it does during the replay * path are atomic. With that if we crash during fast commit replay, after * trying to do recovery again, we will find a file system where fast commit * area is invalid (because new full commit would be found). In order to deal * with that, fast commit replay code should ensure that the "FC_REPLAY" * superblock state is persisted before starting the replay, so that after * the crash, fast commit recovery code can look at that flag and perform * fast commit recovery even if that area is invalidated by later full * commits. * * 1) Handle more ineligible cases. * * 2) Change ext4_fc_commit() to lookup logical to physical mapping using extent * status tree. This would get rid of the need to call ext4_fc_track_inode() * before acquiring i_data_sem. To do that we would need to ensure that * modified extents from the extent status tree are not evicted from memory. */
#include <trace/events/ext4.h>static struct kmem_cache *ext4_fc_dentry_cachep;static struct kmem_cache *ext4_fc_range_cachep;
/* * Avoid spending unbounded time/memory snapshotting highly fragmented files * under jbd2_journal_lock_updates(). If we exceed this limit, fall back to * full commit. */#define EXT4_FC_SNAPSHOT_MAX_INODES 1024#define EXT4_FC_SNAPSHOT_MAX_RANGES 2048
static inline void ext4_fc_set_snap_err(int *snap_err, int err){ if (snap_err && *snap_err == EXT4_FC_SNAP_ERR_NONE) *snap_err = err;}
static void ext4_fc_free_inode_snap(struct inode *inode);
static inline void ext4_fc_reset_inode(struct inode *inode){ struct ext4_inode_info *ei = EXT4_I(inode);
ei->i_fc_lblk_start = 0; ei->i_fc_lblk_len = 0;}
void ext4_fc_init_inode(struct inode *inode){ struct ext4_inode_info *ei = EXT4_I(inode);
ext4_fc_reset_inode(inode); ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING); ext4_clear_inode_state(inode, EXT4_STATE_FC_REQUEUE); INIT_LIST_HEAD(&ei->i_fc_list); INIT_LIST_HEAD(&ei->i_fc_dilist); ei->i_fc_snap = NULL;}
static bool ext4_fc_disabled(struct super_block *sb){ return (!test_opt2(sb, JOURNAL_FAST_COMMIT) || (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY));}
static bool ext4_fc_eligible(struct super_block *sb){ return !ext4_fc_disabled(sb) && !(ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE));}
/* * Wait for an inode fast-commit state bit to clear while dropping the * fast-commit lock around schedule(). */static void ext4_fc_wait_inode_state(struct inode *inode, int bit, int *alloc_ctx){ wait_queue_head_t *wq; unsigned long *wait_word = ext4_inode_state_wait_word(inode); int wait_bit = ext4_inode_state_wait_bit(bit);
while (ext4_test_inode_state(inode, bit)) { DEFINE_WAIT_BIT(wait, wait_word, wait_bit);
wq = bit_waitqueue(wait_word, wait_bit); prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); if (ext4_test_inode_state(inode, bit)) { ext4_fc_unlock(inode->i_sb, *alloc_ctx); schedule(); *alloc_ctx = ext4_fc_lock(inode->i_sb); } finish_wait(wq, &wait.wq_entry); }}
static inline void ext4_fc_wake_inode_state(struct inode *inode, int bit){ wake_up_bit(ext4_inode_state_wait_word(inode), ext4_inode_state_wait_bit(bit));}
static void ext4_fc_snap_stats_update_max(atomic64_t *stat, u64 value){ u64 old = atomic64_read(stat);
while (value > old) { u64 prev = atomic64_cmpxchg(stat, old, value);
if (prev == old) break; old = prev; }}
/* * Remove inode from fast commit list. If the inode is being committed * we wait until inode commit is done. */void ext4_fc_del(struct inode *inode){ struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_fc_dentry_update *fc_dentry; int alloc_ctx;
if (ext4_fc_disabled(inode->i_sb)) return;
alloc_ctx = ext4_fc_lock(inode->i_sb); if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) { ext4_fc_free_inode_snap(inode); ext4_fc_unlock(inode->i_sb, alloc_ctx); return; }
/* * Wait for ongoing fast commit to finish. We cannot remove the inode * from fast commit lists while it is being committed. If we wake from * FC_FLUSHING_DATA, re-check FC_COMMITTING before deleting because the * commit thread sets FC_COMMITTING only after clearing FLUSHING_DATA. */ for (;;) { ext4_fc_wait_inode_state(inode, EXT4_STATE_FC_COMMITTING, &alloc_ctx);
if (!ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA)) break;
ext4_fc_wait_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA, &alloc_ctx); }
ext4_fc_free_inode_snap(inode); list_del_init(&ei->i_fc_list);
/* * Since this inode is getting removed, let's also remove all FC dentry * create references, since it is not needed to log it anyways. */ if (list_empty(&ei->i_fc_dilist)) { ext4_fc_unlock(inode->i_sb, alloc_ctx); return; }
fc_dentry = list_first_entry(&ei->i_fc_dilist, struct ext4_fc_dentry_update, fcd_dilist); WARN_ON(fc_dentry->fcd_op != EXT4_FC_TAG_CREAT); list_del_init(&fc_dentry->fcd_list); list_del_init(&fc_dentry->fcd_dilist);
WARN_ON(!list_empty(&ei->i_fc_dilist)); ext4_fc_unlock(inode->i_sb, alloc_ctx);
release_dentry_name_snapshot(&fc_dentry->fcd_name); kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry);}
/* * Mark file system as fast commit ineligible, and record latest * ineligible transaction tid. This means until the recorded * transaction, commit operation would result in a full jbd2 commit. */void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handle){ struct ext4_sb_info *sbi = EXT4_SB(sb); tid_t tid; bool has_transaction = true; bool is_ineligible; int alloc_ctx;
if (ext4_fc_disabled(sb)) return;
if (!IS_ERR_OR_NULL(handle)) tid = handle->h_transaction->t_tid; else { read_lock(&sbi->s_journal->j_state_lock); if (sbi->s_journal->j_running_transaction) tid = sbi->s_journal->j_running_transaction->t_tid; else has_transaction = false; read_unlock(&sbi->s_journal->j_state_lock); } alloc_ctx = ext4_fc_lock(sb); is_ineligible = ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); if (has_transaction && (!is_ineligible || tid_gt(tid, sbi->s_fc_ineligible_tid))) sbi->s_fc_ineligible_tid = tid; ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); ext4_fc_unlock(sb, alloc_ctx); WARN_ON(reason >= EXT4_FC_REASON_MAX); sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;}
/* * Generic fast commit tracking function. If this is the first time this we are * called after a full commit, we initialize fast commit fields and then call * __fc_track_fn() with update = 0. If we have already been called after a full * commit, we pass update = 1. Based on that, the track function can determine * if it needs to track a field for the first time or if it needs to just * update the previously tracked value. * * If enqueue is set, this function enqueues the inode in fast commit list. */static int ext4_fc_track_template( handle_t *handle, struct inode *inode, int (*__fc_track_fn)(handle_t *handle, struct inode *, void *, bool), void *args, int enqueue){ bool update = false; struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); tid_t tid = 0; int alloc_ctx; int ret;
tid = handle->h_transaction->t_tid; spin_lock(&ei->i_fc_lock); if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) ext4_set_inode_state(inode, EXT4_STATE_FC_REQUEUE); if (tid == ei->i_sync_tid) { update = true; } else { ext4_fc_reset_inode(inode); ei->i_sync_tid = tid; } ret = __fc_track_fn(handle, inode, args, update); spin_unlock(&ei->i_fc_lock); if (!enqueue) return ret;
alloc_ctx = ext4_fc_lock(inode->i_sb); if (list_empty(&EXT4_I(inode)->i_fc_list)) list_add_tail(&EXT4_I(inode)->i_fc_list, (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ? &sbi->s_fc_q[FC_Q_STAGING] : &sbi->s_fc_q[FC_Q_MAIN]); ext4_fc_unlock(inode->i_sb, alloc_ctx);
return ret;}
struct __track_dentry_update_args { struct dentry *dentry; int op;};
/* __track_fn for directory entry updates. Called with ei->i_fc_lock. */static int __track_dentry_update(handle_t *handle, struct inode *inode, void *arg, bool update){ struct ext4_fc_dentry_update *node; struct ext4_inode_info *ei = EXT4_I(inode); struct __track_dentry_update_args *dentry_update = (struct __track_dentry_update_args *)arg; struct dentry *dentry = dentry_update->dentry; struct inode *dir = dentry->d_parent->d_inode; struct super_block *sb = inode->i_sb; struct ext4_sb_info *sbi = EXT4_SB(sb); int alloc_ctx;
spin_unlock(&ei->i_fc_lock);
if (IS_ENCRYPTED(dir)) { ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_ENCRYPTED_FILENAME, handle); spin_lock(&ei->i_fc_lock); return -EOPNOTSUPP; }
node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS); if (!node) { ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM, handle); spin_lock(&ei->i_fc_lock); return -ENOMEM; }
node->fcd_op = dentry_update->op; node->fcd_parent = dir->i_ino; node->fcd_ino = inode->i_ino; take_dentry_name_snapshot(&node->fcd_name, dentry); INIT_LIST_HEAD(&node->fcd_dilist); INIT_LIST_HEAD(&node->fcd_list); alloc_ctx = ext4_fc_lock(sb); if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_STAGING]); else list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]);
/* * This helps us keep a track of all fc_dentry updates which is part of * this ext4 inode. So in case the inode is getting unlinked, before * even we get a chance to fsync, we could remove all fc_dentry * references while evicting the inode in ext4_fc_del(). * Also with this, we don't need to loop over all the inodes in * sbi->s_fc_q to get the corresponding inode in * ext4_fc_commit_dentry_updates(). */ if (dentry_update->op == EXT4_FC_TAG_CREAT) { WARN_ON(!list_empty(&ei->i_fc_dilist)); list_add_tail(&node->fcd_dilist, &ei->i_fc_dilist); } ext4_fc_unlock(sb, alloc_ctx); spin_lock(&ei->i_fc_lock);
return 0;}
void __ext4_fc_track_unlink(handle_t *handle, struct inode *inode, struct dentry *dentry){ struct __track_dentry_update_args args; int ret;
args.dentry = dentry; args.op = EXT4_FC_TAG_UNLINK;
ret = ext4_fc_track_template(handle, inode, __track_dentry_update, (void *)&args, 0); trace_ext4_fc_track_unlink(handle, inode, dentry, ret);}
void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry){ struct inode *inode = d_inode(dentry);
if (ext4_fc_eligible(inode->i_sb)) __ext4_fc_track_unlink(handle, inode, dentry);}
void __ext4_fc_track_link(handle_t *handle, struct inode *inode, struct dentry *dentry){ struct __track_dentry_update_args args; int ret;
args.dentry = dentry; args.op = EXT4_FC_TAG_LINK;
ret = ext4_fc_track_template(handle, inode, __track_dentry_update, (void *)&args, 0); trace_ext4_fc_track_link(handle, inode, dentry, ret);}
void ext4_fc_track_link(handle_t *handle, struct inode *inode, struct dentry *dentry){ if (ext4_fc_eligible(inode->i_sb)) __ext4_fc_track_link(handle, inode, dentry);}
void __ext4_fc_track_create(handle_t *handle, struct inode *inode, struct dentry *dentry){ struct __track_dentry_update_args args; int ret;
args.dentry = dentry; args.op = EXT4_FC_TAG_CREAT;
ret = ext4_fc_track_template(handle, inode, __track_dentry_update, (void *)&args, 0); trace_ext4_fc_track_create(handle, inode, dentry, ret);}
void ext4_fc_track_create(handle_t *handle, struct dentry *dentry){ struct inode *inode = d_inode(dentry);
if (ext4_fc_eligible(inode->i_sb)) __ext4_fc_track_create(handle, inode, dentry);}
/* __track_fn for inode tracking */static int __track_inode(handle_t *handle, struct inode *inode, void *arg, bool update){ if (update) return -EEXIST;
EXT4_I(inode)->i_fc_lblk_len = 0;
return 0;}
void ext4_fc_track_inode(handle_t *handle, struct inode *inode){ int ret;
if (S_ISDIR(inode->i_mode)) return;
if (ext4_should_journal_data(inode)) { ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_INODE_JOURNAL_DATA, handle); return; }
if (!ext4_fc_eligible(inode->i_sb)) return;
/* * Fast commit snapshots inode state at commit time, so there's no need * to wait for EXT4_STATE_FC_COMMITTING here. If the inode is already * on the commit queue, ext4_fc_cleanup() will requeue it for the new * transaction once the current commit finishes. */
/* * From this point on, this inode will not be committed either * by fast or full commit as long as the handle is open. */ ret = ext4_fc_track_template(handle, inode, __track_inode, NULL, 1); trace_ext4_fc_track_inode(handle, inode, ret);}
struct __track_range_args { ext4_lblk_t start, end;};
/* __track_fn for tracking data updates */static int __track_range(handle_t *handle, struct inode *inode, void *arg, bool update){ struct ext4_inode_info *ei = EXT4_I(inode); ext4_lblk_t oldstart; struct __track_range_args *__arg = (struct __track_range_args *)arg;
if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) { ext4_debug("Special inode %llu being modified\n", inode->i_ino); return -ECANCELED; }
oldstart = ei->i_fc_lblk_start;
if (update && ei->i_fc_lblk_len > 0) { ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start); ei->i_fc_lblk_len = max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) - ei->i_fc_lblk_start + 1; } else { ei->i_fc_lblk_start = __arg->start; ei->i_fc_lblk_len = __arg->end - __arg->start + 1; }
return 0;}
void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start, ext4_lblk_t end){ struct __track_range_args args; int ret;
if (S_ISDIR(inode->i_mode)) return;
if (!ext4_fc_eligible(inode->i_sb)) return;
if (ext4_has_inline_data(inode)) { ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR, handle); return; }
args.start = start; args.end = end;
ret = ext4_fc_track_template(handle, inode, __track_range, &args, 1);
trace_ext4_fc_track_range(handle, inode, start, end, ret);}
static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail){ blk_opf_t write_flags = JBD2_JOURNAL_REQ_FLAGS; struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh;
/* Add REQ_FUA | REQ_PREFLUSH only its tail */ if (test_opt(sb, BARRIER) && is_tail) write_flags |= REQ_FUA | REQ_PREFLUSH; lock_buffer(bh); set_buffer_dirty(bh); set_buffer_uptodate(bh); bh_submit(bh, REQ_OP_WRITE | write_flags, bh_end_write); EXT4_SB(sb)->s_fc_bh = NULL;}
/* Ext4 commit path routines */
/* * Allocate len bytes on a fast commit buffer. * * During the commit time this function is used to manage fast commit * block space. We don't split a fast commit log onto different * blocks. So this function makes sure that if there's not enough space * on the current block, the remaining space in the current block is * marked as unused by adding EXT4_FC_TAG_PAD tag. In that case, * new block is from jbd2 and CRC is updated to reflect the padding * we added. */static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc){ struct ext4_fc_tl tl; struct ext4_sb_info *sbi = EXT4_SB(sb); struct buffer_head *bh; int bsize = sbi->s_journal->j_blocksize; int ret, off = sbi->s_fc_bytes % bsize; int remaining; u8 *dst;
/* * If 'len' is too long to fit in any block alongside a PAD tlv, then we * cannot fulfill the request. */ if (len > bsize - EXT4_FC_TAG_BASE_LEN) return NULL;
if (!sbi->s_fc_bh) { ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh); if (ret) return NULL; sbi->s_fc_bh = bh; } dst = sbi->s_fc_bh->b_data + off;
/* * Allocate the bytes in the current block if we can do so while still * leaving enough space for a PAD tlv. */ remaining = bsize - EXT4_FC_TAG_BASE_LEN - off; if (len <= remaining) { sbi->s_fc_bytes += len; return dst; }
/* * Else, terminate the current block with a PAD tlv, then allocate a new * block and allocate the bytes at the start of that new block. */
tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD); tl.fc_len = cpu_to_le16(remaining); memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN); memset(dst + EXT4_FC_TAG_BASE_LEN, 0, remaining); *crc = ext4_chksum(*crc, sbi->s_fc_bh->b_data, bsize);
ext4_fc_submit_bh(sb, false);
ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh); if (ret) return NULL; sbi->s_fc_bh = bh; sbi->s_fc_bytes += bsize - off + len; return sbi->s_fc_bh->b_data;}
/* * Complete a fast commit by writing tail tag. * * Writing tail tag marks the end of a fast commit. In order to guarantee * atomicity, after writing tail tag, even if there's space remaining * in the block, next commit shouldn't use it. That's why tail tag * has the length as that of the remaining space on the block. */static int ext4_fc_write_tail(struct super_block *sb, u32 crc){ struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_fc_tl tl; struct ext4_fc_tail tail; int off, bsize = sbi->s_journal->j_blocksize; u8 *dst;
/* * ext4_fc_reserve_space takes care of allocating an extra block if * there's no enough space on this block for accommodating this tail. */ dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + sizeof(tail), &crc); if (!dst) return -ENOSPC;
off = sbi->s_fc_bytes % bsize;
tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_TAIL); tl.fc_len = cpu_to_le16(bsize - off + sizeof(struct ext4_fc_tail)); sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize);
memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN); dst += EXT4_FC_TAG_BASE_LEN; tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid); memcpy(dst, &tail.fc_tid, sizeof(tail.fc_tid)); dst += sizeof(tail.fc_tid); crc = ext4_chksum(crc, sbi->s_fc_bh->b_data, dst - (u8 *)sbi->s_fc_bh->b_data); tail.fc_crc = cpu_to_le32(crc); memcpy(dst, &tail.fc_crc, sizeof(tail.fc_crc)); dst += sizeof(tail.fc_crc); memset(dst, 0, bsize - off); /* Don't leak uninitialized memory. */
ext4_fc_submit_bh(sb, true);
return 0;}
/* * Adds tag, length, value and updates CRC. Returns true if tlv was added. * Returns false if there's not enough space. */static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val, u32 *crc){ struct ext4_fc_tl tl; u8 *dst;
dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + len, crc); if (!dst) return false;
tl.fc_tag = cpu_to_le16(tag); tl.fc_len = cpu_to_le16(len);
memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN); memcpy(dst + EXT4_FC_TAG_BASE_LEN, val, len);
return true;}
/* Same as above, but adds dentry tlv. */static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc, struct ext4_fc_dentry_update *fc_dentry){ struct ext4_fc_dentry_info fcd; struct ext4_fc_tl tl; int dlen = fc_dentry->fcd_name.name.len; u8 *dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + sizeof(fcd) + dlen, crc);
if (!dst) return false;
fcd.fc_parent_ino = cpu_to_le32(fc_dentry->fcd_parent); fcd.fc_ino = cpu_to_le32(fc_dentry->fcd_ino); tl.fc_tag = cpu_to_le16(fc_dentry->fcd_op); tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen); memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN); dst += EXT4_FC_TAG_BASE_LEN; memcpy(dst, &fcd, sizeof(fcd)); dst += sizeof(fcd); memcpy(dst, fc_dentry->fcd_name.name.name, dlen);
return true;}
struct ext4_fc_range { struct list_head list; u16 tag; ext4_lblk_t lblk; ext4_lblk_t len; ext4_fsblk_t pblk; bool unwritten;};
struct ext4_fc_inode_snap { struct list_head data_list; unsigned int inode_len; u8 inode_buf[];};
/* * Writes inode in the fast commit space under TLV with tag @tag. * Returns 0 on success, error on failure. */static int ext4_fc_write_inode(struct inode *inode, u32 *crc){ struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_fc_inode_snap *snap = ei->i_fc_snap; struct ext4_fc_snap_stats *stats = &EXT4_SB(inode->i_sb)->s_fc_snap_stats; struct ext4_fc_inode fc_inode; struct ext4_fc_tl tl; u8 *dst; u8 *src; int inode_len; int ret;
if (!snap) { atomic64_inc(&stats->snap_fail_no_snap); return -ECANCELED; }
src = snap->inode_buf; inode_len = snap->inode_len; if (!src || inode_len == 0) { atomic64_inc(&stats->snap_fail_no_snap); return -ECANCELED; }
fc_inode.fc_ino = cpu_to_le32(inode->i_ino); tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE); tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino));
ret = -ECANCELED; dst = ext4_fc_reserve_space(inode->i_sb, EXT4_FC_TAG_BASE_LEN + inode_len + sizeof(fc_inode.fc_ino), crc); if (!dst) goto err;
memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN); dst += EXT4_FC_TAG_BASE_LEN; memcpy(dst, &fc_inode, sizeof(fc_inode)); dst += sizeof(fc_inode); memcpy(dst, src, inode_len); ret = 0;err: return ret;}
/* * Writes updated data ranges for the inode in question. Updates CRC. * Returns 0 on success, error otherwise. */static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc){ struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_fc_inode_snap *snap = ei->i_fc_snap; struct ext4_fc_snap_stats *stats = &EXT4_SB(inode->i_sb)->s_fc_snap_stats; struct ext4_fc_add_range fc_ext; struct ext4_fc_del_range lrange; struct ext4_extent *ex; struct ext4_fc_range *range;
if (!snap) { atomic64_inc(&stats->snap_fail_no_snap); return -ECANCELED; }
list_for_each_entry(range, &snap->data_list, list) { if (range->tag == EXT4_FC_TAG_DEL_RANGE) { lrange.fc_ino = cpu_to_le32(inode->i_ino); lrange.fc_lblk = cpu_to_le32(range->lblk); lrange.fc_len = cpu_to_le32(range->len); if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE, sizeof(lrange), (u8 *)&lrange, crc)) return -ENOSPC; continue; }
fc_ext.fc_ino = cpu_to_le32(inode->i_ino); ex = (struct ext4_extent *)&fc_ext.fc_ex; ex->ee_block = cpu_to_le32(range->lblk); ex->ee_len = cpu_to_le16(range->len); ext4_ext_store_pblock(ex, range->pblk); if (range->unwritten) ext4_ext_mark_unwritten(ex); else ext4_ext_mark_initialized(ex);
if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE, sizeof(fc_ext), (u8 *)&fc_ext, crc)) return -ENOSPC; }
return 0;}
static void ext4_fc_free_ranges(struct list_head *head){ struct ext4_fc_range *range, *range_n;
list_for_each_entry_safe(range, range_n, head, list) { list_del(&range->list); kmem_cache_free(ext4_fc_range_cachep, range); }}
static void ext4_fc_free_inode_snap(struct inode *inode){ struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_fc_inode_snap *snap = ei->i_fc_snap;
if (!snap) return;
ext4_fc_free_ranges(&snap->data_list); kfree(snap); ei->i_fc_snap = NULL;}
static int ext4_fc_snapshot_inode_data(struct inode *inode, struct list_head *ranges, unsigned int nr_ranges_total, unsigned int *nr_rangesp, int *snap_err){ struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_fc_snap_stats *stats = &EXT4_SB(inode->i_sb)->s_fc_snap_stats; ext4_lblk_t start_lblk, end_lblk, cur_lblk; unsigned int nr_ranges = 0;
spin_lock(&ei->i_fc_lock); if (ei->i_fc_lblk_len == 0) { spin_unlock(&ei->i_fc_lock); if (nr_rangesp) *nr_rangesp = 0; return 0; } start_lblk = ei->i_fc_lblk_start; end_lblk = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1; ei->i_fc_lblk_len = 0; spin_unlock(&ei->i_fc_lock);
cur_lblk = start_lblk; ext4_debug("snapshot data ranges %u-%u for inode %llu\n", start_lblk, end_lblk, (unsigned long long)inode->i_ino);
while (cur_lblk <= end_lblk) { struct extent_status es; struct ext4_fc_range *range; ext4_lblk_t len; u64 remaining = (u64)end_lblk - cur_lblk + 1;
if (!ext4_es_lookup_extent(inode, cur_lblk, NULL, &es, NULL)) { atomic64_inc(&stats->snap_fail_es_miss); ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_ES_MISS); return -EAGAIN; }
if (ext4_es_is_delayed(&es)) { atomic64_inc(&stats->snap_fail_es_delayed); ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_ES_DELAYED); return -EAGAIN; }
len = es.es_len - (cur_lblk - es.es_lblk); if (len > remaining) len = remaining; if (len == 0) { cur_lblk++; continue; }
if (nr_ranges_total + nr_ranges >= EXT4_FC_SNAPSHOT_MAX_RANGES) { atomic64_inc(&stats->snap_fail_ranges_cap); ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_RANGES_CAP); return -E2BIG; }
range = kmem_cache_alloc(ext4_fc_range_cachep, GFP_NOFS); if (!range) { atomic64_inc(&stats->snap_fail_nomem); ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_NOMEM); return -ENOMEM; } nr_ranges++;
range->lblk = cur_lblk; range->len = len; range->pblk = 0; range->unwritten = false;
if (ext4_es_is_hole(&es)) { range->tag = EXT4_FC_TAG_DEL_RANGE; } else if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) { unsigned int max;
range->tag = EXT4_FC_TAG_ADD_RANGE; range->pblk = ext4_es_pblock(&es) + (cur_lblk - es.es_lblk); range->unwritten = ext4_es_is_unwritten(&es);
max = range->unwritten ? EXT_UNWRITTEN_MAX_LEN : EXT_INIT_MAX_LEN; if (range->len > max) range->len = max; } else { kmem_cache_free(ext4_fc_range_cachep, range); atomic64_inc(&stats->snap_fail_es_other); ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_ES_OTHER); return -EAGAIN; }
INIT_LIST_HEAD(&range->list); list_add_tail(&range->list, ranges);
if ((u64)range->len > (u64)end_lblk - cur_lblk) break;
cur_lblk += range->len; }
if (nr_rangesp) *nr_rangesp = nr_ranges; return 0;}
static int ext4_fc_snapshot_inode(struct inode *inode, unsigned int nr_ranges_total, unsigned int *nr_rangesp, int *snap_err){ struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_fc_snap_stats *stats = &EXT4_SB(inode->i_sb)->s_fc_snap_stats; struct ext4_fc_inode_snap *snap; int inode_len = EXT4_GOOD_OLD_INODE_SIZE; struct ext4_iloc iloc; LIST_HEAD(ranges); unsigned int nr_ranges = 0; int ret; int alloc_ctx;
ret = ext4_get_inode_loc_noio(inode, &iloc); if (ret) { atomic64_inc(&stats->snap_fail_inode_loc); ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_INODE_LOC); return ret; }
if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) inode_len = EXT4_INODE_SIZE(inode->i_sb); else if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) inode_len += ei->i_extra_isize;
snap = kmalloc_flex(*snap, inode_buf, inode_len, GFP_NOFS); if (!snap) { atomic64_inc(&stats->snap_fail_nomem); ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_NOMEM); brelse(iloc.bh); return -ENOMEM; } INIT_LIST_HEAD(&snap->data_list); snap->inode_len = inode_len;
memcpy(snap->inode_buf, (u8 *)ext4_raw_inode(&iloc), inode_len); brelse(iloc.bh);
ret = ext4_fc_snapshot_inode_data(inode, &ranges, nr_ranges_total, &nr_ranges, snap_err); if (ret) { kfree(snap); ext4_fc_free_ranges(&ranges); return ret; }
alloc_ctx = ext4_fc_lock(inode->i_sb); ext4_fc_free_inode_snap(inode); ei->i_fc_snap = snap; list_splice_tail_init(&ranges, &snap->data_list); ext4_fc_unlock(inode->i_sb, alloc_ctx);
atomic64_inc(&stats->snap_inodes); atomic64_add(nr_ranges, &stats->snap_ranges); if (nr_rangesp) *nr_rangesp = nr_ranges; return 0;}
/* Flushes data of all the inodes in the commit queue. */static int ext4_fc_flush_data(journal_t *journal){ struct super_block *sb = journal->j_private; struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_inode_info *ei; int ret = 0;
list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ret = jbd2_submit_inode_data(journal, READ_ONCE(ei->jinode)); if (ret) return ret; }
list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ret = jbd2_wait_inode_data(journal, READ_ONCE(ei->jinode)); if (ret) return ret; }
return 0;}
/* Commit all the directory entry updates */static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc){ struct super_block *sb = journal->j_private; struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_fc_dentry_update *fc_dentry, *fc_dentry_n; struct inode *inode; struct ext4_inode_info *ei; int ret;
if (list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) return 0; list_for_each_entry_safe(fc_dentry, fc_dentry_n, &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) { if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) { if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry)) return -ENOSPC; continue; } /* * With fcd_dilist we need not loop in sbi->s_fc_q to get the * corresponding inode. Also, the corresponding inode could have been * deleted, in which case, we don't need to do anything. */ if (list_empty(&fc_dentry->fcd_dilist)) continue; /* * For EXT4_FC_TAG_CREAT, fcd_dilist is linked on the created * inode's i_fc_dilist list (kept singular), so we can recover the * inode through it. */ ei = list_first_entry(&fc_dentry->fcd_dilist, struct ext4_inode_info, i_fc_dilist); inode = &ei->vfs_inode; WARN_ON(inode->i_ino != fc_dentry->fcd_ino);
/* * We first write the inode and then the create dirent. This * allows the recovery code to create an unnamed inode first * and then link it to a directory entry. This allows us * to use namei.c routines almost as is and simplifies * the recovery code. */ ret = ext4_fc_write_inode(inode, crc); if (ret) return ret; ret = ext4_fc_write_inode_data(inode, crc); if (ret) return ret; if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry)) return -ENOSPC; } return 0;}
static int ext4_fc_alloc_snapshot_inodes(struct super_block *sb, struct inode ***inodesp, unsigned int *nr_inodesp);
static int ext4_fc_snapshot_inodes(journal_t *journal, struct inode **inodes, unsigned int inodes_size, unsigned int *nr_inodesp, unsigned int *nr_rangesp, int *snap_err){ struct super_block *sb = journal->j_private; struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_inode_info *iter; struct ext4_fc_dentry_update *fc_dentry; unsigned int i = 0; unsigned int idx; unsigned int nr_ranges = 0; int ret = 0; int alloc_ctx;
alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { if (i >= inodes_size) { atomic64_inc(&sbi->s_fc_snap_stats.snap_fail_inodes_cap); ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_INODES_CAP); ret = -E2BIG; goto unlock; } inodes[i++] = &iter->vfs_inode; }
list_for_each_entry(fc_dentry, &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) { struct ext4_inode_info *ei; struct inode *inode;
if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) continue; if (list_empty(&fc_dentry->fcd_dilist)) continue;
/* See the comment in ext4_fc_commit_dentry_updates(). */ ei = list_first_entry(&fc_dentry->fcd_dilist, struct ext4_inode_info, i_fc_dilist); inode = &ei->vfs_inode; if (!list_empty(&ei->i_fc_list)) continue;
if (i >= inodes_size) { atomic64_inc(&sbi->s_fc_snap_stats.snap_fail_inodes_cap); ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_INODES_CAP); ret = -E2BIG; goto unlock; } /* * Create-only inodes may only be referenced via fcd_dilist and * not appear on s_fc_q[MAIN]. They may hit the last iput while * we are snapshotting, but inode eviction calls ext4_fc_del(), * which waits for FC_COMMITTING to clear. Mark them FC_COMMITTING * so the inode stays pinned and the snapshot stays valid until * ext4_fc_cleanup(). */ ext4_set_inode_state(inode, EXT4_STATE_FC_COMMITTING); inodes[i++] = inode; }unlock: ext4_fc_unlock(sb, alloc_ctx);
if (ret) return ret;
for (idx = 0; idx < i; idx++) { unsigned int inode_ranges = 0;
ret = ext4_fc_snapshot_inode(inodes[idx], nr_ranges, &inode_ranges, snap_err); if (ret) break; nr_ranges += inode_ranges; }
if (nr_inodesp) *nr_inodesp = idx; if (nr_rangesp) *nr_rangesp = nr_ranges; return ret;}
static int ext4_fc_perform_commit(journal_t *journal, tid_t commit_tid){ struct super_block *sb = journal->j_private; struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_fc_snap_stats *snap_stats = &sbi->s_fc_snap_stats; struct ext4_inode_info *iter; struct ext4_fc_head head; struct inode *inode; struct inode **inodes; unsigned int inodes_size; unsigned int snap_inodes = 0; unsigned int snap_ranges = 0; int snap_err = EXT4_FC_SNAP_ERR_NONE; struct blk_plug plug; int ret = 0; u32 crc = 0; int alloc_ctx; ktime_t lock_start; u64 locked_ns;
/* * Step 1: Mark all inodes on s_fc_q[MAIN] with * EXT4_STATE_FC_FLUSHING_DATA. This prevents these inodes from being * freed until the data flush is over. */ alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_set_inode_state(&iter->vfs_inode, EXT4_STATE_FC_FLUSHING_DATA); } ext4_fc_unlock(sb, alloc_ctx);
/* Step 2: Flush data for all the eligible inodes. */ ret = ext4_fc_flush_data(journal);
/* * Step 3: Clear EXT4_STATE_FC_FLUSHING_DATA flag, before returning * any error from step 2. This ensures that waiters waiting on * EXT4_STATE_FC_FLUSHING_DATA can resume. */ alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_clear_inode_state(&iter->vfs_inode, EXT4_STATE_FC_FLUSHING_DATA); ext4_fc_wake_inode_state(&iter->vfs_inode, EXT4_STATE_FC_FLUSHING_DATA); }
/* * Make sure clearing of EXT4_STATE_FC_FLUSHING_DATA is visible before * the waiter checks the bit. Pairs with implicit barrier in * prepare_to_wait() in ext4_fc_del(). */ smp_mb(); ext4_fc_unlock(sb, alloc_ctx);
/* * If we encountered error in Step 2, return it now after clearing * EXT4_STATE_FC_FLUSHING_DATA bit. */ if (ret) return ret;
ret = ext4_fc_alloc_snapshot_inodes(sb, &inodes, &inodes_size); if (ret) { if (ret == -E2BIG) atomic64_inc(&snap_stats->snap_fail_inodes_cap); else if (ret == -ENOMEM) atomic64_inc(&snap_stats->snap_fail_nomem); return ret; }
/* Step 4: Mark all inodes as being committed. */ jbd2_journal_lock_updates(journal); lock_start = ktime_get(); /* * The journal is now locked. No more handles can start and all the * previous handles are now drained. Snapshotting happens in this * window so log writing can consume only stable snapshots without * doing logical-to-physical mapping. */ alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_set_inode_state(&iter->vfs_inode, EXT4_STATE_FC_COMMITTING); } ext4_fc_unlock(sb, alloc_ctx);
ret = ext4_fc_snapshot_inodes(journal, inodes, inodes_size, &snap_inodes, &snap_ranges, &snap_err); jbd2_journal_unlock_updates(journal); locked_ns = ktime_to_ns(ktime_sub(ktime_get(), lock_start)); atomic64_add(locked_ns, &snap_stats->lock_updates_ns_total); atomic64_inc(&snap_stats->lock_updates_samples); ext4_fc_snap_stats_update_max(&snap_stats->lock_updates_ns_max, locked_ns); if (trace_ext4_fc_lock_updates_enabled()) trace_call__ext4_fc_lock_updates(sb, commit_tid, locked_ns, snap_inodes, snap_ranges, ret, snap_err); kvfree(inodes); if (ret) return ret;
/* * Step 5: If file system device is different from journal device, * issue a cache flush before we start writing fast commit blocks. */ if (journal->j_fs_dev != journal->j_dev) blkdev_issue_flush(journal->j_fs_dev);
blk_start_plug(&plug); alloc_ctx = ext4_fc_lock(sb); /* Step 6: Write fast commit blocks to disk. */ if (sbi->s_fc_bytes == 0) { /* * Step 6.1: Add a head tag only if this is the first fast * commit in this TID. */ head.fc_features = cpu_to_le32(EXT4_FC_SUPPORTED_FEATURES); head.fc_tid = cpu_to_le32( sbi->s_journal->j_running_transaction->t_tid); if (!ext4_fc_add_tlv(sb, EXT4_FC_TAG_HEAD, sizeof(head), (u8 *)&head, &crc)) { ret = -ENOSPC; goto out; } }
/* Step 6.2: Now write all the dentry updates. */ ret = ext4_fc_commit_dentry_updates(journal, &crc); if (ret) goto out;
/* Step 6.3: Now write all the changed inodes to disk. */ list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { inode = &iter->vfs_inode; if (!ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) continue;
ret = ext4_fc_write_inode_data(inode, &crc); if (ret) goto out; ret = ext4_fc_write_inode(inode, &crc); if (ret) goto out; } /* Step 6.4: Finally write tail tag to conclude this fast commit. */ ret = ext4_fc_write_tail(sb, crc);
out: ext4_fc_unlock(sb, alloc_ctx); blk_finish_plug(&plug); return ret;}
static unsigned int ext4_fc_count_snapshot_inodes(struct super_block *sb){ struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_inode_info *iter; struct ext4_fc_dentry_update *fc_dentry; unsigned int nr_inodes = 0; int alloc_ctx;
alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) nr_inodes++;
list_for_each_entry(fc_dentry, &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) { struct ext4_inode_info *ei;
if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) continue; if (list_empty(&fc_dentry->fcd_dilist)) continue;
/* See the comment in ext4_fc_commit_dentry_updates(). */ ei = list_first_entry(&fc_dentry->fcd_dilist, struct ext4_inode_info, i_fc_dilist); if (!list_empty(&ei->i_fc_list)) continue;
nr_inodes++; } ext4_fc_unlock(sb, alloc_ctx);
return nr_inodes;}
static int ext4_fc_alloc_snapshot_inodes(struct super_block *sb, struct inode ***inodesp, unsigned int *nr_inodesp){ unsigned int nr_inodes = ext4_fc_count_snapshot_inodes(sb); struct inode **inodes;
*inodesp = NULL; *nr_inodesp = 0;
if (!nr_inodes) return 0;
if (nr_inodes > EXT4_FC_SNAPSHOT_MAX_INODES) return -E2BIG;
inodes = kvzalloc_objs(*inodes, nr_inodes, GFP_NOFS); if (!inodes) return -ENOMEM;
*inodesp = inodes; *nr_inodesp = nr_inodes; return 0;}
static void ext4_fc_update_stats(struct super_block *sb, int status, u64 commit_time, int nblks, tid_t commit_tid){ struct ext4_fc_stats *stats = &EXT4_SB(sb)->s_fc_stats;
ext4_debug("Fast commit ended with status = %d for tid %u", status, commit_tid); if (status == EXT4_FC_STATUS_OK) { stats->fc_num_commits++; stats->fc_numblks += nblks; if (likely(stats->s_fc_avg_commit_time)) stats->s_fc_avg_commit_time = (commit_time + stats->s_fc_avg_commit_time * 3) / 4; else stats->s_fc_avg_commit_time = commit_time; } else if (status == EXT4_FC_STATUS_FAILED || status == EXT4_FC_STATUS_INELIGIBLE) { if (status == EXT4_FC_STATUS_FAILED) stats->fc_failed_commits++; stats->fc_ineligible_commits++; } else { stats->fc_skipped_commits++; } trace_ext4_fc_commit_stop(sb, nblks, status, commit_tid);}
/* * The main commit entry point. Performs a fast commit for transaction * commit_tid if needed. If it's not possible to perform a fast commit * due to various reasons, we fall back to full commit. Returns 0 * on success, error otherwise. */int ext4_fc_commit(journal_t *journal, tid_t commit_tid){ struct super_block *sb = journal->j_private; struct ext4_sb_info *sbi = EXT4_SB(sb); int nblks = 0, ret, bsize = journal->j_blocksize; int subtid = atomic_read(&sbi->s_fc_subtid); int status = EXT4_FC_STATUS_OK, fc_bufs_before = 0; ktime_t start_time, commit_time; int old_ioprio, journal_ioprio;
if (!test_opt2(sb, JOURNAL_FAST_COMMIT)) return jbd2_complete_transaction(journal, commit_tid);
trace_ext4_fc_commit_start(sb, commit_tid);
start_time = ktime_get(); old_ioprio = get_current_ioprio();
restart_fc: ret = jbd2_fc_begin_commit(journal, commit_tid); if (ret == -EALREADY) { /* There was an ongoing commit, check if we need to restart */ if (atomic_read(&sbi->s_fc_subtid) <= subtid && tid_gt(commit_tid, journal->j_commit_sequence)) goto restart_fc; ext4_fc_update_stats(sb, EXT4_FC_STATUS_SKIPPED, 0, 0, commit_tid); return 0; } else if (ret) { /* * Commit couldn't start. Just update stats and perform a * full commit. */ ext4_fc_update_stats(sb, EXT4_FC_STATUS_FAILED, 0, 0, commit_tid); return jbd2_complete_transaction(journal, commit_tid); }
/* * After establishing journal barrier via jbd2_fc_begin_commit(), check * if we are fast commit ineligible. */ if (ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE)) { status = EXT4_FC_STATUS_INELIGIBLE; goto fallback; }
/* * Now that we know that this thread is going to do a fast commit, * elevate the priority to match that of the journal thread. */ if (journal->j_task->io_context) journal_ioprio = sbi->s_journal->j_task->io_context->ioprio; else journal_ioprio = EXT4_DEF_JOURNAL_IOPRIO; set_task_ioprio(current, journal_ioprio); fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize; ret = ext4_fc_perform_commit(journal, commit_tid); if (ret < 0) { if (ret == -EAGAIN || ret == -E2BIG || ret == -ECANCELED) status = EXT4_FC_STATUS_INELIGIBLE; else status = EXT4_FC_STATUS_FAILED; goto fallback; } nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before; ret = jbd2_fc_wait_bufs(journal, nblks); if (ret < 0) { status = EXT4_FC_STATUS_FAILED; goto fallback; } atomic_inc(&sbi->s_fc_subtid); ret = jbd2_fc_end_commit(journal); set_task_ioprio(current, old_ioprio); /* * weight the commit time higher than the average time so we * don't react too strongly to vast changes in the commit time */ commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time)); ext4_fc_update_stats(sb, status, commit_time, nblks, commit_tid); return ret;
fallback: set_task_ioprio(current, old_ioprio); ret = jbd2_fc_end_commit_fallback(journal); ext4_fc_update_stats(sb, status, 0, 0, commit_tid); return ret;}
/* * Fast commit cleanup routine. This is called after every fast commit and * full commit. full is true if we are called after a full commit. */static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid){ struct super_block *sb = journal->j_private; struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_inode_info *ei; struct ext4_fc_dentry_update *fc_dentry; int alloc_ctx;
if (full && sbi->s_fc_bh) sbi->s_fc_bh = NULL;
trace_ext4_fc_cleanup(journal, full, tid); jbd2_fc_release_bufs(journal);
alloc_ctx = ext4_fc_lock(sb); while (!list_empty(&sbi->s_fc_q[FC_Q_MAIN])) { bool requeue;
ei = list_first_entry(&sbi->s_fc_q[FC_Q_MAIN], struct ext4_inode_info, i_fc_list); list_del_init(&ei->i_fc_list); ext4_fc_free_inode_snap(&ei->vfs_inode); spin_lock(&ei->i_fc_lock); if (full) requeue = !tid_geq(tid, ei->i_sync_tid); else requeue = ext4_test_inode_state(&ei->vfs_inode, EXT4_STATE_FC_REQUEUE); if (!requeue) ext4_fc_reset_inode(&ei->vfs_inode); ext4_clear_inode_state(&ei->vfs_inode, EXT4_STATE_FC_REQUEUE); ext4_clear_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING); spin_unlock(&ei->i_fc_lock); if (requeue) list_add_tail(&ei->i_fc_list, &sbi->s_fc_q[FC_Q_STAGING]); /* * Make sure clearing of EXT4_STATE_FC_COMMITTING is * visible before we send the wakeup. Pairs with implicit * barrier in prepare_to_wait() in ext4_fc_del(). */ smp_mb(); ext4_fc_wake_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING); }
while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) { fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN], struct ext4_fc_dentry_update, fcd_list); list_del_init(&fc_dentry->fcd_list); if (fc_dentry->fcd_op == EXT4_FC_TAG_CREAT && !list_empty(&fc_dentry->fcd_dilist)) { /* See the comment in ext4_fc_commit_dentry_updates(). */ ei = list_first_entry(&fc_dentry->fcd_dilist, struct ext4_inode_info, i_fc_dilist); ext4_fc_free_inode_snap(&ei->vfs_inode); spin_lock(&ei->i_fc_lock); ext4_clear_inode_state(&ei->vfs_inode, EXT4_STATE_FC_REQUEUE); ext4_clear_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING); spin_unlock(&ei->i_fc_lock); /* * Make sure clearing of EXT4_STATE_FC_COMMITTING is * visible before we send the wakeup. Pairs with * implicit barrier in prepare_to_wait() in * ext4_fc_del(). */ smp_mb(); ext4_fc_wake_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING); } list_del_init(&fc_dentry->fcd_dilist);
release_dentry_name_snapshot(&fc_dentry->fcd_name); kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry); }
list_splice_init(&sbi->s_fc_dentry_q[FC_Q_STAGING], &sbi->s_fc_dentry_q[FC_Q_MAIN]); list_splice_init(&sbi->s_fc_q[FC_Q_STAGING], &sbi->s_fc_q[FC_Q_MAIN]);
if (tid_geq(tid, sbi->s_fc_ineligible_tid)) { sbi->s_fc_ineligible_tid = 0; ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); }
if (full) sbi->s_fc_bytes = 0; ext4_fc_unlock(sb, alloc_ctx); trace_ext4_fc_stats(sb);}
/* Ext4 Replay Path Routines */
/* Helper struct for dentry replay routines */struct dentry_info_args { int parent_ino, dname_len, ino, inode_len; char *dname;};
/* Same as struct ext4_fc_tl, but uses native endianness fields */struct ext4_fc_tl_mem { u16 fc_tag; u16 fc_len;};
static inline void tl_to_darg(struct dentry_info_args *darg, struct ext4_fc_tl_mem *tl, u8 *val){ struct ext4_fc_dentry_info fcd;
memcpy(&fcd, val, sizeof(fcd));
darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino); darg->ino = le32_to_cpu(fcd.fc_ino); darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname); darg->dname_len = tl->fc_len - sizeof(struct ext4_fc_dentry_info);}
static inline void ext4_fc_get_tl(struct ext4_fc_tl_mem *tl, u8 *val){ struct ext4_fc_tl tl_disk;
memcpy(&tl_disk, val, EXT4_FC_TAG_BASE_LEN); tl->fc_len = le16_to_cpu(tl_disk.fc_len); tl->fc_tag = le16_to_cpu(tl_disk.fc_tag);}
/* Unlink replay function */static int ext4_fc_replay_unlink(struct super_block *sb, struct ext4_fc_tl_mem *tl, u8 *val){ struct inode *inode, *old_parent; struct qstr entry; struct dentry_info_args darg; int ret = 0;
tl_to_darg(&darg, tl, val);
trace_ext4_fc_replay(sb, EXT4_FC_TAG_UNLINK, darg.ino, darg.parent_ino, darg.dname_len);
entry.name = darg.dname; entry.len = darg.dname_len; inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
if (IS_ERR(inode)) { ext4_debug("Inode %d not found", darg.ino); return 0; }
old_parent = ext4_iget(sb, darg.parent_ino, EXT4_IGET_NORMAL); if (IS_ERR(old_parent)) { ext4_debug("Dir with inode %d not found", darg.parent_ino); iput(inode); return 0; }
ret = __ext4_unlink(old_parent, &entry, inode, NULL); /* -ENOENT ok coz it might not exist anymore. */ if (ret == -ENOENT) ret = 0; iput(old_parent); iput(inode); return ret;}
static int ext4_fc_replay_link_internal(struct super_block *sb, struct dentry_info_args *darg, struct inode *inode){ struct inode *dir = NULL; struct qstr qstr_dname = QSTR_INIT(darg->dname, darg->dname_len); int ret = 0;
dir = ext4_iget(sb, darg->parent_ino, EXT4_IGET_NORMAL); if (IS_ERR(dir)) { ext4_debug("Dir with inode %d not found.", darg->parent_ino); dir = NULL; goto out; }
ret = __ext4_link(dir, inode, &qstr_dname, NULL); /* * It's possible that link already existed since data blocks * for the dir in question got persisted before we crashed OR * we replayed this tag and crashed before the entire replay * could complete. */ if (ret && ret != -EEXIST) { ext4_debug("Failed to link\n"); goto out; }
ret = 0;out: if (dir) iput(dir);
return ret;}
/* Link replay function */static int ext4_fc_replay_link(struct super_block *sb, struct ext4_fc_tl_mem *tl, u8 *val){ struct inode *inode; struct dentry_info_args darg; int ret = 0;
tl_to_darg(&darg, tl, val); trace_ext4_fc_replay(sb, EXT4_FC_TAG_LINK, darg.ino, darg.parent_ino, darg.dname_len);
inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL); if (IS_ERR(inode)) { ext4_debug("Inode not found."); return 0; }
ret = ext4_fc_replay_link_internal(sb, &darg, inode); iput(inode); return ret;}
/* * Record all the modified inodes during replay. We use this later to setup * block bitmaps correctly. */static int ext4_fc_record_modified_inode(struct super_block *sb, int ino){ struct ext4_fc_replay_state *state; int i;
state = &EXT4_SB(sb)->s_fc_replay_state; for (i = 0; i < state->fc_modified_inodes_used; i++) if (state->fc_modified_inodes[i] == ino) return 0; if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) { int *fc_modified_inodes;
fc_modified_inodes = krealloc(state->fc_modified_inodes, sizeof(int) * (state->fc_modified_inodes_size + EXT4_FC_REPLAY_REALLOC_INCREMENT), GFP_KERNEL); if (!fc_modified_inodes) return -ENOMEM; state->fc_modified_inodes = fc_modified_inodes; state->fc_modified_inodes_size += EXT4_FC_REPLAY_REALLOC_INCREMENT; } state->fc_modified_inodes[state->fc_modified_inodes_used++] = ino; return 0;}
/* * Inode replay function */static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl_mem *tl, u8 *val){ struct ext4_fc_inode fc_inode; struct ext4_inode *raw_inode; struct ext4_inode *raw_fc_inode; struct inode *inode = NULL; struct ext4_iloc iloc; int inode_len, ino, ret, tag = tl->fc_tag; struct ext4_extent_header *eh; size_t off_gen = offsetof(struct ext4_inode, i_generation);
memcpy(&fc_inode, val, sizeof(fc_inode));
ino = le32_to_cpu(fc_inode.fc_ino); trace_ext4_fc_replay(sb, tag, ino, 0, 0);
inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL); if (!IS_ERR(inode)) { ext4_ext_clear_bb(inode); iput(inode); } inode = NULL;
ret = ext4_fc_record_modified_inode(sb, ino); if (ret) goto out;
raw_fc_inode = (struct ext4_inode *) (val + offsetof(struct ext4_fc_inode, fc_raw_inode)); ret = ext4_get_fc_inode_loc(sb, ino, &iloc); if (ret) goto out;
inode_len = tl->fc_len - sizeof(struct ext4_fc_inode); raw_inode = ext4_raw_inode(&iloc);
memcpy(raw_inode, raw_fc_inode, offsetof(struct ext4_inode, i_block)); memcpy((u8 *)raw_inode + off_gen, (u8 *)raw_fc_inode + off_gen, inode_len - off_gen); if (le32_to_cpu(raw_inode->i_flags) & EXT4_EXTENTS_FL) { eh = (struct ext4_extent_header *)(&raw_inode->i_block[0]); if (eh->eh_magic != EXT4_EXT_MAGIC) { memset(eh, 0, sizeof(*eh)); eh->eh_magic = EXT4_EXT_MAGIC; eh->eh_max = cpu_to_le16( (sizeof(raw_inode->i_block) - sizeof(struct ext4_extent_header)) / sizeof(struct ext4_extent)); } } else if (le32_to_cpu(raw_inode->i_flags) & EXT4_INLINE_DATA_FL) { memcpy(raw_inode->i_block, raw_fc_inode->i_block, sizeof(raw_inode->i_block)); }
/* Immediately update the inode on disk. */ ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh); if (ret) goto out_brelse; ret = sync_dirty_buffer(iloc.bh); if (ret) goto out_brelse; ret = ext4_mark_inode_used(sb, ino); if (ret) goto out_brelse;
/* Given that we just wrote the inode on disk, this SHOULD succeed. */ inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL); if (IS_ERR(inode)) { ext4_debug("Inode not found."); inode = NULL; ret = -EFSCORRUPTED; goto out_brelse; }
/* * Our allocator could have made different decisions than before * crashing. This should be fixed but until then, we calculate * the number of blocks the inode. */ if (!ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) ext4_ext_replay_set_iblocks(inode);
inode->i_generation = le32_to_cpu(ext4_raw_inode(&iloc)->i_generation); ext4_reset_inode_seed(inode);
ext4_inode_csum_set(inode, ext4_raw_inode(&iloc), EXT4_I(inode)); ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh); sync_dirty_buffer(iloc.bh);out_brelse: brelse(iloc.bh);out: iput(inode); if (!ret) blkdev_issue_flush(sb->s_bdev);
return ret;}
/* * Dentry create replay function. * * EXT4_FC_TAG_CREAT is preceded by EXT4_FC_TAG_INODE_FULL. Which means, the * inode for which we are trying to create a dentry here, should already have * been replayed before we start here. */static int ext4_fc_replay_create(struct super_block *sb, struct ext4_fc_tl_mem *tl, u8 *val){ int ret = 0; struct inode *inode = NULL; struct inode *dir = NULL; struct dentry_info_args darg;
tl_to_darg(&darg, tl, val);
trace_ext4_fc_replay(sb, EXT4_FC_TAG_CREAT, darg.ino, darg.parent_ino, darg.dname_len);
/* This takes care of update group descriptor and other metadata */ ret = ext4_mark_inode_used(sb, darg.ino); if (ret) goto out;
inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL); if (IS_ERR(inode)) { ext4_debug("inode %d not found.", darg.ino); inode = NULL; ret = -EINVAL; goto out; }
if (S_ISDIR(inode->i_mode)) { /* * If we are creating a directory, we need to make sure that the * dot and dot dot dirents are setup properly. */ dir = ext4_iget(sb, darg.parent_ino, EXT4_IGET_NORMAL); if (IS_ERR(dir)) { ext4_debug("Dir %d not found.", darg.ino); goto out; } ret = ext4_init_new_dir(NULL, dir, inode); iput(dir); if (ret) { ret = 0; goto out; } } ret = ext4_fc_replay_link_internal(sb, &darg, inode); if (ret) goto out; set_nlink(inode, 1); ext4_mark_inode_dirty(NULL, inode);out: iput(inode); return ret;}
/* * Record physical disk regions which are in use as per fast commit area, * and used by inodes during replay phase. Our simple replay phase * allocator excludes these regions from allocation. */int ext4_fc_record_regions(struct super_block *sb, int ino, ext4_lblk_t lblk, ext4_fsblk_t pblk, int len, int replay){ struct ext4_fc_replay_state *state; struct ext4_fc_alloc_region *region;
state = &EXT4_SB(sb)->s_fc_replay_state; /* * during replay phase, the fc_regions_valid may not same as * fc_regions_used, update it when do new additions. */ if (replay && state->fc_regions_used != state->fc_regions_valid) state->fc_regions_used = state->fc_regions_valid; if (state->fc_regions_used == state->fc_regions_size) { struct ext4_fc_alloc_region *fc_regions;
fc_regions = krealloc(state->fc_regions, sizeof(struct ext4_fc_alloc_region) * (state->fc_regions_size + EXT4_FC_REPLAY_REALLOC_INCREMENT), GFP_KERNEL); if (!fc_regions) return -ENOMEM; state->fc_regions_size += EXT4_FC_REPLAY_REALLOC_INCREMENT; state->fc_regions = fc_regions; } region = &state->fc_regions[state->fc_regions_used++]; region->ino = ino; region->lblk = lblk; region->pblk = pblk; region->len = len;
if (replay) state->fc_regions_valid++;
return 0;}
/* Replay add range tag */static int ext4_fc_replay_add_range(struct super_block *sb, u8 *val){ struct ext4_fc_add_range fc_add_ex; struct ext4_extent newex, *ex; struct inode *inode; ext4_lblk_t start, cur; int remaining, len; ext4_fsblk_t start_pblk; struct ext4_map_blocks map; struct ext4_ext_path *path = NULL; int ret;
memcpy(&fc_add_ex, val, sizeof(fc_add_ex)); ex = (struct ext4_extent *)&fc_add_ex.fc_ex;
trace_ext4_fc_replay(sb, EXT4_FC_TAG_ADD_RANGE, le32_to_cpu(fc_add_ex.fc_ino), le32_to_cpu(ex->ee_block), ext4_ext_get_actual_len(ex));
inode = ext4_iget(sb, le32_to_cpu(fc_add_ex.fc_ino), EXT4_IGET_NORMAL); if (IS_ERR(inode)) { ext4_debug("Inode not found."); return 0; }
ret = ext4_fc_record_modified_inode(sb, inode->i_ino); if (ret) goto out;
start = le32_to_cpu(ex->ee_block); start_pblk = ext4_ext_pblock(ex); len = ext4_ext_get_actual_len(ex);
cur = start; remaining = len; ext4_debug("ADD_RANGE, lblk %d, pblk %lld, len %d, unwritten %d, inode %llu\n", start, start_pblk, len, ext4_ext_is_unwritten(ex), inode->i_ino);
while (remaining > 0) { map.m_lblk = cur; map.m_len = remaining; map.m_pblk = 0; ret = ext4_map_blocks(NULL, inode, &map, 0);
if (ret < 0) goto out;
if (ret == 0) { /* Range is not mapped */ path = ext4_find_extent(inode, cur, path, 0); if (IS_ERR(path)) { ret = PTR_ERR(path); path = NULL; goto out; } memset(&newex, 0, sizeof(newex)); newex.ee_block = cpu_to_le32(cur); ext4_ext_store_pblock( &newex, start_pblk + cur - start); newex.ee_len = cpu_to_le16(map.m_len); if (ext4_ext_is_unwritten(ex)) ext4_ext_mark_unwritten(&newex); down_write(&EXT4_I(inode)->i_data_sem); path = ext4_ext_insert_extent(NULL, inode, path, &newex, 0); up_write((&EXT4_I(inode)->i_data_sem)); if (IS_ERR(path)) { ret = PTR_ERR(path); path = NULL; goto out; } goto next; }
if (start_pblk + cur - start != map.m_pblk) { /* * Logical to physical mapping changed. This can happen * if this range was removed and then reallocated to * map to new physical blocks during a fast commit. */ ret = ext4_ext_replay_update_ex(inode, cur, map.m_len, ext4_ext_is_unwritten(ex), start_pblk + cur - start); if (ret) goto out; /* * Mark the old blocks as free since they aren't used * anymore. We maintain an array of all the modified * inodes. In case these blocks are still used at either * a different logical range in the same inode or in * some different inode, we will mark them as allocated * at the end of the FC replay using our array of * modified inodes. */ ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, false); goto next; }
/* Range is mapped and needs a state change */ ext4_debug("Converting from %ld to %d %lld", map.m_flags & EXT4_MAP_UNWRITTEN, ext4_ext_is_unwritten(ex), map.m_pblk); ret = ext4_ext_replay_update_ex(inode, cur, map.m_len, ext4_ext_is_unwritten(ex), map.m_pblk); if (ret) goto out; /* * We may have split the extent tree while toggling the state. * Try to shrink the extent tree now. */ ext4_ext_replay_shrink_inode(inode, start + len);next: cur += map.m_len; remaining -= map.m_len; } ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >> sb->s_blocksize_bits); ret = 0;out: ext4_free_ext_path(path); iput(inode); return ret;}
/* Replay DEL_RANGE tag */static intext4_fc_replay_del_range(struct super_block *sb, u8 *val){ struct inode *inode; struct ext4_fc_del_range lrange; struct ext4_map_blocks map; ext4_lblk_t cur, remaining; int ret;
memcpy(&lrange, val, sizeof(lrange)); cur = le32_to_cpu(lrange.fc_lblk); remaining = le32_to_cpu(lrange.fc_len);
trace_ext4_fc_replay(sb, EXT4_FC_TAG_DEL_RANGE, le32_to_cpu(lrange.fc_ino), cur, remaining);
inode = ext4_iget(sb, le32_to_cpu(lrange.fc_ino), EXT4_IGET_NORMAL); if (IS_ERR(inode)) { ext4_debug("Inode %d not found", le32_to_cpu(lrange.fc_ino)); return 0; }
ret = ext4_fc_record_modified_inode(sb, inode->i_ino); if (ret) goto out;
ext4_debug("DEL_RANGE, inode %llu, lblk %d, len %d\n", inode->i_ino, le32_to_cpu(lrange.fc_lblk), le32_to_cpu(lrange.fc_len)); while (remaining > 0) { map.m_lblk = cur; map.m_len = remaining;
ret = ext4_map_blocks(NULL, inode, &map, 0); if (ret < 0) goto out; if (ret > 0) { remaining -= ret; cur += ret; ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, false); } else { remaining -= map.m_len; cur += map.m_len; } }
down_write(&EXT4_I(inode)->i_data_sem); ret = ext4_ext_remove_space(inode, le32_to_cpu(lrange.fc_lblk), le32_to_cpu(lrange.fc_lblk) + le32_to_cpu(lrange.fc_len) - 1); up_write(&EXT4_I(inode)->i_data_sem); if (ret) goto out; ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >> sb->s_blocksize_bits); ext4_mark_inode_dirty(NULL, inode); ret = 0;out: iput(inode); return ret;}
static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb){ struct ext4_fc_replay_state *state; struct inode *inode; struct ext4_ext_path *path = NULL; struct ext4_map_blocks map; int i, ret, j; ext4_lblk_t cur, end;
state = &EXT4_SB(sb)->s_fc_replay_state; for (i = 0; i < state->fc_modified_inodes_used; i++) { inode = ext4_iget(sb, state->fc_modified_inodes[i], EXT4_IGET_NORMAL); if (IS_ERR(inode)) { ext4_debug("Inode %d not found.", state->fc_modified_inodes[i]); continue; } cur = 0; end = EXT_MAX_BLOCKS; if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) { iput(inode); continue; } while (cur < end) { map.m_lblk = cur; map.m_len = end - cur;
ret = ext4_map_blocks(NULL, inode, &map, 0); if (ret < 0) break;
if (ret > 0) { path = ext4_find_extent(inode, map.m_lblk, path, 0); if (!IS_ERR(path)) { for (j = 0; j < path->p_depth; j++) ext4_mb_mark_bb(inode->i_sb, path[j].p_block, 1, true); } else { path = NULL; } cur += ret; ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, true); } else { cur = cur + (map.m_len ? map.m_len : 1); } } iput(inode); }
ext4_free_ext_path(path);}
/* * Check if block is in excluded regions for block allocation. The simple * allocator that runs during replay phase is calls this function to see * if it is okay to use a block. */bool ext4_fc_replay_check_excluded(struct super_block *sb, ext4_fsblk_t blk){ int i; struct ext4_fc_replay_state *state;
state = &EXT4_SB(sb)->s_fc_replay_state; for (i = 0; i < state->fc_regions_valid; i++) { if (state->fc_regions[i].ino == 0 || state->fc_regions[i].len == 0) continue; if (in_range(blk, state->fc_regions[i].pblk, state->fc_regions[i].len)) return true; } return false;}
/* Cleanup function called after replay */void ext4_fc_replay_cleanup(struct super_block *sb){ struct ext4_sb_info *sbi = EXT4_SB(sb);
sbi->s_mount_state &= ~EXT4_FC_REPLAY; kfree(sbi->s_fc_replay_state.fc_regions); kfree(sbi->s_fc_replay_state.fc_modified_inodes);}
static bool ext4_fc_value_len_isvalid(struct ext4_sb_info *sbi, int tag, int len){ switch (tag) { case EXT4_FC_TAG_ADD_RANGE: return len == sizeof(struct ext4_fc_add_range); case EXT4_FC_TAG_DEL_RANGE: return len == sizeof(struct ext4_fc_del_range); case EXT4_FC_TAG_CREAT: case EXT4_FC_TAG_LINK: case EXT4_FC_TAG_UNLINK: len -= sizeof(struct ext4_fc_dentry_info); return len >= 1 && len <= EXT4_NAME_LEN; case EXT4_FC_TAG_INODE: len -= sizeof(struct ext4_fc_inode); return len >= EXT4_GOOD_OLD_INODE_SIZE && len <= sbi->s_inode_size; case EXT4_FC_TAG_PAD: return true; /* padding can have any length */ case EXT4_FC_TAG_TAIL: return len >= sizeof(struct ext4_fc_tail); case EXT4_FC_TAG_HEAD: return len == sizeof(struct ext4_fc_head); } return false;}
/* * Recovery Scan phase handler * * This function is called during the scan phase and is responsible * for doing following things: * - Make sure the fast commit area has valid tags for replay * - Count number of tags that need to be replayed by the replay handler * - Verify CRC * - Create a list of excluded blocks for allocation during replay phase * * This function returns JBD2_FC_REPLAY_CONTINUE to indicate that SCAN is * incomplete and JBD2 should send more blocks. It returns JBD2_FC_REPLAY_STOP * to indicate that scan has finished and JBD2 can now start replay phase. * It returns a negative error to indicate that there was an error. At the end * of a successful scan phase, sbi->s_fc_replay_state.fc_replay_num_tags is set * to indicate the number of tags that need to replayed during the replay phase. */static int ext4_fc_replay_scan(journal_t *journal, struct buffer_head *bh, int off, tid_t expected_tid){ struct super_block *sb = journal->j_private; struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_fc_replay_state *state; int ret = JBD2_FC_REPLAY_CONTINUE; struct ext4_fc_add_range ext; struct ext4_fc_tl_mem tl; struct ext4_fc_tail tail; __u8 *start, *end, *cur, *val; struct ext4_fc_head head; struct ext4_extent *ex;
state = &sbi->s_fc_replay_state;
start = (u8 *)bh->b_data; end = start + journal->j_blocksize;
if (state->fc_replay_expected_off == 0) { state->fc_cur_tag = 0; state->fc_replay_num_tags = 0; state->fc_crc = 0; state->fc_regions = NULL; state->fc_regions_valid = state->fc_regions_used = state->fc_regions_size = 0; /* Check if we can stop early */ if (le16_to_cpu(((struct ext4_fc_tl *)start)->fc_tag) != EXT4_FC_TAG_HEAD) return 0; }
if (off != state->fc_replay_expected_off) { ret = -EFSCORRUPTED; goto out_err; }
state->fc_replay_expected_off++; for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN; cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) { ext4_fc_get_tl(&tl, cur); val = cur + EXT4_FC_TAG_BASE_LEN; if (tl.fc_len > end - val || !ext4_fc_value_len_isvalid(sbi, tl.fc_tag, tl.fc_len)) { ret = state->fc_replay_num_tags ? JBD2_FC_REPLAY_STOP : -ECANCELED; goto out_err; } ext4_debug("Scan phase, tag:%s, blk %lld\n", tag2str(tl.fc_tag), bh->b_blocknr); switch (tl.fc_tag) { case EXT4_FC_TAG_ADD_RANGE: memcpy(&ext, val, sizeof(ext)); ex = (struct ext4_extent *)&ext.fc_ex; ret = ext4_fc_record_regions(sb, le32_to_cpu(ext.fc_ino), le32_to_cpu(ex->ee_block), ext4_ext_pblock(ex), ext4_ext_get_actual_len(ex), 0); if (ret < 0) break; ret = JBD2_FC_REPLAY_CONTINUE; fallthrough; case EXT4_FC_TAG_DEL_RANGE: case EXT4_FC_TAG_LINK: case EXT4_FC_TAG_UNLINK: case EXT4_FC_TAG_CREAT: case EXT4_FC_TAG_INODE: case EXT4_FC_TAG_PAD: state->fc_cur_tag++; state->fc_crc = ext4_chksum(state->fc_crc, cur, EXT4_FC_TAG_BASE_LEN + tl.fc_len); break; case EXT4_FC_TAG_TAIL: state->fc_cur_tag++; memcpy(&tail, val, sizeof(tail)); state->fc_crc = ext4_chksum(state->fc_crc, cur, EXT4_FC_TAG_BASE_LEN + offsetof(struct ext4_fc_tail, fc_crc)); if (le32_to_cpu(tail.fc_tid) == expected_tid && le32_to_cpu(tail.fc_crc) == state->fc_crc) { state->fc_replay_num_tags = state->fc_cur_tag; state->fc_regions_valid = state->fc_regions_used; } else { ret = state->fc_replay_num_tags ? JBD2_FC_REPLAY_STOP : -EFSBADCRC; } state->fc_crc = 0; break; case EXT4_FC_TAG_HEAD: memcpy(&head, val, sizeof(head)); if (le32_to_cpu(head.fc_features) & ~EXT4_FC_SUPPORTED_FEATURES) { ret = -EOPNOTSUPP; break; } if (le32_to_cpu(head.fc_tid) != expected_tid) { ret = JBD2_FC_REPLAY_STOP; break; } state->fc_cur_tag++; state->fc_crc = ext4_chksum(state->fc_crc, cur, EXT4_FC_TAG_BASE_LEN + tl.fc_len); break; default: ret = state->fc_replay_num_tags ? JBD2_FC_REPLAY_STOP : -ECANCELED; } if (ret < 0 || ret == JBD2_FC_REPLAY_STOP) break; }
out_err: trace_ext4_fc_replay_scan(sb, ret, off); return ret;}
/* * Main recovery path entry point. * The meaning of return codes is similar as above. */static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh, enum passtype pass, int off, tid_t expected_tid){ struct super_block *sb = journal->j_private; struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_fc_tl_mem tl; __u8 *start, *end, *cur, *val; int ret = JBD2_FC_REPLAY_CONTINUE; struct ext4_fc_replay_state *state = &sbi->s_fc_replay_state; struct ext4_fc_tail tail;
if (pass == PASS_SCAN) { state->fc_current_pass = PASS_SCAN; return ext4_fc_replay_scan(journal, bh, off, expected_tid); }
if (state->fc_current_pass != pass) { state->fc_current_pass = pass; sbi->s_mount_state |= EXT4_FC_REPLAY; } if (!sbi->s_fc_replay_state.fc_replay_num_tags) { ext4_debug("Replay stops\n"); ext4_fc_set_bitmaps_and_counters(sb); return 0; }
#ifdef CONFIG_EXT4_DEBUG if (sbi->s_fc_debug_max_replay && off >= sbi->s_fc_debug_max_replay) { pr_warn("Dropping fc block %d because max_replay set\n", off); return JBD2_FC_REPLAY_STOP; }#endif
start = (u8 *)bh->b_data; end = start + journal->j_blocksize;
for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN; cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) { ext4_fc_get_tl(&tl, cur); val = cur + EXT4_FC_TAG_BASE_LEN;
if (state->fc_replay_num_tags == 0) { ret = JBD2_FC_REPLAY_STOP; ext4_fc_set_bitmaps_and_counters(sb); break; }
ext4_debug("Replay phase, tag:%s\n", tag2str(tl.fc_tag)); state->fc_replay_num_tags--; switch (tl.fc_tag) { case EXT4_FC_TAG_LINK: ret = ext4_fc_replay_link(sb, &tl, val); break; case EXT4_FC_TAG_UNLINK: ret = ext4_fc_replay_unlink(sb, &tl, val); break; case EXT4_FC_TAG_ADD_RANGE: ret = ext4_fc_replay_add_range(sb, val); break; case EXT4_FC_TAG_CREAT: ret = ext4_fc_replay_create(sb, &tl, val); break; case EXT4_FC_TAG_DEL_RANGE: ret = ext4_fc_replay_del_range(sb, val); break; case EXT4_FC_TAG_INODE: ret = ext4_fc_replay_inode(sb, &tl, val); break; case EXT4_FC_TAG_PAD: trace_ext4_fc_replay(sb, EXT4_FC_TAG_PAD, 0, tl.fc_len, 0); break; case EXT4_FC_TAG_TAIL: trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL, 0, tl.fc_len, 0); memcpy(&tail, val, sizeof(tail)); WARN_ON(le32_to_cpu(tail.fc_tid) != expected_tid); break; case EXT4_FC_TAG_HEAD: break; default: trace_ext4_fc_replay(sb, tl.fc_tag, 0, tl.fc_len, 0); ret = -ECANCELED; break; } if (ret < 0) break; ret = JBD2_FC_REPLAY_CONTINUE; } return ret;}
void ext4_fc_init(struct super_block *sb, journal_t *journal){ /* * We set replay callback even if fast commit disabled because we may * could still have fast commit blocks that need to be replayed even if * fast commit has now been turned off. */ journal->j_fc_replay_callback = ext4_fc_replay; if (!test_opt2(sb, JOURNAL_FAST_COMMIT)) return; journal->j_fc_cleanup_callback = ext4_fc_cleanup;}
static const char * const fc_ineligible_reasons[] = { [EXT4_FC_REASON_XATTR] = "Extended attributes changed", [EXT4_FC_REASON_CROSS_RENAME] = "Cross rename", [EXT4_FC_REASON_JOURNAL_FLAG_CHANGE] = "Journal flag changed", [EXT4_FC_REASON_NOMEM] = "Insufficient memory", [EXT4_FC_REASON_SWAP_BOOT] = "Swap boot", [EXT4_FC_REASON_RESIZE] = "Resize", [EXT4_FC_REASON_RENAME_DIR] = "Dir renamed", [EXT4_FC_REASON_FALLOC_RANGE] = "Falloc range op", [EXT4_FC_REASON_INODE_JOURNAL_DATA] = "Data journalling", [EXT4_FC_REASON_ENCRYPTED_FILENAME] = "Encrypted filename", [EXT4_FC_REASON_MIGRATE] = "Inode format migration", [EXT4_FC_REASON_VERITY] = "fs-verity enable", [EXT4_FC_REASON_MOVE_EXT] = "Move extents",};
int ext4_fc_info_show(struct seq_file *seq, void *v){ struct ext4_sb_info *sbi = EXT4_SB((struct super_block *)seq->private); struct ext4_fc_stats *stats = &sbi->s_fc_stats; struct ext4_fc_snap_stats *snap_stats = &sbi->s_fc_snap_stats; u64 lock_avg_ns = 0; u64 lock_updates_samples; u64 lock_updates_ns_total; u64 lock_updates_ns_max; int i;
if (v != SEQ_START_TOKEN) return 0;
lock_updates_samples = atomic64_read(&snap_stats->lock_updates_samples); lock_updates_ns_total = atomic64_read(&snap_stats->lock_updates_ns_total); lock_updates_ns_max = atomic64_read(&snap_stats->lock_updates_ns_max); if (lock_updates_samples) lock_avg_ns = div64_u64(lock_updates_ns_total, lock_updates_samples);
seq_printf(seq, "fc stats:\n%ld commits\n%ld ineligible\n%ld numblks\n%lluus avg_commit_time\n", stats->fc_num_commits, stats->fc_ineligible_commits, stats->fc_numblks, div_u64(stats->s_fc_avg_commit_time, 1000)); seq_puts(seq, "Ineligible reasons:\n"); for (i = 0; i < EXT4_FC_REASON_MAX; i++) seq_printf(seq, "\"%s\":\t%d\n", fc_ineligible_reasons[i], stats->fc_ineligible_reason_count[i]);
seq_printf(seq, "Snapshot stats:\n%llu inodes\n%llu ranges\n%lluus lock_updates_avg\n%lluus lock_updates_max\n", atomic64_read(&snap_stats->snap_inodes), atomic64_read(&snap_stats->snap_ranges), div_u64(lock_avg_ns, 1000), div_u64(lock_updates_ns_max, 1000)); seq_printf(seq, "Snapshot failures:\n%llu es_miss\n%llu es_delayed\n%llu es_other\n%llu inodes_cap\n%llu ranges_cap\n%llu nomem\n%llu inode_loc\n%llu no_snap\n", atomic64_read(&snap_stats->snap_fail_es_miss), atomic64_read(&snap_stats->snap_fail_es_delayed), atomic64_read(&snap_stats->snap_fail_es_other), atomic64_read(&snap_stats->snap_fail_inodes_cap), atomic64_read(&snap_stats->snap_fail_ranges_cap), atomic64_read(&snap_stats->snap_fail_nomem), atomic64_read(&snap_stats->snap_fail_inode_loc), atomic64_read(&snap_stats->snap_fail_no_snap));
return 0;}
int __init ext4_fc_init_dentry_cache(void){ ext4_fc_dentry_cachep = KMEM_CACHE(ext4_fc_dentry_update, SLAB_RECLAIM_ACCOUNT);
if (!ext4_fc_dentry_cachep) return -ENOMEM;
ext4_fc_range_cachep = KMEM_CACHE(ext4_fc_range, SLAB_RECLAIM_ACCOUNT); if (!ext4_fc_range_cachep) { kmem_cache_destroy(ext4_fc_dentry_cachep); return -ENOMEM; }
return 0;}
void ext4_fc_destroy_dentry_cache(void){ kmem_cache_destroy(ext4_fc_range_cachep); kmem_cache_destroy(ext4_fc_dentry_cachep);}