Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282// SPDX-License-Identifier: GPL-2.0-or-later
#include <linux/bio.h>
#include "cache.h"#include "backing_dev.h"#include "cache_dev.h"#include "dm_pcache.h"
static void writeback_ctx_end(struct pcache_cache *cache, int ret){ if (ret && !cache->writeback_ctx.ret) { pcache_dev_err(CACHE_TO_PCACHE(cache), "writeback error: %d", ret); cache->writeback_ctx.ret = ret; }
if (!atomic_dec_and_test(&cache->writeback_ctx.pending)) return;
if (!cache->writeback_ctx.ret) { backing_dev_flush(cache->backing_dev);
mutex_lock(&cache->dirty_tail_lock); cache_pos_advance(&cache->dirty_tail, cache->writeback_ctx.advance); cache_encode_dirty_tail(cache); mutex_unlock(&cache->dirty_tail_lock); } queue_delayed_work(cache_get_wq(cache), &cache->writeback_work, 0);}
static void writeback_end_req(struct pcache_backing_dev_req *backing_req, int ret){ struct pcache_cache *cache = backing_req->priv_data;
mutex_lock(&cache->writeback_lock); writeback_ctx_end(cache, ret); mutex_unlock(&cache->writeback_lock);}
static inline bool is_cache_clean(struct pcache_cache *cache, struct pcache_cache_pos *dirty_tail){ struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); struct pcache_cache_kset_onmedia *kset_onmedia; u32 to_copy; void *addr; int ret;
addr = cache_pos_addr(dirty_tail); kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->wb_kset_onmedia_buf;
to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(dirty_tail)); ret = copy_mc_to_kernel(kset_onmedia, addr, to_copy); if (ret) { pcache_dev_err(pcache, "error to read kset: %d", ret); return true; }
/* Reject a corrupted or out-of-bounds kset before reading its keys */ if (!kset_onmedia_valid(kset_onmedia)) { pcache_dev_debug(pcache, "dirty_tail: %u:%u invalid kset magic: %llx, key_num: %u\n", dirty_tail->cache_seg->cache_seg_id, dirty_tail->seg_off, kset_onmedia->magic, kset_onmedia->key_num); return true; }
/* Verify the CRC checksum for data integrity */ if (kset_onmedia->crc != cache_kset_crc(kset_onmedia)) { pcache_dev_debug(pcache, "dirty_tail: %u:%u crc: %x, not expected: %x\n", dirty_tail->cache_seg->cache_seg_id, dirty_tail->seg_off, cache_kset_crc(kset_onmedia), kset_onmedia->crc); return true; }
return false;}
void cache_writeback_exit(struct pcache_cache *cache){ cancel_delayed_work_sync(&cache->writeback_work); backing_dev_flush(cache->backing_dev); cache_tree_exit(&cache->writeback_key_tree);}
int cache_writeback_init(struct pcache_cache *cache){ int ret;
ret = cache_tree_init(cache, &cache->writeback_key_tree, 1); if (ret) goto err;
atomic_set(&cache->writeback_ctx.pending, 0);
/* Queue delayed work to start writeback handling */ queue_delayed_work(cache_get_wq(cache), &cache->writeback_work, 0);
return 0;err: return ret;}
static void cache_key_writeback(struct pcache_cache *cache, struct pcache_cache_key *key){ struct pcache_backing_dev_req *writeback_req; struct pcache_backing_dev_req_opts writeback_req_opts = { 0 }; struct pcache_cache_pos *pos; void *addr; u32 seg_remain, req_len, done = 0;
if (cache_key_clean(key)) return;
pos = &key->cache_pos;
seg_remain = cache_seg_remain(pos); BUG_ON(seg_remain < key->len);next_req: addr = cache_pos_addr(pos) + done; req_len = backing_dev_req_coalesced_max_len(addr, key->len - done);
writeback_req_opts.type = BACKING_DEV_REQ_TYPE_KMEM; writeback_req_opts.gfp_mask = GFP_NOIO; writeback_req_opts.end_fn = writeback_end_req; writeback_req_opts.priv_data = cache;
writeback_req_opts.kmem.data = addr; writeback_req_opts.kmem.opf = REQ_OP_WRITE; writeback_req_opts.kmem.len = req_len; writeback_req_opts.kmem.backing_off = key->off + done;
writeback_req = backing_dev_req_create(cache->backing_dev, &writeback_req_opts);
atomic_inc(&cache->writeback_ctx.pending); backing_dev_req_submit(writeback_req, true);
done += req_len; if (done < key->len) goto next_req;}
static void cache_wb_tree_writeback(struct pcache_cache *cache, u32 advance){ struct pcache_cache_tree *cache_tree = &cache->writeback_key_tree; struct pcache_cache_subtree *cache_subtree; struct rb_node *node; struct pcache_cache_key *key; u32 i;
cache->writeback_ctx.ret = 0; cache->writeback_ctx.advance = advance; atomic_set(&cache->writeback_ctx.pending, 1);
for (i = 0; i < cache_tree->n_subtrees; i++) { cache_subtree = &cache_tree->subtrees[i];
node = rb_first(&cache_subtree->root); while (node) { key = CACHE_KEY(node); node = rb_next(node);
cache_key_writeback(cache, key); cache_key_delete(key); } } writeback_ctx_end(cache, 0);}
static int cache_kset_insert_tree(struct pcache_cache *cache, struct pcache_cache_kset_onmedia *kset_onmedia){ struct pcache_cache_key_onmedia *key_onmedia; struct pcache_cache_subtree *cache_subtree; struct pcache_cache_key *key; int ret; u32 i;
/* Iterate through all keys in the kset and write each back to storage */ for (i = 0; i < kset_onmedia->key_num; i++) { key_onmedia = &kset_onmedia->data[i];
key = cache_key_alloc(&cache->writeback_key_tree, GFP_NOIO); ret = cache_key_decode(cache, key_onmedia, key); if (ret) { cache_key_put(key); goto clear_tree; }
cache_subtree = get_subtree(&cache->writeback_key_tree, key->off); spin_lock(&cache_subtree->tree_lock); cache_key_insert(&cache->writeback_key_tree, key, true); spin_unlock(&cache_subtree->tree_lock); }
return 0;clear_tree: cache_tree_clear(&cache->writeback_key_tree); return ret;}
static int last_kset_writeback(struct pcache_cache *cache, struct pcache_cache_kset_onmedia *last_kset_onmedia){ struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); struct pcache_cache_segment *next_seg;
if (!cache_seg_id_valid(cache, last_kset_onmedia->next_cache_seg_id)) { pcache_dev_err(pcache, "invalid next_cache_seg_id %u in writeback (n_segs %u)\n", last_kset_onmedia->next_cache_seg_id, cache->n_segs); return -EIO; }
pcache_dev_debug(pcache, "last kset, next: %u\n", last_kset_onmedia->next_cache_seg_id);
next_seg = &cache->segments[last_kset_onmedia->next_cache_seg_id];
mutex_lock(&cache->dirty_tail_lock); cache->dirty_tail.cache_seg = next_seg; cache->dirty_tail.seg_off = 0; cache_encode_dirty_tail(cache); mutex_unlock(&cache->dirty_tail_lock);
return 0;}
void cache_writeback_fn(struct work_struct *work){ struct pcache_cache *cache = container_of(work, struct pcache_cache, writeback_work.work); struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); struct pcache_cache_pos dirty_tail; struct pcache_cache_kset_onmedia *kset_onmedia; u32 delay; int ret;
mutex_lock(&cache->writeback_lock); if (atomic_read(&cache->writeback_ctx.pending)) goto unlock;
if (pcache_is_stopping(pcache)) goto unlock;
if (atomic_read(&cache->writeback_errors)) goto unlock;
kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->wb_kset_onmedia_buf;
mutex_lock(&cache->dirty_tail_lock); cache_pos_copy(&dirty_tail, &cache->dirty_tail); mutex_unlock(&cache->dirty_tail_lock);
if (is_cache_clean(cache, &dirty_tail)) { delay = PCACHE_CACHE_WRITEBACK_INTERVAL; goto queue_work; }
if (kset_onmedia->flags & PCACHE_KSET_FLAGS_LAST) { ret = last_kset_writeback(cache, kset_onmedia); if (ret) { atomic_inc(&cache->writeback_errors); goto unlock; } delay = 0; goto queue_work; }
if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(&dirty_tail)) { atomic_inc(&cache->writeback_errors); goto unlock; }
ret = cache_kset_insert_tree(cache, kset_onmedia); if (ret) { atomic_inc(&cache->writeback_errors); goto unlock; }
cache_wb_tree_writeback(cache, get_kset_onmedia_size(kset_onmedia)); delay = 0;queue_work: queue_delayed_work(cache_get_wq(cache), &cache->writeback_work, delay);unlock: mutex_unlock(&cache->writeback_lock);}