Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324// SPDX-License-Identifier: GPL-2.0-or-later
#include <linux/blkdev.h>#include <linux/dax.h>#include <linux/vmalloc.h>#include <linux/parser.h>
#include "cache_dev.h"#include "backing_dev.h"#include "cache.h"#include "dm_pcache.h"
static void cache_dev_dax_exit(struct pcache_cache_dev *cache_dev){ if (cache_dev->use_vmap) vunmap(cache_dev->mapping);}
static int build_vmap(struct dax_device *dax_dev, long total_pages, void **vaddr){ struct page **pages; long i = 0, chunk; unsigned long pfn; int ret;
pages = vmalloc_array(total_pages, sizeof(struct page *)); if (!pages) return -ENOMEM;
do { chunk = dax_direct_access(dax_dev, i, total_pages - i, DAX_ACCESS, NULL, &pfn); if (chunk <= 0) { ret = chunk ? chunk : -EINVAL; goto out_free; }
if (!pfn_valid(pfn)) { ret = -EOPNOTSUPP; goto out_free; }
while (chunk-- && i < total_pages) { pages[i++] = pfn_to_page(pfn); pfn++; if (!(i & 15)) cond_resched(); } } while (i < total_pages);
*vaddr = vmap(pages, total_pages, VM_MAP, PAGE_KERNEL); if (!*vaddr) { ret = -ENOMEM; goto out_free; }
ret = 0;
out_free: vfree(pages); return ret;}
static int cache_dev_dax_init(struct pcache_cache_dev *cache_dev){ struct dm_pcache *pcache = CACHE_DEV_TO_PCACHE(cache_dev); struct dax_device *dax_dev; long total_pages, mapped_pages; u64 bdev_size; void *vaddr; int ret; int id; unsigned long pfn;
dax_dev = cache_dev->dm_dev->dax_dev; /* total size check */ bdev_size = bdev_nr_bytes(cache_dev->dm_dev->bdev); if (bdev_size < PCACHE_CACHE_DEV_SIZE_MIN) { pcache_dev_err(pcache, "dax device is too small, required at least %llu", PCACHE_CACHE_DEV_SIZE_MIN); ret = -ENOSPC; goto out; }
total_pages = bdev_size >> PAGE_SHIFT; /* attempt: direct-map the whole range */ id = dax_read_lock(); mapped_pages = dax_direct_access(dax_dev, 0, total_pages, DAX_ACCESS, &vaddr, &pfn); if (mapped_pages < 0) { pcache_dev_err(pcache, "dax_direct_access failed: %ld\n", mapped_pages); ret = mapped_pages; goto unlock; }
if (!pfn_valid(pfn)) { ret = -EOPNOTSUPP; goto unlock; }
if (mapped_pages == total_pages) { /* success: contiguous direct mapping */ cache_dev->mapping = vaddr; } else { /* need vmap fallback */ ret = build_vmap(dax_dev, total_pages, &vaddr); if (ret) { pcache_dev_err(pcache, "vmap fallback failed: %d\n", ret); goto unlock; }
cache_dev->mapping = vaddr; cache_dev->use_vmap = true; } dax_read_unlock(id);
return 0;unlock: dax_read_unlock(id);out: return ret;}
void cache_dev_zero_range(struct pcache_cache_dev *cache_dev, void *pos, u32 size){ memset(pos, 0, size); dax_flush(cache_dev->dm_dev->dax_dev, pos, size);}
static int sb_read(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb){ struct pcache_sb *sb_addr = CACHE_DEV_SB(cache_dev);
if (copy_mc_to_kernel(sb, sb_addr, sizeof(struct pcache_sb))) return -EIO;
return 0;}
static void sb_write(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb){ struct pcache_sb *sb_addr = CACHE_DEV_SB(cache_dev);
memcpy_flushcache(sb_addr, sb, sizeof(struct pcache_sb)); pmem_wmb();}
static int sb_init(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb){ struct dm_pcache *pcache = CACHE_DEV_TO_PCACHE(cache_dev); u64 nr_segs; u64 cache_dev_size; u64 magic; u32 flags = 0;
magic = le64_to_cpu(sb->magic); if (magic) return -EEXIST;
cache_dev_size = bdev_nr_bytes(file_bdev(cache_dev->dm_dev->bdev_file)); if (cache_dev_size < PCACHE_CACHE_DEV_SIZE_MIN) { pcache_dev_err(pcache, "dax device is too small, required at least %llu", PCACHE_CACHE_DEV_SIZE_MIN); return -ENOSPC; }
nr_segs = (cache_dev_size - PCACHE_SEGMENTS_OFF) / ((PCACHE_SEG_SIZE));
#if defined(__BYTE_ORDER) ? (__BIG_ENDIAN == __BYTE_ORDER) : defined(__BIG_ENDIAN) flags |= PCACHE_SB_F_BIGENDIAN;#endif sb->flags = cpu_to_le32(flags); sb->magic = cpu_to_le64(PCACHE_MAGIC); sb->seg_num = cpu_to_le32(nr_segs); sb->crc = cpu_to_le32(crc32c(PCACHE_CRC_SEED, (void *)(sb) + 4, sizeof(struct pcache_sb) - 4));
cache_dev_zero_range(cache_dev, CACHE_DEV_CACHE_INFO(cache_dev), PCACHE_CACHE_INFO_SIZE * PCACHE_META_INDEX_MAX + PCACHE_CACHE_CTRL_SIZE);
return 0;}
static int sb_validate(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb){ struct dm_pcache *pcache = CACHE_DEV_TO_PCACHE(cache_dev); u32 flags; u32 crc;
if (le64_to_cpu(sb->magic) != PCACHE_MAGIC) { pcache_dev_err(pcache, "unexpected magic: %llx\n", le64_to_cpu(sb->magic)); return -EINVAL; }
crc = crc32c(PCACHE_CRC_SEED, (void *)(sb) + 4, sizeof(struct pcache_sb) - 4); if (crc != le32_to_cpu(sb->crc)) { pcache_dev_err(pcache, "corrupted sb: %u, expected: %u\n", crc, le32_to_cpu(sb->crc)); return -EINVAL; }
flags = le32_to_cpu(sb->flags);#if defined(__BYTE_ORDER) ? (__BIG_ENDIAN == __BYTE_ORDER) : defined(__BIG_ENDIAN) if (!(flags & PCACHE_SB_F_BIGENDIAN)) { pcache_dev_err(pcache, "cache_dev is not big endian\n"); return -EINVAL; }#else if (flags & PCACHE_SB_F_BIGENDIAN) { pcache_dev_err(pcache, "cache_dev is big endian\n"); return -EINVAL; }#endif return 0;}
static int cache_dev_init(struct pcache_cache_dev *cache_dev, u32 seg_num){ cache_dev->seg_num = seg_num; cache_dev->seg_bitmap = kvcalloc(BITS_TO_LONGS(cache_dev->seg_num), sizeof(unsigned long), GFP_KERNEL); if (!cache_dev->seg_bitmap) return -ENOMEM;
return 0;}
static void cache_dev_exit(struct pcache_cache_dev *cache_dev){ kvfree(cache_dev->seg_bitmap);}
void cache_dev_stop(struct dm_pcache *pcache){ struct pcache_cache_dev *cache_dev = &pcache->cache_dev;
cache_dev_exit(cache_dev); cache_dev_dax_exit(cache_dev);}
int cache_dev_start(struct dm_pcache *pcache){ struct pcache_cache_dev *cache_dev = &pcache->cache_dev; struct pcache_sb sb; bool format = false; u32 seg_num; u64 max_segs; int ret;
mutex_init(&cache_dev->seg_lock);
ret = cache_dev_dax_init(cache_dev); if (ret) { pcache_dev_err(pcache, "failed to init cache_dev %s via dax way: %d.", cache_dev->dm_dev->name, ret); goto err; }
ret = sb_read(cache_dev, &sb); if (ret) goto dax_release;
if (le64_to_cpu(sb.magic) == 0) { format = true; ret = sb_init(cache_dev, &sb); if (ret < 0) goto dax_release; }
ret = sb_validate(cache_dev, &sb); if (ret) goto dax_release;
cache_dev->sb_flags = le32_to_cpu(sb.flags);
/* * seg_num is read from the crc32c-only superblock, so whoever supplies * the cache device controls it. It is the ceiling every later on-media * segment id is validated against, so bound it against what the device * physically holds before it is trusted, or a forged seg_num lets a * segment id address past the DAX mapping. */ seg_num = le32_to_cpu(sb.seg_num); max_segs = (bdev_nr_bytes(cache_dev->dm_dev->bdev) - PCACHE_SEGMENTS_OFF) / PCACHE_SEG_SIZE; if (seg_num == 0 || seg_num > max_segs || seg_num > PCACHE_CACHE_SEGS_MAX) { pcache_dev_err(pcache, "invalid seg_num %u from cache device (device holds %llu, max %u)\n", seg_num, max_segs, (u32)PCACHE_CACHE_SEGS_MAX); ret = -EIO; goto dax_release; }
ret = cache_dev_init(cache_dev, seg_num); if (ret) goto dax_release;
if (format) sb_write(cache_dev, &sb);
return 0;
dax_release: cache_dev_dax_exit(cache_dev);err: return ret;}
int cache_dev_get_empty_segment_id(struct pcache_cache_dev *cache_dev, u32 *seg_id){ int ret;
mutex_lock(&cache_dev->seg_lock); *seg_id = find_next_zero_bit(cache_dev->seg_bitmap, cache_dev->seg_num, 0); if (*seg_id == cache_dev->seg_num) { ret = -ENOSPC; goto unlock; }
__set_bit(*seg_id, cache_dev->seg_bitmap); ret = 0;unlock: mutex_unlock(&cache_dev->seg_lock); return ret;}