Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360// SPDX-License-Identifier: GPL-2.0-only/* Copyright(c) 2020 Intel Corporation. */
#include <linux/io-64-nonatomic-lo-hi.h>#include <linux/firmware.h>#include <linux/device.h>#include <linux/slab.h>#include <linux/idr.h>#include <linux/pci.h>#include <cxlmem.h>#include "trace.h"#include "core.h"
static DECLARE_RWSEM(cxl_memdev_rwsem);
/* * An entire PCI topology full of devices should be enough for any * config */#define CXL_MEM_MAX_DEVS 65536
static int cxl_mem_major;static DEFINE_IDA(cxl_memdev_ida);
static void cxl_memdev_release(struct device *dev){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct device *parent = dev->parent;
ida_free(&cxl_memdev_ida, cxlmd->id); kfree(cxlmd); put_device(parent);}
static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, kgid_t *gid){ return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev));}
static ssize_t firmware_version_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
if (!mds) return sysfs_emit(buf, "\n"); return sysfs_emit(buf, "%.16s\n", mds->firmware_version);}static DEVICE_ATTR_RO(firmware_version);
static ssize_t payload_max_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
if (!mds) return sysfs_emit(buf, "\n"); return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size);}static DEVICE_ATTR_RO(payload_max);
static ssize_t label_storage_size_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
if (!mds) return sysfs_emit(buf, "\n"); return sysfs_emit(buf, "%zu\n", mds->lsa_size);}static DEVICE_ATTR_RO(label_storage_size);
static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; unsigned long long len = cxl_part_size(cxlds, CXL_PARTMODE_RAM);
return sysfs_emit(buf, "%#llx\n", len);}
static struct device_attribute dev_attr_ram_size = __ATTR(size, 0444, ram_size_show, NULL);
static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; unsigned long long len = cxl_part_size(cxlds, CXL_PARTMODE_PMEM);
return sysfs_emit(buf, "%#llx\n", len);}
static struct device_attribute dev_attr_pmem_size = __ATTR(size, 0444, pmem_size_show, NULL);
static ssize_t serial_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds;
return sysfs_emit(buf, "%#llx\n", cxlds->serial);}static DEVICE_ATTR_RO(serial);
static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, char *buf){ return sysfs_emit(buf, "%d\n", dev_to_node(dev));}static DEVICE_ATTR_RO(numa_node);
static ssize_t security_state_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); unsigned long state = mds->security.state; int rc = 0;
/* sync with latest submission state */ mutex_lock(&cxl_mbox->mbox_mutex); if (mds->security.sanitize_active) rc = sysfs_emit(buf, "sanitize\n"); mutex_unlock(&cxl_mbox->mbox_mutex); if (rc) return rc;
if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET)) return sysfs_emit(buf, "disabled\n"); if (state & CXL_PMEM_SEC_STATE_FROZEN || state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT || state & CXL_PMEM_SEC_STATE_USER_PLIMIT) return sysfs_emit(buf, "frozen\n"); if (state & CXL_PMEM_SEC_STATE_LOCKED) return sysfs_emit(buf, "locked\n");
return sysfs_emit(buf, "unlocked\n");}static struct device_attribute dev_attr_security_state = __ATTR(state, 0444, security_state_show, NULL);
static ssize_t security_sanitize_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); bool sanitize; ssize_t rc;
if (kstrtobool(buf, &sanitize) || !sanitize) return -EINVAL;
rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE); if (rc) return rc;
return len;}static struct device_attribute dev_attr_security_sanitize = __ATTR(sanitize, 0200, NULL, security_sanitize_store);
static ssize_t security_erase_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); ssize_t rc; bool erase;
if (kstrtobool(buf, &erase) || !erase) return -EINVAL;
rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE); if (rc) return rc;
return len;}static struct device_attribute dev_attr_security_erase = __ATTR(erase, 0200, NULL, security_erase_store);
bool cxl_memdev_has_poison_cmd(struct cxl_memdev *cxlmd, enum poison_cmd_enabled_bits cmd){ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
if (!mds) return 0;
return test_bit(cmd, mds->poison.enabled_cmds);}
static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd){ struct cxl_dev_state *cxlds = cxlmd->cxlds; u64 offset, length; int rc = 0;
/* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */ for (int i = 0; i < cxlds->nr_partitions; i++) { const struct resource *res = &cxlds->part[i].res;
offset = res->start; length = resource_size(res); rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); /* * Invalid Physical Address is not an error for * volatile addresses. Device support is optional. */ if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM) rc = 0; } return rc;}
int cxl_trigger_poison_list(struct cxl_memdev *cxlmd){ struct cxl_port *port; int rc;
port = cxlmd->endpoint; if (!port || !is_cxl_endpoint(port)) return -EINVAL;
ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region); if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem))) return rc;
ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa); if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem))) return rc;
if (cxl_num_decoders_committed(port) == 0) { /* No regions mapped to this memdev */ rc = cxl_get_poison_by_memdev(cxlmd); } else { /* Regions mapped, collect poison by endpoint */ rc = cxl_get_poison_by_endpoint(port); }
return rc;}EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, "CXL");
static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa){ struct cxl_dev_state *cxlds = cxlmd->cxlds;
if (!IS_ENABLED(CONFIG_DEBUG_FS)) return 0;
if (!resource_size(&cxlds->dpa_res)) { dev_dbg(cxlds->dev, "device has no dpa resource\n"); return -EINVAL; } if (!cxl_resource_contains_addr(&cxlds->dpa_res, dpa)) { dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n", dpa, &cxlds->dpa_res); return -EINVAL; } if (!IS_ALIGNED(dpa, 64)) { dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa); return -EINVAL; }
return 0;}
int cxl_inject_poison_locked(struct cxl_memdev *cxlmd, u64 dpa){ struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; struct cxl_mbox_inject_poison inject; struct cxl_poison_record record; struct cxl_mbox_cmd mbox_cmd; struct cxl_region *cxlr; int rc;
if (!IS_ENABLED(CONFIG_DEBUG_FS)) return 0;
lockdep_assert_held(&cxl_rwsem.dpa); lockdep_assert_held(&cxl_rwsem.region);
rc = cxl_validate_poison_dpa(cxlmd, dpa); if (rc) return rc;
inject.address = cpu_to_le64(dpa); mbox_cmd = (struct cxl_mbox_cmd) { .opcode = CXL_MBOX_OP_INJECT_POISON, .size_in = sizeof(inject), .payload_in = &inject, }; rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc) return rc;
cxlr = cxl_dpa_to_region(cxlmd, dpa); if (cxlr) dev_warn_once(cxl_mbox->host, "poison inject dpa:%#llx region: %s\n", dpa, dev_name(&cxlr->dev));
record = (struct cxl_poison_record) { .address = cpu_to_le64(dpa), .length = cpu_to_le32(1), }; trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT);
return 0;}
int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa){ int rc;
ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region); if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem))) return rc;
ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa); if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem))) return rc;
return cxl_inject_poison_locked(cxlmd, dpa);}EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, "CXL");
int cxl_clear_poison_locked(struct cxl_memdev *cxlmd, u64 dpa){ struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; struct cxl_mbox_clear_poison clear; struct cxl_poison_record record; struct cxl_mbox_cmd mbox_cmd; struct cxl_region *cxlr; int rc;
if (!IS_ENABLED(CONFIG_DEBUG_FS)) return 0;
lockdep_assert_held(&cxl_rwsem.dpa); lockdep_assert_held(&cxl_rwsem.region);
rc = cxl_validate_poison_dpa(cxlmd, dpa); if (rc) return rc;
/* * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command * is defined to accept 64 bytes of write-data, along with the * address to clear. This driver uses zeroes as write-data. */ clear = (struct cxl_mbox_clear_poison) { .address = cpu_to_le64(dpa) };
mbox_cmd = (struct cxl_mbox_cmd) { .opcode = CXL_MBOX_OP_CLEAR_POISON, .size_in = sizeof(clear), .payload_in = &clear, };
rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc) return rc;
cxlr = cxl_dpa_to_region(cxlmd, dpa); if (cxlr) dev_warn_once(cxl_mbox->host, "poison clear dpa:%#llx region: %s\n", dpa, dev_name(&cxlr->dev));
record = (struct cxl_poison_record) { .address = cpu_to_le64(dpa), .length = cpu_to_le32(1), }; trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR);
return 0;}
int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa){ int rc;
ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region); if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem))) return rc;
ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa); if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem))) return rc;
return cxl_clear_poison_locked(cxlmd, dpa);}EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, "CXL");
static struct attribute *cxl_memdev_attributes[] = { &dev_attr_serial.attr, &dev_attr_firmware_version.attr, &dev_attr_payload_max.attr, &dev_attr_label_storage_size.attr, &dev_attr_numa_node.attr, NULL,};
static struct cxl_dpa_perf *part_perf(struct cxl_dev_state *cxlds, enum cxl_partition_mode mode){ for (int i = 0; i < cxlds->nr_partitions; i++) if (cxlds->part[i].mode == mode) return &cxlds->part[i].perf; return NULL;}
static ssize_t pmem_qos_class_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds;
return sysfs_emit(buf, "%d\n", part_perf(cxlds, CXL_PARTMODE_PMEM)->qos_class);}
static struct device_attribute dev_attr_pmem_qos_class = __ATTR(qos_class, 0444, pmem_qos_class_show, NULL);
static struct attribute *cxl_memdev_pmem_attributes[] = { &dev_attr_pmem_size.attr, &dev_attr_pmem_qos_class.attr, NULL,};
static ssize_t ram_qos_class_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds;
return sysfs_emit(buf, "%d\n", part_perf(cxlds, CXL_PARTMODE_RAM)->qos_class);}
static struct device_attribute dev_attr_ram_qos_class = __ATTR(qos_class, 0444, ram_qos_class_show, NULL);
static struct attribute *cxl_memdev_ram_attributes[] = { &dev_attr_ram_size.attr, &dev_attr_ram_qos_class.attr, NULL,};
static struct attribute *cxl_memdev_security_attributes[] = { &dev_attr_security_state.attr, &dev_attr_security_sanitize.attr, &dev_attr_security_erase.attr, NULL,};
static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a, int n){ if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr) return 0; return a->mode;}
static struct attribute_group cxl_memdev_attribute_group = { .attrs = cxl_memdev_attributes, .is_visible = cxl_memdev_visible,};
static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n){ struct device *dev = kobj_to_dev(kobj); struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dpa_perf *perf = part_perf(cxlmd->cxlds, CXL_PARTMODE_RAM);
if (a == &dev_attr_ram_qos_class.attr && (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID)) return 0;
return a->mode;}
static struct attribute_group cxl_memdev_ram_attribute_group = { .name = "ram", .attrs = cxl_memdev_ram_attributes, .is_visible = cxl_ram_visible,};
static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n){ struct device *dev = kobj_to_dev(kobj); struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dpa_perf *perf = part_perf(cxlmd->cxlds, CXL_PARTMODE_PMEM);
if (a == &dev_attr_pmem_qos_class.attr && (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID)) return 0;
return a->mode;}
static struct attribute_group cxl_memdev_pmem_attribute_group = { .name = "pmem", .attrs = cxl_memdev_pmem_attributes, .is_visible = cxl_pmem_visible,};
static umode_t cxl_memdev_security_visible(struct kobject *kobj, struct attribute *a, int n){ struct device *dev = kobj_to_dev(kobj); struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
if (a == &dev_attr_security_sanitize.attr && !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) return 0;
if (a == &dev_attr_security_erase.attr && !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds)) return 0;
return a->mode;}
static struct attribute_group cxl_memdev_security_attribute_group = { .name = "security", .attrs = cxl_memdev_security_attributes, .is_visible = cxl_memdev_security_visible,};
static const struct attribute_group *cxl_memdev_attribute_groups[] = { &cxl_memdev_attribute_group, &cxl_memdev_ram_attribute_group, &cxl_memdev_pmem_attribute_group, &cxl_memdev_security_attribute_group, NULL,};
void cxl_memdev_update_perf(struct cxl_memdev *cxlmd){ sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group); sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group);}EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, "CXL");
static const struct device_type cxl_class_memdev_type = { .name = "cxl_memdev", .release = cxl_memdev_release, .devnode = cxl_memdev_devnode, .groups = cxl_memdev_attribute_groups,};
static const struct device_type cxl_memdev_type = { .name = "cxl_memdev", .release = cxl_memdev_release, .devnode = cxl_memdev_devnode,};
bool is_cxl_memdev(const struct device *dev){ return (dev->type == &cxl_class_memdev_type || dev->type == &cxl_memdev_type);}EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL");
static void add_part(struct cxl_dpa_info *info, u64 start, u64 size, enum cxl_partition_mode mode, u8 handle){ int i = info->nr_partitions;
if (size == 0) return;
info->part[i].range = (struct range) { .start = start, .end = start + size - 1, }; info->part[i].mode = mode; info->part[i].handle = handle; info->nr_partitions++;}
int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info){ struct cxl_dev_state *cxlds = &mds->cxlds; struct device *dev = cxlds->dev; int rc;
if (!cxlds->media_ready) { info->size = 0; return 0; }
info->size = mds->total_bytes;
if (mds->partition_align_bytes == 0) { add_part(info, 0, mds->volatile_only_bytes, CXL_PARTMODE_RAM, 0); add_part(info, mds->volatile_only_bytes, mds->persistent_only_bytes, CXL_PARTMODE_PMEM, 0); return 0; }
rc = cxl_mem_get_partition_info(mds); if (rc) { dev_err(dev, "Failed to query partition information\n"); return rc; }
add_part(info, 0, mds->active_volatile_bytes, CXL_PARTMODE_RAM, 0); add_part(info, mds->active_volatile_bytes, mds->active_persistent_bytes, CXL_PARTMODE_PMEM, 0);
return 0;}EXPORT_SYMBOL_NS_GPL(cxl_mem_dpa_fetch, "CXL");
int cxl_configure_dcd(struct cxl_memdev_state *mds, struct cxl_dpa_info *info){ struct cxl_dc_partition_info dc_info = { }; struct device *dev = mds->cxlds.dev; u64 static_end = 0; u64 dc_end; int rc;
if (!mds->cxlds.media_ready) return 0;
rc = cxl_dev_dc_identify(&mds->cxlds.cxl_mbox, &dc_info); if (rc) { dev_warn(dev, "Failed to read Dynamic Capacity config: %d\n", rc); return rc; }
if (info->nr_partitions) static_end = info->part[info->nr_partitions - 1].range.end + 1;
if (dc_info.start < static_end) { dev_err(dev, "DC partition 0 base %#llx overlaps static capacity ending at %#llx\n", dc_info.start, static_end); return -EINVAL; }
/* A gap between static capacity and the DC partition is not supported */ if (dc_info.start > static_end) { dev_warn(dev, "DC partition 0 base %#llx leaves a gap from static capacity ending at %#llx\n", dc_info.start, static_end); return -EOPNOTSUPP; }
if (check_add_overflow(dc_info.start, dc_info.size, &dc_end)) { dev_err(dev, "DC partition 0 size %#llx overflows DPA space at base %#llx\n", dc_info.size, dc_info.start); return -EINVAL; }
/* The DPA space must reach the end of the DC partition */ if (dc_end > info->size) info->size = dc_end; dev_dbg(dev, "Adding dynamic ram partition; %#llx size %#llx\n", dc_info.start, dc_info.size); add_part(info, dc_info.start, dc_info.size, CXL_PARTMODE_DYNAMIC_RAM, dc_info.handle);
return 0;}EXPORT_SYMBOL_NS_GPL(cxl_configure_dcd, "CXL");
/** * cxl_set_capacity: initialize dpa by a driver without a mailbox. * * @cxlds: pointer to cxl_dev_state * @capacity: device volatile memory size */int cxl_set_capacity(struct cxl_dev_state *cxlds, u64 capacity){ struct cxl_dpa_info range_info = { .size = capacity, };
add_part(&range_info, 0, capacity, CXL_PARTMODE_RAM, 0); return cxl_dpa_setup(cxlds, &range_info);}EXPORT_SYMBOL_NS_GPL(cxl_set_capacity, "CXL");
/** * set_exclusive_cxl_commands() - atomically disable user cxl commands * @mds: The device state to operate on * @cmds: bitmap of commands to mark exclusive * * Grab the cxl_memdev_rwsem in write mode to flush in-flight * invocations of the ioctl path and then disable future execution of * commands with the command ids set in @cmds. */void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, unsigned long *cmds){ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
guard(rwsem_write)(&cxl_memdev_rwsem); bitmap_or(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds, cmds, CXL_MEM_COMMAND_ID_MAX);}EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, "CXL");
/** * clear_exclusive_cxl_commands() - atomically enable user cxl commands * @mds: The device state to modify * @cmds: bitmap of commands to mark available for userspace */void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds, unsigned long *cmds){ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
guard(rwsem_write)(&cxl_memdev_rwsem); bitmap_andnot(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds, cmds, CXL_MEM_COMMAND_ID_MAX);}EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, "CXL");
static void cxl_memdev_shutdown(struct device *dev){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
guard(rwsem_write)(&cxl_memdev_rwsem); cxlmd->cxlds = NULL;}
static void cxl_memdev_unregister(void *_cxlmd){ struct cxl_memdev *cxlmd = _cxlmd; struct device *dev = &cxlmd->dev;
cdev_device_del(&cxlmd->cdev, dev); cxl_memdev_shutdown(dev); put_device(dev);}
static void detach_memdev(struct work_struct *work){ struct cxl_memdev *cxlmd;
cxlmd = container_of(work, typeof(*cxlmd), detach_work);
/* * When the creator of @cxlmd sets ->attach it indicates CXL operation * is required. In that case, @cxlmd detach escalates to parent device * detach. */ if (cxlmd->attach) device_release_driver(cxlmd->dev.parent); else device_release_driver(&cxlmd->dev); put_device(&cxlmd->dev);}
static struct lock_class_key cxl_memdev_key;
struct cxl_dev_state *_devm_cxl_dev_state_create(struct device *dev, enum cxl_devtype type, u64 serial, u16 dvsec, size_t size, bool has_mbox){ struct cxl_dev_state *cxlds = devm_kzalloc(dev, size, GFP_KERNEL);
if (!cxlds) return NULL;
cxlds->dev = dev; cxlds->type = type; cxlds->serial = serial; cxlds->cxl_dvsec = dvsec; cxlds->reg_map.host = dev; cxlds->reg_map.resource = CXL_RESOURCE_NONE;
if (has_mbox) cxlds->cxl_mbox.host = dev;
return cxlds;}EXPORT_SYMBOL_NS_GPL(_devm_cxl_dev_state_create, "CXL");
static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds, const struct file_operations *fops, const struct cxl_memdev_attach *attach){ struct cxl_memdev *cxlmd; struct device *dev; struct cdev *cdev; int rc;
cxlmd = kzalloc_obj(*cxlmd); if (!cxlmd) return ERR_PTR(-ENOMEM);
rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL); if (rc < 0) goto err; cxlmd->id = rc; cxlmd->depth = -1; cxlmd->attach = attach; cxlmd->endpoint = ERR_PTR(-ENXIO);
dev = &cxlmd->dev; device_initialize(dev); lockdep_set_class(&dev->mutex, &cxl_memdev_key); dev->parent = get_device(cxlds->dev); dev->bus = &cxl_bus_type; dev->devt = MKDEV(cxl_mem_major, cxlmd->id); if (cxlds->type == CXL_DEVTYPE_DEVMEM) dev->type = &cxl_memdev_type; else dev->type = &cxl_class_memdev_type; device_set_pm_not_required(dev); INIT_WORK(&cxlmd->detach_work, detach_memdev);
cdev = &cxlmd->cdev; cdev_init(cdev, fops); return cxlmd;
err: kfree(cxlmd); return ERR_PTR(rc);}
static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, unsigned long arg){ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
switch (cmd) { case CXL_MEM_QUERY_COMMANDS: return cxl_query_cmd(cxl_mbox, (void __user *)arg); case CXL_MEM_SEND_COMMAND: return cxl_send_cmd(cxl_mbox, (void __user *)arg); default: return -ENOTTY; }}
static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg){ struct cxl_memdev *cxlmd = file->private_data; struct cxl_dev_state *cxlds;
guard(rwsem_read)(&cxl_memdev_rwsem); cxlds = cxlmd->cxlds; if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM) return __cxl_memdev_ioctl(cxlmd, cmd, arg);
return -ENXIO;}
static int cxl_memdev_open(struct inode *inode, struct file *file){ struct cxl_memdev *cxlmd = container_of(inode->i_cdev, typeof(*cxlmd), cdev);
get_device(&cxlmd->dev); file->private_data = cxlmd;
return 0;}
static int cxl_memdev_release_file(struct inode *inode, struct file *file){ struct cxl_memdev *cxlmd = container_of(inode->i_cdev, typeof(*cxlmd), cdev);
put_device(&cxlmd->dev);
return 0;}
/** * cxl_mem_get_fw_info - Get Firmware info * @mds: The device data for the operation * * Retrieve firmware info for the device specified. * * Return: 0 if no error: or the result of the mailbox command. * * See CXL-3.0 8.2.9.3.1 Get FW Info */static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds){ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_get_fw_info info; struct cxl_mbox_cmd mbox_cmd; int rc;
mbox_cmd = (struct cxl_mbox_cmd) { .opcode = CXL_MBOX_OP_GET_FW_INFO, .size_out = sizeof(info), .payload_out = &info, };
rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc < 0) return rc;
mds->fw.num_slots = info.num_slots; mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK, info.slot_info);
return 0;}
/** * cxl_mem_activate_fw - Activate Firmware * @mds: The device data for the operation * @slot: slot number to activate * * Activate firmware in a given slot for the device specified. * * Return: 0 if no error: or the result of the mailbox command. * * See CXL-3.0 8.2.9.3.3 Activate FW */static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot){ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_activate_fw activate; struct cxl_mbox_cmd mbox_cmd;
if (slot == 0 || slot > mds->fw.num_slots) return -EINVAL;
mbox_cmd = (struct cxl_mbox_cmd) { .opcode = CXL_MBOX_OP_ACTIVATE_FW, .size_in = sizeof(activate), .payload_in = &activate, };
/* Only offline activation supported for now */ activate.action = CXL_FW_ACTIVATE_OFFLINE; activate.slot = slot;
return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);}
/** * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer * @mds: The device data for the operation * * Abort an in-progress firmware transfer for the device specified. * * Return: 0 if no error: or the result of the mailbox command. * * See CXL-3.0 8.2.9.3.2 Transfer FW */static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds){ struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_transfer_fw *transfer; struct cxl_mbox_cmd mbox_cmd; int rc;
transfer = kzalloc_flex(*transfer, data, 0); if (!transfer) return -ENOMEM;
/* Set a 1s poll interval and a total wait time of 30s */ mbox_cmd = (struct cxl_mbox_cmd) { .opcode = CXL_MBOX_OP_TRANSFER_FW, .size_in = sizeof(*transfer), .payload_in = transfer, .poll_interval_ms = 1000, .poll_count = 30, };
transfer->action = CXL_FW_TRANSFER_ACTION_ABORT;
rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); kfree(transfer); return rc;}
static void cxl_fw_cleanup(struct fw_upload *fwl){ struct cxl_memdev_state *mds = fwl->dd_handle;
mds->fw.next_slot = 0;}
static int cxl_fw_do_cancel(struct fw_upload *fwl){ struct cxl_memdev_state *mds = fwl->dd_handle; struct cxl_dev_state *cxlds = &mds->cxlds; struct cxl_memdev *cxlmd = cxlds->cxlmd; int rc;
rc = cxl_mem_abort_fw_xfer(mds); if (rc < 0) dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc);
return FW_UPLOAD_ERR_CANCELED;}
static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, u32 size){ struct cxl_memdev_state *mds = fwl->dd_handle; struct cxl_mbox_transfer_fw *transfer; struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
if (!size) return FW_UPLOAD_ERR_INVALID_SIZE;
mds->fw.oneshot = struct_size(transfer, data, size) <= cxl_mbox->payload_size;
if (cxl_mem_get_fw_info(mds)) return FW_UPLOAD_ERR_HW_ERROR;
/* * So far no state has been changed, hence no other cleanup is * necessary. Simply return the cancelled status. */ if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) return FW_UPLOAD_ERR_CANCELED;
return FW_UPLOAD_ERR_NONE;}
static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, u32 offset, u32 size, u32 *written){ struct cxl_memdev_state *mds = fwl->dd_handle; struct cxl_dev_state *cxlds = &mds->cxlds; struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; struct cxl_memdev *cxlmd = cxlds->cxlmd; struct cxl_mbox_transfer_fw *transfer; struct cxl_mbox_cmd mbox_cmd; u32 cur_size, remaining; size_t size_in; int rc;
*written = 0;
/* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */ if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) { dev_err(&cxlmd->dev, "misaligned offset for FW transfer slice (%u)\n", offset); return FW_UPLOAD_ERR_RW_ERROR; }
/* * Pick transfer size based on mds->payload_size @size must bw 128-byte * aligned, ->payload_size is a power of 2 starting at 256 bytes, and * sizeof(*transfer) is 128. These constraints imply that @cur_size * will always be 128b aligned. */ cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer));
remaining = size - cur_size; size_in = struct_size(transfer, data, cur_size);
if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) return cxl_fw_do_cancel(fwl);
/* * Slot numbers are 1-indexed * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1') * Check for rollover using modulo, and 1-index it by adding 1 */ mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1;
/* Do the transfer via mailbox cmd */ transfer = kzalloc(size_in, GFP_KERNEL); if (!transfer) return FW_UPLOAD_ERR_RW_ERROR;
transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT); memcpy(transfer->data, data + offset, cur_size); if (mds->fw.oneshot) { transfer->action = CXL_FW_TRANSFER_ACTION_FULL; transfer->slot = mds->fw.next_slot; } else { if (offset == 0) { transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE; } else if (remaining == 0) { transfer->action = CXL_FW_TRANSFER_ACTION_END; transfer->slot = mds->fw.next_slot; } else { transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE; } }
mbox_cmd = (struct cxl_mbox_cmd) { .opcode = CXL_MBOX_OP_TRANSFER_FW, .size_in = size_in, .payload_in = transfer, .poll_interval_ms = 1000, .poll_count = 30, };
rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc < 0) { rc = FW_UPLOAD_ERR_RW_ERROR; goto out_free; }
*written = cur_size;
/* Activate FW if oneshot or if the last slice was written */ if (mds->fw.oneshot || remaining == 0) { dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n", mds->fw.next_slot); rc = cxl_mem_activate_fw(mds, mds->fw.next_slot); if (rc < 0) { dev_err(&cxlmd->dev, "Error activating firmware: %d\n", rc); rc = FW_UPLOAD_ERR_HW_ERROR; goto out_free; } }
rc = FW_UPLOAD_ERR_NONE;
out_free: kfree(transfer); return rc;}
static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl){ struct cxl_memdev_state *mds = fwl->dd_handle;
/* * cxl_internal_send_cmd() handles background operations synchronously. * No need to wait for completions here - any errors would've been * reported and handled during the ->write() call(s). * Just check if a cancel request was received, and return success. */ if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) return cxl_fw_do_cancel(fwl);
return FW_UPLOAD_ERR_NONE;}
static void cxl_fw_cancel(struct fw_upload *fwl){ struct cxl_memdev_state *mds = fwl->dd_handle;
set_bit(CXL_FW_CANCEL, mds->fw.state);}
static const struct fw_upload_ops cxl_memdev_fw_ops = { .prepare = cxl_fw_prepare, .write = cxl_fw_write, .poll_complete = cxl_fw_poll_complete, .cancel = cxl_fw_cancel, .cleanup = cxl_fw_cleanup,};
static void cxl_remove_fw_upload(void *fwl){ firmware_upload_unregister(fwl);}
int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds){ struct cxl_dev_state *cxlds = &mds->cxlds; struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; struct device *dev = &cxlds->cxlmd->dev; struct fw_upload *fwl;
if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, cxl_mbox->enabled_cmds)) return 0;
fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev), &cxl_memdev_fw_ops, mds); if (IS_ERR(fwl)) return PTR_ERR(fwl); return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl);}EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, "CXL");
static const struct file_operations cxl_memdev_fops = { .owner = THIS_MODULE, .unlocked_ioctl = cxl_memdev_ioctl, .open = cxl_memdev_open, .release = cxl_memdev_release_file, .compat_ioctl = compat_ptr_ioctl, .llseek = noop_llseek,};
/* * Activate ioctl operations, no cxl_memdev_rwsem manipulation needed as this is * ordered with cdev_add() publishing the device. */static int cxlmd_add(struct cxl_memdev *cxlmd, struct cxl_dev_state *cxlds){ int rc;
cxlmd->cxlds = cxlds; cxlds->cxlmd = cxlmd;
rc = cdev_device_add(&cxlmd->cdev, &cxlmd->dev); if (rc) { /* * The cdev was briefly live, shutdown any ioctl operations that * saw that state. */ cxl_memdev_shutdown(&cxlmd->dev); return rc; }
return 0;}
DEFINE_FREE(put_cxlmd, struct cxl_memdev *, if (!IS_ERR_OR_NULL(_T)) put_device(&_T->dev))
static bool cxl_memdev_attach_failed(struct cxl_memdev *cxlmd){ /* * If @attach is provided fail if the driver is not attached upon * return. Note that failure here could be the result of a race to * teardown the CXL port topology. I.e. cxl_mem_probe() could have * succeeded and then cxl_mem unbound before the lock is acquired. */ guard(device)(&cxlmd->dev); return (cxlmd->attach && !cxlmd->dev.driver);}
static struct cxl_memdev *cxl_memdev_autoremove(struct cxl_memdev *cxlmd){ int rc;
if (cxl_memdev_attach_failed(cxlmd)) { cxl_memdev_unregister(cxlmd); return ERR_PTR(-ENXIO); }
rc = devm_add_action_or_reset(cxlmd->cxlds->dev, cxl_memdev_unregister, cxlmd); if (rc) return ERR_PTR(rc);
return cxlmd;}
/* * Core helper for devm_cxl_add_memdev() that wants to both create a device and * assert to the caller that upon return cxl_mem::probe() has been invoked. */struct cxl_memdev *__devm_cxl_add_memdev(struct cxl_dev_state *cxlds, const struct cxl_memdev_attach *attach){ struct device *dev; int rc;
struct cxl_memdev *cxlmd __free(put_cxlmd) = cxl_memdev_alloc(cxlds, &cxl_memdev_fops, attach); if (IS_ERR(cxlmd)) return cxlmd;
dev = &cxlmd->dev; rc = dev_set_name(dev, "mem%d", cxlmd->id); if (rc) return ERR_PTR(rc);
rc = cxlmd_add(cxlmd, cxlds); if (rc) return ERR_PTR(rc);
return cxl_memdev_autoremove(no_free_ptr(cxlmd));}EXPORT_SYMBOL_FOR_MODULES(__devm_cxl_add_memdev, "cxl_mem");
static void sanitize_teardown_notifier(void *data){ struct cxl_memdev_state *mds = data; struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct kernfs_node *state;
/* * Prevent new irq triggered invocations of the workqueue and * flush inflight invocations. */ mutex_lock(&cxl_mbox->mbox_mutex); state = mds->security.sanitize_node; mds->security.sanitize_node = NULL; mutex_unlock(&cxl_mbox->mbox_mutex);
cancel_delayed_work_sync(&mds->security.poll_dwork); sysfs_put(state);}
int devm_cxl_sanitize_setup_notifier(struct device *host, struct cxl_memdev *cxlmd){ struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); struct kernfs_node *sec;
if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) return 0;
/* * Note, the expectation is that @cxlmd would have failed to be * created if these sysfs_get_dirent calls fail. */ sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security"); if (!sec) return -ENOENT; mds->security.sanitize_node = sysfs_get_dirent(sec, "state"); sysfs_put(sec); if (!mds->security.sanitize_node) return -ENOENT;
return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds);}EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, "CXL");
__init int cxl_memdev_init(void){ dev_t devt; int rc;
rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); if (rc) return rc;
cxl_mem_major = MAJOR(devt);
return 0;}
void cxl_memdev_exit(void){ unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS);}