Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475// SPDX-License-Identifier: GPL-2.0// Copyright (C) 2025 Arm Ltd.
#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__
#include <linux/acpi.h>#include <linux/atomic.h>#include <linux/arm_mpam.h>#include <linux/bitfield.h>#include <linux/bitmap.h>#include <linux/cacheinfo.h>#include <linux/cpu.h>#include <linux/cpumask.h>#include <linux/device.h>#include <linux/errname.h>#include <linux/errno.h>#include <linux/gfp.h>#include <linux/interrupt.h>#include <linux/irq.h>#include <linux/irqdesc.h>#include <linux/list.h>#include <linux/lockdep.h>#include <linux/mailbox_client.h>#include <linux/mutex.h>#include <linux/platform_device.h>#include <linux/printk.h>#include <linux/property.h>#include <linux/srcu.h>#include <linux/spinlock.h>#include <linux/types.h>#include <linux/workqueue.h>
#include <acpi/pcc.h>#include <acpi/acpi_io.h>
#include "mpam_internal.h"
/* Values for the T241 errata workaround */#define T241_CHIPS_MAX 4#define T241_CHIP_NSLICES 12#define T241_SPARE_REG0_OFF 0x1b0000#define T241_SPARE_REG1_OFF 0x1c0000#define T241_CHIP_ID(phys) FIELD_GET(GENMASK_ULL(44, 43), phys)#define T241_SHADOW_REG_OFF(sidx, pid) (0x360048 + (sidx) * 0x10000 + (pid) * 8)#define SMCCC_SOC_ID_T241 0x036b0241static void __iomem *t241_scratch_regs[T241_CHIPS_MAX];
/* * mpam_list_lock protects the SRCU lists when writing. Once the * mpam_enabled key is enabled these lists are read-only, * unless the error interrupt disables the driver. */static DEFINE_MUTEX(mpam_list_lock);static LIST_HEAD(mpam_all_msc);
struct srcu_struct mpam_srcu;
/* PCC channels might be serving multiple MSCs, so keep a refcounted list. */static DEFINE_MUTEX(pcc_chan_list_lock);static LIST_HEAD(pcc_chan_list);
static void mpam_pcc_chan_release(struct kref *ref){ struct mpam_pcc_chan *cur = container_of(ref, struct mpam_pcc_chan, refcount);
pcc_mbox_free_channel(cur->pcc_chan); list_del(&cur->pcc_chans); mutex_destroy(&cur->pcc_chan_lock); kfree(cur);}
static struct mpam_pcc_chan *mpam_pcc_chan_get(struct device *dev, int subspace_id){ struct mpam_pcc_chan *cur;
guard(mutex)(&pcc_chan_list_lock);
list_for_each_entry(cur, &pcc_chan_list, pcc_chans) { if (cur->subspace_id == subspace_id) { kref_get(&cur->refcount);
return cur; } }
cur = kzalloc_obj(*cur); if (!cur) return ERR_PTR(-ENOMEM);
cur->pcc_cl.tx_block = true;
cur->pcc_chan = pcc_mbox_request_channel(&cur->pcc_cl, subspace_id); if (IS_ERR(cur->pcc_chan)) { long err = PTR_ERR(cur->pcc_chan);
kfree(cur); return ERR_PTR(err); }
/* * Timeout based on the "nominal latency" in us, from the * PCC ACPI table. tx_tout is in ms. * Add some margin here to be on the safe side. */ cur->pcc_cl.tx_tout = DIV_ROUND_UP(cur->pcc_chan->latency * 5, 1000);
mutex_init(&cur->pcc_chan_lock);
cur->subspace_id = subspace_id; kref_init(&cur->refcount);
list_add_tail(&cur->pcc_chans, &pcc_chan_list);
return cur;}
static int mpam_pcc_chan_put(struct mpam_pcc_chan *pcc_chan){ struct mpam_pcc_chan *cur, *tmp;
if (!pcc_chan) return 0;
guard(mutex)(&pcc_chan_list_lock);
list_for_each_entry_safe(cur, tmp, &pcc_chan_list, pcc_chans) { if (cur == pcc_chan) { kref_put(&cur->refcount, mpam_pcc_chan_release);
return 0; } }
return -ENOENT;}
/* * Number of MSCs that have been probed. Once all MSCs have been probed MPAM * can be enabled. */static atomic_t mpam_num_msc;
static int mpam_cpuhp_state;static DEFINE_MUTEX(mpam_cpuhp_state_lock);
/* * The smallest common values for any CPU or MSC in the system. * Generating traffic outside this range will result in screaming interrupts. */u16 mpam_partid_max;u8 mpam_pmg_max;static bool partid_max_init, partid_max_published;static DEFINE_SPINLOCK(partid_max_lock);
/* * mpam is enabled once all devices have been probed from CPU online callbacks, * scheduled via this work_struct. If access to an MSC depends on a CPU that * was not brought online at boot, this can happen surprisingly late. */static DECLARE_WORK(mpam_enable_work, &mpam_enable);
/* * All mpam error interrupts indicate a software bug. On receipt, disable the * driver. */static DECLARE_WORK(mpam_broken_work, &mpam_disable);
/* When mpam is disabled, the printed reason to aid debugging */static char *mpam_disable_reason;static int mpam_disable_errno;static int mpam_disable_mpam_fb_err;
void mpam_fb_disable_mpam(int err, int mpam_fb_err){ /* Prevent repeated calls when mpam_disable() does MSC accesses. */ if (!mpam_is_enabled()) return;
mpam_disable_errno = err; mpam_disable_mpam_fb_err = mpam_fb_err;
mpam_disable_reason = "MPAM-Fb error"; schedule_work(&mpam_broken_work);}
/* * Whether resctrl has been setup. Used by cpuhp in preference to * mpam_is_enabled(). The disable call after an error interrupt makes * mpam_is_enabled() false before the cpuhp callbacks are made. * Reads/writes should hold mpam_cpuhp_state_lock, (or be cpuhp callbacks). */static bool mpam_resctrl_enabled;
/* * An MSC is a physical container for controls and monitors, each identified by * their RIS index. These share a base-address, interrupts and some MMIO * registers. A vMSC is a virtual container for RIS in an MSC that control or * monitor the same thing. Members of a vMSC are all RIS in the same MSC, but * not all RIS in an MSC share a vMSC. * * Components are a group of vMSC that control or monitor the same thing but * are from different MSC, so have different base-address, interrupts etc. * Classes are the set components of the same type. * * The features of a vMSC is the union of the RIS it contains. * The features of a Class and Component are the common subset of the vMSC * they contain. * * e.g. The system cache may have bandwidth controls on multiple interfaces, * for regulating traffic from devices independently of traffic from CPUs. * If these are two RIS in one MSC, they will be treated as controlling * different things, and will not share a vMSC/component/class. * * e.g. The L2 may have one MSC and two RIS, one for cache-controls another * for bandwidth. These two RIS are members of the same vMSC. * * e.g. The set of RIS that make up the L2 are grouped as a component. These * are sometimes termed slices. They should be configured the same, as if there * were only one. * * e.g. The SoC probably has more than one L2, each attached to a distinct set * of CPUs. All the L2 components are grouped as a class. * * When creating an MSC, struct mpam_msc is added to the all mpam_all_msc list, * then linked via struct mpam_ris to a vmsc, component and class. * The same MSC may exist under different class->component->vmsc paths, but the * RIS index will be unique. */LIST_HEAD(mpam_classes);
/* List of all objects that can be free()d after synchronise_srcu() */static LLIST_HEAD(mpam_garbage);
static inline void init_garbage(struct mpam_garbage *garbage){ init_llist_node(&garbage->llist);}
#define add_to_garbage(x) \do { \ __typeof__(x) _x = (x); \ _x->garbage.to_free = _x; \ llist_add(&_x->garbage.llist, &mpam_garbage); \} while (0)
static void mpam_free_garbage(void){ struct mpam_garbage *iter, *tmp; struct llist_node *to_free = llist_del_all(&mpam_garbage);
if (!to_free) return;
synchronize_srcu(&mpam_srcu);
llist_for_each_entry_safe(iter, tmp, to_free, llist) { if (iter->pdev) devm_kfree(&iter->pdev->dev, iter->to_free); else kvfree(iter->to_free); }}
/* * Once mpam is enabled, new requestors cannot further reduce the available * partid. Assert that the size is fixed, and new requestors will be turned * away. This is needed when walking over structures sized by PARTID. * * During mpam_disable() these structures are not fixed, but the MSC state * is still reset using whatever sizes have been discovered so far. As only * PARTID 0 will be used after mpam_disable(), any race would be benign. * Skip the check if a mpam_disable_reason has been set. */static void mpam_assert_partid_sizes_fixed(void){ if (!mpam_disable_reason) WARN_ON_ONCE(!partid_max_published);}
static int __mpam_read_reg(struct mpam_msc *msc, u16 reg, u32 *res){
if (msc->iface == MPAM_IFACE_PCC) return mpam_fb_send_read_request(msc, reg, res);
WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); *res = readl_relaxed(msc->mapped_hwpage + reg);
return 0;}
static inline int _mpam_read_partsel_reg(struct mpam_msc *msc, u16 reg, u32 *res){ lockdep_assert_held_once(&msc->part_sel_lock); return __mpam_read_reg(msc, reg, res);}
#define mpam_read_partsel_reg(msc, reg, res) _mpam_read_partsel_reg(msc, MPAMF_##reg, res)
static int __mpam_write_reg(struct mpam_msc *msc, u16 reg, u32 val){
if (msc->iface == MPAM_IFACE_PCC) return mpam_fb_send_write_request(msc, reg, val);
WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); WARN_ON_ONCE(reg + sizeof(u32) > msc->mapped_hwpage_sz); writel_relaxed(val, msc->mapped_hwpage + reg);
return 0;}
static inline int _mpam_write_partsel_reg(struct mpam_msc *msc, u16 reg, u32 val){ lockdep_assert_held_once(&msc->part_sel_lock); return __mpam_write_reg(msc, reg, val);}
#define mpam_write_partsel_reg(msc, reg, val) _mpam_write_partsel_reg(msc, MPAMCFG_##reg, val)
static inline int _mpam_read_monsel_reg(struct mpam_msc *msc, u16 reg, u32 *res){ mpam_mon_sel_lock_held(msc); return __mpam_read_reg(msc, reg, res);}
#define mpam_read_monsel_reg(msc, reg, res) _mpam_read_monsel_reg(msc, MSMON_##reg, res)
static inline int _mpam_write_monsel_reg(struct mpam_msc *msc, u16 reg, u32 val){ mpam_mon_sel_lock_held(msc); return __mpam_write_reg(msc, reg, val);}
#define mpam_write_monsel_reg(msc, reg, val) _mpam_write_monsel_reg(msc, MSMON_##reg, val)
static bool mpam_msc_check_aidr(struct mpam_msc *msc){ u32 aidr, major, minor;
if (__mpam_read_reg(msc, MPAMF_AIDR, &aidr)) return false;
major = FIELD_GET(MPAMF_AIDR_ARCH_MAJOR_REV, aidr); minor = FIELD_GET(MPAMF_AIDR_ARCH_MINOR_REV, aidr); /* * v0.0 and >v2.x aren't supported, but anything else should be backward * compatible to v0.1 or v1.0. */ if (!major && !minor) return false; if (major > 1) return false;
return true;}
static int mpam_msc_read_idr(struct mpam_msc *msc, u64 *res){ u32 idr_high, idr_low; int ret;
lockdep_assert_held(&msc->part_sel_lock);
ret = mpam_read_partsel_reg(msc, IDR, &idr_low); if (ret) return ret;
if (FIELD_GET(MPAMF_IDR_EXT, idr_low)) { ret = mpam_read_partsel_reg(msc, IDR + 4, &idr_high); if (ret) return ret; } else { idr_high = 0; }
*res = ((u64)idr_high << 32) | idr_low;
return 0;}
static int mpam_msc_clear_esr(struct mpam_msc *msc){ u32 esr_low; int ret;
ret = __mpam_read_reg(msc, MPAMF_ESR, &esr_low); if (ret) return ret;
if (!esr_low) return 0;
/* * Clearing the high/low bits of MPAMF_ESR can not be atomic. * Clear the top half first, so that the pending error bits in the * lower half prevent hardware from updating either half of the * register. */ if (msc->has_extd_esr) { ret = __mpam_write_reg(msc, MPAMF_ESR + 4, 0); if (ret) return ret; }
return __mpam_write_reg(msc, MPAMF_ESR, 0);}
static int mpam_msc_read_esr(struct mpam_msc *msc, u64 *res){ u32 esr_high, esr_low; int ret;
ret = __mpam_read_reg(msc, MPAMF_ESR, &esr_low); if (ret) return ret;
if (msc->has_extd_esr) { ret = __mpam_read_reg(msc, MPAMF_ESR + 4, &esr_high); if (ret) return ret; } else { esr_high = 0; }
*res = ((u64)esr_high << 32) | esr_low;
return 0;}
static int __mpam_part_sel_raw(u32 partsel, struct mpam_msc *msc){ lockdep_assert_held(&msc->part_sel_lock);
return mpam_write_partsel_reg(msc, PART_SEL, partsel);}
static int __mpam_part_sel(u8 ris_idx, u16 partid, struct mpam_msc *msc){ u32 partsel = FIELD_PREP(MPAMCFG_PART_SEL_RIS, ris_idx) | FIELD_PREP(MPAMCFG_PART_SEL_PARTID_SEL, partid);
return __mpam_part_sel_raw(partsel, msc);}
static int __mpam_intpart_sel(u8 ris_idx, u16 intpartid, struct mpam_msc *msc){ u32 partsel = FIELD_PREP(MPAMCFG_PART_SEL_RIS, ris_idx) | FIELD_PREP(MPAMCFG_PART_SEL_PARTID_SEL, intpartid) | MPAMCFG_PART_SEL_INTERNAL;
return __mpam_part_sel_raw(partsel, msc);}
int mpam_register_requestor(u16 partid_max, u8 pmg_max){ guard(spinlock)(&partid_max_lock); if (!partid_max_init) { mpam_partid_max = partid_max; mpam_pmg_max = pmg_max; partid_max_init = true; } else if (!partid_max_published) { mpam_partid_max = min(mpam_partid_max, partid_max); mpam_pmg_max = min(mpam_pmg_max, pmg_max); } else { /* New requestors can't lower the values */ if (partid_max < mpam_partid_max || pmg_max < mpam_pmg_max) return -EBUSY; }
return 0;}EXPORT_SYMBOL(mpam_register_requestor);
static struct mpam_class *mpam_class_alloc(u8 level_idx, enum mpam_class_types type){ struct mpam_class *class;
lockdep_assert_held(&mpam_list_lock);
class = kzalloc_obj(*class); if (!class) return ERR_PTR(-ENOMEM); init_garbage(&class->garbage);
INIT_LIST_HEAD_RCU(&class->components); /* Affinity is updated when ris are added */ class->level = level_idx; class->type = type; INIT_LIST_HEAD_RCU(&class->classes_list); ida_init(&class->ida_csu_mon); ida_init(&class->ida_mbwu_mon);
list_add_rcu(&class->classes_list, &mpam_classes);
return class;}
static void mpam_class_destroy(struct mpam_class *class){ lockdep_assert_held(&mpam_list_lock);
list_del_rcu(&class->classes_list); add_to_garbage(class);}
static struct mpam_class *mpam_class_find(u8 level_idx, enum mpam_class_types type){ struct mpam_class *class;
lockdep_assert_held(&mpam_list_lock);
list_for_each_entry(class, &mpam_classes, classes_list) { if (class->type == type && class->level == level_idx) return class; }
return mpam_class_alloc(level_idx, type);}
static struct mpam_component *mpam_component_alloc(struct mpam_class *class, int id){ struct mpam_component *comp;
lockdep_assert_held(&mpam_list_lock);
comp = kzalloc_obj(*comp); if (!comp) return ERR_PTR(-ENOMEM); init_garbage(&comp->garbage);
comp->comp_id = id; INIT_LIST_HEAD_RCU(&comp->vmsc); /* Affinity is updated when RIS are added */ INIT_LIST_HEAD_RCU(&comp->class_list); comp->class = class;
list_add_rcu(&comp->class_list, &class->components);
return comp;}
static void __destroy_component_cfg(struct mpam_component *comp);
static void mpam_component_destroy(struct mpam_component *comp){ struct mpam_class *class = comp->class;
lockdep_assert_held(&mpam_list_lock);
__destroy_component_cfg(comp);
list_del_rcu(&comp->class_list); add_to_garbage(comp);
if (list_empty(&class->components)) mpam_class_destroy(class);}
static struct mpam_component *mpam_component_find(struct mpam_class *class, int id){ struct mpam_component *comp;
lockdep_assert_held(&mpam_list_lock);
list_for_each_entry(comp, &class->components, class_list) { if (comp->comp_id == id) return comp; }
return mpam_component_alloc(class, id);}
static struct mpam_vmsc *mpam_vmsc_alloc(struct mpam_component *comp, struct mpam_msc *msc){ struct mpam_vmsc *vmsc;
lockdep_assert_held(&mpam_list_lock);
vmsc = kzalloc_obj(*vmsc); if (!vmsc) return ERR_PTR(-ENOMEM); init_garbage(&vmsc->garbage);
INIT_LIST_HEAD_RCU(&vmsc->ris); INIT_LIST_HEAD_RCU(&vmsc->comp_list); vmsc->comp = comp; vmsc->msc = msc;
list_add_rcu(&vmsc->comp_list, &comp->vmsc);
return vmsc;}
static void mpam_vmsc_destroy(struct mpam_vmsc *vmsc){ struct mpam_component *comp = vmsc->comp;
lockdep_assert_held(&mpam_list_lock);
list_del_rcu(&vmsc->comp_list); add_to_garbage(vmsc);
if (list_empty(&comp->vmsc)) mpam_component_destroy(comp);}
static struct mpam_vmsc *mpam_vmsc_find(struct mpam_component *comp, struct mpam_msc *msc){ struct mpam_vmsc *vmsc;
lockdep_assert_held(&mpam_list_lock);
list_for_each_entry(vmsc, &comp->vmsc, comp_list) { if (vmsc->msc->id == msc->id) return vmsc; }
return mpam_vmsc_alloc(comp, msc);}
/* * The cacheinfo structures are only populated when CPUs are online. * This helper walks the acpi tables to include offline CPUs too. */int mpam_get_cpumask_from_cache_id(unsigned long cache_id, u32 cache_level, cpumask_t *affinity){ return acpi_pptt_get_cpumask_from_cache_id(cache_id, affinity);}
/* * cpumask_of_node() only knows about online CPUs. This can't tell us whether * a class is represented on all possible CPUs. */static void get_cpumask_from_node_id(u32 node_id, cpumask_t *affinity){ int cpu;
for_each_possible_cpu(cpu) { if (node_id == cpu_to_node(cpu)) cpumask_set_cpu(cpu, affinity); }}
static int mpam_ris_get_affinity(struct mpam_msc *msc, cpumask_t *affinity, enum mpam_class_types type, struct mpam_class *class, struct mpam_component *comp){ int err;
switch (type) { case MPAM_CLASS_CACHE: err = mpam_get_cpumask_from_cache_id(comp->comp_id, class->level, affinity); if (err) { dev_warn_once(&msc->pdev->dev, "Failed to determine CPU affinity\n"); return err; }
if (cpumask_empty(affinity)) dev_warn_once(&msc->pdev->dev, "no CPUs associated with cache node\n");
break; case MPAM_CLASS_MEMORY: get_cpumask_from_node_id(comp->comp_id, affinity); /* affinity may be empty for CPU-less memory nodes */ break; case MPAM_CLASS_UNKNOWN: return 0; }
cpumask_and(affinity, affinity, &msc->accessibility);
return 0;}
static int mpam_ris_create_locked(struct mpam_msc *msc, u8 ris_idx, enum mpam_class_types type, u8 class_id, int component_id){ int err; struct mpam_vmsc *vmsc; struct mpam_msc_ris *ris; struct mpam_class *class; struct mpam_component *comp; struct platform_device *pdev = msc->pdev;
lockdep_assert_held(&mpam_list_lock);
if (ris_idx > MPAM_MSC_MAX_NUM_RIS) return -EINVAL;
if (test_and_set_bit(ris_idx, &msc->ris_idxs)) return -EBUSY;
ris = devm_kzalloc(&msc->pdev->dev, sizeof(*ris), GFP_KERNEL); if (!ris) return -ENOMEM; init_garbage(&ris->garbage); ris->garbage.pdev = pdev;
class = mpam_class_find(class_id, type); if (IS_ERR(class)) return PTR_ERR(class);
comp = mpam_component_find(class, component_id); if (IS_ERR(comp)) { if (list_empty(&class->components)) mpam_class_destroy(class); return PTR_ERR(comp); }
vmsc = mpam_vmsc_find(comp, msc); if (IS_ERR(vmsc)) { if (list_empty(&comp->vmsc)) mpam_component_destroy(comp); return PTR_ERR(vmsc); }
err = mpam_ris_get_affinity(msc, &ris->affinity, type, class, comp); if (err) { if (list_empty(&vmsc->ris)) mpam_vmsc_destroy(vmsc); return err; }
ris->ris_idx = ris_idx; INIT_LIST_HEAD_RCU(&ris->msc_list); INIT_LIST_HEAD_RCU(&ris->vmsc_list); ris->vmsc = vmsc;
cpumask_or(&comp->affinity, &comp->affinity, &ris->affinity); cpumask_or(&class->affinity, &class->affinity, &ris->affinity); list_add_rcu(&ris->vmsc_list, &vmsc->ris); list_add_rcu(&ris->msc_list, &msc->ris);
return 0;}
static void mpam_ris_destroy(struct mpam_msc_ris *ris){ struct mpam_vmsc *vmsc = ris->vmsc; struct mpam_msc *msc = vmsc->msc; struct mpam_component *comp = vmsc->comp; struct mpam_class *class = comp->class;
lockdep_assert_held(&mpam_list_lock);
/* * It is assumed affinities don't overlap. If they do the class becomes * unusable immediately. */ cpumask_andnot(&class->affinity, &class->affinity, &ris->affinity); cpumask_andnot(&comp->affinity, &comp->affinity, &ris->affinity); clear_bit(ris->ris_idx, &msc->ris_idxs); list_del_rcu(&ris->msc_list); list_del_rcu(&ris->vmsc_list); add_to_garbage(ris);
if (list_empty(&vmsc->ris)) mpam_vmsc_destroy(vmsc);}
int mpam_ris_create(struct mpam_msc *msc, u8 ris_idx, enum mpam_class_types type, u8 class_id, int component_id){ int err;
mutex_lock(&mpam_list_lock); err = mpam_ris_create_locked(msc, ris_idx, type, class_id, component_id); mutex_unlock(&mpam_list_lock); if (err) mpam_free_garbage();
return err;}
static struct mpam_msc_ris *mpam_get_or_create_ris(struct mpam_msc *msc, u8 ris_idx){ int err; struct mpam_msc_ris *ris;
lockdep_assert_held(&mpam_list_lock);
if (!test_bit(ris_idx, &msc->ris_idxs)) { err = mpam_ris_create_locked(msc, ris_idx, MPAM_CLASS_UNKNOWN, 0, 0); if (err) return ERR_PTR(err); }
list_for_each_entry(ris, &msc->ris, msc_list) { if (ris->ris_idx == ris_idx) return ris; }
return ERR_PTR(-ENOENT);}
static int mpam_enable_quirk_nvidia_t241_1(struct mpam_msc *msc, const struct mpam_quirk *quirk){ s32 soc_id = arm_smccc_get_soc_id_version(); struct resource *r; phys_addr_t phys;
/* * A mapping to a device other than the MSC is needed, check * SOC_ID is NVIDIA T241 chip (036b:0241) */ if (soc_id < 0 || soc_id != SMCCC_SOC_ID_T241) return -EINVAL;
r = platform_get_resource(msc->pdev, IORESOURCE_MEM, 0); if (!r) return -EINVAL;
/* Find the internal registers base addr from the CHIP ID */ msc->t241_id = T241_CHIP_ID(r->start); phys = FIELD_PREP(GENMASK_ULL(45, 44), msc->t241_id) | 0x19000000ULL;
t241_scratch_regs[msc->t241_id] = ioremap(phys, SZ_8M); if (WARN_ON_ONCE(!t241_scratch_regs[msc->t241_id])) return -EINVAL;
pr_info_once("Enabled workaround for NVIDIA T241 erratum T241-MPAM-1\n");
return 0;}
static const struct mpam_quirk mpam_quirks[] = { { /* NVIDIA t241 erratum T241-MPAM-1 */ .init = mpam_enable_quirk_nvidia_t241_1, .iidr = MPAM_IIDR_NVIDIA_T241, .iidr_mask = MPAM_IIDR_MATCH_ONE, .workaround = T241_SCRUB_SHADOW_REGS, }, { /* NVIDIA t241 erratum T241-MPAM-4 */ .iidr = MPAM_IIDR_NVIDIA_T241, .iidr_mask = MPAM_IIDR_MATCH_ONE, .workaround = T241_FORCE_MBW_MIN_TO_ONE, }, { /* NVIDIA t241 erratum T241-MPAM-6 */ .iidr = MPAM_IIDR_NVIDIA_T241, .iidr_mask = MPAM_IIDR_MATCH_ONE, .workaround = T241_MBW_COUNTER_SCALE_64, }, { /* ARM CMN-650 CSU erratum 3642720 */ .iidr = MPAM_IIDR_ARM_CMN_650, .iidr_mask = MPAM_IIDR_MATCH_ONE, .workaround = IGNORE_CSU_NRDY, }, { NULL } /* Sentinel */};
static void mpam_enable_quirks(struct mpam_msc *msc){ const struct mpam_quirk *quirk;
for (quirk = &mpam_quirks[0]; quirk->iidr_mask; quirk++) { int err = 0;
if (quirk->iidr != (msc->iidr & quirk->iidr_mask)) continue;
if (quirk->init) err = quirk->init(msc, quirk);
if (err) continue;
mpam_set_quirk(quirk->workaround, msc); }}
/* * IHI009A.a has this nugget: "If a monitor does not support automatic behaviour * of NRDY, software can use this bit for any purpose" - so hardware might not * implement this - but it isn't RES0. * * Try and see what values stick in this bit. If we can write either value, * its probably not implemented by hardware. */static int mpam_ris_hw_probe_csu_nrdy(struct mpam_msc_ris *ris){ u32 now, mon_sel, ctl_val; bool can_set, can_clear; struct mpam_msc *msc = ris->vmsc->msc; int ret;
ACQUIRE(mon_sel_lock, guard)(msc); ret = ACQUIRE_ERR(mon_sel_lock, &guard); if (ret) return ret;
mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, 0) | FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx); ret = mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel); if (ret) return ret;
/* Hardware might ignore nrdy if it's not enabled */ ctl_val = MSMON_CFG_CSU_CTL_TYPE_CSU; ctl_val |= MSMON_CFG_x_CTL_MATCH_PARTID; ctl_val |= MSMON_CFG_x_CTL_MATCH_PMG; ctl_val |= MSMON_CFG_x_CTL_EN; ret = mpam_write_monsel_reg(msc, CFG_CSU_FLT, 0); if (ret) return ret; ret = mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val); if (ret) return ret;
ret = _mpam_write_monsel_reg(msc, MSMON_CSU, MSMON___NRDY); if (ret) return ret; ret = _mpam_read_monsel_reg(msc, MSMON_CSU, &now); if (ret) return ret; can_set = now & MSMON___NRDY;
ret = _mpam_write_monsel_reg(msc, MSMON_CSU, 0); if (ret) return ret; /* Configuration change to try and coax hardware into setting nrdy */ ret = mpam_write_monsel_reg(msc, CFG_CSU_FLT, 0x1); if (ret) return ret; ret = _mpam_read_monsel_reg(msc, MSMON_CSU, &now); if (ret) return ret; can_clear = !(now & MSMON___NRDY);
return (!can_set || !can_clear);}
static int mpam_ris_hw_probe(struct mpam_msc_ris *ris){ int fw_has_nrdy_err, err; struct mpam_msc *msc = ris->vmsc->msc; struct device *dev = &msc->pdev->dev; struct mpam_props *props = &ris->props; struct mpam_class *class = ris->vmsc->comp->class;
lockdep_assert_held(&msc->probe_lock); lockdep_assert_held(&msc->part_sel_lock);
/* Cache Capacity Partitioning */ if (FIELD_GET(MPAMF_IDR_HAS_CCAP_PART, ris->idr)) { u32 ccap_features;
err = mpam_read_partsel_reg(msc, CCAP_IDR, &ccap_features); if (err) return err;
props->cmax_wd = FIELD_GET(MPAMF_CCAP_IDR_CMAX_WD, ccap_features); if (props->cmax_wd && FIELD_GET(MPAMF_CCAP_IDR_HAS_CMAX_SOFTLIM, ccap_features)) mpam_set_feature(mpam_feat_cmax_softlim, props);
if (props->cmax_wd && !FIELD_GET(MPAMF_CCAP_IDR_NO_CMAX, ccap_features)) mpam_set_feature(mpam_feat_cmax_cmax, props);
if (props->cmax_wd && FIELD_GET(MPAMF_CCAP_IDR_HAS_CMIN, ccap_features)) mpam_set_feature(mpam_feat_cmax_cmin, props);
props->cassoc_wd = FIELD_GET(MPAMF_CCAP_IDR_CASSOC_WD, ccap_features); if (props->cassoc_wd && FIELD_GET(MPAMF_CCAP_IDR_HAS_CASSOC, ccap_features)) mpam_set_feature(mpam_feat_cmax_cassoc, props); }
/* Cache Portion partitioning */ if (FIELD_GET(MPAMF_IDR_HAS_CPOR_PART, ris->idr)) { u32 cpor_features;
err = mpam_read_partsel_reg(msc, CPOR_IDR, &cpor_features); if (err) return err;
props->cpbm_wd = FIELD_GET(MPAMF_CPOR_IDR_CPBM_WD, cpor_features); if (props->cpbm_wd) mpam_set_feature(mpam_feat_cpor_part, props); }
/* Memory bandwidth partitioning */ if (FIELD_GET(MPAMF_IDR_HAS_MBW_PART, ris->idr)) { u32 mbw_features; err = mpam_read_partsel_reg(msc, MBW_IDR, &mbw_features); if (err) return err;
/* portion bitmap resolution */ props->mbw_pbm_bits = FIELD_GET(MPAMF_MBW_IDR_BWPBM_WD, mbw_features); if (props->mbw_pbm_bits && FIELD_GET(MPAMF_MBW_IDR_HAS_PBM, mbw_features)) mpam_set_feature(mpam_feat_mbw_part, props);
props->bwa_wd = FIELD_GET(MPAMF_MBW_IDR_BWA_WD, mbw_features);
/* * The BWA_WD field can represent 0-63, but the control fields it * describes have a maximum of 16 bits. */ props->bwa_wd = min(props->bwa_wd, 16);
if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MAX, mbw_features)) mpam_set_feature(mpam_feat_mbw_max, props);
if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MIN, mbw_features)) mpam_set_feature(mpam_feat_mbw_min, props);
if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_PROP, mbw_features)) mpam_set_feature(mpam_feat_mbw_prop, props); }
/* Priority partitioning */ if (FIELD_GET(MPAMF_IDR_HAS_PRI_PART, ris->idr)) { u32 pri_features; err = mpam_read_partsel_reg(msc, PRI_IDR, &pri_features); if (err) return err;
props->intpri_wd = FIELD_GET(MPAMF_PRI_IDR_INTPRI_WD, pri_features); if (props->intpri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_INTPRI, pri_features)) { mpam_set_feature(mpam_feat_intpri_part, props); if (FIELD_GET(MPAMF_PRI_IDR_INTPRI_0_IS_LOW, pri_features)) mpam_set_feature(mpam_feat_intpri_part_0_low, props); }
props->dspri_wd = FIELD_GET(MPAMF_PRI_IDR_DSPRI_WD, pri_features); if (props->dspri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_DSPRI, pri_features)) { mpam_set_feature(mpam_feat_dspri_part, props); if (FIELD_GET(MPAMF_PRI_IDR_DSPRI_0_IS_LOW, pri_features)) mpam_set_feature(mpam_feat_dspri_part_0_low, props); } }
/* Performance Monitoring */ if (FIELD_GET(MPAMF_IDR_HAS_MSMON, ris->idr)) { u32 msmon_features;
err = mpam_read_partsel_reg(msc, MSMON_IDR, &msmon_features); if (err) return err;
/* * If the firmware max-nrdy-us property is missing, the * CSU counters can't be used. Should we wait forever? */ fw_has_nrdy_err = device_property_read_u32(&msc->pdev->dev, "arm,not-ready-us", &msc->nrdy_usec);
if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_CSU, msmon_features)) { u32 csumonidr;
err = mpam_read_partsel_reg(msc, CSUMON_IDR, &csumonidr); if (err) return err;
props->num_csu_mon = FIELD_GET(MPAMF_CSUMON_IDR_NUM_MON, csumonidr); if (props->num_csu_mon) { bool hw_managed;
mpam_set_feature(mpam_feat_msmon_csu, props);
if (FIELD_GET(MPAMF_CSUMON_IDR_HAS_XCL, csumonidr)) mpam_set_feature(mpam_feat_msmon_csu_xcl, props);
/* Is NRDY hardware managed? */ err = mpam_ris_hw_probe_csu_nrdy(ris); if (err < 0) return err; hw_managed = err;
/* * Accept the missing firmware property if NRDY appears * un-implemented. */ if (fw_has_nrdy_err && hw_managed) dev_err_once(dev, "Counters are not usable because not-ready timeout was not provided by firmware."); } } if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_MBWU, msmon_features)) { bool has_long; u32 mbwumon_idr;
err = mpam_read_partsel_reg(msc, MBWUMON_IDR, &mbwumon_idr); if (err) return err;
props->num_mbwu_mon = FIELD_GET(MPAMF_MBWUMON_IDR_NUM_MON, mbwumon_idr); if (props->num_mbwu_mon) { mpam_set_feature(mpam_feat_msmon_mbwu, props);
if (FIELD_GET(MPAMF_MBWUMON_IDR_HAS_RWBW, mbwumon_idr)) mpam_set_feature(mpam_feat_msmon_mbwu_rwbw, props);
has_long = FIELD_GET(MPAMF_MBWUMON_IDR_HAS_LONG, mbwumon_idr); if (has_long) { if (FIELD_GET(MPAMF_MBWUMON_IDR_LWD, mbwumon_idr)) mpam_set_feature(mpam_feat_msmon_mbwu_63counter, props); else mpam_set_feature(mpam_feat_msmon_mbwu_44counter, props); }
mpam_set_feature(mpam_feat_msmon_mbwu_31counter, props); } } }
/* * RIS with PARTID narrowing don't have enough storage for one * configuration per PARTID. If these are in a class we could use, * reduce the supported partid_max to match the number of intpartid. * If the class is unknown, just ignore it. */ if (FIELD_GET(MPAMF_IDR_HAS_PARTID_NRW, ris->idr) && class->type != MPAM_CLASS_UNKNOWN) { u16 partid_max; u32 nrwidr;
err = mpam_read_partsel_reg(msc, PARTID_NRW_IDR, &nrwidr); if (err) return err;
partid_max = FIELD_GET(MPAMF_PARTID_NRW_IDR_INTPARTID_MAX, nrwidr);
mpam_set_feature(mpam_feat_partid_nrw, props); msc->partid_max = min(msc->partid_max, partid_max); }
return 0;}
static int mpam_msc_hw_probe(struct mpam_msc *msc){ int ret; u64 idr; u16 partid_max; u8 ris_idx, pmg_max; struct mpam_msc_ris *ris; struct device *dev = &msc->pdev->dev;
lockdep_assert_held(&msc->probe_lock);
if (!mpam_msc_check_aidr(msc)) { dev_err_once(dev, "MSC does not match architecture v1.x\n"); return -EIO; }
/* Grab an IDR value to find out how many RIS there are */ scoped_guard(mutex, &msc->part_sel_lock) { ret = mpam_msc_read_idr(msc, &idr); if (ret) return ret;
ret = mpam_read_partsel_reg(msc, IIDR, &msc->iidr); if (ret) return ret; }
mpam_enable_quirks(msc);
msc->ris_max = FIELD_GET(MPAMF_IDR_RIS_MAX, idr);
/* Use these values so partid/pmg always starts with a valid value */ msc->partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr); msc->pmg_max = FIELD_GET(MPAMF_IDR_PMG_MAX, idr);
for (ris_idx = 0; ris_idx <= msc->ris_max; ris_idx++) { scoped_guard(mutex, &msc->part_sel_lock) { ret = __mpam_part_sel(ris_idx, 0, msc); if (ret) return ret;
ret = mpam_msc_read_idr(msc, &idr); if (ret) return ret; }
partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr); pmg_max = FIELD_GET(MPAMF_IDR_PMG_MAX, idr); msc->partid_max = min(msc->partid_max, partid_max); msc->pmg_max = min(msc->pmg_max, pmg_max); msc->has_extd_esr = FIELD_GET(MPAMF_IDR_HAS_EXTD_ESR, idr);
scoped_guard(mutex, &mpam_list_lock) { ris = mpam_get_or_create_ris(msc, ris_idx); if (IS_ERR(ris)) return PTR_ERR(ris); } ris->idr = idr;
scoped_guard(mutex, &msc->part_sel_lock) { ret = __mpam_part_sel(ris_idx, 0, msc); if (ret) return ret;
ret = mpam_ris_hw_probe(ris); if (ret) return ret; } }
/* Clear any stale errors */ ret = mpam_msc_clear_esr(msc); if (ret) return ret;
spin_lock(&partid_max_lock); mpam_partid_max = min(mpam_partid_max, msc->partid_max); mpam_pmg_max = min(mpam_pmg_max, msc->pmg_max); spin_unlock(&partid_max_lock);
msc->probed = true;
return 0;}
struct mon_read { struct mpam_msc_ris *ris; struct mon_cfg *ctx; enum mpam_device_features type; u64 *val; int err; bool waited_timeout;};
static int mpam_msc_read_mbwu_l(struct mpam_msc *msc, u64 *res){ int ret; int retry = 3; u32 mbwu_l_low; u32 mbwu_l_high1, mbwu_l_high2;
mpam_mon_sel_lock_held(msc);
if (msc->iface == MPAM_IFACE_MMIO) { WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz); WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); }
ret = __mpam_read_reg(msc, MSMON_MBWU_L + 4, &mbwu_l_high2); if (ret) return ret;
do { mbwu_l_high1 = mbwu_l_high2; ret = __mpam_read_reg(msc, MSMON_MBWU_L, &mbwu_l_low); if (ret) return ret; ret = __mpam_read_reg(msc, MSMON_MBWU_L + 4, &mbwu_l_high2); if (ret) return ret;
retry--; } while (mbwu_l_high1 != mbwu_l_high2 && retry > 0);
if (mbwu_l_high1 == mbwu_l_high2) { *res = ((u64)mbwu_l_high1 << 32) | mbwu_l_low; } else { pr_warn("Failed to read a stable value\n"); *res = MSMON___L_NRDY; }
return 0;}
static int mpam_msc_zero_mbwu_l(struct mpam_msc *msc){ int ret;
mpam_mon_sel_lock_held(msc);
if (msc->iface == MPAM_IFACE_MMIO) { WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz); WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); }
ret = __mpam_write_reg(msc, MSMON_MBWU_L, 0); if (ret) return ret;
return __mpam_write_reg(msc, MSMON_MBWU_L + 4, 0);}
static void gen_msmon_ctl_flt_vals(struct mon_read *m, u32 *ctl_val, u32 *flt_val){ struct mon_cfg *ctx = m->ctx;
/* * For CSU counters its implementation-defined what happens when not * filtering by partid. */ *ctl_val = MSMON_CFG_x_CTL_MATCH_PARTID;
*flt_val = FIELD_PREP(MSMON_CFG_x_FLT_PARTID, ctx->partid);
if (m->ctx->match_pmg) { *ctl_val |= MSMON_CFG_x_CTL_MATCH_PMG; *flt_val |= FIELD_PREP(MSMON_CFG_x_FLT_PMG, ctx->pmg); }
switch (m->type) { case mpam_feat_msmon_csu: *ctl_val |= MSMON_CFG_CSU_CTL_TYPE_CSU;
if (mpam_has_feature(mpam_feat_msmon_csu_xcl, &m->ris->props)) *flt_val |= FIELD_PREP(MSMON_CFG_CSU_FLT_XCL, ctx->csu_exclude_clean);
break; case mpam_feat_msmon_mbwu_31counter: case mpam_feat_msmon_mbwu_44counter: case mpam_feat_msmon_mbwu_63counter: *ctl_val |= MSMON_CFG_MBWU_CTL_TYPE_MBWU;
if (mpam_has_feature(mpam_feat_msmon_mbwu_rwbw, &m->ris->props)) *flt_val |= FIELD_PREP(MSMON_CFG_MBWU_FLT_RWBW, ctx->opts);
break; default: pr_warn("Unexpected monitor type %d\n", m->type); }}
static int read_msmon_ctl_flt_vals(struct mon_read *m, u32 *ctl_val, u32 *flt_val){ struct mpam_msc *msc = m->ris->vmsc->msc; int ret;
switch (m->type) { case mpam_feat_msmon_csu: ret = mpam_read_monsel_reg(msc, CFG_CSU_CTL, ctl_val); if (ret) return ret; return mpam_read_monsel_reg(msc, CFG_CSU_FLT, flt_val); case mpam_feat_msmon_mbwu_31counter: case mpam_feat_msmon_mbwu_44counter: case mpam_feat_msmon_mbwu_63counter: ret = mpam_read_monsel_reg(msc, CFG_MBWU_CTL, ctl_val); if (ret) return ret; return mpam_read_monsel_reg(msc, CFG_MBWU_FLT, flt_val); default: pr_warn("Unexpected monitor type %d\n", m->type); return -EINVAL; }}
/* Remove values set by the hardware to prevent apparent mismatches. */static inline void clean_msmon_ctl_val(u32 *cur_ctl){ *cur_ctl &= ~MSMON_CFG_x_CTL_OFLOW_STATUS;
if (FIELD_GET(MSMON_CFG_x_CTL_TYPE, *cur_ctl) == MSMON_CFG_MBWU_CTL_TYPE_MBWU) *cur_ctl &= ~MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L;}
static int write_msmon_ctl_flt_vals(struct mon_read *m, u32 ctl_val, u32 flt_val){ struct mpam_msc *msc = m->ris->vmsc->msc; int ret;
/* * Write the ctl_val with the enable bit cleared, reset the counter, * then enable counter. */ switch (m->type) { case mpam_feat_msmon_csu: ret = mpam_write_monsel_reg(msc, CFG_CSU_FLT, flt_val); if (ret) return ret; ret = mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val); if (ret) return ret; ret = mpam_write_monsel_reg(msc, CSU, 0); if (ret) return ret; return mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val | MSMON_CFG_x_CTL_EN); case mpam_feat_msmon_mbwu_31counter: case mpam_feat_msmon_mbwu_44counter: case mpam_feat_msmon_mbwu_63counter: ret = mpam_write_monsel_reg(msc, CFG_MBWU_FLT, flt_val); if (ret) return ret; ret = mpam_write_monsel_reg(msc, CFG_MBWU_CTL, ctl_val); if (ret) return ret; ret = mpam_write_monsel_reg(msc, CFG_MBWU_CTL, ctl_val | MSMON_CFG_x_CTL_EN); if (ret) return ret; /* Counting monitors require NRDY to be reset by software */ if (m->type == mpam_feat_msmon_mbwu_31counter) return mpam_write_monsel_reg(msc, MBWU, 0);
return mpam_msc_zero_mbwu_l(m->ris->vmsc->msc); default: pr_warn("Unexpected monitor type %d\n", m->type); return -EINVAL; }}
static u64 __mpam_msmon_overflow_val(enum mpam_device_features type){ /* TODO: implement scaling counters */ switch (type) { case mpam_feat_msmon_mbwu_63counter: return BIT_ULL(hweight_long(MSMON___LWD_VALUE)); case mpam_feat_msmon_mbwu_44counter: return BIT_ULL(hweight_long(MSMON___L_VALUE)); case mpam_feat_msmon_mbwu_31counter: return BIT_ULL(hweight_long(MSMON___VALUE)); default: return 0; }}
static u64 mpam_msmon_overflow_val(enum mpam_device_features type, struct mpam_msc *msc){ u64 overflow_val = __mpam_msmon_overflow_val(type);
if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc)) overflow_val *= 64;
return overflow_val;}
/* * This function might be called via smp_call_function_any(), so propagate * errors inside the arg struct. */static void __ris_msmon_read_locked(struct mon_read *m){ u64 now; u32 now32; bool nrdy = false; bool config_mismatch; bool overflow = false; struct mon_cfg *ctx = m->ctx; bool reset_on_next_read = false; struct mpam_msc_ris *ris = m->ris; struct msmon_mbwu_state *mbwu_state; struct mpam_msc *msc = m->ris->vmsc->msc; u32 mon_sel, ctl_val, flt_val, cur_ctl, cur_flt;
mpam_mon_sel_lock_held(msc);
mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, ctx->mon) | FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx); m->err = mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel); if (m->err) return;
switch (m->type) { case mpam_feat_msmon_mbwu_31counter: case mpam_feat_msmon_mbwu_44counter: case mpam_feat_msmon_mbwu_63counter: mbwu_state = &ris->mbwu_state[ctx->mon]; if (mbwu_state) { reset_on_next_read = mbwu_state->reset_on_next_read; mbwu_state->reset_on_next_read = false; } break; default: break; }
/* * Read the existing configuration to avoid re-writing the same values. * This saves waiting for 'nrdy' on subsequent reads. */ m->err = read_msmon_ctl_flt_vals(m, &cur_ctl, &cur_flt); if (m->err) return;
if (mpam_feat_msmon_mbwu_31counter == m->type) overflow = cur_ctl & MSMON_CFG_x_CTL_OFLOW_STATUS; else if (mpam_feat_msmon_mbwu_44counter == m->type || mpam_feat_msmon_mbwu_63counter == m->type) overflow = cur_ctl & MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L;
clean_msmon_ctl_val(&cur_ctl); gen_msmon_ctl_flt_vals(m, &ctl_val, &flt_val); config_mismatch = cur_flt != flt_val || cur_ctl != (ctl_val | MSMON_CFG_x_CTL_EN);
if (config_mismatch || reset_on_next_read) { m->err = write_msmon_ctl_flt_vals(m, ctl_val, flt_val); if (m->err) return; overflow = false; } else if (overflow) { m->err = mpam_write_monsel_reg(msc, CFG_MBWU_CTL, cur_ctl & ~(MSMON_CFG_x_CTL_OFLOW_STATUS | MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L)); if (m->err) return; }
switch (m->type) { case mpam_feat_msmon_csu: m->err = mpam_read_monsel_reg(msc, CSU, &now32); if (m->err) return; nrdy = now32 & MSMON___NRDY; now = FIELD_GET(MSMON___VALUE, now32);
if (mpam_has_quirk(IGNORE_CSU_NRDY, msc) && m->waited_timeout) nrdy = false;
break; case mpam_feat_msmon_mbwu_31counter: case mpam_feat_msmon_mbwu_44counter: case mpam_feat_msmon_mbwu_63counter: if (m->type != mpam_feat_msmon_mbwu_31counter) { m->err = mpam_msc_read_mbwu_l(msc, &now); if (m->err) return; nrdy = now & MSMON___L_NRDY;
if (m->type == mpam_feat_msmon_mbwu_63counter) now = FIELD_GET(MSMON___LWD_VALUE, now); else now = FIELD_GET(MSMON___L_VALUE, now); } else { m->err = mpam_read_monsel_reg(msc, MBWU, &now32); if (m->err) return; nrdy = now32 & MSMON___NRDY; now = FIELD_GET(MSMON___VALUE, now32); }
if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc)) now *= 64;
if (nrdy) break;
mbwu_state = &ris->mbwu_state[ctx->mon];
if (overflow) mbwu_state->correction += mpam_msmon_overflow_val(m->type, msc);
/* * Include bandwidth consumed before the last hardware reset and * a counter size increment for each overflow. */ now += mbwu_state->correction; break; default: m->err = -EINVAL; }
if (nrdy) m->err = -EBUSY;
if (m->err) return;
*m->val += now;}
static void __ris_msmon_read(void *arg){ struct mon_read *m = arg; struct mpam_msc *msc = m->ris->vmsc->msc;
if (!mpam_mon_sel_lock(msc)) { m->err = -EIO; return; }
__ris_msmon_read_locked(m);
mpam_mon_sel_unlock(msc);}
static int _msmon_read(struct mpam_component *comp, struct mon_read *arg){ int err, any_err = 0; struct mpam_vmsc *vmsc;
guard(srcu)(&mpam_srcu); list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list, srcu_read_lock_held(&mpam_srcu)) { struct mpam_msc *msc = vmsc->msc; struct mpam_msc_ris *ris;
list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list, srcu_read_lock_held(&mpam_srcu)) { arg->ris = ris;
if (msc->iface == MPAM_IFACE_MMIO) { err = smp_call_function_any(&msc->accessibility, __ris_msmon_read, arg, true); if (!err) err = arg->err; } else { __ris_msmon_read(arg); err = arg->err; }
/* * Save one error to be returned to the caller, but * keep reading counters so that get reprogrammed. On * platforms with NRDY this lets us wait once. */ if (err) any_err = err; } }
return any_err;}
static enum mpam_device_features mpam_msmon_choose_counter(struct mpam_class *class){ struct mpam_props *cprops = &class->props;
if (mpam_has_feature(mpam_feat_msmon_mbwu_63counter, cprops)) return mpam_feat_msmon_mbwu_63counter; if (mpam_has_feature(mpam_feat_msmon_mbwu_44counter, cprops)) return mpam_feat_msmon_mbwu_44counter;
return mpam_feat_msmon_mbwu_31counter;}
int mpam_msmon_read(struct mpam_component *comp, struct mon_cfg *ctx, enum mpam_device_features type, u64 *val){ int err; struct mon_read arg; u64 wait_jiffies = 0; struct mpam_class *class = comp->class; struct mpam_props *cprops = &class->props;
might_sleep();
if (!mpam_is_enabled()) return -EIO;
if (!mpam_has_feature(type, cprops)) return -EOPNOTSUPP;
if (type == mpam_feat_msmon_mbwu) type = mpam_msmon_choose_counter(class);
arg = (struct mon_read) { .ctx = ctx, .type = type, .val = val, }; *val = 0;
err = _msmon_read(comp, &arg); if (err == -EBUSY && class->nrdy_usec) wait_jiffies = usecs_to_jiffies(class->nrdy_usec);
while (wait_jiffies) wait_jiffies = schedule_timeout_uninterruptible(wait_jiffies);
if (err == -EBUSY) { arg = (struct mon_read) { .ctx = ctx, .type = type, .val = val, .waited_timeout = true, }; *val = 0;
err = _msmon_read(comp, &arg); }
return err;}
void mpam_msmon_reset_mbwu(struct mpam_component *comp, struct mon_cfg *ctx){ struct mpam_msc *msc; struct mpam_vmsc *vmsc; struct mpam_msc_ris *ris;
if (!mpam_is_enabled()) return;
guard(srcu)(&mpam_srcu); list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list, srcu_read_lock_held(&mpam_srcu)) { if (!mpam_has_feature(mpam_feat_msmon_mbwu, &vmsc->props)) continue;
msc = vmsc->msc; list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list, srcu_read_lock_held(&mpam_srcu)) { if (!mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props)) continue;
if (WARN_ON_ONCE(!mpam_mon_sel_lock(msc))) continue;
ris->mbwu_state[ctx->mon].correction = 0; ris->mbwu_state[ctx->mon].reset_on_next_read = true; mpam_mon_sel_unlock(msc); } }}
static int mpam_reset_msc_bitmap(struct mpam_msc *msc, u16 reg, u16 wd){ u32 num_words, msb; u32 bm = ~0; int i;
lockdep_assert_held(&msc->part_sel_lock);
if (wd == 0) return 0;
/* * Write all ~0 to all but the last 32bit-word, which may * have fewer bits... */ num_words = DIV_ROUND_UP(wd, 32); for (i = 0; i < num_words - 1; i++, reg += sizeof(bm)) { int ret;
ret = __mpam_write_reg(msc, reg, bm); if (ret) return ret; }
/* * ....and then the last (maybe) partial 32bit word. When wd is a * multiple of 32, msb should be 31 to write a full 32bit word. */ msb = (wd - 1) % 32; bm = GENMASK(msb, 0); return __mpam_write_reg(msc, reg, bm);}
static void mpam_apply_t241_erratum(struct mpam_msc_ris *ris, u16 partid){ int sidx, i, lcount = 1000; void __iomem *regs; u64 val0, val;
regs = t241_scratch_regs[ris->vmsc->msc->t241_id];
for (i = 0; i < lcount; i++) { /* Read the shadow register at index 0 */ val0 = readq_relaxed(regs + T241_SHADOW_REG_OFF(0, partid));
/* Check if all the shadow registers have the same value */ for (sidx = 1; sidx < T241_CHIP_NSLICES; sidx++) { val = readq_relaxed(regs + T241_SHADOW_REG_OFF(sidx, partid)); if (val != val0) break; } if (sidx == T241_CHIP_NSLICES) break; }
if (i == lcount) pr_warn_once("t241: inconsistent values in shadow regs");
/* Write a value zero to spare registers to take effect of MBW conf */ writeq_relaxed(0, regs + T241_SPARE_REG0_OFF); writeq_relaxed(0, regs + T241_SPARE_REG1_OFF);}
static void mpam_quirk_post_config_change(struct mpam_msc_ris *ris, u16 partid, struct mpam_config *cfg){ if (mpam_has_quirk(T241_SCRUB_SHADOW_REGS, ris->vmsc->msc)) mpam_apply_t241_erratum(ris, partid);}
static u16 mpam_wa_t241_force_mbw_min_to_one(struct mpam_props *props){ u16 max_hw_value, min_hw_granule, res0_bits;
res0_bits = 16 - props->bwa_wd; max_hw_value = ((1 << props->bwa_wd) - 1) << res0_bits; min_hw_granule = ~max_hw_value;
return min_hw_granule + 1;}
static u16 mpam_wa_t241_calc_min_from_max(struct mpam_props *props, struct mpam_config *cfg){ u16 val = 0; u16 max; u16 delta = ((5 * MPAMCFG_MBW_MAX_MAX) / 100) - 1;
if (mpam_has_feature(mpam_feat_mbw_max, cfg)) { max = cfg->mbw_max; } else { /* Resetting. Hence, use the ris specific default. */ max = GENMASK(15, 16 - props->bwa_wd); }
if (max > delta) val = max - delta;
return val;}
/* Called via IPI. Call while holding an SRCU reference */static int mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, struct mpam_config *cfg){ u16 cmax = MPAMCFG_CMAX_CMAX; struct mpam_msc *msc = ris->vmsc->msc; struct mpam_props *rprops = &ris->props; int ret;
guard(mutex)(&msc->part_sel_lock); ret = __mpam_part_sel(ris->ris_idx, partid, msc); if (ret) return ret;
if (mpam_has_feature(mpam_feat_partid_nrw, rprops)) { /* Update the intpartid mapping */ ret = mpam_write_partsel_reg(msc, INTPARTID, MPAMCFG_INTPARTID_INTERNAL | partid); if (ret) return ret;
/* * Then switch to the 'internal' partid to update the * configuration. */ ret = __mpam_intpart_sel(ris->ris_idx, partid, msc); if (ret) return ret; }
if (mpam_has_feature(mpam_feat_cpor_part, rprops)) { if (mpam_has_feature(mpam_feat_cpor_part, cfg)) { ret = mpam_write_partsel_reg(msc, CPBM, cfg->cpbm); if (ret) return ret; } else { ret = mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM, rprops->cpbm_wd); if (ret) return ret; } }
if (mpam_has_feature(mpam_feat_mbw_part, rprops)) { if (mpam_has_feature(mpam_feat_mbw_part, cfg)) { ret = mpam_write_partsel_reg(msc, MBW_PBM, cfg->mbw_pbm); if (ret) return ret; } else { ret = mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM, rprops->mbw_pbm_bits); if (ret) return ret; } }
if (mpam_has_feature(mpam_feat_mbw_min, rprops)) { u16 val = 0;
if (mpam_has_quirk(T241_FORCE_MBW_MIN_TO_ONE, msc)) { u16 min = mpam_wa_t241_force_mbw_min_to_one(rprops);
val = mpam_wa_t241_calc_min_from_max(rprops, cfg); val = max(val, min); }
ret = mpam_write_partsel_reg(msc, MBW_MIN, val); if (ret) return ret; }
if (mpam_has_feature(mpam_feat_mbw_max, rprops)) { if (mpam_has_feature(mpam_feat_mbw_max, cfg)) { ret = mpam_write_partsel_reg(msc, MBW_MAX, cfg->mbw_max); if (ret) return ret; } else { ret = mpam_write_partsel_reg(msc, MBW_MAX, MPAMCFG_MBW_MAX_MAX); if (ret) return ret; } }
if (mpam_has_feature(mpam_feat_mbw_prop, rprops)) { ret = mpam_write_partsel_reg(msc, MBW_PROP, 0); if (ret) return ret; }
if (mpam_has_feature(mpam_feat_cmax_cmax, rprops)) { ret = mpam_write_partsel_reg(msc, CMAX, cmax); if (ret) return ret; }
if (mpam_has_feature(mpam_feat_cmax_cmin, rprops)) { ret = mpam_write_partsel_reg(msc, CMIN, 0); if (ret) return ret; }
if (mpam_has_feature(mpam_feat_cmax_cassoc, rprops)) { ret = mpam_write_partsel_reg(msc, CASSOC, MPAMCFG_CASSOC_CASSOC); if (ret) return ret; }
if (mpam_has_feature(mpam_feat_intpri_part, rprops) || mpam_has_feature(mpam_feat_dspri_part, rprops)) { u32 pri_val = 0;
if (mpam_has_feature(mpam_feat_intpri_part, rprops)) { u16 intpri = GENMASK(rprops->intpri_wd - 1, 0);
/* aces high? */ if (!mpam_has_feature(mpam_feat_intpri_part_0_low, rprops)) intpri = 0;
pri_val |= FIELD_PREP(MPAMCFG_PRI_INTPRI, intpri); } if (mpam_has_feature(mpam_feat_dspri_part, rprops)) { u16 dspri = GENMASK(rprops->dspri_wd - 1, 0);
if (!mpam_has_feature(mpam_feat_dspri_part_0_low, rprops)) dspri = 0;
pri_val |= FIELD_PREP(MPAMCFG_PRI_DSPRI, dspri); }
ret = mpam_write_partsel_reg(msc, PRI, pri_val); if (ret) return ret; }
mpam_quirk_post_config_change(ris, partid, cfg);
return 0;}
/* Call with msc cfg_lock held */static int mpam_restore_mbwu_state(void *_ris){ int i; u64 val; struct mon_read mbwu_arg; struct mpam_msc_ris *ris = _ris; struct msmon_mbwu_state *mbwu_state; struct mpam_msc *msc = ris->vmsc->msc; struct mpam_class *class = ris->vmsc->comp->class;
for (i = 0; i < ris->props.num_mbwu_mon; i++) { if (WARN_ON_ONCE(!mpam_mon_sel_lock(msc))) return -EIO;
mbwu_state = &ris->mbwu_state[i];
if (!mbwu_state->enabled) { mpam_mon_sel_unlock(msc); continue; }
val = 0; mbwu_arg = (struct mon_read) { .ris = ris, .ctx = &mbwu_state->cfg, .type = mpam_msmon_choose_counter(class), .val = &val, };
mbwu_state->reset_on_next_read = true;
__ris_msmon_read_locked(&mbwu_arg);
mpam_mon_sel_unlock(msc);
if (mbwu_arg.err && mbwu_arg.err != -EBUSY) return mbwu_arg.err; }
return 0;}
/* Call with MSC cfg_lock held */static int mpam_save_mbwu_state(void *arg){ int i; int ret; u64 val; struct mon_cfg *cfg; struct mon_read mbwu_arg; u32 cur_flt, cur_ctl, mon_sel; struct mpam_msc_ris *ris = arg; struct msmon_mbwu_state *mbwu_state; struct mpam_msc *msc = ris->vmsc->msc; struct mpam_class *class = ris->vmsc->comp->class;
for (i = 0; i < ris->props.num_mbwu_mon; i++) { if (WARN_ON_ONCE(!mpam_mon_sel_lock(msc))) return -EIO;
mbwu_state = &ris->mbwu_state[i]; cfg = &mbwu_state->cfg;
mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, i) | FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx); ret = mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel); if (ret) return ret; ret = mpam_read_monsel_reg(msc, CFG_MBWU_FLT, &cur_flt); if (ret) return ret; ret = mpam_read_monsel_reg(msc, CFG_MBWU_CTL, &cur_ctl); if (ret) return ret;
cfg->mon = i; cfg->pmg = FIELD_GET(MSMON_CFG_x_FLT_PMG, cur_flt); cfg->match_pmg = FIELD_GET(MSMON_CFG_x_CTL_MATCH_PMG, cur_ctl); cfg->partid = FIELD_GET(MSMON_CFG_x_FLT_PARTID, cur_flt); mbwu_state->enabled = FIELD_GET(MSMON_CFG_x_CTL_EN, cur_ctl);
if (!mbwu_state->enabled) { mpam_mon_sel_unlock(msc); continue; }
val = 0; mbwu_arg = (struct mon_read) { .ris = ris, .ctx = cfg, .type = mpam_msmon_choose_counter(class), .val = &val, };
__ris_msmon_read_locked(&mbwu_arg);
mbwu_state->reset_on_next_read = true; if (!mbwu_arg.err) { /* * __ris_msmon_read_locked() already included the * previous correction value. */ mbwu_state->correction = val; }
ret = mpam_write_monsel_reg(msc, CFG_MBWU_CTL, 0); if (ret) return ret;
mpam_mon_sel_unlock(msc); }
return 0;}
/* * Called via smp_call_on_cpu() to prevent migration, while still being * pre-emptible. Caller must hold mpam_srcu. */static int mpam_reset_ris(void *arg){ u16 partid_max; struct mpam_config reset_cfg = {}; struct mpam_msc_ris *ris = arg; int ret;
if (ris->in_reset_state) return 0;
spin_lock(&partid_max_lock); partid_max = mpam_partid_max; spin_unlock(&partid_max_lock); for (u32 partid = 0; partid <= partid_max; partid++) { ret = mpam_reprogram_ris_partid(ris, partid, &reset_cfg); if (ret) return ret; }
return 0;}
/* * Get the preferred CPU for this MSC. If it is accessible from this CPU, * this CPU is preferred. This can be preempted/migrated, it will only result * in more work. */static int mpam_get_msc_preferred_cpu(struct mpam_msc *msc){ int cpu = raw_smp_processor_id();
if (cpumask_test_cpu(cpu, &msc->accessibility)) return cpu;
return cpumask_first_and(&msc->accessibility, cpu_online_mask);}
static int mpam_touch_msc(struct mpam_msc *msc, int (*fn)(void *a), void *arg){ if (msc->iface != MPAM_IFACE_MMIO) return fn(arg);
lockdep_assert_irqs_enabled(); lockdep_assert_cpus_held(); WARN_ON_ONCE(!srcu_read_lock_held((&mpam_srcu)));
return smp_call_on_cpu(mpam_get_msc_preferred_cpu(msc), fn, arg, true);}
struct mpam_write_config_arg { struct mpam_msc_ris *ris; struct mpam_component *comp; u16 partid;};
static int __write_config(void *arg){ struct mpam_write_config_arg *c = arg;
return mpam_reprogram_ris_partid(c->ris, c->partid, &c->comp->cfg[c->partid]);}
static void mpam_reprogram_msc(struct mpam_msc *msc){ bool reset; struct mpam_config *cfg; struct mpam_msc_ris *ris; struct mpam_write_config_arg arg;
/* * No lock for mpam_partid_max as partid_max_published has been * set by mpam_enabled(), so the values can no longer change. */ mpam_assert_partid_sizes_fixed();
mutex_lock(&msc->cfg_lock); list_for_each_entry_srcu(ris, &msc->ris, msc_list, srcu_read_lock_held(&mpam_srcu)) { if (!mpam_is_enabled() && !ris->in_reset_state) { mpam_touch_msc(msc, &mpam_reset_ris, ris); ris->in_reset_state = true; continue; }
arg.comp = ris->vmsc->comp; arg.ris = ris; reset = true; for (u32 partid = 0; partid <= mpam_partid_max; partid++) { cfg = &ris->vmsc->comp->cfg[partid]; if (!bitmap_empty(cfg->features, MPAM_FEATURE_LAST)) reset = false;
arg.partid = partid; mpam_touch_msc(msc, __write_config, &arg); } ris->in_reset_state = reset;
if (mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props)) mpam_touch_msc(msc, &mpam_restore_mbwu_state, ris); } mutex_unlock(&msc->cfg_lock);}
static int mpam_enable_msc_ecr(void *_msc){ struct mpam_msc *msc = _msc;
return __mpam_write_reg(msc, MPAMF_ECR, MPAMF_ECR_INTEN);}
/* This can run in mpam_disable(), and the interrupt handler on the same CPU */static int mpam_disable_msc_ecr(void *_msc){ struct mpam_msc *msc = _msc;
return __mpam_write_reg(msc, MPAMF_ECR, 0);}
static void _enable_percpu_irq(void *_irq){ int *irq = _irq;
enable_percpu_irq(*irq, IRQ_TYPE_NONE);}
static int mpam_cpu_online(unsigned int cpu){ struct mpam_msc *msc;
guard(srcu)(&mpam_srcu); list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list, srcu_read_lock_held(&mpam_srcu)) { if (!cpumask_test_cpu(cpu, &msc->accessibility)) continue;
if (msc->reenable_error_ppi) _enable_percpu_irq(&msc->reenable_error_ppi);
if (atomic_fetch_inc(&msc->online_refs) == 0) { mutex_lock(&msc->error_irq_lock); if (msc->error_irq_hw_enabled) mpam_touch_msc(msc, mpam_enable_msc_ecr, msc); mutex_unlock(&msc->error_irq_lock);
mpam_reprogram_msc(msc); } }
if (mpam_resctrl_enabled) return mpam_resctrl_online_cpu(cpu);
return 0;}
/* Before mpam is enabled, try to probe new MSC */static int mpam_discovery_cpu_online(unsigned int cpu){ int err = 0; struct mpam_msc *msc; bool new_device_probed = false;
if (mpam_is_enabled()) return 0;
guard(srcu)(&mpam_srcu); list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list, srcu_read_lock_held(&mpam_srcu)) { if (!cpumask_test_cpu(cpu, &msc->accessibility)) continue;
mutex_lock(&msc->probe_lock); if (!msc->probed) err = mpam_msc_hw_probe(msc); mutex_unlock(&msc->probe_lock);
if (err) break; new_device_probed = true; }
if (new_device_probed && !err) schedule_work(&mpam_enable_work); if (err) { mpam_disable_reason = "error during probing"; schedule_work(&mpam_broken_work); }
return err;}
static int mpam_cpu_offline(unsigned int cpu){ struct mpam_msc *msc;
if (mpam_resctrl_enabled) mpam_resctrl_offline_cpu(cpu);
guard(srcu)(&mpam_srcu); list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list, srcu_read_lock_held(&mpam_srcu)) { if (!cpumask_test_cpu(cpu, &msc->accessibility)) continue;
if (msc->reenable_error_ppi) disable_percpu_irq(msc->reenable_error_ppi);
if (atomic_dec_and_test(&msc->online_refs)) { struct mpam_msc_ris *ris;
mutex_lock(&msc->cfg_lock); list_for_each_entry_srcu(ris, &msc->ris, msc_list, srcu_read_lock_held(&mpam_srcu)) { mpam_touch_msc(msc, &mpam_reset_ris, ris);
/* * The reset state for non-zero partid may be * lost while the CPUs are offline. */ ris->in_reset_state = false;
if (mpam_is_enabled()) mpam_touch_msc(msc, &mpam_save_mbwu_state, ris); } mutex_unlock(&msc->cfg_lock); } }
return 0;}
static void mpam_register_cpuhp_callbacks(int (*online)(unsigned int online), int (*offline)(unsigned int offline), char *name){ mutex_lock(&mpam_cpuhp_state_lock); if (mpam_cpuhp_state) { cpuhp_remove_state(mpam_cpuhp_state); mpam_cpuhp_state = 0; }
mpam_cpuhp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, name, online, offline); if (mpam_cpuhp_state <= 0) { pr_err("Failed to register cpuhp callbacks"); mpam_cpuhp_state = 0; } mutex_unlock(&mpam_cpuhp_state_lock);}
static int __setup_ppi(struct mpam_msc *msc){ int cpu;
msc->error_dev_id = alloc_percpu(struct mpam_msc *); if (!msc->error_dev_id) return -ENOMEM;
for_each_cpu(cpu, &msc->accessibility) *per_cpu_ptr(msc->error_dev_id, cpu) = msc;
return 0;}
static int mpam_msc_setup_error_irq(struct mpam_msc *msc){ int irq;
irq = platform_get_irq_byname_optional(msc->pdev, "error"); if (irq <= 0) return 0;
/* Allocate and initialise the percpu device pointer for PPI */ if (irq_is_percpu(irq)) return __setup_ppi(msc);
/* sanity check: shared interrupts can be routed anywhere? */ if (!cpumask_equal(&msc->accessibility, cpu_possible_mask)) { pr_err_once("msc:%u is a private resource with a shared error interrupt", msc->id); return -EINVAL; }
return 0;}
/* * An MSC can control traffic from a set of CPUs, but may only be accessible * from a (hopefully wider) set of CPUs. The common reason for this is power * management. If all the CPUs in a cluster are in PSCI:CPU_SUSPEND, the * corresponding cache may also be powered off. By making accesses from * one of those CPUs, we ensure we don't access a cache that's powered off. */static void update_msc_accessibility(struct mpam_msc *msc){ u32 affinity_id; int err;
err = device_property_read_u32(&msc->pdev->dev, "cpu_affinity", &affinity_id); if (err) cpumask_copy(&msc->accessibility, cpu_possible_mask); else acpi_pptt_get_cpus_from_container(affinity_id, &msc->accessibility);}
/* * There are two ways of reaching a struct mpam_msc_ris. Via the * class->component->vmsc->ris, or via the msc. * When destroying the msc, the other side needs unlinking and cleaning up too. */static void mpam_msc_destroy(struct mpam_msc *msc){ struct platform_device *pdev = msc->pdev; struct mpam_msc_ris *ris, *tmp;
lockdep_assert_held(&mpam_list_lock);
list_for_each_entry_safe(ris, tmp, &msc->ris, msc_list) mpam_ris_destroy(ris);
list_del_rcu(&msc->all_msc_list); synchronize_srcu(&mpam_srcu); mpam_pcc_chan_put(msc->pcc_chan); platform_set_drvdata(pdev, NULL);
add_to_garbage(msc);}
static void mpam_msc_drv_remove(struct platform_device *pdev){ struct mpam_msc *msc = platform_get_drvdata(pdev);
if (!msc) return;
mutex_lock(&mpam_list_lock); mpam_msc_destroy(msc); mutex_unlock(&mpam_list_lock);
mpam_free_garbage();}
static struct mpam_msc *do_mpam_msc_drv_probe(struct platform_device *pdev){ int err; u32 pcc_subspace_id; struct mpam_msc *msc; struct resource *msc_res; struct device *dev = &pdev->dev;
lockdep_assert_held(&mpam_list_lock);
msc = devm_kzalloc(&pdev->dev, sizeof(*msc), GFP_KERNEL); if (!msc) return ERR_PTR(-ENOMEM); init_garbage(&msc->garbage); msc->garbage.pdev = pdev;
err = devm_mutex_init(dev, &msc->probe_lock); if (err) return ERR_PTR(err);
err = devm_mutex_init(dev, &msc->part_sel_lock); if (err) return ERR_PTR(err);
err = devm_mutex_init(dev, &msc->error_irq_lock); if (err) return ERR_PTR(err);
err = devm_mutex_init(dev, &msc->cfg_lock); if (err) return ERR_PTR(err);
msc->id = pdev->id; msc->pdev = pdev; INIT_LIST_HEAD_RCU(&msc->all_msc_list); INIT_LIST_HEAD_RCU(&msc->ris);
update_msc_accessibility(msc); if (cpumask_empty(&msc->accessibility)) { dev_err_once(dev, "MSC is not accessible from any CPU!"); return ERR_PTR(-EINVAL); }
err = mpam_msc_setup_error_irq(msc); if (err) return ERR_PTR(err);
if (device_property_read_u32(dev, "pcc-channel", &pcc_subspace_id)) msc->iface = MPAM_IFACE_MMIO; else msc->iface = MPAM_IFACE_PCC;
/* Lock type depends on MSC interface used */ err = mpam_mon_sel_lock_init(dev, msc); if (err) return ERR_PTR(err);
if (msc->iface == MPAM_IFACE_MMIO) { void __iomem *io;
io = devm_platform_get_and_ioremap_resource(pdev, 0, &msc_res); if (IS_ERR(io)) { dev_err_once(dev, "Failed to map MSC base address\n"); return ERR_CAST(io); } msc->mapped_hwpage_sz = msc_res->end - msc_res->start; msc->mapped_hwpage = io; } else if (msc->iface == MPAM_IFACE_PCC) { u32 mpam_fb_msc_id;
msc->pcc_chan = mpam_pcc_chan_get(dev, pcc_subspace_id); if (IS_ERR(msc->pcc_chan)) { pr_err("Failed to request MSC PCC channel\n"); return ERR_CAST(msc->pcc_chan); }
err = mpam_fb_check_shared_buffer_size(msc); if (err) { mpam_pcc_chan_put(msc->pcc_chan);
return ERR_PTR(err); }
err = mpam_fb_check_protocol_version(msc); if (err) { mpam_pcc_chan_put(msc->pcc_chan);
return ERR_PTR(err); }
if (device_property_read_u32(&pdev->dev, "mpam-fb-msc-id", &mpam_fb_msc_id)) mpam_fb_msc_id = msc->id; msc->fb_id = mpam_fb_msc_id; } else { return ERR_PTR(-EINVAL); }
list_add_rcu(&msc->all_msc_list, &mpam_all_msc); platform_set_drvdata(pdev, msc);
return msc;}
static int fw_num_msc;
static int mpam_msc_drv_probe(struct platform_device *pdev){ int err; struct mpam_msc *msc = NULL; void *plat_data = pdev->dev.platform_data;
mutex_lock(&mpam_list_lock); msc = do_mpam_msc_drv_probe(pdev); mutex_unlock(&mpam_list_lock);
if (IS_ERR(msc)) return PTR_ERR(msc);
/* Create RIS entries described by firmware */ err = acpi_mpam_parse_resources(msc, plat_data); if (err) { mpam_msc_drv_remove(pdev); return err; }
if (atomic_add_return(1, &mpam_num_msc) == fw_num_msc) mpam_register_cpuhp_callbacks(mpam_discovery_cpu_online, NULL, "mpam:drv_probe");
return 0;}
static struct platform_driver mpam_msc_driver = { .driver = { .name = "mpam_msc", .suppress_bind_attrs = true, }, .probe = mpam_msc_drv_probe, .remove = mpam_msc_drv_remove,};
/* Any of these features mean the BWA_WD field is valid. */static bool mpam_has_bwa_wd_feature(struct mpam_props *props){ if (mpam_has_feature(mpam_feat_mbw_min, props)) return true; if (mpam_has_feature(mpam_feat_mbw_max, props)) return true; if (mpam_has_feature(mpam_feat_mbw_prop, props)) return true; return false;}
/* Any of these features mean the CMAX_WD field is valid. */static bool mpam_has_cmax_wd_feature(struct mpam_props *props){ if (mpam_has_feature(mpam_feat_cmax_cmax, props)) return true; if (mpam_has_feature(mpam_feat_cmax_cmin, props)) return true; return false;}
#define MISMATCHED_HELPER(parent, child, helper, field, alias) \ helper(parent) && \ ((helper(child) && (parent)->field != (child)->field) || \ (!helper(child) && !(alias)))
#define MISMATCHED_FEAT(parent, child, feat, field, alias) \ mpam_has_feature((feat), (parent)) && \ ((mpam_has_feature((feat), (child)) && (parent)->field != (child)->field) || \ (!mpam_has_feature((feat), (child)) && !(alias)))
#define CAN_MERGE_FEAT(parent, child, feat, alias) \ (alias) && !mpam_has_feature((feat), (parent)) && \ mpam_has_feature((feat), (child))
/* * Combine two props fields. * If this is for controls that alias the same resource, it is safe to just * copy the values over. If two aliasing controls implement the same scheme * a safe value must be picked. * For non-aliasing controls, these control different resources, and the * resulting safe value must be compatible with both. When merging values in * the tree, all the aliasing resources must be handled first. * On mismatch, parent is modified. * Quirks on an MSC will apply to all MSC in that class. */static void __props_mismatch(struct mpam_props *parent, struct mpam_props *child, bool alias){ if (CAN_MERGE_FEAT(parent, child, mpam_feat_cpor_part, alias)) { parent->cpbm_wd = child->cpbm_wd; } else if (MISMATCHED_FEAT(parent, child, mpam_feat_cpor_part, cpbm_wd, alias)) { pr_debug("cleared cpor_part\n"); mpam_clear_feature(mpam_feat_cpor_part, parent); parent->cpbm_wd = 0; }
if (CAN_MERGE_FEAT(parent, child, mpam_feat_mbw_part, alias)) { parent->mbw_pbm_bits = child->mbw_pbm_bits; } else if (MISMATCHED_FEAT(parent, child, mpam_feat_mbw_part, mbw_pbm_bits, alias)) { pr_debug("cleared mbw_part\n"); mpam_clear_feature(mpam_feat_mbw_part, parent); parent->mbw_pbm_bits = 0; }
/* bwa_wd is a count of bits, fewer bits means less precision */ if (alias && !mpam_has_bwa_wd_feature(parent) && mpam_has_bwa_wd_feature(child)) { parent->bwa_wd = child->bwa_wd; } else if (MISMATCHED_HELPER(parent, child, mpam_has_bwa_wd_feature, bwa_wd, alias)) { pr_debug("took the min bwa_wd\n"); parent->bwa_wd = min(parent->bwa_wd, child->bwa_wd); }
if (alias && !mpam_has_cmax_wd_feature(parent) && mpam_has_cmax_wd_feature(child)) { parent->cmax_wd = child->cmax_wd; } else if (MISMATCHED_HELPER(parent, child, mpam_has_cmax_wd_feature, cmax_wd, alias)) { pr_debug("%s took the min cmax_wd\n", __func__); parent->cmax_wd = min(parent->cmax_wd, child->cmax_wd); }
if (CAN_MERGE_FEAT(parent, child, mpam_feat_cmax_cassoc, alias)) { parent->cassoc_wd = child->cassoc_wd; } else if (MISMATCHED_FEAT(parent, child, mpam_feat_cmax_cassoc, cassoc_wd, alias)) { pr_debug("%s cleared cassoc_wd\n", __func__); mpam_clear_feature(mpam_feat_cmax_cassoc, parent); parent->cassoc_wd = 0; }
/* For num properties, take the minimum */ if (CAN_MERGE_FEAT(parent, child, mpam_feat_msmon_csu, alias)) { parent->num_csu_mon = child->num_csu_mon; } else if (MISMATCHED_FEAT(parent, child, mpam_feat_msmon_csu, num_csu_mon, alias)) { pr_debug("took the min num_csu_mon\n"); parent->num_csu_mon = min(parent->num_csu_mon, child->num_csu_mon); }
if (CAN_MERGE_FEAT(parent, child, mpam_feat_msmon_mbwu, alias)) { parent->num_mbwu_mon = child->num_mbwu_mon; } else if (MISMATCHED_FEAT(parent, child, mpam_feat_msmon_mbwu, num_mbwu_mon, alias)) { pr_debug("took the min num_mbwu_mon\n"); parent->num_mbwu_mon = min(parent->num_mbwu_mon, child->num_mbwu_mon); }
if (CAN_MERGE_FEAT(parent, child, mpam_feat_intpri_part, alias)) { parent->intpri_wd = child->intpri_wd; } else if (MISMATCHED_FEAT(parent, child, mpam_feat_intpri_part, intpri_wd, alias)) { pr_debug("%s took the min intpri_wd\n", __func__); parent->intpri_wd = min(parent->intpri_wd, child->intpri_wd); }
if (CAN_MERGE_FEAT(parent, child, mpam_feat_dspri_part, alias)) { parent->dspri_wd = child->dspri_wd; } else if (MISMATCHED_FEAT(parent, child, mpam_feat_dspri_part, dspri_wd, alias)) { pr_debug("%s took the min dspri_wd\n", __func__); parent->dspri_wd = min(parent->dspri_wd, child->dspri_wd); }
/* TODO: alias support for these two */ /* {int,ds}pri may not have differing 0-low behaviour */ if (mpam_has_feature(mpam_feat_intpri_part, parent) && (!mpam_has_feature(mpam_feat_intpri_part, child) || mpam_has_feature(mpam_feat_intpri_part_0_low, parent) != mpam_has_feature(mpam_feat_intpri_part_0_low, child))) { pr_debug("%s cleared intpri_part\n", __func__); mpam_clear_feature(mpam_feat_intpri_part, parent); mpam_clear_feature(mpam_feat_intpri_part_0_low, parent); } if (mpam_has_feature(mpam_feat_dspri_part, parent) && (!mpam_has_feature(mpam_feat_dspri_part, child) || mpam_has_feature(mpam_feat_dspri_part_0_low, parent) != mpam_has_feature(mpam_feat_dspri_part_0_low, child))) { pr_debug("%s cleared dspri_part\n", __func__); mpam_clear_feature(mpam_feat_dspri_part, parent); mpam_clear_feature(mpam_feat_dspri_part_0_low, parent); }
if (alias) { /* Merge features for aliased resources */ bitmap_or(parent->features, parent->features, child->features, MPAM_FEATURE_LAST); } else { /* Clear missing features for non aliasing */ bitmap_and(parent->features, parent->features, child->features, MPAM_FEATURE_LAST); }}
/* * If a vmsc doesn't match class feature/configuration, do the right thing(tm). * For 'num' properties we can just take the minimum. * For properties where the mismatched unused bits would make a difference, we * nobble the class feature, as we can't configure all the resources. * e.g. The L3 cache is composed of two resources with 13 and 17 portion * bitmaps respectively. * Quirks on an MSC will apply to all MSC in that class. */static void__class_props_mismatch(struct mpam_class *class, struct mpam_vmsc *vmsc){ struct mpam_props *cprops = &class->props; struct mpam_props *vprops = &vmsc->props; struct device *dev = &vmsc->msc->pdev->dev;
lockdep_assert_held(&mpam_list_lock); /* we modify class */
dev_dbg(dev, "Merging features for class:0x%lx &= vmsc:0x%lx\n", (long)cprops->features, (long)vprops->features);
/* Merge quirks */ class->quirks |= vmsc->msc->quirks;
/* Take the safe value for any common features */ __props_mismatch(cprops, vprops, false);}
static void__vmsc_props_mismatch(struct mpam_vmsc *vmsc, struct mpam_msc_ris *ris){ struct mpam_props *rprops = &ris->props; struct mpam_props *vprops = &vmsc->props; struct device *dev = &vmsc->msc->pdev->dev;
lockdep_assert_held(&mpam_list_lock); /* we modify vmsc */
dev_dbg(dev, "Merging features for vmsc:0x%lx |= ris:0x%lx\n", (long)vprops->features, (long)rprops->features);
/* * Merge mismatched features - Copy any features that aren't common, * but take the safe value for any common features. */ __props_mismatch(vprops, rprops, true);}
/* * Copy the first component's first vMSC's properties and features to the * class. __class_props_mismatch() will remove conflicts. * It is not possible to have a class with no components, or a component with * no resources. The vMSC properties have already been built. */static void mpam_enable_init_class_features(struct mpam_class *class){ struct mpam_vmsc *vmsc; struct mpam_component *comp;
comp = list_first_entry(&class->components, struct mpam_component, class_list); vmsc = list_first_entry(&comp->vmsc, struct mpam_vmsc, comp_list);
class->props = vmsc->props;}
static void mpam_enable_merge_vmsc_features(struct mpam_component *comp){ struct mpam_vmsc *vmsc; struct mpam_msc_ris *ris; struct mpam_class *class = comp->class;
list_for_each_entry(vmsc, &comp->vmsc, comp_list) { list_for_each_entry(ris, &vmsc->ris, vmsc_list) { __vmsc_props_mismatch(vmsc, ris); class->nrdy_usec = max(class->nrdy_usec, vmsc->msc->nrdy_usec); } }}
static void mpam_enable_merge_class_features(struct mpam_component *comp){ struct mpam_vmsc *vmsc; struct mpam_class *class = comp->class;
list_for_each_entry(vmsc, &comp->vmsc, comp_list) __class_props_mismatch(class, vmsc);
if (mpam_has_quirk(T241_FORCE_MBW_MIN_TO_ONE, class)) mpam_clear_feature(mpam_feat_mbw_min, &class->props);}
/* * Merge all the common resource features into class. * vmsc features are bitwise-or'd together by mpam_enable_merge_vmsc_features() * as the first step so that mpam_enable_init_class_features() can initialise * the class with a representative set of features. * Next the mpam_enable_merge_class_features() bitwise-and's all the vmsc * features to form the class features. * Other features are the min/max as appropriate. * * To avoid walking the whole tree twice, the class->nrdy_usec property is * updated when working with the vmsc as it is a max(), and doesn't need * initialising first. */static void mpam_enable_merge_features(struct list_head *all_classes_list){ struct mpam_class *class; struct mpam_component *comp;
lockdep_assert_held(&mpam_list_lock);
list_for_each_entry(class, all_classes_list, classes_list) { list_for_each_entry(comp, &class->components, class_list) mpam_enable_merge_vmsc_features(comp);
mpam_enable_init_class_features(class);
list_for_each_entry(comp, &class->components, class_list) mpam_enable_merge_class_features(comp); }}
static char *mpam_errcode_names[16] = { [MPAM_ERRCODE_NONE] = "No error", [MPAM_ERRCODE_PARTID_SEL_RANGE] = "PARTID_SEL_Range", [MPAM_ERRCODE_REQ_PARTID_RANGE] = "Req_PARTID_Range", [MPAM_ERRCODE_MSMONCFG_ID_RANGE] = "MSMONCFG_ID_RANGE", [MPAM_ERRCODE_REQ_PMG_RANGE] = "Req_PMG_Range", [MPAM_ERRCODE_MONITOR_RANGE] = "Monitor_Range", [MPAM_ERRCODE_INTPARTID_RANGE] = "intPARTID_Range", [MPAM_ERRCODE_UNEXPECTED_INTERNAL] = "Unexpected_INTERNAL", [MPAM_ERRCODE_UNDEFINED_RIS_PART_SEL] = "Undefined_RIS_PART_SEL", [MPAM_ERRCODE_RIS_NO_CONTROL] = "RIS_No_Control", [MPAM_ERRCODE_UNDEFINED_RIS_MON_SEL] = "Undefined_RIS_MON_SEL", [MPAM_ERRCODE_RIS_NO_MONITOR] = "RIS_No_Monitor", [12 ... 15] = "Reserved"};
/* * This will run as the threaded IRQ handler part when using MPAM-Fb, but * as the sole hard-IRQ handler for MMIO based accesses. */static irqreturn_t __mpam_irq_handler(int irq, struct mpam_msc *msc){ u64 reg; int ret; u16 partid; u8 errcode, pmg, ris;
if (WARN_ON_ONCE(!msc)) return IRQ_NONE;
if (msc->iface == MPAM_IFACE_MMIO && WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility))) return IRQ_NONE;
ret = mpam_msc_read_esr(msc, ®); if (ret) { pr_err_ratelimited("unknown error irq from msc:%u\n", msc->id);
/* Try out best here ... */ goto out_disable; }
errcode = FIELD_GET(MPAMF_ESR_ERRCODE, reg); if (!errcode) return IRQ_NONE;
/* * Clear the level triggered IRQ. If that fails, we cannot do anything * about it, so ignore any errors. We will disable the IRQ either on * the device side or on the irqchip level next anyway. */ mpam_msc_clear_esr(msc);
partid = FIELD_GET(MPAMF_ESR_PARTID_MON, reg); pmg = FIELD_GET(MPAMF_ESR_PMG, reg); ris = FIELD_GET(MPAMF_ESR_RIS, reg);
pr_err_ratelimited("error irq from msc:%u '%s', partid:%u, pmg: %u, ris: %u\n", msc->id, mpam_errcode_names[errcode], partid, pmg, ris);
out_disable: /* * Disable this interrupt on the device side. If that fails, disable * the IRQ on the irqchip level, as we must prevent further handler * invocations. We only take the interrupt once anyway, as we * are going to free the IRQ next, in mpam_disable(). */ ret = mpam_disable_msc_ecr(msc); if (ret) disable_irq_nosync(irq);
/* Check whether we are racing with the thread disabling MPAM. */ if (!mpam_is_enabled()) return IRQ_HANDLED;
/* * Schedule the teardown work. We have to defer it as we can't * unregister the interrupt from the threaded part of a handler. */ mpam_disable_reason = "hardware error interrupt"; schedule_work(&mpam_broken_work);
return IRQ_HANDLED;}
static irqreturn_t mpam_ppi_handler(int irq, void *dev_id){ struct mpam_msc *msc = *(struct mpam_msc **)dev_id;
return __mpam_irq_handler(irq, msc);}
/* * MMIO based MSC accesses must run in non-preemptible context, as they * might have affinity requirements to check. * MPAM-Fb based MSC accesses must NOT run in hard-IRQ context, as they * can sleep. * So split the IRQ handling up, depending on the MSC access type. */static irqreturn_t mpam_shared_hard_irq(int irq, void *dev_id){ struct mpam_msc *msc = dev_id;
if (msc->iface == MPAM_IFACE_MMIO) return __mpam_irq_handler(irq, msc);
return IRQ_WAKE_THREAD;}
static irqreturn_t mpam_shared_thread_irq(int irq, void *dev_id){ struct mpam_msc *msc = dev_id;
if (msc->iface == MPAM_IFACE_MMIO) return IRQ_HANDLED;
return __mpam_irq_handler(irq, msc);}
static int mpam_register_irqs(void){ int err, irq; struct mpam_msc *msc;
lockdep_assert_cpus_held();
guard(srcu)(&mpam_srcu); list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list, srcu_read_lock_held(&mpam_srcu)) { irq = platform_get_irq_byname_optional(msc->pdev, "error"); if (irq <= 0) continue;
/* The MPAM spec says the interrupt can be SPI, PPI or LPI */ /* We anticipate sharing the interrupt with other MSCs */ if (irq_is_percpu(irq)) { if (msc->iface != MPAM_IFACE_MMIO) { dev_err(&msc->pdev->dev, "Only MMIO MSCs can use per-CPU interrupts\n"); return -EINVAL; } err = request_percpu_irq(irq, &mpam_ppi_handler, "mpam:msc:error", msc->error_dev_id); if (err) return err;
msc->reenable_error_ppi = irq; smp_call_function_many(&msc->accessibility, &_enable_percpu_irq, &irq, true); } else { err = devm_request_threaded_irq(&msc->pdev->dev, irq, &mpam_shared_hard_irq, &mpam_shared_thread_irq, IRQF_SHARED | IRQF_ONESHOT, "mpam:msc:error", msc); if (err) return err; }
guard(mutex)(&msc->error_irq_lock); msc->error_irq_req = true; err = mpam_touch_msc(msc, mpam_enable_msc_ecr, msc); if (err) return err; msc->error_irq_hw_enabled = true; }
return 0;}
static void mpam_unregister_irqs(void){ int irq; struct mpam_msc *msc;
guard(cpus_read_lock)(); guard(srcu)(&mpam_srcu); list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list, srcu_read_lock_held(&mpam_srcu)) { irq = platform_get_irq_byname_optional(msc->pdev, "error"); if (irq <= 0) continue;
mutex_lock(&msc->error_irq_lock); if (msc->error_irq_hw_enabled) { if (!mpam_touch_msc(msc, mpam_disable_msc_ecr, msc)) msc->error_irq_hw_enabled = false; }
if (msc->error_irq_req) { if (irq_is_percpu(irq)) { msc->reenable_error_ppi = 0; free_percpu_irq(irq, msc->error_dev_id); } else { devm_free_irq(&msc->pdev->dev, irq, msc); } msc->error_irq_req = false; } mutex_unlock(&msc->error_irq_lock); }}
static void __destroy_component_cfg(struct mpam_component *comp){ struct mpam_msc *msc; struct mpam_vmsc *vmsc; struct mpam_msc_ris *ris;
lockdep_assert_held(&mpam_list_lock);
if (!comp->cfg) return;
add_to_garbage(comp->cfg); list_for_each_entry(vmsc, &comp->vmsc, comp_list) { msc = vmsc->msc;
if (mpam_mon_sel_lock(msc)) { list_for_each_entry(ris, &vmsc->ris, vmsc_list) { if (ris->mbwu_state) add_to_garbage(ris->mbwu_state); } mpam_mon_sel_unlock(msc); } }}
static void mpam_reset_component_cfg(struct mpam_component *comp){ int i; struct mpam_props *cprops = &comp->class->props;
mpam_assert_partid_sizes_fixed();
if (!comp->cfg) return;
for (i = 0; i <= mpam_partid_max; i++) { comp->cfg[i] = (struct mpam_config) {}; if (cprops->cpbm_wd) comp->cfg[i].cpbm = GENMASK(cprops->cpbm_wd - 1, 0); if (cprops->mbw_pbm_bits) comp->cfg[i].mbw_pbm = GENMASK(cprops->mbw_pbm_bits - 1, 0); if (cprops->bwa_wd) comp->cfg[i].mbw_max = GENMASK(15, 16 - cprops->bwa_wd); }}
static int __allocate_component_cfg(struct mpam_component *comp){ struct mpam_vmsc *vmsc;
mpam_assert_partid_sizes_fixed();
if (comp->cfg) return 0;
comp->cfg = kvzalloc_objs(*comp->cfg, mpam_partid_max + 1); if (!comp->cfg) return -ENOMEM;
/* * The array is free()d in one go, so only cfg[0]'s structure needs * to be initialised. */ init_garbage(&comp->cfg[0].garbage);
mpam_reset_component_cfg(comp);
list_for_each_entry(vmsc, &comp->vmsc, comp_list) { struct mpam_msc *msc; struct mpam_msc_ris *ris; struct msmon_mbwu_state *mbwu_state;
if (!vmsc->props.num_mbwu_mon) continue;
msc = vmsc->msc; list_for_each_entry(ris, &vmsc->ris, vmsc_list) { if (!ris->props.num_mbwu_mon) continue;
mbwu_state = kzalloc_objs(*ris->mbwu_state, ris->props.num_mbwu_mon); if (!mbwu_state) { __destroy_component_cfg(comp); return -ENOMEM; }
init_garbage(&mbwu_state[0].garbage);
if (mpam_mon_sel_lock(msc)) { ris->mbwu_state = mbwu_state; mpam_mon_sel_unlock(msc); } } }
return 0;}
static int mpam_allocate_config(void){ struct mpam_class *class; struct mpam_component *comp;
lockdep_assert_held(&mpam_list_lock);
list_for_each_entry(class, &mpam_classes, classes_list) { list_for_each_entry(comp, &class->components, class_list) { int err = __allocate_component_cfg(comp); if (err) return err; } }
return 0;}
static void mpam_enable_once(void){ int err;
/* * Once the cpuhp callbacks have been changed, mpam_partid_max can no * longer change. */ spin_lock(&partid_max_lock); partid_max_published = true; spin_unlock(&partid_max_lock);
/* * If all the MSC have been probed, enabling the IRQs happens next. * That involves cross-calling to a CPU that can reach the MSC, and * the locks must be taken in this order: */ cpus_read_lock(); mutex_lock(&mpam_list_lock); do { mpam_enable_merge_features(&mpam_classes);
err = mpam_register_irqs(); if (err) { pr_warn("Failed to register irqs: %d\n", err); break; }
err = mpam_allocate_config(); if (err) { pr_err("Failed to allocate configuration arrays.\n"); break; } } while (0); mutex_unlock(&mpam_list_lock); cpus_read_unlock();
if (!err) { err = mpam_resctrl_setup(); if (err) pr_err("Failed to initialise resctrl: %d\n", err); }
if (err) { mpam_disable_reason = "Failed to enable."; schedule_work(&mpam_broken_work); return; }
static_branch_enable(&mpam_enabled); mpam_resctrl_enabled = true; mpam_register_cpuhp_callbacks(mpam_cpu_online, mpam_cpu_offline, "mpam:online");
/* Use printk() to avoid the pr_fmt adding the function name. */ printk(KERN_INFO "MPAM enabled with %u PARTIDs and %u PMGs\n", mpam_partid_max + 1, mpam_pmg_max + 1);}
static int mpam_reset_component_locked(struct mpam_component *comp){ struct mpam_vmsc *vmsc;
lockdep_assert_cpus_held(); mpam_assert_partid_sizes_fixed();
mpam_reset_component_cfg(comp);
guard(srcu)(&mpam_srcu); list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list, srcu_read_lock_held(&mpam_srcu)) { struct mpam_msc *msc = vmsc->msc; struct mpam_msc_ris *ris; int ret;
list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list, srcu_read_lock_held(&mpam_srcu)) { if (!ris->in_reset_state) { ret = mpam_touch_msc(msc, mpam_reset_ris, ris); if (ret) return ret; } ris->in_reset_state = true; } }
return 0;}
int mpam_reset_class_locked(struct mpam_class *class){ struct mpam_component *comp; int ret;
lockdep_assert_cpus_held();
guard(srcu)(&mpam_srcu); list_for_each_entry_srcu(comp, &class->components, class_list, srcu_read_lock_held(&mpam_srcu)) { ret = mpam_reset_component_locked(comp); if (ret) return ret; }
return 0;}
static void mpam_reset_class(struct mpam_class *class){ cpus_read_lock(); mpam_reset_class_locked(class); cpus_read_unlock();}
/* * Called in response to an error IRQ. * All of MPAMs errors indicate a software bug, restore any modified * controls to their reset values. */void mpam_disable(struct work_struct *ignored){ int idx; bool do_resctrl_exit; struct mpam_class *class; struct mpam_msc *msc, *tmp;
if (mpam_is_enabled()) static_branch_disable(&mpam_enabled);
mutex_lock(&mpam_cpuhp_state_lock); if (mpam_cpuhp_state) { cpuhp_remove_state(mpam_cpuhp_state); mpam_cpuhp_state = 0; }
/* * Removing the cpuhp state called mpam_cpu_offline() and told resctrl * all the CPUs are offline. */ do_resctrl_exit = mpam_resctrl_enabled; mpam_resctrl_enabled = false; mutex_unlock(&mpam_cpuhp_state_lock);
if (do_resctrl_exit) mpam_resctrl_exit();
mpam_unregister_irqs();
idx = srcu_read_lock(&mpam_srcu); list_for_each_entry_srcu(class, &mpam_classes, classes_list, srcu_read_lock_held(&mpam_srcu)) { mpam_reset_class(class); if (do_resctrl_exit) mpam_resctrl_teardown_class(class); } srcu_read_unlock(&mpam_srcu, idx);
mutex_lock(&mpam_list_lock); list_for_each_entry_safe(msc, tmp, &mpam_all_msc, all_msc_list) mpam_msc_destroy(msc); mutex_unlock(&mpam_list_lock); mpam_free_garbage();
if (mpam_disable_errno || mpam_disable_mpam_fb_err) pr_err_once("MPAM disabled due to %s: %s, MPAM-Fb error %d\n", mpam_disable_reason, errname(mpam_disable_errno), mpam_disable_mpam_fb_err); else pr_err_once("MPAM disabled due to %s\n", mpam_disable_reason);}
/* * Enable mpam once all devices have been probed. * Scheduled by mpam_discovery_cpu_online() once all devices have been created. * Also scheduled when new devices are probed when new CPUs come online. */void mpam_enable(struct work_struct *work){ static atomic_t once; struct mpam_msc *msc; bool all_devices_probed = true;
/* Have we probed all the hw devices? */ guard(srcu)(&mpam_srcu); list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list, srcu_read_lock_held(&mpam_srcu)) { mutex_lock(&msc->probe_lock); if (!msc->probed) all_devices_probed = false; mutex_unlock(&msc->probe_lock);
if (!all_devices_probed) break; }
if (all_devices_probed && !atomic_fetch_inc(&once)) mpam_enable_once();}
#define maybe_update_config(cfg, feature, newcfg, member, changes) do { \ if (mpam_has_feature(feature, newcfg) && \ (newcfg)->member != (cfg)->member) { \ (cfg)->member = (newcfg)->member; \ mpam_set_feature(feature, cfg); \ \ (changes) = true; \ } \} while (0)
static bool mpam_update_config(struct mpam_config *cfg, const struct mpam_config *newcfg){ bool has_changes = false;
maybe_update_config(cfg, mpam_feat_cpor_part, newcfg, cpbm, has_changes); maybe_update_config(cfg, mpam_feat_mbw_part, newcfg, mbw_pbm, has_changes); maybe_update_config(cfg, mpam_feat_mbw_max, newcfg, mbw_max, has_changes);
return has_changes;}
int mpam_apply_config(struct mpam_component *comp, u16 partid, struct mpam_config *cfg){ struct mpam_write_config_arg arg; struct mpam_msc_ris *ris; struct mpam_vmsc *vmsc; struct mpam_msc *msc; int ret;
lockdep_assert_cpus_held();
/* Don't pass in the current config! */ WARN_ON_ONCE(&comp->cfg[partid] == cfg);
if (!mpam_update_config(&comp->cfg[partid], cfg)) return 0;
arg.comp = comp; arg.partid = partid;
guard(srcu)(&mpam_srcu); list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list, srcu_read_lock_held(&mpam_srcu)) { msc = vmsc->msc;
guard(mutex)(&msc->cfg_lock); list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list, srcu_read_lock_held(&mpam_srcu)) { arg.ris = ris; ret = mpam_touch_msc(msc, __write_config, &arg); if (ret) return ret; ris->in_reset_state = false; } }
return 0;}
static int __init mpam_msc_driver_init(void){ if (!system_supports_mpam()) return -EOPNOTSUPP;
init_srcu_struct(&mpam_srcu);
fw_num_msc = acpi_mpam_count_msc(); if (fw_num_msc <= 0) { pr_err("No MSC devices found in firmware\n"); return -EINVAL; }
return platform_driver_register(&mpam_msc_driver);}
/* Must occur after arm64_mpam_register_cpus() from arch_initcall() */subsys_initcall(mpam_msc_driver_init);
#ifdef CONFIG_MPAM_KUNIT_TEST#include "test_mpam_devices.c"#endif