Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (c) 2021, NVIDIA Corporation. */
#include <linux/completion.h>#include <linux/device.h>#include <linux/host1x_context_bus.h>#include <linux/kref.h>#include <linux/list.h>#include <linux/moduleparam.h>#include <linux/of.h>#include <linux/of_device.h>#include <linux/pid.h>#include <linux/slab.h>
#include "context.h"#include "dev.h"
static bool static_context_alloc;module_param(static_context_alloc, bool, 0644);MODULE_PARM_DESC(static_context_alloc, "If enabled, memory contexts are allocated immediately on channel open and cannot be relinquished while a channel is open");
static void host1x_memory_context_release(struct device *dev){ /* context device is freed in host1x_memory_context_list_free() */}
int host1x_memory_context_list_init(struct host1x *host1x){ struct host1x_memory_context_list *cdl = &host1x->context_list; struct device_node *node = host1x->dev->of_node; struct host1x_hw_memory_context *ctx; unsigned int devs, i; int err;
cdl->devs = NULL; cdl->len = 0; mutex_init(&cdl->lock); INIT_LIST_HEAD(&cdl->waiters);
err = of_property_count_u32_elems(node, "iommu-map"); if (err < 0) return 0;
devs = 0;
for (i = 0; i < err / 4; i++) { u32 length;
of_property_read_u32_index(node, "iommu-map", i * 4 + 3, &length); devs += length; }
cdl->len = devs; cdl->devs = kzalloc_objs(*cdl->devs, cdl->len); if (!cdl->devs) return -ENOMEM;
for (i = 0; i < cdl->len; i++) { ctx = &cdl->devs[i];
ctx->host = host1x; INIT_LIST_HEAD(&ctx->owners);
device_initialize(&ctx->dev);
/* * Due to an issue with T194 NVENC, only 38 bits can be used. * Anyway, 256GiB of IOVA ought to be enough for anyone. */ ctx->dma_mask = DMA_BIT_MASK(38); ctx->dev.dma_mask = &ctx->dma_mask; ctx->dev.coherent_dma_mask = ctx->dma_mask; dev_set_name(&ctx->dev, "host1x-ctx.%d", i); ctx->dev.bus = &host1x_context_device_bus_type; ctx->dev.parent = host1x->dev; ctx->dev.release = host1x_memory_context_release;
ctx->dev.dma_parms = &ctx->dma_parms; dma_set_max_seg_size(&ctx->dev, UINT_MAX);
err = device_add(&ctx->dev); if (err) { dev_err(host1x->dev, "could not add context device %d: %d\n", i, err); put_device(&ctx->dev); goto unreg_devices; }
err = of_dma_configure_id(&ctx->dev, node, true, &i); if (err) { dev_err(host1x->dev, "IOMMU configuration failed for context device %d: %d\n", i, err); device_unregister(&ctx->dev); goto unreg_devices; }
if (!tegra_dev_iommu_get_stream_id(&ctx->dev, &ctx->stream_id) || !device_iommu_mapped(&ctx->dev)) { dev_err(host1x->dev, "Context device %d has no IOMMU!\n", i); device_unregister(&ctx->dev);
/* * This means that if IOMMU is disabled but context devices * are defined in the device tree, Host1x will fail to probe. * That's probably OK in this time and age. */ err = -EINVAL;
goto unreg_devices; } }
return 0;
unreg_devices: while (i--) device_unregister(&cdl->devs[i].dev);
kfree(cdl->devs); cdl->devs = NULL; cdl->len = 0;
return err;}
void host1x_memory_context_list_free(struct host1x_memory_context_list *cdl){ unsigned int i;
for (i = 0; i < cdl->len; i++) device_unregister(&cdl->devs[i].dev);
kfree(cdl->devs); cdl->len = 0;}
static bool hw_usable_for_dev(struct host1x_hw_memory_context *hw, struct device *dev){ return hw->dev.iommu->iommu_dev == dev->iommu->iommu_dev;}
static struct host1x_hw_memory_context *host1x_memory_context_alloc_hw_locked(struct host1x *host1x, struct device *dev, struct pid *pid){ struct host1x_memory_context_list *cdl = &host1x->context_list; struct host1x_hw_memory_context *free = NULL, *can_steal = NULL; struct host1x_memory_context *ctx; int i;
if (!cdl->len) return ERR_PTR(-EOPNOTSUPP);
for (i = 0; i < cdl->len; i++) { struct host1x_hw_memory_context *cd = &cdl->devs[i];
if (!hw_usable_for_dev(cd, dev)) continue;
if (cd->owner == pid) { refcount_inc(&cd->ref); return cd; } else if (!cd->owner && !free) { free = cd; } else if (refcount_read(&cd->active) == 0) { can_steal = cd; } }
if (free) goto found;
/* Steal */
if (!can_steal || static_context_alloc) return ERR_PTR(-EBUSY);
list_for_each_entry(ctx, &can_steal->owners, entry) { struct host1x_context_mapping *mapping;
ctx->hw = NULL;
list_for_each_entry(mapping, &ctx->mappings, entry) { host1x_bo_unpin(mapping->mapping); mapping->mapping = NULL; } }
put_pid(can_steal->owner);
free = can_steal;
found: refcount_set(&free->ref, 1); free->owner = get_pid(pid); INIT_LIST_HEAD(&free->owners);
return free;}
static void host1x_memory_context_hw_put(struct host1x_hw_memory_context *cd){ if (refcount_dec_and_test(&cd->ref)) { put_pid(cd->owner); cd->owner = NULL; }}
/** * host1x_memory_context_alloc() - allocate a logical memory context * @host1x: host1x instance * @dev: engine device that will own the context * @pid: PID (TGID) of the process the context is allocated for * * Allocate a logical memory context. The context has no backing hardware memory * context until host1x_memory_context_active() is called unless static context * allocation is enabled. * * Return: pointer to the new context, or an ERR_PTR() on failure. * ERR_PTR(-EOPNOTSUPP) is returned if the platform has no hardware memory * contexts. */struct host1x_memory_context *host1x_memory_context_alloc(struct host1x *host1x, struct device *dev, struct pid *pid){ struct host1x_memory_context_list *cdl = &host1x->context_list; struct host1x_memory_context *ctx; int err;
if (!cdl->len) return ERR_PTR(-EOPNOTSUPP);
ctx = kzalloc_obj(*ctx, GFP_KERNEL); if (!ctx) return ERR_PTR(-ENOMEM);
ctx->host = host1x; ctx->dev = dev; ctx->pid = get_pid(pid);
refcount_set(&ctx->ref, 1); INIT_LIST_HEAD(&ctx->mappings);
if (static_context_alloc) { err = host1x_memory_context_active(ctx); if (err) { put_pid(ctx->pid); kfree(ctx); return ERR_PTR(err); }
ctx->static_alloc = true; }
return ctx;}EXPORT_SYMBOL_GPL(host1x_memory_context_alloc);
struct hw_alloc_waiter { struct completion wait; /* Completion to wait for free hw context */ struct list_head entry; struct device *dev;};
/** * host1x_memory_context_active() - mark a memory context as in use * @ctx: memory context * * Bind a hardware memory context to @ctx and map all of its stored mappings, * allocating or stealing a hardware context from an idle context if needed. * While the context is active its hardware context cannot be stolen away. * * Each successful call must be balanced by exactly one * host1x_memory_context_inactive() call. * * May sleep, and may block until a hardware context becomes available unless * static allocation is in use. Must be called in process context. * * Return: 0 on success or a negative error code. -EBUSY is returned if no * hardware context could be obtained and static context allocation is enabled. */int host1x_memory_context_active(struct host1x_memory_context *ctx){ struct host1x_memory_context_list *cdl = &ctx->host->context_list; struct host1x_context_mapping *mapping; struct host1x_hw_memory_context *hw; struct hw_alloc_waiter waiter; bool retrying = false; int err = 0;
mutex_lock(&cdl->lock);
retry: if (ctx->hw) goto have_hw;
hw = host1x_memory_context_alloc_hw_locked(ctx->host, ctx->dev, ctx->pid); if (PTR_ERR(hw) == -EBUSY) { /* All contexts busy. Wait for free context. */ if (static_context_alloc) { dev_warn(ctx->dev, "%s: all memory contexts are busy\n", current->comm); err = -EBUSY; goto unlock; } if (!retrying) dev_warn(ctx->dev, "%s: all memory contexts are busy, waiting\n", current->comm);
init_completion(&waiter.wait); waiter.dev = ctx->dev; list_add(&waiter.entry, &cdl->waiters);
mutex_unlock(&cdl->lock); err = wait_for_completion_interruptible(&waiter.wait); mutex_lock(&cdl->lock);
list_del(&waiter.entry); if (err) goto unlock;
retrying = true; goto retry; } if (IS_ERR(hw)) { err = PTR_ERR(hw); goto unlock; }
ctx->hw = hw; list_add(&ctx->entry, &hw->owners);
list_for_each_entry(mapping, &ctx->mappings, entry) { mapping->mapping = host1x_bo_pin(&hw->dev, mapping->bo, mapping->direction, NULL); if (IS_ERR(mapping->mapping)) { err = PTR_ERR(mapping->mapping); mapping->mapping = NULL; goto unpin; } }
have_hw: /* We're just counting active users. No need to do anything on 0->1 transition.*/ if (!refcount_inc_not_zero(&ctx->hw->active)) refcount_set(&ctx->hw->active, 1);
mutex_unlock(&cdl->lock);
return 0;
unpin: list_for_each_entry(mapping, &ctx->mappings, entry) { if (mapping->mapping) { host1x_bo_unpin(mapping->mapping); mapping->mapping = NULL; } }
host1x_memory_context_hw_put(ctx->hw); list_del(&ctx->entry); ctx->hw = NULL;unlock: mutex_unlock(&cdl->lock); return err;}EXPORT_SYMBOL_GPL(host1x_memory_context_active);
/** * host1x_memory_context_map() - add a buffer mapping to a memory context * @ctx: memory context * @bo: buffer object to map * @direction: DMA direction of the mapping * * Register a mapping of @bo on @ctx. If @ctx currently has a hardware context * bound, the buffer is mapped immediately; otherwise it is pinned the next time * the context is made active. The mapping is unmapped and freed by * host1x_memory_context_unmap(). * * Return: pointer to the mapping, or an ERR_PTR() on failure. */struct host1x_context_mapping *host1x_memory_context_map(struct host1x_memory_context *ctx, struct host1x_bo *bo, enum dma_data_direction direction){ struct host1x_memory_context_list *cdl = &ctx->host->context_list; struct host1x_context_mapping *m; struct host1x_bo_mapping *bo_m;
m = kzalloc_obj(*m, GFP_KERNEL); if (!m) return ERR_PTR(-ENOMEM);
m->host = ctx->host; m->bo = bo; m->direction = direction;
mutex_lock(&cdl->lock);
if (ctx->hw) { bo_m = host1x_bo_pin(&ctx->hw->dev, bo, direction, NULL); if (IS_ERR(bo_m)) { mutex_unlock(&cdl->lock); kfree(m);
return ERR_CAST(bo_m); }
m->mapping = bo_m; }
list_add(&m->entry, &ctx->mappings);
mutex_unlock(&cdl->lock);
return m;}EXPORT_SYMBOL_GPL(host1x_memory_context_map);
/** * host1x_memory_context_unmap() - remove a buffer mapping from a memory context * @m: mapping previously returned by host1x_memory_context_map() * * Unmap the buffer if it is currently mapped, and free the mapping. */void host1x_memory_context_unmap(struct host1x_context_mapping *m){ struct host1x_memory_context_list *cdl = &m->host->context_list;
mutex_lock(&cdl->lock);
list_del(&m->entry);
mutex_unlock(&cdl->lock);
if (m->mapping) host1x_bo_unpin(m->mapping);
kfree(m);}EXPORT_SYMBOL_GPL(host1x_memory_context_unmap);
static void host1x_memory_context_inactive_locked(struct host1x_memory_context *ctx){ struct host1x_memory_context_list *cdl = &ctx->host->context_list; struct hw_alloc_waiter *waiter;
if (refcount_dec_and_test(&ctx->hw->active)) { /* Hardware context becomes eligible for stealing */ list_for_each_entry(waiter, &cdl->waiters, entry) { if (!hw_usable_for_dev(ctx->hw, waiter->dev)) continue;
complete(&waiter->wait);
/* * Need to wake up all waiters -- there could be multiple from * the same process that can use the same freed hardware context. */ } }}
/** * host1x_memory_context_inactive() - mark a memory context as no longer in use * @ctx: memory context * * Drop one active reference taken by host1x_memory_context_active(). When the * last active reference is dropped, the backing hardware context becomes * eligible to be stolen by another context, and any waiters are woken. * * Must be called exactly once for each successful * host1x_memory_context_active(). */void host1x_memory_context_inactive(struct host1x_memory_context *ctx){ struct host1x_memory_context_list *cdl = &ctx->host->context_list;
mutex_lock(&cdl->lock);
host1x_memory_context_inactive_locked(ctx);
mutex_unlock(&cdl->lock);}EXPORT_SYMBOL_GPL(host1x_memory_context_inactive);
/** * host1x_memory_context_get() - take a reference to a memory context * @ctx: memory context */void host1x_memory_context_get(struct host1x_memory_context *ctx){ refcount_inc(&ctx->ref);}EXPORT_SYMBOL_GPL(host1x_memory_context_get);
/** * host1x_memory_context_put() - release a reference to a memory context * @ctx: memory context * * Drop a reference taken by host1x_memory_context_alloc() or * host1x_memory_context_get(). When the last reference is dropped the context * is detached from its hardware context and freed. */void host1x_memory_context_put(struct host1x_memory_context *ctx){ struct host1x_memory_context_list *cdl = &ctx->host->context_list;
if (refcount_dec_and_mutex_lock(&ctx->ref, &cdl->lock)) { if (ctx->static_alloc) host1x_memory_context_inactive_locked(ctx);
if (ctx->hw) { list_del(&ctx->entry);
host1x_memory_context_hw_put(ctx->hw); ctx->hw = NULL;
WARN_ON(!list_empty(&ctx->mappings)); }
put_pid(ctx->pid); mutex_unlock(&cdl->lock); kfree(ctx); }}EXPORT_SYMBOL_GPL(host1x_memory_context_put);