Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2020 NVIDIA Corporation */
#include <linux/dma-fence-unwrap.h>#include <linux/dma-mapping.h>#include <linux/file.h>#include <linux/host1x.h>#include <linux/iommu.h>#include <linux/kref.h>#include <linux/list.h>#include <linux/nospec.h>#include <linux/pm_runtime.h>#include <linux/scatterlist.h>#include <linux/slab.h>
#include <drm/drm_drv.h>#include <drm/drm_file.h>#include <drm/drm_syncobj.h>
#include "drm.h"#include "gem.h"#include "submit.h"#include "uapi.h"
#define SUBMIT_ERR(context, fmt, ...) \ dev_err_ratelimited(context->client->base.dev, \ "%s: job submission failed: " fmt "\n", \ current->comm, ##__VA_ARGS__)
struct gather_bo { struct host1x_bo base;
struct kref ref;
struct device *dev; u32 *gather_data; dma_addr_t gather_data_dma; size_t gather_data_words;};
static struct host1x_bo *gather_bo_get(struct host1x_bo *host_bo){ struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
kref_get(&bo->ref);
return host_bo;}
static void gather_bo_release(struct kref *ref){ struct gather_bo *bo = container_of(ref, struct gather_bo, ref);
dma_free_attrs(bo->dev, bo->gather_data_words * 4, bo->gather_data, bo->gather_data_dma, 0); kfree(bo);}
static void gather_bo_put(struct host1x_bo *host_bo){ struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
kref_put(&bo->ref, gather_bo_release);}
static struct host1x_bo_mapping *gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction){ struct gather_bo *gather = container_of(bo, struct gather_bo, base); struct host1x_bo_mapping *map; int err;
map = kzalloc_obj(*map); if (!map) return ERR_PTR(-ENOMEM);
kref_init(&map->ref); map->bo = bo; map->direction = direction; map->dev = dev;
map->sgt = kzalloc_obj(*map->sgt); if (!map->sgt) { err = -ENOMEM; goto free; }
err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma, gather->gather_data_words * 4); if (err) goto free_sgt;
err = dma_map_sgtable(dev, map->sgt, direction, 0); if (err) goto free_sgt;
map->phys = sg_dma_address(map->sgt->sgl); map->size = gather->gather_data_words * 4; map->chunks = err;
return map;
free_sgt: sg_free_table(map->sgt); kfree(map->sgt);free: kfree(map); return ERR_PTR(err);}
static void gather_bo_unpin(struct host1x_bo_mapping *map){ if (!map) return;
dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); sg_free_table(map->sgt); kfree(map->sgt);
kfree(map);}
static void *gather_bo_mmap(struct host1x_bo *host_bo){ struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
return bo->gather_data;}
static void gather_bo_munmap(struct host1x_bo *host_bo, void *addr){}
static const struct host1x_bo_ops gather_bo_ops = { .get = gather_bo_get, .put = gather_bo_put, .pin = gather_bo_pin, .unpin = gather_bo_unpin, .mmap = gather_bo_mmap, .munmap = gather_bo_munmap,};
static struct tegra_drm_mapping *tegra_drm_mapping_get(struct tegra_drm_context *context, u32 id){ struct tegra_drm_mapping *mapping;
xa_lock(&context->mappings);
mapping = xa_load(&context->mappings, id); if (mapping) kref_get(&mapping->ref);
xa_unlock(&context->mappings);
return mapping;}
static void *alloc_copy_user_array(void __user *from, size_t count, size_t size){ size_t copy_len; void *data;
if (check_mul_overflow(count, size, ©_len)) return ERR_PTR(-EINVAL);
if (copy_len > 0x4000) return ERR_PTR(-E2BIG);
data = vmemdup_user(from, copy_len); if (IS_ERR(data)) return ERR_CAST(data);
return data;}
static int submit_copy_gather_data(struct gather_bo **pbo, struct device *dev, struct tegra_drm_context *context, struct drm_tegra_channel_submit *args){ struct gather_bo *bo; size_t copy_len;
if (args->gather_data_words == 0) { SUBMIT_ERR(context, "gather_data_words cannot be zero"); return -EINVAL; }
if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, ©_len)) { SUBMIT_ERR(context, "gather_data_words is too large"); return -EINVAL; }
bo = kzalloc_obj(*bo); if (!bo) { SUBMIT_ERR(context, "failed to allocate memory for bo info"); return -ENOMEM; }
host1x_bo_init(&bo->base, &gather_bo_ops); kref_init(&bo->ref); bo->dev = dev;
bo->gather_data = dma_alloc_attrs(dev, copy_len, &bo->gather_data_dma, GFP_KERNEL | __GFP_NOWARN, 0); if (!bo->gather_data) { SUBMIT_ERR(context, "failed to allocate memory for gather data"); kfree(bo); return -ENOMEM; }
if (copy_from_user(bo->gather_data, u64_to_user_ptr(args->gather_data_ptr), copy_len)) { SUBMIT_ERR(context, "failed to copy gather data from userspace"); dma_free_attrs(dev, copy_len, bo->gather_data, bo->gather_data_dma, 0); kfree(bo); return -EFAULT; }
bo->gather_data_words = args->gather_data_words;
*pbo = bo;
return 0;}
static int submit_write_reloc(struct tegra_drm_context *context, struct gather_bo *bo, struct drm_tegra_submit_buf *buf, struct tegra_drm_mapping *mapping){ /* TODO check that target_offset is within bounds */ dma_addr_t iova = buf->reloc.target_offset; u32 written_ptr;
if (mapping->bo_map) iova += mapping->iova; else iova += mapping->ctx_map->mapping->phys;
#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT if (buf->flags & DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) iova |= BIT_ULL(39);#endif
written_ptr = iova >> buf->reloc.shift;
if (buf->reloc.gather_offset_words >= bo->gather_data_words) { SUBMIT_ERR(context, "relocation has too large gather offset (%u vs gather length %zu)", buf->reloc.gather_offset_words, bo->gather_data_words); return -EINVAL; }
buf->reloc.gather_offset_words = array_index_nospec(buf->reloc.gather_offset_words, bo->gather_data_words);
bo->gather_data[buf->reloc.gather_offset_words] = written_ptr;
return 0;}
static int submit_process_bufs(struct tegra_drm_context *context, struct gather_bo *bo, struct drm_tegra_channel_submit *args, struct tegra_drm_submit_data *job_data){ struct tegra_drm_used_mapping *mappings; struct drm_tegra_submit_buf *bufs; int err; u32 i;
bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr), args->num_bufs, sizeof(*bufs)); if (IS_ERR(bufs)) { SUBMIT_ERR(context, "failed to copy bufs array from userspace"); return PTR_ERR(bufs); }
mappings = kzalloc_objs(*mappings, args->num_bufs); if (!mappings) { SUBMIT_ERR(context, "failed to allocate memory for mapping info"); err = -ENOMEM; goto done; }
for (i = 0; i < args->num_bufs; i++) { struct drm_tegra_submit_buf *buf = &bufs[i]; struct tegra_drm_mapping *mapping;
if (buf->flags & ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) { SUBMIT_ERR(context, "invalid flag specified for buffer"); err = -EINVAL; goto drop_refs; }
mapping = tegra_drm_mapping_get(context, buf->mapping); if (!mapping) { SUBMIT_ERR(context, "invalid mapping ID '%u' for buffer", buf->mapping); err = -EINVAL; goto drop_refs; }
err = submit_write_reloc(context, bo, buf, mapping); if (err) { tegra_drm_mapping_put(mapping); goto drop_refs; }
mappings[i].mapping = mapping; mappings[i].flags = buf->flags; }
job_data->used_mappings = mappings; job_data->num_used_mappings = i;
err = 0;
goto done;
drop_refs: while (i--) tegra_drm_mapping_put(mappings[i].mapping);
kfree(mappings); job_data->used_mappings = NULL;
done: kvfree(bufs);
return err;}
static int submit_get_syncpt(struct tegra_drm_context *context, struct host1x_job *job, struct xarray *syncpoints, struct drm_tegra_channel_submit *args){ struct host1x_syncpt *sp;
if (args->syncpt.flags) { SUBMIT_ERR(context, "invalid flag specified for syncpt"); return -EINVAL; }
/* Syncpt ref will be dropped on job release */ sp = xa_load(syncpoints, args->syncpt.id); if (!sp) { SUBMIT_ERR(context, "syncpoint specified in syncpt was not allocated"); return -EINVAL; }
job->syncpt = host1x_syncpt_get(sp); job->syncpt_incrs = args->syncpt.increments;
return 0;}
static int submit_handle_in_fence(struct tegra_drm_context *context, struct host1x *host1x, struct dma_fence *fence, u32 *num_waits){ struct dma_fence_unwrap iter; struct host1x *fence_host1x; struct dma_fence *f; int err = 0;
*num_waits = 0;
dma_fence_unwrap_for_each(f, &iter, fence) { long wait_err;
if (err) continue;
if (dma_fence_is_signaled(f)) continue;
err = host1x_fence_extract(f, &fence_host1x, NULL, NULL); if (!err && fence_host1x == host1x) { (*num_waits)++; continue; }
wait_err = dma_fence_wait_timeout(f, true, msecs_to_jiffies(10000)); if (wait_err == 0) { SUBMIT_ERR(context, "wait for syncobj_in timed out"); err = -ETIMEDOUT; continue; } else if (wait_err < 0) { /* In practice, -ERESTARTSYS */ err = wait_err; continue; } else { err = 0; } }
return err;}
static void submit_job_add_prefence(struct host1x_job *job, struct dma_fence *fence, u32 class){ struct dma_fence_unwrap iter; struct dma_fence *f;
dma_fence_unwrap_for_each(f, &iter, fence) { u32 id, threshold;
if (dma_fence_is_signaled(f)) continue;
if (host1x_fence_extract(f, NULL, &id, &threshold)) { /* * The fence may have signalled since the check above, and * the dma_fence framework then detached its ops. */ continue; }
host1x_job_add_wait(job, id, threshold, false, class); }}
static int submit_job_add_gather(struct host1x_job *job, struct tegra_drm_context *context, struct drm_tegra_submit_cmd_gather_uptr *cmd, struct gather_bo *bo, u32 *offset, struct tegra_drm_submit_data *job_data, u32 *class){ u32 next_offset;
if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2]) { SUBMIT_ERR(context, "non-zero reserved field in GATHER_UPTR command"); return -EINVAL; }
/* Check for maximum gather size */ if (cmd->words > 16383) { SUBMIT_ERR(context, "too many words in GATHER_UPTR command"); return -EINVAL; }
if (check_add_overflow(*offset, cmd->words, &next_offset)) { SUBMIT_ERR(context, "too many total words in job"); return -EINVAL; }
if (next_offset > bo->gather_data_words) { SUBMIT_ERR(context, "GATHER_UPTR command overflows gather data"); return -EINVAL; }
if (tegra_drm_fw_validate(context->client, bo->gather_data, *offset, cmd->words, job_data, class)) { SUBMIT_ERR(context, "job was rejected by firewall"); return -EINVAL; }
host1x_job_add_gather(job, &bo->base, cmd->words, *offset * 4);
*offset = next_offset;
return 0;}
static struct host1x_job *submit_create_job(struct tegra_drm_context *context, struct gather_bo *bo, struct drm_tegra_channel_submit *args, struct tegra_drm_submit_data *job_data, struct xarray *syncpoints, struct dma_fence *in_fence, u32 num_prefence_waits){ struct drm_tegra_submit_cmd *cmds; u32 i, gather_offset = 0, class; struct host1x_job *job; int err;
/* Set initial class for firewall. */ class = context->client->base.class;
cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr), args->num_cmds, sizeof(*cmds)); if (IS_ERR(cmds)) { SUBMIT_ERR(context, "failed to copy cmds array from userspace"); return ERR_CAST(cmds); }
job = host1x_job_alloc(context->channel, args->num_cmds + num_prefence_waits, 0, true); if (!job) { SUBMIT_ERR(context, "failed to allocate memory for job"); job = ERR_PTR(-ENOMEM); goto done; }
err = submit_get_syncpt(context, job, syncpoints, args); if (err < 0) goto free_job;
job->client = &context->client->base; job->class = context->client->base.class; job->serialize = true;
if (in_fence) submit_job_add_prefence(job, in_fence, class);
for (i = 0; i < args->num_cmds; i++) { struct drm_tegra_submit_cmd *cmd = &cmds[i];
if (cmd->flags) { SUBMIT_ERR(context, "unknown flags given for cmd"); err = -EINVAL; goto free_job; }
if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) { err = submit_job_add_gather(job, context, &cmd->gather_uptr, bo, &gather_offset, job_data, &class); if (err) goto free_job; } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) { if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { SUBMIT_ERR(context, "non-zero reserved value"); err = -EINVAL; goto free_job; }
host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, false, class); } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE) { if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { SUBMIT_ERR(context, "non-zero reserved value"); err = -EINVAL; goto free_job; }
if (cmd->wait_syncpt.id != args->syncpt.id) { SUBMIT_ERR(context, "syncpoint ID in CMD_WAIT_SYNCPT_RELATIVE is not used by the job"); err = -EINVAL; goto free_job; }
host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, true, class); } else { SUBMIT_ERR(context, "unknown cmd type"); err = -EINVAL; goto free_job; } }
if (gather_offset == 0) { SUBMIT_ERR(context, "job must have at least one gather"); err = -EINVAL; goto free_job; }
goto done;
free_job: host1x_job_put(job); job = ERR_PTR(err);
done: kvfree(cmds);
return job;}
static void release_job(struct host1x_job *job){ struct tegra_drm_client *client = container_of(job->client, struct tegra_drm_client, base); struct tegra_drm_submit_data *job_data = job->user_data; u32 i;
for (i = 0; i < job_data->num_used_mappings; i++) tegra_drm_mapping_put(job_data->used_mappings[i].mapping);
if (job->memory_context) { host1x_memory_context_inactive(job->memory_context); host1x_memory_context_put(job->memory_context); }
kfree(job_data->used_mappings); kfree(job_data);
pm_runtime_mark_last_busy(client->base.dev); pm_runtime_put_autosuspend(client->base.dev);}
int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data, struct drm_file *file){ struct host1x *host1x = dev_get_drvdata(drm->dev->parent); struct host1x_memory_context *active_memctx = NULL; struct tegra_drm_file *fpriv = file->driver_priv; struct drm_tegra_channel_submit *args = data; struct tegra_drm_submit_data *job_data; struct drm_syncobj *syncobj = NULL; struct dma_fence *in_fence = NULL; struct tegra_drm_context *context; u32 num_prefence_waits = 0; struct host1x_job *job; struct gather_bo *bo; u32 i; int err;
mutex_lock(&fpriv->lock);
context = xa_load(&fpriv->contexts, args->context); if (!context) { mutex_unlock(&fpriv->lock); pr_err_ratelimited("%s: %s: invalid channel context '%#x'", __func__, current->comm, args->context); return -EINVAL; }
if (context->memory_context) { err = host1x_memory_context_active(context->memory_context); if (err) { mutex_unlock(&fpriv->lock); SUBMIT_ERR(context, "failed to activate memory context"); return err; }
active_memctx = context->memory_context; }
if (args->syncobj_in) { err = drm_syncobj_find_fence(file, args->syncobj_in, 0, 0, &in_fence); if (err) { SUBMIT_ERR(context, "invalid syncobj_in '%#x'", args->syncobj_in); goto unlock; }
err = submit_handle_in_fence(context, host1x, in_fence, &num_prefence_waits); if (err) goto put_in_fence; }
if (args->syncobj_out) { syncobj = drm_syncobj_find(file, args->syncobj_out); if (!syncobj) { SUBMIT_ERR(context, "invalid syncobj_out '%#x'", args->syncobj_out); err = -ENOENT; goto put_in_fence; } }
/* Allocate gather BO and copy gather words in. */ err = submit_copy_gather_data(&bo, drm->dev, context, args); if (err) goto put_in_fence;
job_data = kzalloc_obj(*job_data); if (!job_data) { SUBMIT_ERR(context, "failed to allocate memory for job data"); err = -ENOMEM; goto put_bo; }
/* Get data buffer mappings and do relocation patching. */ err = submit_process_bufs(context, bo, args, job_data); if (err) goto free_job_data;
/* Allocate host1x_job and add gathers and waits to it. */ job = submit_create_job(context, bo, args, job_data, &fpriv->syncpoints, in_fence, num_prefence_waits); if (IS_ERR(job)) { err = PTR_ERR(job); goto free_job_data; }
/* * The prefence has been recorded as plain syncpoint waits, so the fence * itself is not needed anymore. */ dma_fence_put(in_fence); in_fence = NULL;
/* Map gather data for Host1x. */ err = host1x_job_pin(job, context->client->base.dev); if (err) { SUBMIT_ERR(context, "failed to pin job: %d", err); goto put_job; }
if (context->client->ops->get_streamid_offset) { err = context->client->ops->get_streamid_offset( context->client, &job->engine_streamid_offset); if (err) { SUBMIT_ERR(context, "failed to get streamid offset: %d", err); goto unpin_job; } }
if (context->memory_context && context->client->ops->can_use_memory_ctx) { bool supported;
err = context->client->ops->can_use_memory_ctx(context->client, &supported); if (err) { SUBMIT_ERR(context, "failed to detect if engine can use memory context: %d", err); goto unpin_job; }
if (supported) { job->memory_context = active_memctx; active_memctx = NULL; host1x_memory_context_get(job->memory_context); } } else if (context->client->ops->get_streamid_offset) { /* * Job submission will need to temporarily change stream ID, * so need to tell it what to change it back to. */ if (!tegra_dev_iommu_get_stream_id(context->client->base.dev, &job->engine_fallback_streamid)) job->engine_fallback_streamid = TEGRA_STREAM_ID_BYPASS; }
/* Boot engine. */ err = pm_runtime_resume_and_get(context->client->base.dev); if (err < 0) { SUBMIT_ERR(context, "could not power up engine: %d", err); goto put_memory_context; }
job->user_data = job_data; job->release = release_job; job->timeout = 10000;
/* * job_data is now part of job reference counting, so don't release * it from here. */ job_data = NULL;
/* Submit job to hardware. */ err = host1x_job_submit(job); if (err) { SUBMIT_ERR(context, "host1x job submission failed: %d", err); goto unpin_job; }
/* Return postfences to userspace and add fences to DMA reservations. */ args->syncpt.value = job->syncpt_end;
if (syncobj) { struct dma_fence *fence = host1x_fence_create(job->syncpt, job->syncpt_end, true); if (IS_ERR(fence)) { err = PTR_ERR(fence); SUBMIT_ERR(context, "failed to create postfence: %d", err); goto put_job; }
drm_syncobj_replace_fence(syncobj, fence); dma_fence_put(fence); }
goto put_job;
put_memory_context: if (job->memory_context) { host1x_memory_context_inactive(job->memory_context); host1x_memory_context_put(job->memory_context); }unpin_job: host1x_job_unpin(job);put_job: host1x_job_put(job);free_job_data: if (job_data && job_data->used_mappings) { for (i = 0; i < job_data->num_used_mappings; i++) tegra_drm_mapping_put(job_data->used_mappings[i].mapping);
kfree(job_data->used_mappings); }
kfree(job_data);put_bo: gather_bo_put(&bo->base);put_in_fence: dma_fence_put(in_fence);unlock: if (syncobj) drm_syncobj_put(syncobj); if (active_memctx) host1x_memory_context_inactive(active_memctx);
mutex_unlock(&fpriv->lock); return err;}