Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526// SPDX-License-Identifier: GPL-2.0 OR MIT/* * Copyright 2011 Red Hat Inc. * Copyright 2023 Intel Corporation. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * *//* Algorithm: * * We store the last allocated bo in "hole", we always try to allocate * after the last allocated bo. Principle is that in a linear GPU ring * progression was is after last is the oldest bo we allocated and thus * the first one that should no longer be in use by the GPU. * * If it's not the case we skip over the bo after last to the closest * done bo if such one exist. If none exist and we are not asked to * block we report failure to allocate. * * If we are asked to block we wait on all the oldest fence of all * rings. We just wait for any of those fence to complete. */
#include <drm/drm_suballoc.h>#include <drm/drm_print.h>
#include <linux/export.h>#include <linux/slab.h>#include <linux/sched.h>#include <linux/wait.h>#include <linux/dma-fence.h>
static void drm_suballoc_remove_locked(struct drm_suballoc *sa);static void drm_suballoc_try_free(struct drm_suballoc_manager *sa_manager);
/** * drm_suballoc_manager_init() - Initialise the drm_suballoc_manager * @sa_manager: pointer to the sa_manager * @size: number of bytes we want to suballocate * @align: alignment for each suballocated chunk * * Prepares the suballocation manager for suballocations. */void drm_suballoc_manager_init(struct drm_suballoc_manager *sa_manager, size_t size, size_t align){ unsigned int i;
BUILD_BUG_ON(!is_power_of_2(DRM_SUBALLOC_MAX_QUEUES));
if (!align) align = 1;
/* alignment must be a power of 2 */ if (WARN_ON_ONCE(align & (align - 1))) align = roundup_pow_of_two(align);
init_waitqueue_head(&sa_manager->wq); sa_manager->size = size; sa_manager->align = align; sa_manager->hole = &sa_manager->olist; INIT_LIST_HEAD(&sa_manager->olist); for (i = 0; i < DRM_SUBALLOC_MAX_QUEUES; ++i) INIT_LIST_HEAD(&sa_manager->flist[i]);}EXPORT_SYMBOL(drm_suballoc_manager_init);
/** * drm_suballoc_manager_fini() - Destroy the drm_suballoc_manager * @sa_manager: pointer to the sa_manager * * Cleans up the suballocation manager after use. All fences added * with drm_suballoc_free() must be signaled, or we cannot clean up * the entire manager. */void drm_suballoc_manager_fini(struct drm_suballoc_manager *sa_manager){ struct drm_suballoc *sa, *tmp;
if (!sa_manager->size) return;
if (!list_empty(&sa_manager->olist)) { sa_manager->hole = &sa_manager->olist; drm_suballoc_try_free(sa_manager); if (!list_empty(&sa_manager->olist)) DRM_ERROR("sa_manager is not empty, clearing anyway\n"); } list_for_each_entry_safe(sa, tmp, &sa_manager->olist, olist) { drm_suballoc_remove_locked(sa); }
sa_manager->size = 0;}EXPORT_SYMBOL(drm_suballoc_manager_fini);
static void drm_suballoc_remove_locked(struct drm_suballoc *sa){ struct drm_suballoc_manager *sa_manager = sa->manager;
if (sa_manager->hole == &sa->olist) sa_manager->hole = sa->olist.prev;
list_del_init(&sa->olist); list_del_init(&sa->flist); dma_fence_put(sa->fence); kfree(sa);}
static void drm_suballoc_try_free(struct drm_suballoc_manager *sa_manager){ struct drm_suballoc *sa, *tmp;
if (sa_manager->hole->next == &sa_manager->olist) return;
sa = list_entry(sa_manager->hole->next, struct drm_suballoc, olist); list_for_each_entry_safe_from(sa, tmp, &sa_manager->olist, olist) { if (!sa->fence || !dma_fence_is_signaled(sa->fence)) return;
drm_suballoc_remove_locked(sa); }}
static size_t drm_suballoc_hole_soffset(struct drm_suballoc_manager *sa_manager){ struct list_head *hole = sa_manager->hole;
if (hole != &sa_manager->olist) return list_entry(hole, struct drm_suballoc, olist)->eoffset;
return 0;}
static size_t drm_suballoc_hole_eoffset(struct drm_suballoc_manager *sa_manager){ struct list_head *hole = sa_manager->hole;
if (hole->next != &sa_manager->olist) return list_entry(hole->next, struct drm_suballoc, olist)->soffset; return sa_manager->size;}
static bool drm_suballoc_try_alloc(struct drm_suballoc_manager *sa_manager, struct drm_suballoc *sa, size_t size, size_t align){ size_t soffset, eoffset, wasted;
soffset = drm_suballoc_hole_soffset(sa_manager); eoffset = drm_suballoc_hole_eoffset(sa_manager); wasted = round_up(soffset, align) - soffset;
if ((eoffset - soffset) >= (size + wasted)) { soffset += wasted;
sa->manager = sa_manager; sa->soffset = soffset; sa->eoffset = soffset + size; list_add(&sa->olist, sa_manager->hole); INIT_LIST_HEAD(&sa->flist); sa_manager->hole = &sa->olist; return true; } return false;}
static bool __drm_suballoc_event(struct drm_suballoc_manager *sa_manager, size_t size, size_t align){ size_t soffset, eoffset, wasted; unsigned int i;
for (i = 0; i < DRM_SUBALLOC_MAX_QUEUES; ++i) if (!list_empty(&sa_manager->flist[i])) return true;
soffset = drm_suballoc_hole_soffset(sa_manager); eoffset = drm_suballoc_hole_eoffset(sa_manager); wasted = round_up(soffset, align) - soffset;
return ((eoffset - soffset) >= (size + wasted));}
/** * drm_suballoc_event() - Check if we can stop waiting * @sa_manager: pointer to the sa_manager * @size: number of bytes we want to allocate * @align: alignment we need to match * * Return: true if either there is a fence we can wait for or * enough free memory to satisfy the allocation directly. * false otherwise. */static bool drm_suballoc_event(struct drm_suballoc_manager *sa_manager, size_t size, size_t align){ bool ret;
spin_lock(&sa_manager->wq.lock); ret = __drm_suballoc_event(sa_manager, size, align); spin_unlock(&sa_manager->wq.lock); return ret;}
static bool drm_suballoc_next_hole(struct drm_suballoc_manager *sa_manager, struct dma_fence **fences, unsigned int *tries){ struct drm_suballoc *best_bo = NULL; unsigned int i, best_idx; size_t soffset, best, tmp;
/* if hole points to the end of the buffer */ if (sa_manager->hole->next == &sa_manager->olist) { /* try again with its beginning */ sa_manager->hole = &sa_manager->olist; return true; }
soffset = drm_suballoc_hole_soffset(sa_manager); /* to handle wrap around we add sa_manager->size */ best = sa_manager->size * 2; /* go over all fence list and try to find the closest sa * of the current last */ for (i = 0; i < DRM_SUBALLOC_MAX_QUEUES; ++i) { struct drm_suballoc *sa;
fences[i] = NULL;
if (list_empty(&sa_manager->flist[i])) continue;
sa = list_first_entry(&sa_manager->flist[i], struct drm_suballoc, flist);
if (!dma_fence_is_signaled(sa->fence)) { fences[i] = sa->fence; continue; }
/* limit the number of tries each freelist gets */ if (tries[i] > 2) continue;
tmp = sa->soffset; if (tmp < soffset) { /* wrap around, pretend it's after */ tmp += sa_manager->size; } tmp -= soffset; if (tmp < best) { /* this sa bo is the closest one */ best = tmp; best_idx = i; best_bo = sa; } }
if (best_bo) { ++tries[best_idx]; sa_manager->hole = best_bo->olist.prev;
/* * We know that this one is signaled, * so it's safe to remove it. */ drm_suballoc_remove_locked(best_bo); return true; } return false;}
/** * drm_suballoc_alloc() - Allocate uninitialized suballoc object. * @gfp: gfp flags used for memory allocation. * * Allocate memory for an uninitialized suballoc object. Intended usage is * allocate memory for suballoc object outside of a reclaim tainted context * and then be initialized at a later time in a reclaim tainted context. * * @drm_suballoc_free() should be used to release the memory if returned * suballoc object is in uninitialized state. * * Return: a new uninitialized suballoc object, or an ERR_PTR(-ENOMEM). */struct drm_suballoc *drm_suballoc_alloc(gfp_t gfp){ struct drm_suballoc *sa;
sa = kmalloc_obj(*sa, gfp); if (!sa) return ERR_PTR(-ENOMEM);
sa->manager = NULL;
return sa;}EXPORT_SYMBOL(drm_suballoc_alloc);
/** * drm_suballoc_insert() - Initialize a suballocation and insert a hole. * @sa_manager: pointer to the sa_manager * @sa: The struct drm_suballoc. * @size: number of bytes we want to suballocate. * @intr: Whether to perform waits interruptible. This should typically * always be true, unless the caller needs to propagate a * non-interruptible context from above layers. * @align: Alignment. Must not exceed the default manager alignment. * If @align is zero, then the manager alignment is used. * * Try to make a suballocation on a pre-allocated suballoc object of size @size, * which will be rounded up to the alignment specified in specified in * drm_suballoc_manager_init(). * * Return: zero on success, errno on failure. */int drm_suballoc_insert(struct drm_suballoc_manager *sa_manager, struct drm_suballoc *sa, size_t size, bool intr, size_t align){ struct dma_fence *fences[DRM_SUBALLOC_MAX_QUEUES]; unsigned int tries[DRM_SUBALLOC_MAX_QUEUES]; unsigned int count; int i, r;
if (WARN_ON_ONCE(align > sa_manager->align)) return -EINVAL; if (WARN_ON_ONCE(size > sa_manager->size || !size)) return -EINVAL;
if (!align) align = sa_manager->align;
sa->manager = sa_manager; sa->fence = NULL; INIT_LIST_HEAD(&sa->olist); INIT_LIST_HEAD(&sa->flist);
spin_lock(&sa_manager->wq.lock); do { for (i = 0; i < DRM_SUBALLOC_MAX_QUEUES; ++i) tries[i] = 0;
do { drm_suballoc_try_free(sa_manager);
if (drm_suballoc_try_alloc(sa_manager, sa, size, align)) { spin_unlock(&sa_manager->wq.lock); return 0; }
/* see if we can skip over some allocations */ } while (drm_suballoc_next_hole(sa_manager, fences, tries));
for (i = 0, count = 0; i < DRM_SUBALLOC_MAX_QUEUES; ++i) if (fences[i]) fences[count++] = dma_fence_get(fences[i]);
if (count) { long t;
spin_unlock(&sa_manager->wq.lock); t = dma_fence_wait_any_timeout(fences, count, intr, MAX_SCHEDULE_TIMEOUT, NULL); for (i = 0; i < count; ++i) dma_fence_put(fences[i]);
r = (t > 0) ? 0 : t; spin_lock(&sa_manager->wq.lock); } else if (intr) { /* if we have nothing to wait for block */ r = wait_event_interruptible_locked (sa_manager->wq, __drm_suballoc_event(sa_manager, size, align)); } else { spin_unlock(&sa_manager->wq.lock); wait_event(sa_manager->wq, drm_suballoc_event(sa_manager, size, align)); r = 0; spin_lock(&sa_manager->wq.lock); } } while (!r);
spin_unlock(&sa_manager->wq.lock); sa->manager = NULL; return r;}EXPORT_SYMBOL(drm_suballoc_insert);
/** * drm_suballoc_new() - Make a suballocation. * @sa_manager: pointer to the sa_manager * @size: number of bytes we want to suballocate. * @gfp: gfp flags used for memory allocation. Typically GFP_KERNEL but * the argument is provided for suballocations from reclaim context or * where the caller wants to avoid pipelining rather than wait for * reclaim. * @intr: Whether to perform waits interruptible. This should typically * always be true, unless the caller needs to propagate a * non-interruptible context from above layers. * @align: Alignment. Must not exceed the default manager alignment. * If @align is zero, then the manager alignment is used. * * Try to make a suballocation of size @size, which will be rounded * up to the alignment specified in specified in drm_suballoc_manager_init(). * * Return: a new suballocated bo, or an ERR_PTR. */struct drm_suballoc *drm_suballoc_new(struct drm_suballoc_manager *sa_manager, size_t size, gfp_t gfp, bool intr, size_t align){ struct drm_suballoc *sa; int err;
sa = drm_suballoc_alloc(gfp); if (IS_ERR(sa)) return sa;
err = drm_suballoc_insert(sa_manager, sa, size, intr, align); if (err) { drm_suballoc_free(sa, NULL); return ERR_PTR(err); }
return sa;}EXPORT_SYMBOL(drm_suballoc_new);
/** * drm_suballoc_free - Free a suballocation * @suballoc: pointer to the suballocation * @fence: fence that signals when suballocation is idle * * Free the suballocation. The suballocation can be re-used after @fence signals. */void drm_suballoc_free(struct drm_suballoc *suballoc, struct dma_fence *fence){ struct drm_suballoc_manager *sa_manager;
if (!suballoc) return;
if (!suballoc->manager) { kfree(suballoc); return; }
sa_manager = suballoc->manager;
spin_lock(&sa_manager->wq.lock); if (fence && !dma_fence_is_signaled(fence)) { u32 idx;
suballoc->fence = dma_fence_get(fence); idx = fence->context & (DRM_SUBALLOC_MAX_QUEUES - 1); list_add_tail(&suballoc->flist, &sa_manager->flist[idx]); } else { drm_suballoc_remove_locked(suballoc); } wake_up_all_locked(&sa_manager->wq); spin_unlock(&sa_manager->wq.lock);}EXPORT_SYMBOL(drm_suballoc_free);
#ifdef CONFIG_DEBUG_FSvoid drm_suballoc_dump_debug_info(struct drm_suballoc_manager *sa_manager, struct drm_printer *p, unsigned long long suballoc_base){ struct drm_suballoc *i;
spin_lock(&sa_manager->wq.lock); list_for_each_entry(i, &sa_manager->olist, olist) { unsigned long long soffset = i->soffset; unsigned long long eoffset = i->eoffset;
if (&i->olist == sa_manager->hole) drm_puts(p, ">"); else drm_puts(p, " ");
drm_printf(p, "[0x%010llx 0x%010llx] size %8lld", suballoc_base + soffset, suballoc_base + eoffset, eoffset - soffset);
if (i->fence) drm_printf(p, " protected by 0x%016llx on context %llu", (unsigned long long)i->fence->seqno, (unsigned long long)i->fence->context);
drm_puts(p, "\n"); } spin_unlock(&sa_manager->wq.lock);}EXPORT_SYMBOL(drm_suballoc_dump_debug_info);#endifMODULE_AUTHOR("Multiple");MODULE_DESCRIPTION("Range suballocator helper");MODULE_LICENSE("Dual MIT/GPL");