Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250// SPDX-License-Identifier: GPL-2.0-or-later/* Rolling buffer helpers * * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */
#include <linux/bitops.h>#include <linux/mempool.h>#include <linux/pagemap.h>#include <linux/rolling_buffer.h>#include <linux/slab.h>#include "internal.h"
static atomic_t debug_ids;
/** * netfs_folioq_alloc - Allocate a folio_queue struct * @rreq_id: Associated debugging ID for tracing purposes * @gfp: Allocation constraints * @trace: Trace tag to indicate the purpose of the allocation * * Allocate, initialise and account the folio_queue struct and log a trace line * to mark the allocation. */struct folio_queue *netfs_folioq_alloc(unsigned int rreq_id, gfp_t gfp, unsigned int /*enum netfs_folioq_trace*/ trace){ struct folio_queue *fq;
if (gfp == GFP_KERNEL) fq = mempool_alloc_noreserve(&netfs_folioq_pool, gfp); else fq = mempool_alloc(&netfs_folioq_pool, gfp); if (fq) { netfs_stat(&netfs_n_folioq); folioq_init(fq, rreq_id); fq->debug_id = atomic_inc_return(&debug_ids); trace_netfs_folioq(fq, trace); } return fq;}EXPORT_SYMBOL(netfs_folioq_alloc);
/** * netfs_folioq_free - Free a folio_queue struct * @folioq: The object to free * @trace: Trace tag to indicate which free * * Free and unaccount the folio_queue struct. */void netfs_folioq_free(struct folio_queue *folioq, unsigned int /*enum netfs_trace_folioq*/ trace){ trace_netfs_folioq(folioq, trace); netfs_stat_d(&netfs_n_folioq); mempool_free(folioq, &netfs_folioq_pool);}EXPORT_SYMBOL(netfs_folioq_free);
/* * Initialise a rolling buffer. We allocate an empty folio queue struct to so * that the pointers can be independently driven by the producer and the * consumer. */int rolling_buffer_init(struct rolling_buffer *roll, unsigned int rreq_id, unsigned int direction, gfp_t gfp){ struct folio_queue *fq;
fq = netfs_folioq_alloc(rreq_id, gfp, netfs_trace_folioq_rollbuf_init); if (!fq) return -ENOMEM;
roll->head = fq; roll->tail = fq; iov_iter_folio_queue(&roll->iter, direction, fq, 0, 0, 0); return 0;}
/* * Add another folio_queue to a rolling buffer if there's no space left. */int rolling_buffer_make_space(struct rolling_buffer *roll, gfp_t gfp){ struct folio_queue *fq, *head = roll->head;
if (!folioq_full(head)) return 0;
fq = netfs_folioq_alloc(head->rreq_id, gfp, netfs_trace_folioq_make_space); if (!fq) return -ENOMEM; fq->prev = head;
roll->head = fq; if (folioq_full(head)) { /* Make sure we don't leave the master iterator pointing to a * block that might get immediately consumed. */ if (roll->iter.folioq == head && roll->iter.folioq_slot == folioq_nr_slots(head)) { roll->iter.folioq = fq; roll->iter.folioq_slot = 0; } }
/* Make sure the initialisation is stored before the next pointer. * * [!] NOTE: After we set head->next, the consumer is at liberty to * immediately delete the old head. */ smp_store_release(&head->next, fq); return 0;}
/* * Decant the entire list of folios to read into a rolling buffer. */ssize_t rolling_buffer_bulk_load_from_ra(struct rolling_buffer *roll, struct readahead_control *ractl, unsigned int rreq_id, gfp_t gfp){ struct folio_queue *fq; ssize_t loaded = 0;
while (ractl->_nr_pages - ractl->_batch_count > 0) { unsigned int nr;
/* Allocate a folioq to put some folios into and attach it to * the rolling buffer. */ fq = netfs_folioq_alloc(rreq_id, gfp, netfs_trace_folioq_make_space); if (!fq) goto nomem_unlock; fq->prev = roll->head; if (!roll->tail) roll->tail = fq; else roll->head->next = fq; roll->head = fq;
/* Get a batch of folios and note their orders. */ nr = __readahead_batch(ractl, (struct page **)fq->vec.folios, folioq_nr_slots(fq)); if (WARN_ON_ONCE(!nr)) break; fq->vec.nr = nr;
for (int slot = 0; slot < nr; slot++) { struct folio *folio = folioq_folio(fq, slot); unsigned int order;
order = folio_order(folio); fq->orders[slot] = order; loaded += PAGE_SIZE << order; trace_netfs_folio(folio, netfs_folio_trace_read); } }
WRITE_ONCE(roll->iter.count, loaded); iov_iter_folio_queue(&roll->iter, ITER_DEST, roll->tail, 0, 0, loaded); return loaded;
nomem_unlock: for (fq = roll->tail; fq; fq = fq->next) { for (int slot = 0; slot < folioq_count(fq); slot++) { folio_unlock(fq->vec.folios[slot]); folioq_mark(fq, slot); } } rolling_buffer_clear(roll); roll->head = NULL; roll->tail = NULL; return -ENOMEM;}
/* * Append a folio to the rolling buffer. */ssize_t rolling_buffer_append(struct rolling_buffer *roll, struct folio *folio, unsigned int flags, gfp_t gfp){ ssize_t size = folio_size(folio); int slot;
if (rolling_buffer_make_space(roll, gfp) < 0) return -ENOMEM;
slot = folioq_append(roll->head, folio); if (flags & ROLLBUF_MARK_1) folioq_mark(roll->head, slot); if (flags & ROLLBUF_MARK_2) folioq_mark2(roll->head, slot);
WRITE_ONCE(roll->iter.count, roll->iter.count + size);
/* Store the counter after setting the slot. */ smp_store_release(&roll->next_head_slot, slot); return size;}
/* * Delete a spent buffer from a rolling queue and return the next in line. We * don't return the last buffer to keep the pointers independent, but return * NULL instead. */struct folio_queue *rolling_buffer_delete_spent(struct rolling_buffer *roll){ struct folio_queue *spent = roll->tail, *next = READ_ONCE(spent->next);
if (!next) return NULL; next->prev = NULL; netfs_folioq_free(spent, netfs_trace_folioq_delete); roll->tail = next; return next;}
/* * Clear out a rolling queue. Folios that have mark 1 set are put. */void rolling_buffer_clear(struct rolling_buffer *roll){ struct folio_batch fbatch; struct folio_queue *p;
folio_batch_init(&fbatch);
while ((p = roll->tail)) { roll->tail = p->next; for (int slot = 0; slot < folioq_count(p); slot++) { struct folio *folio = folioq_folio(p, slot);
if (!folio) continue; if (folioq_is_marked(p, slot)) { trace_netfs_folio(folio, netfs_folio_trace_put); if (!folio_batch_add(&fbatch, folio)) folio_batch_release(&fbatch); } }
netfs_folioq_free(p, netfs_trace_folioq_clear); }
folio_batch_release(&fbatch);}