Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739// SPDX-License-Identifier: GPL-2.0+/* * comedi_buf.c * * COMEDI - Linux Control and Measurement Device Interface * Copyright (C) 1997-2000 David A. Schleef <ds@schleef.org> * Copyright (C) 2002 Frank Mori Hess <fmhess@users.sourceforge.net> */
#include <linux/vmalloc.h>#include <linux/slab.h>#include <linux/comedi/comedidev.h>#include "comedi_internal.h"
#ifdef PAGE_KERNEL_NOCACHE#define COMEDI_PAGE_PROTECTION PAGE_KERNEL_NOCACHE#else#define COMEDI_PAGE_PROTECTION PAGE_KERNEL#endif
static void comedi_buf_map_kref_release(struct kref *kref){ struct comedi_buf_map *bm = container_of(kref, struct comedi_buf_map, refcount); struct comedi_buf_page *buf; unsigned int i;
if (bm->page_list) { if (bm->dma_dir != DMA_NONE) { for (i = 0; i < bm->n_pages; i++) { buf = &bm->page_list[i]; dma_free_coherent(bm->dma_hw_dev, PAGE_SIZE, buf->virt_addr, buf->dma_addr); } } else { for (i = 0; i < bm->n_pages; i++) { buf = &bm->page_list[i]; ClearPageReserved(virt_to_page(buf->virt_addr)); free_page((unsigned long)buf->virt_addr); } } vfree(bm->page_list); } if (bm->dma_dir != DMA_NONE) put_device(bm->dma_hw_dev); kfree(bm);}
static void __comedi_buf_free(struct comedi_device *dev, struct comedi_subdevice *s){ struct comedi_async *async = s->async; struct comedi_buf_map *bm; unsigned long flags;
async->prealloc_bufsz = 0; spin_lock_irqsave(&s->spin_lock, flags); bm = async->buf_map; async->buf_map = NULL; spin_unlock_irqrestore(&s->spin_lock, flags); comedi_buf_map_put(bm);}
static struct comedi_buf_map *comedi_buf_map_alloc(struct comedi_device *dev, enum dma_data_direction dma_dir, unsigned int n_pages){ struct comedi_buf_map *bm; struct comedi_buf_page *buf; unsigned int i;
bm = kzalloc_obj(*bm); if (!bm) return NULL;
kref_init(&bm->refcount); bm->dma_dir = dma_dir; if (bm->dma_dir != DMA_NONE) { /* Need ref to hardware device to free buffer later. */ bm->dma_hw_dev = get_device(dev->hw_dev); }
bm->page_list = vzalloc(sizeof(*buf) * n_pages); if (!bm->page_list) goto err;
if (bm->dma_dir != DMA_NONE) { for (i = 0; i < n_pages; i++) { buf = &bm->page_list[i]; buf->virt_addr = dma_alloc_coherent(bm->dma_hw_dev, PAGE_SIZE, &buf->dma_addr, GFP_KERNEL); if (!buf->virt_addr) break; } } else { for (i = 0; i < n_pages; i++) { buf = &bm->page_list[i]; buf->virt_addr = (void *)get_zeroed_page(GFP_KERNEL); if (!buf->virt_addr) break;
SetPageReserved(virt_to_page(buf->virt_addr)); } } bm->n_pages = i; if (i < n_pages) goto err;
return bm;
err: comedi_buf_map_put(bm); return NULL;}
static void __comedi_buf_alloc(struct comedi_device *dev, struct comedi_subdevice *s, unsigned int n_pages){ struct comedi_async *async = s->async; struct comedi_buf_map *bm; unsigned long flags;
if (!IS_ENABLED(CONFIG_HAS_DMA) && s->async_dma_dir != DMA_NONE) { dev_err(dev->class_dev, "dma buffer allocation not supported\n"); return; }
bm = comedi_buf_map_alloc(dev, s->async_dma_dir, n_pages); if (!bm) return;
spin_lock_irqsave(&s->spin_lock, flags); async->buf_map = bm; spin_unlock_irqrestore(&s->spin_lock, flags); async->prealloc_bufsz = n_pages << PAGE_SHIFT;}
void comedi_buf_map_get(struct comedi_buf_map *bm){ if (bm) kref_get(&bm->refcount);}
int comedi_buf_map_put(struct comedi_buf_map *bm){ if (bm) return kref_put(&bm->refcount, comedi_buf_map_kref_release); return 1;}
/* helper for "access" vm operation */int comedi_buf_map_access(struct comedi_buf_map *bm, unsigned long offset, void *buf, int len, int write){ unsigned int pgoff = offset_in_page(offset); unsigned long pg = offset >> PAGE_SHIFT; int done = 0;
while (done < len && pg < bm->n_pages) { int l = min_t(int, len - done, PAGE_SIZE - pgoff); void *b = bm->page_list[pg].virt_addr + pgoff;
if (write) memcpy(b, buf, l); else memcpy(buf, b, l); buf += l; done += l; pg++; pgoff = 0; } return done;}
/* returns s->async->buf_map and increments its kref refcount */struct comedi_buf_map *comedi_buf_map_from_subdev_get(struct comedi_subdevice *s){ struct comedi_async *async = s->async; struct comedi_buf_map *bm = NULL; unsigned long flags;
if (!async) return NULL;
spin_lock_irqsave(&s->spin_lock, flags); bm = async->buf_map; /* only want it if buffer pages allocated */ if (bm && bm->n_pages) comedi_buf_map_get(bm); else bm = NULL; spin_unlock_irqrestore(&s->spin_lock, flags);
return bm;}
bool comedi_buf_is_mmapped(struct comedi_subdevice *s){ struct comedi_buf_map *bm = s->async->buf_map;
return bm && (kref_read(&bm->refcount) > 1);}
int comedi_buf_alloc(struct comedi_device *dev, struct comedi_subdevice *s, unsigned long new_size){ struct comedi_async *async = s->async;
lockdep_assert_held(&dev->mutex);
/* Round up new_size to multiple of PAGE_SIZE */ new_size = (new_size + PAGE_SIZE - 1) & PAGE_MASK;
/* if no change is required, do nothing */ if (async->prealloc_bufsz == new_size) return 0;
/* deallocate old buffer */ __comedi_buf_free(dev, s);
/* allocate new buffer */ if (new_size) { unsigned int n_pages = new_size >> PAGE_SHIFT;
__comedi_buf_alloc(dev, s, n_pages); if (!async->prealloc_bufsz) return -ENOMEM; }
return 0;}
void comedi_buf_reset(struct comedi_subdevice *s){ struct comedi_async *async = s->async;
async->buf_write_alloc_count = 0; async->buf_write_count = 0; async->buf_read_alloc_count = 0; async->buf_read_count = 0;
async->buf_write_ptr = 0; async->buf_read_ptr = 0;
async->cur_chan = 0; async->scans_done = 0; async->scan_progress = 0; async->munge_chan = 0; async->munge_count = 0; async->munge_ptr = 0;
async->events = 0;}
static unsigned int comedi_buf_write_n_unalloc(struct comedi_subdevice *s){ struct comedi_async *async = s->async; unsigned int free_end = async->buf_read_count + async->prealloc_bufsz;
return free_end - async->buf_write_alloc_count;}
unsigned int comedi_buf_write_n_available(struct comedi_subdevice *s){ struct comedi_async *async = s->async; unsigned int free_end = async->buf_read_count + async->prealloc_bufsz;
return free_end - async->buf_write_count;}
unsigned int _comedi_buf_write_alloc(struct comedi_subdevice *s, unsigned int nbytes){ struct comedi_async *async = s->async; unsigned int unalloc = comedi_buf_write_n_unalloc(s);
if (nbytes > unalloc) nbytes = unalloc;
async->buf_write_alloc_count += nbytes;
/* * ensure the async buffer 'counts' are read and updated * before we write data to the write-alloc'ed buffer space */ smp_mb();
return nbytes;}
/** * comedi_buf_write_alloc() - Reserve buffer space for writing * @s: COMEDI subdevice. * @nbytes: Maximum space to reserve in bytes. * * Reserve up to @nbytes bytes of space to be written in the COMEDI acquisition * data buffer associated with the subdevice. The amount reserved is limited * by the space available. * * Return: The amount of space reserved in bytes. */unsigned int comedi_buf_write_alloc(struct comedi_subdevice *s, unsigned int nbytes){ if (comedi_get_is_subdevice_running(s)) { nbytes = _comedi_buf_write_alloc(s, nbytes); comedi_put_is_subdevice_running(s); } else { nbytes = 0; } return nbytes;}EXPORT_SYMBOL_GPL(comedi_buf_write_alloc);
/* * munging is applied to data by core as it passes between user * and kernel space */static unsigned int comedi_buf_munge(struct comedi_subdevice *s, unsigned int num_bytes){ struct comedi_async *async = s->async; struct comedi_buf_page *buf_page_list = async->buf_map->page_list; unsigned int count = 0; const unsigned int num_sample_bytes = comedi_bytes_per_sample(s);
if (!s->munge || (async->cmd.flags & CMDF_RAWDATA) || async->cmd.chanlist_len == 0) { async->munge_count += num_bytes; return num_bytes; }
/* don't munge partial samples */ num_bytes -= num_bytes % num_sample_bytes; while (count < num_bytes) { /* * Do not munge beyond page boundary. * Note: prealloc_bufsz is a multiple of PAGE_SIZE. */ unsigned int page = async->munge_ptr >> PAGE_SHIFT; unsigned int offset = offset_in_page(async->munge_ptr); unsigned int block_size = min(num_bytes - count, PAGE_SIZE - offset);
s->munge(s->device, s, buf_page_list[page].virt_addr + offset, block_size, async->munge_chan);
/* * ensure data is munged in buffer before the * async buffer munge_count is incremented */ smp_wmb();
async->munge_chan += block_size / num_sample_bytes; async->munge_chan %= async->cmd.chanlist_len; async->munge_count += block_size; async->munge_ptr += block_size; if (async->munge_ptr == async->prealloc_bufsz) async->munge_ptr = 0; count += block_size; }
return count;}
unsigned int comedi_buf_write_n_allocated(struct comedi_subdevice *s){ struct comedi_async *async = s->async;
return async->buf_write_alloc_count - async->buf_write_count;}
unsigned int _comedi_buf_write_free(struct comedi_subdevice *s, unsigned int nbytes){ struct comedi_async *async = s->async; unsigned int allocated = comedi_buf_write_n_allocated(s);
if (nbytes > allocated) nbytes = allocated;
async->buf_write_count += nbytes; async->buf_write_ptr += nbytes; comedi_buf_munge(s, async->buf_write_count - async->munge_count); if (async->buf_write_ptr >= async->prealloc_bufsz) async->buf_write_ptr %= async->prealloc_bufsz;
return nbytes;}
/** * comedi_buf_write_free() - Free buffer space after it is written * @s: COMEDI subdevice. * @nbytes: Maximum space to free in bytes. * * Free up to @nbytes bytes of space previously reserved for writing in the * COMEDI acquisition data buffer associated with the subdevice. The amount of * space freed is limited to the amount that was reserved. The freed space is * assumed to have been filled with sample data by the writer. * * If the samples in the freed space need to be "munged", do so here. The * freed space becomes available for allocation by the reader. * * Return: The amount of space freed in bytes. */unsigned int comedi_buf_write_free(struct comedi_subdevice *s, unsigned int nbytes){ if (comedi_get_is_subdevice_running(s)) { nbytes = _comedi_buf_write_free(s, nbytes); comedi_put_is_subdevice_running(s); } else { nbytes = 0; } return nbytes;}EXPORT_SYMBOL_GPL(comedi_buf_write_free);
unsigned int _comedi_buf_read_n_available(struct comedi_subdevice *s){ struct comedi_async *async = s->async; unsigned int num_bytes;
if (!async) return 0;
num_bytes = async->munge_count - async->buf_read_count;
/* * ensure the async buffer 'counts' are read before we * attempt to read data from the buffer */ smp_rmb();
return num_bytes;}
/** * comedi_buf_read_n_available() - Determine amount of readable buffer space * @s: COMEDI subdevice. * * Determine the amount of readable buffer space in the COMEDI acquisition data * buffer associated with the subdevice. The readable buffer space is that * which has been freed by the writer and "munged" to the sample data format * expected by COMEDI if necessary. * * Return: The amount of readable buffer space. */unsigned int comedi_buf_read_n_available(struct comedi_subdevice *s){ unsigned int num_bytes;
if (comedi_get_is_subdevice_running(s)) { num_bytes = _comedi_buf_read_n_available(s); comedi_put_is_subdevice_running(s); } else { num_bytes = 0; } return num_bytes;}EXPORT_SYMBOL_GPL(comedi_buf_read_n_available);
unsigned int _comedi_buf_read_alloc(struct comedi_subdevice *s, unsigned int nbytes){ struct comedi_async *async = s->async; unsigned int available;
available = async->munge_count - async->buf_read_alloc_count; if (nbytes > available) nbytes = available;
async->buf_read_alloc_count += nbytes;
/* * ensure the async buffer 'counts' are read before we * attempt to read data from the read-alloc'ed buffer space */ smp_rmb();
return nbytes;}
/** * comedi_buf_read_alloc() - Reserve buffer space for reading * @s: COMEDI subdevice. * @nbytes: Maximum space to reserve in bytes. * * Reserve up to @nbytes bytes of previously written and "munged" buffer space * for reading in the COMEDI acquisition data buffer associated with the * subdevice. The amount reserved is limited to the space available. The * reader can read from the reserved space and then free it. A reader is also * allowed to read from the space before reserving it as long as it determines * the amount of readable data available, but the space needs to be marked as * reserved before it can be freed. * * Return: The amount of space reserved in bytes. */unsigned int comedi_buf_read_alloc(struct comedi_subdevice *s, unsigned int nbytes){ if (comedi_get_is_subdevice_running(s)) { nbytes = _comedi_buf_read_alloc(s, nbytes); comedi_put_is_subdevice_running(s); } else { nbytes = 0; } return nbytes;}EXPORT_SYMBOL_GPL(comedi_buf_read_alloc);
static unsigned int comedi_buf_read_n_allocated(struct comedi_async *async){ return async->buf_read_alloc_count - async->buf_read_count;}
unsigned int _comedi_buf_read_free(struct comedi_subdevice *s, unsigned int nbytes){ struct comedi_async *async = s->async; unsigned int allocated;
/* * ensure data has been read out of buffer before * the async read count is incremented */ smp_mb();
allocated = comedi_buf_read_n_allocated(async); if (nbytes > allocated) nbytes = allocated;
async->buf_read_count += nbytes; async->buf_read_ptr += nbytes; async->buf_read_ptr %= async->prealloc_bufsz; return nbytes;}
/** * comedi_buf_read_free() - Free buffer space after it has been read * @s: COMEDI subdevice. * @nbytes: Maximum space to free in bytes. * * Free up to @nbytes bytes of buffer space previously reserved for reading in * the COMEDI acquisition data buffer associated with the subdevice. The * amount of space freed is limited to the amount that was reserved. * * The freed space becomes available for allocation by the writer. * * Return: The amount of space freed in bytes. */unsigned int comedi_buf_read_free(struct comedi_subdevice *s, unsigned int nbytes){ if (comedi_get_is_subdevice_running(s)) { nbytes = _comedi_buf_read_free(s, nbytes); comedi_put_is_subdevice_running(s); } else { nbytes = 0; } return nbytes;}EXPORT_SYMBOL_GPL(comedi_buf_read_free);
static void comedi_buf_memcpy_to(struct comedi_subdevice *s, const void *data, unsigned int num_bytes){ struct comedi_async *async = s->async; struct comedi_buf_page *buf_page_list = async->buf_map->page_list; unsigned int write_ptr = async->buf_write_ptr;
while (num_bytes) { /* * Do not copy beyond page boundary. * Note: prealloc_bufsz is a multiple of PAGE_SIZE. */ unsigned int page = write_ptr >> PAGE_SHIFT; unsigned int offset = offset_in_page(write_ptr); unsigned int block_size = min(num_bytes, PAGE_SIZE - offset);
memcpy(buf_page_list[page].virt_addr + offset, data, block_size);
data += block_size; num_bytes -= block_size; write_ptr += block_size; if (write_ptr == async->prealloc_bufsz) write_ptr = 0; }}
static void comedi_buf_memcpy_from(struct comedi_subdevice *s, void *dest, unsigned int nbytes){ struct comedi_async *async = s->async; struct comedi_buf_page *buf_page_list = async->buf_map->page_list; unsigned int read_ptr = async->buf_read_ptr;
while (nbytes) { /* * Do not copy beyond page boundary. * Note: prealloc_bufsz is a multiple of PAGE_SIZE. */ unsigned int page = read_ptr >> PAGE_SHIFT; unsigned int offset = offset_in_page(read_ptr); unsigned int block_size = min(nbytes, PAGE_SIZE - offset);
memcpy(dest, buf_page_list[page].virt_addr + offset, block_size); nbytes -= block_size; dest += block_size; read_ptr += block_size; if (read_ptr == async->prealloc_bufsz) read_ptr = 0; }}
static unsigned int _comedi_buf_write_samples(struct comedi_subdevice *s, const void *data, unsigned int nsamples){ unsigned int max_samples; unsigned int nbytes;
/* * Make sure there is enough room in the buffer for all the samples. * If not, clamp the nsamples to the number that will fit, flag the * buffer overrun and add the samples that fit. */ max_samples = comedi_bytes_to_samples(s, comedi_buf_write_n_unalloc(s)); if (nsamples > max_samples) { dev_warn(s->device->class_dev, "buffer overrun\n"); s->async->events |= COMEDI_CB_OVERFLOW; nsamples = max_samples; }
if (nsamples == 0) return 0;
nbytes = comedi_samples_to_bytes(s, nsamples); nbytes = _comedi_buf_write_alloc(s, nbytes); comedi_buf_memcpy_to(s, data, nbytes); _comedi_buf_write_free(s, nbytes); _comedi_inc_scan_progress(s, nbytes); s->async->events |= COMEDI_CB_BLOCK;
return nbytes;}
/** * comedi_buf_write_samples() - Write sample data to COMEDI buffer * @s: COMEDI subdevice. * @data: Pointer to source samples. * @nsamples: Number of samples to write. * * Write up to @nsamples samples to the COMEDI acquisition data buffer * associated with the subdevice, mark it as written and update the * acquisition scan progress. If there is not enough room for the specified * number of samples, the number of samples written is limited to the number * that will fit and the %COMEDI_CB_OVERFLOW event flag is set to cause the * acquisition to terminate with an overrun error. Set the %COMEDI_CB_BLOCK * event flag if any samples are written to cause waiting tasks to be woken * when the event flags are processed. * * Return: The amount of data written in bytes. */unsigned int comedi_buf_write_samples(struct comedi_subdevice *s, const void *data, unsigned int nsamples){ unsigned int nbytes;
if (comedi_get_is_subdevice_running(s)) { nbytes = _comedi_buf_write_samples(s, data, nsamples); comedi_put_is_subdevice_running(s); } else { nbytes = 0; } return nbytes;}EXPORT_SYMBOL_GPL(comedi_buf_write_samples);
static unsigned int _comedi_buf_read_samples(struct comedi_subdevice *s, void *data, unsigned int nsamples){ unsigned int max_samples; unsigned int nbytes;
/* clamp nsamples to the number of full samples available */ max_samples = comedi_bytes_to_samples(s, _comedi_buf_read_n_available(s)); if (nsamples > max_samples) nsamples = max_samples;
if (nsamples == 0) return 0;
nbytes = _comedi_buf_read_alloc(s, comedi_samples_to_bytes(s, nsamples)); comedi_buf_memcpy_from(s, data, nbytes); _comedi_buf_read_free(s, nbytes); _comedi_inc_scan_progress(s, nbytes); s->async->events |= COMEDI_CB_BLOCK;
return nbytes;}
/** * comedi_buf_read_samples() - Read sample data from COMEDI buffer * @s: COMEDI subdevice. * @data: Pointer to destination. * @nsamples: Maximum number of samples to read. * * Read up to @nsamples samples from the COMEDI acquisition data buffer * associated with the subdevice, mark it as read and update the acquisition * scan progress. Limit the number of samples read to the number available. * Set the %COMEDI_CB_BLOCK event flag if any samples are read to cause waiting * tasks to be woken when the event flags are processed. * * Return: The amount of data read in bytes. */unsigned int comedi_buf_read_samples(struct comedi_subdevice *s, void *data, unsigned int nsamples){ unsigned int nbytes;
if (comedi_get_is_subdevice_running(s)) { nbytes = _comedi_buf_read_samples(s, data, nsamples); comedi_put_is_subdevice_running(s); } else { nbytes = 0; } return nbytes;}EXPORT_SYMBOL_GPL(comedi_buf_read_samples);