Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368// SPDX-License-Identifier: GPL-2.0/* * Functions for dibs loopback/loopback-ism device. * * Copyright (c) 2024, Alibaba Inc. * * Author: Wen Gu <guwen@linux.alibaba.com> * Tony Lu <tonylu@linux.alibaba.com> * */
#include <linux/bitops.h>#include <linux/device.h>#include <linux/dibs.h>#include <linux/mm.h>#include <linux/slab.h>#include <linux/spinlock.h>#include <linux/types.h>
#include "dibs_loopback.h"
#define DIBS_LO_SUPPORT_NOCOPY 0x1#define DIBS_DMA_ADDR_INVALID (~(dma_addr_t)0)
static const char dibs_lo_dev_name[] = "lo";/* global loopback device */static struct dibs_lo_dev *lo_dev;
static u16 dibs_lo_get_fabric_id(struct dibs_dev *dibs){ return DIBS_LOOPBACK_FABRIC;}
static int dibs_lo_query_rgid(struct dibs_dev *dibs, const uuid_t *rgid, u32 vid_valid, u32 vid){ /* rgid should be the same as lgid */ if (!uuid_equal(rgid, &dibs->gid)) return -ENETUNREACH; return 0;}
static int dibs_lo_max_dmbs(void){ return DIBS_LO_MAX_DMBS;}
static int dibs_lo_register_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb, struct dibs_client *client){ struct dibs_lo_dmb_node *dmb_node, *tmp_node; struct dibs_lo_dev *ldev; struct folio *folio; unsigned long flags; int sba_idx, rc;
ldev = dibs->drv_priv; sba_idx = dmb->idx; /* check space for new dmb */ for_each_clear_bit(sba_idx, ldev->sba_idx_mask, DIBS_LO_MAX_DMBS) { if (!test_and_set_bit(sba_idx, ldev->sba_idx_mask)) break; } if (sba_idx == DIBS_LO_MAX_DMBS) return -ENOSPC;
dmb_node = kzalloc_obj(*dmb_node); if (!dmb_node) { rc = -ENOMEM; goto err_bit; }
dmb_node->sba_idx = sba_idx; dmb_node->len = dmb->dmb_len;
/* not critical; fail under memory pressure and fallback to TCP */ folio = folio_alloc(GFP_KERNEL | __GFP_NOWARN | __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_ZERO, get_order(dmb_node->len)); if (!folio) { rc = -ENOMEM; goto err_node; } dmb_node->cpu_addr = folio_address(folio); dmb_node->dma_addr = DIBS_DMA_ADDR_INVALID; refcount_set(&dmb_node->refcnt, 1);
again: /* add new dmb into hash table */ get_random_bytes(&dmb_node->token, sizeof(dmb_node->token)); write_lock_bh(&ldev->dmb_ht_lock); hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_node->token) { if (tmp_node->token == dmb_node->token) { write_unlock_bh(&ldev->dmb_ht_lock); goto again; } } hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token); write_unlock_bh(&ldev->dmb_ht_lock); atomic_inc(&ldev->dmb_cnt);
dmb->idx = dmb_node->sba_idx; dmb->dmb_tok = dmb_node->token; dmb->cpu_addr = dmb_node->cpu_addr; dmb->dma_addr = dmb_node->dma_addr; dmb->dmb_len = dmb_node->len;
spin_lock_irqsave(&dibs->lock, flags); dibs->dmb_clientid_arr[sba_idx] = client->id; spin_unlock_irqrestore(&dibs->lock, flags);
return 0;
err_node: kfree(dmb_node);err_bit: clear_bit(sba_idx, ldev->sba_idx_mask); return rc;}
static void dibs_lo_free_dmb(struct dibs_lo_dev *ldev, struct dibs_lo_dmb_node *dmb_node){ clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask); folio_put(virt_to_folio(dmb_node->cpu_addr)); kfree(dmb_node);
if (atomic_dec_and_test(&ldev->dmb_cnt)) wake_up(&ldev->ldev_release);}
static int dibs_lo_unregister_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb){ struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node; struct dibs_lo_dev *ldev; unsigned long flags; bool last;
ldev = dibs->drv_priv;
/* find dmb from hash table */ write_lock_bh(&ldev->dmb_ht_lock); hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) { if (tmp_node->token == dmb->dmb_tok) { dmb_node = tmp_node; break; } } if (!dmb_node) { write_unlock_bh(&ldev->dmb_ht_lock); return -EINVAL; } last = refcount_dec_and_test(&dmb_node->refcnt); if (last) hash_del(&dmb_node->list); write_unlock_bh(&ldev->dmb_ht_lock);
if (last) { spin_lock_irqsave(&dibs->lock, flags); dibs->dmb_clientid_arr[dmb_node->sba_idx] = NO_DIBS_CLIENT; spin_unlock_irqrestore(&dibs->lock, flags);
dibs_lo_free_dmb(ldev, dmb_node); } return 0;}
static int dibs_lo_support_dmb_nocopy(struct dibs_dev *dibs){ return DIBS_LO_SUPPORT_NOCOPY;}
static int dibs_lo_attach_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb){ struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node; struct dibs_lo_dev *ldev;
ldev = dibs->drv_priv;
/* find dmb_node according to dmb->dmb_tok */ read_lock_bh(&ldev->dmb_ht_lock); hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) { if (tmp_node->token == dmb->dmb_tok) { dmb_node = tmp_node; break; } } if (!dmb_node) { read_unlock_bh(&ldev->dmb_ht_lock); return -EINVAL; } refcount_inc(&dmb_node->refcnt); read_unlock_bh(&ldev->dmb_ht_lock);
/* provide dmb information */ dmb->idx = dmb_node->sba_idx; dmb->dmb_tok = dmb_node->token; dmb->cpu_addr = dmb_node->cpu_addr; dmb->dma_addr = dmb_node->dma_addr; dmb->dmb_len = dmb_node->len; return 0;}
static int dibs_lo_detach_dmb(struct dibs_dev *dibs, u64 token){ struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node; struct dibs_lo_dev *ldev; bool last;
ldev = dibs->drv_priv;
/* find dmb_node according to dmb->dmb_tok */ write_lock_bh(&ldev->dmb_ht_lock); hash_for_each_possible(ldev->dmb_ht, tmp_node, list, token) { if (tmp_node->token == token) { dmb_node = tmp_node; break; } } if (!dmb_node) { write_unlock_bh(&ldev->dmb_ht_lock); return -EINVAL; } last = refcount_dec_and_test(&dmb_node->refcnt); if (last) hash_del(&dmb_node->list); write_unlock_bh(&ldev->dmb_ht_lock);
if (last) dibs_lo_free_dmb(ldev, dmb_node);
return 0;}
static int dibs_lo_move_data(struct dibs_dev *dibs, u64 dmb_tok, unsigned int idx, bool sf, unsigned int offset, void *data, unsigned int size){ struct dibs_lo_dmb_node *rmb_node = NULL, *tmp_node; struct dibs_lo_dev *ldev; unsigned long flags; u16 s_mask; u8 client_id; u32 sba_idx;
ldev = dibs->drv_priv;
read_lock_bh(&ldev->dmb_ht_lock); hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_tok) { if (tmp_node->token == dmb_tok) { rmb_node = tmp_node; break; } } if (!rmb_node) { read_unlock_bh(&ldev->dmb_ht_lock); return -EINVAL; } if ((u64)offset + size > rmb_node->len) { read_unlock_bh(&ldev->dmb_ht_lock); return -EINVAL; }
memcpy((char *)rmb_node->cpu_addr + offset, data, size); sba_idx = rmb_node->sba_idx; read_unlock_bh(&ldev->dmb_ht_lock);
if (!sf) return 0;
spin_lock_irqsave(&dibs->lock, flags); client_id = dibs->dmb_clientid_arr[sba_idx]; s_mask = ror16(0x1000, idx); if (likely(client_id != NO_DIBS_CLIENT && dibs->subs[client_id])) dibs->subs[client_id]->ops->handle_irq(dibs, sba_idx, s_mask); spin_unlock_irqrestore(&dibs->lock, flags);
return 0;}
static const struct dibs_dev_ops dibs_lo_ops = { .get_fabric_id = dibs_lo_get_fabric_id, .query_remote_gid = dibs_lo_query_rgid, .max_dmbs = dibs_lo_max_dmbs, .register_dmb = dibs_lo_register_dmb, .unregister_dmb = dibs_lo_unregister_dmb, .move_data = dibs_lo_move_data, .support_mmapped_rdmb = dibs_lo_support_dmb_nocopy, .attach_dmb = dibs_lo_attach_dmb, .detach_dmb = dibs_lo_detach_dmb,};
static void dibs_lo_dev_init(struct dibs_lo_dev *ldev){ rwlock_init(&ldev->dmb_ht_lock); hash_init(ldev->dmb_ht); atomic_set(&ldev->dmb_cnt, 0); init_waitqueue_head(&ldev->ldev_release);}
static void dibs_lo_dev_exit(struct dibs_lo_dev *ldev){ if (atomic_read(&ldev->dmb_cnt)) wait_event(ldev->ldev_release, !atomic_read(&ldev->dmb_cnt));}
static int dibs_lo_dev_probe(void){ struct dibs_lo_dev *ldev; struct dibs_dev *dibs; int ret;
ldev = kzalloc_obj(*ldev); if (!ldev) return -ENOMEM;
dibs = dibs_dev_alloc(); if (!dibs) { kfree(ldev); return -ENOMEM; }
ldev->dibs = dibs; dibs->drv_priv = ldev; dibs_lo_dev_init(ldev); uuid_gen(&dibs->gid); dibs->ops = &dibs_lo_ops;
dibs->dev.parent = NULL; dev_set_name(&dibs->dev, "%s", dibs_lo_dev_name);
ret = dibs_dev_add(dibs); if (ret) goto err_reg; lo_dev = ldev; return 0;
err_reg: /* pairs with dibs_dev_alloc() */ put_device(&dibs->dev); kfree(ldev);
return ret;}
static void dibs_lo_dev_remove(void){ if (!lo_dev) return;
dibs_dev_del(lo_dev->dibs); dibs_lo_dev_exit(lo_dev); /* pairs with dibs_dev_alloc() */ put_device(&lo_dev->dibs->dev); kfree(lo_dev); lo_dev = NULL;}
int dibs_loopback_init(void){ return dibs_lo_dev_probe();}
void dibs_loopback_exit(void){ dibs_lo_dev_remove();}