Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509// SPDX-License-Identifier: GPL-2.0-or-later/* * Core IEEE1394 transaction logic * * Copyright (C) 2004-2006 Kristian Hoegsberg <krh@bitplanet.net> */
#include <linux/bug.h>#include <linux/completion.h>#include <linux/device.h>#include <linux/errno.h>#include <linux/firewire.h>#include <linux/firewire-constants.h>#include <linux/fs.h>#include <linux/init.h>#include <linux/jiffies.h>#include <linux/kernel.h>#include <linux/list.h>#include <linux/module.h>#include <linux/rculist.h>#include <linux/slab.h>#include <linux/spinlock.h>#include <linux/string.h>#include <linux/timer.h>#include <linux/types.h>#include <linux/workqueue.h>
#include <asm/byteorder.h>
#include "core.h"#include "packet-header-definitions.h"#include "phy-packet-definitions.h"#include <trace/events/firewire.h>
#define HEADER_DESTINATION_IS_BROADCAST(header) \ ((async_header_get_destination(header) & 0x3f) == 0x3f)
/* returns 0 if the split timeout handler is already running */static int try_cancel_split_timeout(struct fw_transaction *t){ if (t->is_split_transaction) return timer_delete(&t->split_timeout_timer); else return 1;}
// card->transactions.lock must be acquired in advance.static void remove_transaction_entry(struct fw_card *card, struct fw_transaction *entry){ list_del_init(&entry->link); card->transactions.tlabel_mask &= ~(1ULL << entry->tlabel);}
// Must be called without holding card->transactions.lock.void fw_cancel_pending_transactions(struct fw_card *card){ struct fw_transaction *t, *tmp; LIST_HEAD(pending_list);
// NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for // local destination never runs in any type of IRQ context. scoped_guard(spinlock_irqsave, &card->transactions.lock) { list_for_each_entry_safe(t, tmp, &card->transactions.list, link) { if (try_cancel_split_timeout(t)) list_move(&t->link, &pending_list); } }
list_for_each_entry_safe(t, tmp, &pending_list, link) { list_del(&t->link);
if (!t->with_tstamp) { t->callback.without_tstamp(card, RCODE_CANCELLED, NULL, 0, t->callback_data); } else { t->callback.with_tstamp(card, RCODE_CANCELLED, t->packet.timestamp, 0, NULL, 0, t->callback_data); } }}
// card->transactions.lock must be acquired in advance.#define find_and_pop_transaction_entry(card, condition) \({ \ struct fw_transaction *iter, *t = NULL; \ list_for_each_entry(iter, &card->transactions.list, link) { \ if (condition) { \ t = iter; \ break; \ } \ } \ if (t && try_cancel_split_timeout(t)) \ remove_transaction_entry(card, t); \ t; \})
static int close_transaction(struct fw_transaction *transaction, struct fw_card *card, int rcode, u32 response_tstamp){ struct fw_transaction *t;
// NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for // local destination never runs in any type of IRQ context. scoped_guard(spinlock_irqsave, &card->transactions.lock) { t = find_and_pop_transaction_entry(card, iter == transaction); if (!t) return -ENOENT; }
if (!t->with_tstamp) { t->callback.without_tstamp(card, rcode, NULL, 0, t->callback_data); } else { t->callback.with_tstamp(card, rcode, t->packet.timestamp, response_tstamp, NULL, 0, t->callback_data); }
return 0;}
/* * Only valid for transactions that are potentially pending (ie have * been sent). */int fw_cancel_transaction(struct fw_card *card, struct fw_transaction *transaction){ u32 tstamp;
/* * Cancel the packet transmission if it's still queued. That * will call the packet transmission callback which cancels * the transaction. */
if (card->driver->cancel_packet(card, &transaction->packet) == 0) return 0;
/* * If the request packet has already been sent, we need to see * if the transaction is still pending and remove it in that case. */
if (transaction->packet.ack == 0) { // The timestamp is reused since it was just read now. tstamp = transaction->packet.timestamp; } else { u32 curr_cycle_time = 0;
(void)fw_card_read_cycle_time(card, &curr_cycle_time); tstamp = cycle_time_to_ohci_tstamp(curr_cycle_time); }
return close_transaction(transaction, card, RCODE_CANCELLED, tstamp);}EXPORT_SYMBOL(fw_cancel_transaction);
static void split_transaction_timeout_callback(struct timer_list *timer){ struct fw_transaction *t = timer_container_of(t, timer, split_timeout_timer); struct fw_card *card = t->card;
scoped_guard(spinlock_irqsave, &card->transactions.lock) { if (list_empty(&t->link)) return; remove_transaction_entry(card, t); }
if (!t->with_tstamp) { t->callback.without_tstamp(card, RCODE_CANCELLED, NULL, 0, t->callback_data); } else { t->callback.with_tstamp(card, RCODE_CANCELLED, t->packet.timestamp, t->split_timeout_cycle, NULL, 0, t->callback_data); }}
// card->transactions.lock should be acquired in advance for the linked list.static void start_split_transaction_timeout(struct fw_transaction *t, unsigned int delta){ if (list_empty(&t->link) || WARN_ON(t->is_split_transaction)) return;
t->is_split_transaction = true;
mod_timer(&t->split_timeout_timer, jiffies + delta);}
static u32 compute_split_timeout_timestamp(struct fw_card *card, u32 request_timestamp);
static void transmit_complete_callback(struct fw_packet *packet, struct fw_card *card, int status){ struct fw_transaction *t = container_of(packet, struct fw_transaction, packet);
trace_async_request_outbound_complete((uintptr_t)t, card->index, packet->generation, packet->speed, status, packet->timestamp);
switch (status) { case ACK_COMPLETE: close_transaction(t, card, RCODE_COMPLETE, packet->timestamp); break; case ACK_PENDING: { unsigned int delta;
// NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for // local destination never runs in any type of IRQ context. scoped_guard(spinlock_irqsave, &card->split_timeout.lock) { t->split_timeout_cycle = compute_split_timeout_timestamp(card, packet->timestamp) & 0xffff; delta = card->split_timeout.jiffies; }
// NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for // local destination never runs in any type of IRQ context. scoped_guard(spinlock_irqsave, &card->transactions.lock) start_split_transaction_timeout(t, delta); break; } case ACK_BUSY_X: case ACK_BUSY_A: case ACK_BUSY_B: close_transaction(t, card, RCODE_BUSY, packet->timestamp); break; case ACK_DATA_ERROR: close_transaction(t, card, RCODE_DATA_ERROR, packet->timestamp); break; case ACK_TYPE_ERROR: close_transaction(t, card, RCODE_TYPE_ERROR, packet->timestamp); break; default: /* * In this case the ack is really a juju specific * rcode, so just forward that to the callback. */ close_transaction(t, card, status, packet->timestamp); break; }}
static void fw_fill_request(struct fw_packet *packet, int tcode, int tlabel, int destination_id, int source_id, int generation, int speed, unsigned long long offset, void *payload, size_t length){ int ext_tcode;
if (tcode == TCODE_STREAM_DATA) { // The value of destination_id argument should include tag, channel, and sy fields // as isochronous packet header has. packet->header[0] = destination_id; isoc_header_set_data_length(packet->header, length); isoc_header_set_tcode(packet->header, TCODE_STREAM_DATA); packet->header_length = 4; packet->payload = payload; packet->payload_length = length;
goto common; }
if (tcode > 0x10) { ext_tcode = tcode & ~0x10; tcode = TCODE_LOCK_REQUEST; } else ext_tcode = 0;
async_header_set_retry(packet->header, RETRY_X); async_header_set_tlabel(packet->header, tlabel); async_header_set_tcode(packet->header, tcode); async_header_set_destination(packet->header, destination_id); async_header_set_source(packet->header, source_id); async_header_set_offset(packet->header, offset);
switch (tcode) { case TCODE_WRITE_QUADLET_REQUEST: async_header_set_quadlet_data(packet->header, *(u32 *)payload); packet->header_length = 16; packet->payload_length = 0; break;
case TCODE_LOCK_REQUEST: case TCODE_WRITE_BLOCK_REQUEST: async_header_set_data_length(packet->header, length); async_header_set_extended_tcode(packet->header, ext_tcode); packet->header_length = 16; packet->payload = payload; packet->payload_length = length; break;
case TCODE_READ_QUADLET_REQUEST: packet->header_length = 12; packet->payload_length = 0; break;
case TCODE_READ_BLOCK_REQUEST: async_header_set_data_length(packet->header, length); async_header_set_extended_tcode(packet->header, ext_tcode); packet->header_length = 16; packet->payload_length = 0; break;
default: WARN(1, "wrong tcode %d\n", tcode); } common: packet->speed = speed; packet->generation = generation; packet->ack = 0; packet->payload_mapped = false;}
static int allocate_tlabel(struct fw_card *card)__must_hold(&card->transactions.lock){ int tlabel;
lockdep_assert_held(&card->transactions.lock);
tlabel = card->transactions.current_tlabel; while (card->transactions.tlabel_mask & (1ULL << tlabel)) { tlabel = (tlabel + 1) & 0x3f; if (tlabel == card->transactions.current_tlabel) return -EBUSY; }
card->transactions.current_tlabel = (tlabel + 1) & 0x3f; card->transactions.tlabel_mask |= 1ULL << tlabel;
return tlabel;}
/** * __fw_send_request() - submit a request packet for transmission to generate callback for response * subaction with or without time stamp. * @card: interface to send the request at * @t: transaction instance to which the request belongs * @tcode: transaction code * @destination_id: destination node ID, consisting of bus_ID and phy_ID * @generation: bus generation in which request and response are valid * @speed: transmission speed * @offset: 48bit wide offset into destination's address space * @payload: data payload for the request subaction * @length: length of the payload, in bytes * @callback: union of two functions whether to receive time stamp or not for response * subaction. * @with_tstamp: Whether to receive time stamp or not for response subaction. * @callback_data: data to be passed to the transaction completion callback * * Submit a request packet into the asynchronous request transmission queue. * Can be called from atomic context. If you prefer a blocking API, use * fw_run_transaction() in a context that can sleep. * * In case of lock requests, specify one of the firewire-core specific %TCODE_ * constants instead of %TCODE_LOCK_REQUEST in @tcode. * * Make sure that the value in @destination_id is not older than the one in * @generation. Otherwise the request is in danger to be sent to a wrong node. * * In case of asynchronous stream packets i.e. %TCODE_STREAM_DATA, the caller * needs to synthesize @destination_id with fw_stream_packet_destination_id(). * It will contain tag, channel, and sy data instead of a node ID then. * * The payload buffer at @data is going to be DMA-mapped except in case of * @length <= 8 or of local (loopback) requests. Hence make sure that the * buffer complies with the restrictions of the streaming DMA mapping API. * @payload must not be freed before the @callback is called. * * In case of request types without payload, @data is NULL and @length is 0. * * After the transaction is completed successfully or unsuccessfully, the * @callback will be called. Among its parameters is the response code which * is either one of the rcodes per IEEE 1394 or, in case of internal errors, * the firewire-core specific %RCODE_SEND_ERROR. The other firewire-core * specific rcodes (%RCODE_CANCELLED, %RCODE_BUSY, %RCODE_GENERATION, * %RCODE_NO_ACK) denote transaction timeout, busy responder, stale request * generation, or missing ACK respectively. * * Note some timing corner cases: fw_send_request() may complete much earlier * than when the request packet actually hits the wire. On the other hand, * transaction completion and hence execution of @callback may happen even * before fw_send_request() returns. */void __fw_send_request(struct fw_card *card, struct fw_transaction *t, int tcode, int destination_id, int generation, int speed, unsigned long long offset, void *payload, size_t length, union fw_transaction_callback callback, bool with_tstamp, void *callback_data){ int tlabel;
/* * Allocate tlabel from the bitmap and put the transaction on * the list while holding the card spinlock. */
// NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for // local destination never runs in any type of IRQ context. scoped_guard(spinlock_irqsave, &card->transactions.lock) tlabel = allocate_tlabel(card); if (tlabel < 0) { if (!with_tstamp) { callback.without_tstamp(card, RCODE_SEND_ERROR, NULL, 0, callback_data); } else { // Timestamping on behalf of hardware. u32 curr_cycle_time = 0; u32 tstamp;
(void)fw_card_read_cycle_time(card, &curr_cycle_time); tstamp = cycle_time_to_ohci_tstamp(curr_cycle_time);
callback.with_tstamp(card, RCODE_SEND_ERROR, tstamp, tstamp, NULL, 0, callback_data); } return; }
t->node_id = destination_id; t->tlabel = tlabel; t->card = card; t->is_split_transaction = false; timer_setup(&t->split_timeout_timer, split_transaction_timeout_callback, 0); t->callback = callback; t->with_tstamp = with_tstamp; t->callback_data = callback_data; t->packet.callback = transmit_complete_callback;
// NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for // local destination never runs in any type of IRQ context. scoped_guard(spinlock_irqsave, &card->lock) { // The node_id field of fw_card can be updated when handling SelfIDComplete. fw_fill_request(&t->packet, tcode, t->tlabel, destination_id, card->node_id, generation, speed, offset, payload, length); }
// NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for // local destination never runs in any type of IRQ context. scoped_guard(spinlock_irqsave, &card->transactions.lock) list_add_tail(&t->link, &card->transactions.list);
// Safe with no lock, since the index field of fw_card is immutable once assigned. trace_async_request_outbound_initiate((uintptr_t)t, card->index, generation, speed, t->packet.header, payload, tcode_is_read_request(tcode) ? 0 : length / 4);
card->driver->send_request(card, &t->packet);}EXPORT_SYMBOL_GPL(__fw_send_request);
struct transaction_callback_data { struct completion done; void *payload; int rcode;};
static void transaction_callback(struct fw_card *card, int rcode, void *payload, size_t length, void *data){ struct transaction_callback_data *d = data;
if (rcode == RCODE_COMPLETE) memcpy(d->payload, payload, length); d->rcode = rcode; complete(&d->done);}
/** * fw_run_transaction() - send request and sleep until transaction is completed * @card: card interface for this request * @tcode: transaction code * @destination_id: destination node ID, consisting of bus_ID and phy_ID * @generation: bus generation in which request and response are valid * @speed: transmission speed * @offset: 48bit wide offset into destination's address space * @payload: data payload for the request subaction * @length: length of the payload, in bytes * * Returns the RCODE. See fw_send_request() for parameter documentation. * Unlike fw_send_request(), @data points to the payload of the request or/and * to the payload of the response. DMA mapping restrictions apply to outbound * request payloads of >= 8 bytes but not to inbound response payloads. */int fw_run_transaction(struct fw_card *card, int tcode, int destination_id, int generation, int speed, unsigned long long offset, void *payload, size_t length){ struct transaction_callback_data d; struct fw_transaction t;
timer_setup_on_stack(&t.split_timeout_timer, NULL, 0); init_completion(&d.done); d.payload = payload; fw_send_request(card, &t, tcode, destination_id, generation, speed, offset, payload, length, transaction_callback, &d); wait_for_completion(&d.done); timer_destroy_on_stack(&t.split_timeout_timer);
return d.rcode;}EXPORT_SYMBOL(fw_run_transaction);
static DEFINE_MUTEX(phy_config_mutex);static DECLARE_COMPLETION(phy_config_done);
static void transmit_phy_packet_callback(struct fw_packet *packet, struct fw_card *card, int status){ trace_async_phy_outbound_complete((uintptr_t)packet, card->index, packet->generation, status, packet->timestamp); complete(&phy_config_done);}
static struct fw_packet phy_config_packet = { .header_length = 12, .payload_length = 0, .speed = SCODE_100, .callback = transmit_phy_packet_callback,};
void fw_send_phy_config(struct fw_card *card, int node_id, int generation, int gap_count){ long timeout = msecs_to_jiffies(100); u32 data = 0;
phy_packet_set_packet_identifier(&data, PHY_PACKET_PACKET_IDENTIFIER_PHY_CONFIG);
if (node_id != FW_PHY_CONFIG_NO_NODE_ID) { phy_packet_phy_config_set_root_id(&data, node_id); phy_packet_phy_config_set_force_root_node(&data, true); }
if (gap_count == FW_PHY_CONFIG_CURRENT_GAP_COUNT) { gap_count = card->driver->read_phy_reg(card, 1); if (gap_count < 0) return;
gap_count &= 63; if (gap_count == 63) return; } phy_packet_phy_config_set_gap_count(&data, gap_count); phy_packet_phy_config_set_gap_count_optimization(&data, true);
guard(mutex)(&phy_config_mutex);
async_header_set_tcode(phy_config_packet.header, TCODE_LINK_INTERNAL); phy_config_packet.header[1] = data; phy_config_packet.header[2] = ~data; phy_config_packet.generation = generation; reinit_completion(&phy_config_done);
trace_async_phy_outbound_initiate((uintptr_t)&phy_config_packet, card->index, phy_config_packet.generation, phy_config_packet.header[1], phy_config_packet.header[2]);
card->driver->send_request(card, &phy_config_packet); wait_for_completion_timeout(&phy_config_done, timeout);}
static struct fw_address_handler *lookup_overlapping_address_handler( struct list_head *list, unsigned long long offset, size_t length){ struct fw_address_handler *handler;
list_for_each_entry_rcu(handler, list, link) { if (handler->offset < offset + length && offset < handler->offset + handler->length) return handler; }
return NULL;}
static bool is_enclosing_handler(struct fw_address_handler *handler, unsigned long long offset, size_t length){ return handler->offset <= offset && offset + length <= handler->offset + handler->length;}
static struct fw_address_handler *lookup_enclosing_address_handler( struct list_head *list, unsigned long long offset, size_t length){ struct fw_address_handler *handler;
list_for_each_entry_rcu(handler, list, link) { if (is_enclosing_handler(handler, offset, length)) return handler; }
return NULL;}
static DEFINE_SPINLOCK(address_handler_list_lock);static LIST_HEAD(address_handler_list);
const struct fw_address_region fw_high_memory_region = { .start = FW_MAX_PHYSICAL_RANGE, .end = 0xffffe0000000ULL, };EXPORT_SYMBOL(fw_high_memory_region);
static const struct fw_address_region low_memory_region = { .start = 0x000000000000ULL, .end = FW_MAX_PHYSICAL_RANGE, };
#if 0const struct fw_address_region fw_private_region = { .start = 0xffffe0000000ULL, .end = 0xfffff0000000ULL, };const struct fw_address_region fw_csr_region = { .start = CSR_REGISTER_BASE, .end = CSR_REGISTER_BASE | CSR_CONFIG_ROM_END, };const struct fw_address_region fw_unit_space_region = { .start = 0xfffff0000900ULL, .end = 0x1000000000000ULL, };#endif /* 0 */
static void complete_address_handler(struct kref *kref){ struct fw_address_handler *handler = container_of(kref, struct fw_address_handler, kref);
complete(&handler->done);}
static void get_address_handler(struct fw_address_handler *handler){ kref_get(&handler->kref);}
static int put_address_handler(struct fw_address_handler *handler){ return kref_put(&handler->kref, complete_address_handler);}
/** * fw_core_add_address_handler() - register for incoming requests * @handler: callback * @region: region in the IEEE 1212 node space address range * * region->start, ->end, and handler->length have to be quadlet-aligned. * * When a request is received that falls within the specified address range, the specified callback * is invoked. The parameters passed to the callback give the details of the particular request. * The callback is invoked in the workqueue context in most cases. However, if the request is * initiated by the local node, the callback is invoked in the initiator's context. * * To be called in process context. * Return value: 0 on success, non-zero otherwise. * * The start offset of the handler's address region is determined by * fw_core_add_address_handler() and is returned in handler->offset. * * Address allocations are exclusive, except for the FCP registers. */int fw_core_add_address_handler(struct fw_address_handler *handler, const struct fw_address_region *region){ struct fw_address_handler *other; int ret = -EBUSY;
if (region->start & 0xffff000000000003ULL || region->start >= region->end || region->end > 0x0001000000000000ULL || handler->length & 3 || handler->length == 0) return -EINVAL;
guard(spinlock)(&address_handler_list_lock);
handler->offset = region->start; while (handler->offset + handler->length <= region->end) { if (is_in_fcp_region(handler->offset, handler->length)) other = NULL; else other = lookup_overlapping_address_handler (&address_handler_list, handler->offset, handler->length); if (other != NULL) { handler->offset += other->length; } else { init_completion(&handler->done); kref_init(&handler->kref); list_add_tail_rcu(&handler->link, &address_handler_list); ret = 0; break; } }
return ret;}EXPORT_SYMBOL(fw_core_add_address_handler);
/** * fw_core_remove_address_handler() - unregister an address handler * @handler: callback * * To be called in process context. * * When fw_core_remove_address_handler() returns, @handler->callback() is * guaranteed to not run on any CPU anymore. */void fw_core_remove_address_handler(struct fw_address_handler *handler){ scoped_guard(spinlock, &address_handler_list_lock) list_del_rcu(&handler->link);
synchronize_rcu();
if (!put_address_handler(handler)) wait_for_completion(&handler->done);}EXPORT_SYMBOL(fw_core_remove_address_handler);
struct fw_request { struct kref kref; struct fw_packet response; u32 request_header[ASYNC_HEADER_QUADLET_COUNT]; int ack; u32 timestamp; u32 length; u32 data[];};
void fw_request_get(struct fw_request *request){ kref_get(&request->kref);}
static void release_request(struct kref *kref){ struct fw_request *request = container_of(kref, struct fw_request, kref);
kfree(request);}
void fw_request_put(struct fw_request *request){ kref_put(&request->kref, release_request);}
static void free_response_callback(struct fw_packet *packet, struct fw_card *card, int status){ struct fw_request *request = container_of(packet, struct fw_request, response);
trace_async_response_outbound_complete((uintptr_t)request, card->index, packet->generation, packet->speed, status, packet->timestamp);
// Decrease the reference count since not at in-flight. fw_request_put(request);
// Decrease the reference count to release the object. fw_request_put(request);}
int fw_get_response_length(struct fw_request *r){ int tcode, ext_tcode, data_length;
tcode = async_header_get_tcode(r->request_header);
switch (tcode) { case TCODE_WRITE_QUADLET_REQUEST: case TCODE_WRITE_BLOCK_REQUEST: return 0;
case TCODE_READ_QUADLET_REQUEST: return 4;
case TCODE_READ_BLOCK_REQUEST: data_length = async_header_get_data_length(r->request_header); return data_length;
case TCODE_LOCK_REQUEST: ext_tcode = async_header_get_extended_tcode(r->request_header); data_length = async_header_get_data_length(r->request_header); switch (ext_tcode) { case EXTCODE_FETCH_ADD: case EXTCODE_LITTLE_ADD: return data_length; default: return data_length / 2; }
default: WARN(1, "wrong tcode %d\n", tcode); return 0; }}
void fw_fill_response(struct fw_packet *response, u32 *request_header, int rcode, void *payload, size_t length){ int tcode, tlabel, extended_tcode, source, destination;
tcode = async_header_get_tcode(request_header); tlabel = async_header_get_tlabel(request_header); source = async_header_get_destination(request_header); // Exchange. destination = async_header_get_source(request_header); // Exchange. extended_tcode = async_header_get_extended_tcode(request_header);
async_header_set_retry(response->header, RETRY_1); async_header_set_tlabel(response->header, tlabel); async_header_set_destination(response->header, destination); async_header_set_source(response->header, source); async_header_set_rcode(response->header, rcode); response->header[2] = 0; // The field is reserved.
switch (tcode) { case TCODE_WRITE_QUADLET_REQUEST: case TCODE_WRITE_BLOCK_REQUEST: async_header_set_tcode(response->header, TCODE_WRITE_RESPONSE); response->header_length = 12; response->payload_length = 0; break;
case TCODE_READ_QUADLET_REQUEST: async_header_set_tcode(response->header, TCODE_READ_QUADLET_RESPONSE); if (payload != NULL) async_header_set_quadlet_data(response->header, *(u32 *)payload); else async_header_set_quadlet_data(response->header, 0); response->header_length = 16; response->payload_length = 0; break;
case TCODE_READ_BLOCK_REQUEST: case TCODE_LOCK_REQUEST: async_header_set_tcode(response->header, tcode + 2); async_header_set_data_length(response->header, length); async_header_set_extended_tcode(response->header, extended_tcode); response->header_length = 16; response->payload = payload; response->payload_length = length; break;
default: WARN(1, "wrong tcode %d\n", tcode); }
response->payload_mapped = false;}EXPORT_SYMBOL(fw_fill_response);
static u32 compute_split_timeout_timestamp(struct fw_card *card, u32 request_timestamp)__must_hold(&card->split_timeout.lock){ unsigned int cycles; u32 timestamp;
lockdep_assert_held(&card->split_timeout.lock);
cycles = card->split_timeout.cycles; cycles += request_timestamp & 0x1fff;
timestamp = request_timestamp & ~0x1fff; timestamp += (cycles / 8000) << 13; timestamp |= cycles % 8000;
return timestamp;}
static struct fw_request *allocate_request(struct fw_card *card, struct fw_packet *p){ struct fw_request *request; u32 *data, length; int request_tcode;
request_tcode = async_header_get_tcode(p->header); switch (request_tcode) { case TCODE_WRITE_QUADLET_REQUEST: data = &p->header[3]; length = 4; break;
case TCODE_WRITE_BLOCK_REQUEST: case TCODE_LOCK_REQUEST: data = p->payload; length = async_header_get_data_length(p->header); break;
case TCODE_READ_QUADLET_REQUEST: data = NULL; length = 4; break;
case TCODE_READ_BLOCK_REQUEST: data = NULL; length = async_header_get_data_length(p->header); break;
default: fw_notice(card, "ERROR - corrupt request received - %08x %08x %08x\n", p->header[0], p->header[1], p->header[2]); return NULL; }
request = kmalloc(sizeof(*request) + length, GFP_ATOMIC); if (request == NULL) return NULL; kref_init(&request->kref);
// NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for // local destination never runs in any type of IRQ context. scoped_guard(spinlock_irqsave, &card->split_timeout.lock) request->response.timestamp = compute_split_timeout_timestamp(card, p->timestamp);
request->response.speed = p->speed; request->response.generation = p->generation; request->response.ack = 0; request->response.callback = free_response_callback; request->ack = p->ack; request->timestamp = p->timestamp; request->length = length; if (data) memcpy(request->data, data, length);
memcpy(request->request_header, p->header, sizeof(p->header));
return request;}
/** * fw_send_response: - send response packet for asynchronous transaction. * @card: interface to send the response at. * @request: firewire request data for the transaction. * @rcode: response code to send. * * Submit a response packet into the asynchronous response transmission queue. The @request * is going to be released when the transmission successfully finishes later. */void fw_send_response(struct fw_card *card, struct fw_request *request, int rcode){ u32 *data = NULL; unsigned int data_length = 0;
/* unified transaction or broadcast transaction: don't respond */ if (request->ack != ACK_PENDING || HEADER_DESTINATION_IS_BROADCAST(request->request_header)) { fw_request_put(request); return; }
if (rcode == RCODE_COMPLETE) { data = request->data; data_length = fw_get_response_length(request); }
fw_fill_response(&request->response, request->request_header, rcode, data, data_length);
// Increase the reference count so that the object is kept during in-flight. fw_request_get(request);
trace_async_response_outbound_initiate((uintptr_t)request, card->index, request->response.generation, request->response.speed, request->response.header, data, data ? data_length / 4 : 0);
card->driver->send_response(card, &request->response);}EXPORT_SYMBOL(fw_send_response);
/** * fw_get_request_speed() - returns speed at which the @request was received * @request: firewire request data */int fw_get_request_speed(struct fw_request *request){ return request->response.speed;}EXPORT_SYMBOL(fw_get_request_speed);
/** * fw_request_get_timestamp: Get timestamp of the request. * @request: The opaque pointer to request structure. * * Get timestamp when 1394 OHCI controller receives the asynchronous request subaction. The * timestamp consists of the low order 3 bits of second field and the full 13 bits of count * field of isochronous cycle time register. * * Returns: timestamp of the request. */u32 fw_request_get_timestamp(const struct fw_request *request){ return request->timestamp;}EXPORT_SYMBOL_GPL(fw_request_get_timestamp);
static void handle_exclusive_region_request(struct fw_card *card, struct fw_packet *p, struct fw_request *request, unsigned long long offset){ struct fw_address_handler *handler; int tcode, destination, source;
destination = async_header_get_destination(p->header); source = async_header_get_source(p->header); tcode = async_header_get_tcode(p->header); if (tcode == TCODE_LOCK_REQUEST) tcode = 0x10 + async_header_get_extended_tcode(p->header);
scoped_guard(rcu) { handler = lookup_enclosing_address_handler(&address_handler_list, offset, request->length); if (handler) get_address_handler(handler); }
if (!handler) { fw_send_response(card, request, RCODE_ADDRESS_ERROR); return; }
// Outside the RCU read-side critical section. Without spinlock. With reference count. handler->address_callback(card, request, tcode, destination, source, p->generation, offset, request->data, request->length, handler->callback_data); put_address_handler(handler);}
// To use kmalloc allocator efficiently, this should be power of two.#define BUFFER_ON_KERNEL_STACK_SIZE 4
static void handle_fcp_region_request(struct fw_card *card, struct fw_packet *p, struct fw_request *request, unsigned long long offset){ struct fw_address_handler *buffer_on_kernel_stack[BUFFER_ON_KERNEL_STACK_SIZE]; struct fw_address_handler *handler, **handlers; int tcode, destination, source, i, count, buffer_size;
if ((offset != (CSR_REGISTER_BASE | CSR_FCP_COMMAND) && offset != (CSR_REGISTER_BASE | CSR_FCP_RESPONSE)) || request->length > 0x200) { fw_send_response(card, request, RCODE_ADDRESS_ERROR);
return; }
tcode = async_header_get_tcode(p->header); destination = async_header_get_destination(p->header); source = async_header_get_source(p->header);
if (tcode != TCODE_WRITE_QUADLET_REQUEST && tcode != TCODE_WRITE_BLOCK_REQUEST) { fw_send_response(card, request, RCODE_TYPE_ERROR);
return; }
count = 0; handlers = buffer_on_kernel_stack; buffer_size = ARRAY_SIZE(buffer_on_kernel_stack); scoped_guard(rcu) { list_for_each_entry_rcu(handler, &address_handler_list, link) { if (is_enclosing_handler(handler, offset, request->length)) { if (count >= buffer_size) { int next_size = buffer_size * 2; struct fw_address_handler **buffer_on_kernel_heap;
if (handlers == buffer_on_kernel_stack) buffer_on_kernel_heap = NULL; else buffer_on_kernel_heap = handlers;
buffer_on_kernel_heap = krealloc_array(buffer_on_kernel_heap, next_size, sizeof(*buffer_on_kernel_heap), GFP_ATOMIC); // FCP is used for purposes unrelated to significant system // resources (e.g. storage or networking), so allocation // failures are not considered so critical. if (!buffer_on_kernel_heap) break;
if (handlers == buffer_on_kernel_stack) { memcpy(buffer_on_kernel_heap, buffer_on_kernel_stack, sizeof(buffer_on_kernel_stack)); }
handlers = buffer_on_kernel_heap; buffer_size = next_size; } get_address_handler(handler); handlers[count++] = handler; } } }
for (i = 0; i < count; ++i) { handler = handlers[i]; handler->address_callback(card, request, tcode, destination, source, p->generation, offset, request->data, request->length, handler->callback_data); put_address_handler(handler); }
if (handlers != buffer_on_kernel_stack) kfree(handlers);
fw_send_response(card, request, RCODE_COMPLETE);}
void fw_core_handle_request(struct fw_card *card, struct fw_packet *p){ struct fw_request *request; unsigned long long offset; unsigned int tcode;
if (p->ack != ACK_PENDING && p->ack != ACK_COMPLETE) return;
tcode = async_header_get_tcode(p->header); if (tcode_is_link_internal(tcode)) { trace_async_phy_inbound((uintptr_t)p, card->index, p->generation, p->ack, p->timestamp, p->header[1], p->header[2]); fw_cdev_handle_phy_packet(card, p); return; }
request = allocate_request(card, p); if (request == NULL) { /* FIXME: send statically allocated busy packet. */ return; }
trace_async_request_inbound((uintptr_t)request, card->index, p->generation, p->speed, p->ack, p->timestamp, p->header, request->data, tcode_is_read_request(tcode) ? 0 : request->length / 4);
offset = async_header_get_offset(p->header);
if (!is_in_fcp_region(offset, request->length)) handle_exclusive_region_request(card, p, request, offset); else handle_fcp_region_request(card, p, request, offset);
}EXPORT_SYMBOL(fw_core_handle_request);
void fw_core_handle_response(struct fw_card *card, struct fw_packet *p){ struct fw_transaction *t = NULL; u32 *data; size_t data_length; int tcode, tlabel, source, rcode;
tcode = async_header_get_tcode(p->header); tlabel = async_header_get_tlabel(p->header); source = async_header_get_source(p->header); rcode = async_header_get_rcode(p->header);
// FIXME: sanity check packet, is length correct, does tcodes // and addresses match to the transaction request queried later. // // For the tracepoints event, let us decode the header here against the concern.
switch (tcode) { case TCODE_READ_QUADLET_RESPONSE: data = (u32 *) &p->header[3]; data_length = 4; break;
case TCODE_WRITE_RESPONSE: data = NULL; data_length = 0; break;
case TCODE_READ_BLOCK_RESPONSE: case TCODE_LOCK_RESPONSE: data = p->payload; data_length = async_header_get_data_length(p->header); break;
default: /* Should never happen, this is just to shut up gcc. */ data = NULL; data_length = 0; break; }
// NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for // local destination never runs in any type of IRQ context. scoped_guard(spinlock_irqsave, &card->transactions.lock) { t = find_and_pop_transaction_entry(card, iter->node_id == source && iter->tlabel == tlabel); }
trace_async_response_inbound((uintptr_t)t, card->index, p->generation, p->speed, p->ack, p->timestamp, p->header, data, data_length / 4);
if (!t) { fw_notice(card, "unsolicited response (source %x, tlabel %x)\n", source, tlabel); return; }
/* * The response handler may be executed while the request handler * is still pending. Cancel the request handler. */ card->driver->cancel_packet(card, &t->packet);
if (!t->with_tstamp) { t->callback.without_tstamp(card, rcode, data, data_length, t->callback_data); } else { t->callback.with_tstamp(card, rcode, t->packet.timestamp, p->timestamp, data, data_length, t->callback_data); }}EXPORT_SYMBOL(fw_core_handle_response);
/** * fw_rcode_string - convert a firewire result code to an error description * @rcode: the result code */const char *fw_rcode_string(int rcode){ static const char *const names[] = { [RCODE_COMPLETE] = "no error", [RCODE_CONFLICT_ERROR] = "conflict error", [RCODE_DATA_ERROR] = "data error", [RCODE_TYPE_ERROR] = "type error", [RCODE_ADDRESS_ERROR] = "address error", [RCODE_SEND_ERROR] = "send error", [RCODE_CANCELLED] = "timeout", [RCODE_BUSY] = "busy", [RCODE_GENERATION] = "bus reset", [RCODE_NO_ACK] = "no ack", };
if ((unsigned int)rcode < ARRAY_SIZE(names) && names[rcode]) return names[rcode]; else return "unknown";}EXPORT_SYMBOL(fw_rcode_string);
static const struct fw_address_region topology_map_region = { .start = CSR_REGISTER_BASE | CSR_TOPOLOGY_MAP, .end = CSR_REGISTER_BASE | CSR_TOPOLOGY_MAP_END, };
static void handle_topology_map(struct fw_card *card, struct fw_request *request, int tcode, int destination, int source, int generation, unsigned long long offset, void *payload, size_t length, void *callback_data){ int start;
if (!tcode_is_read_request(tcode)) { fw_send_response(card, request, RCODE_TYPE_ERROR); return; }
if ((offset & 3) > 0 || (length & 3) > 0) { fw_send_response(card, request, RCODE_ADDRESS_ERROR); return; }
start = (offset - topology_map_region.start) / 4;
// NOTE: This can be without irqsave when we can guarantee that fw_send_request() for local // destination never runs in any type of IRQ context. scoped_guard(spinlock_irqsave, &card->topology_map.lock) memcpy(payload, &card->topology_map.buffer[start], length);
fw_send_response(card, request, RCODE_COMPLETE);}
static struct fw_address_handler topology_map = { .length = 0x400, .address_callback = handle_topology_map,};
static const struct fw_address_region registers_region = { .start = CSR_REGISTER_BASE, .end = CSR_REGISTER_BASE | CSR_CONFIG_ROM, };
static void update_split_timeout(struct fw_card *card)__must_hold(&card->split_timeout.lock){ unsigned int cycles;
cycles = card->split_timeout.hi * 8000 + (card->split_timeout.lo >> 19);
/* minimum per IEEE 1394, maximum which doesn't overflow OHCI */ cycles = clamp(cycles, 800u, 3u * 8000u);
card->split_timeout.cycles = cycles; card->split_timeout.jiffies = isoc_cycles_to_jiffies(cycles);}
static void handle_registers(struct fw_card *card, struct fw_request *request, int tcode, int destination, int source, int generation, unsigned long long offset, void *payload, size_t length, void *callback_data){ int reg = offset & ~CSR_REGISTER_BASE; __be32 *data = payload; int rcode = RCODE_COMPLETE;
switch (reg) { case CSR_PRIORITY_BUDGET: if (!card->priority_budget_implemented) { rcode = RCODE_ADDRESS_ERROR; break; } fallthrough;
case CSR_NODE_IDS: /* * per IEEE 1394-2008 8.3.22.3, not IEEE 1394.1-2004 3.2.8 * and 9.6, but interoperable with IEEE 1394.1-2004 bridges */ fallthrough;
case CSR_STATE_CLEAR: case CSR_STATE_SET: case CSR_CYCLE_TIME: case CSR_BUS_TIME: case CSR_BUSY_TIMEOUT: if (tcode == TCODE_READ_QUADLET_REQUEST) *data = cpu_to_be32(card->driver->read_csr(card, reg)); else if (tcode == TCODE_WRITE_QUADLET_REQUEST) card->driver->write_csr(card, reg, be32_to_cpu(*data)); else rcode = RCODE_TYPE_ERROR; break;
case CSR_RESET_START: if (tcode == TCODE_WRITE_QUADLET_REQUEST) card->driver->write_csr(card, CSR_STATE_CLEAR, CSR_STATE_BIT_ABDICATE); else rcode = RCODE_TYPE_ERROR; break;
case CSR_SPLIT_TIMEOUT_HI: if (tcode == TCODE_READ_QUADLET_REQUEST) { *data = cpu_to_be32(card->split_timeout.hi); } else if (tcode == TCODE_WRITE_QUADLET_REQUEST) { // NOTE: This can be without irqsave when we can guarantee that // __fw_send_request() for local destination never runs in any type of IRQ // context. scoped_guard(spinlock_irqsave, &card->split_timeout.lock) { card->split_timeout.hi = be32_to_cpu(*data) & 7; update_split_timeout(card); } } else { rcode = RCODE_TYPE_ERROR; } break;
case CSR_SPLIT_TIMEOUT_LO: if (tcode == TCODE_READ_QUADLET_REQUEST) { *data = cpu_to_be32(card->split_timeout.lo); } else if (tcode == TCODE_WRITE_QUADLET_REQUEST) { // NOTE: This can be without irqsave when we can guarantee that // __fw_send_request() for local destination never runs in any type of IRQ // context. scoped_guard(spinlock_irqsave, &card->split_timeout.lock) { card->split_timeout.lo = be32_to_cpu(*data) & 0xfff80000; update_split_timeout(card); } } else { rcode = RCODE_TYPE_ERROR; } break;
case CSR_MAINT_UTILITY: if (tcode == TCODE_READ_QUADLET_REQUEST) *data = card->maint_utility_register; else if (tcode == TCODE_WRITE_QUADLET_REQUEST) card->maint_utility_register = *data; else rcode = RCODE_TYPE_ERROR; break;
case CSR_BROADCAST_CHANNEL: if (tcode == TCODE_READ_QUADLET_REQUEST) *data = cpu_to_be32(card->broadcast_channel); else if (tcode == TCODE_WRITE_QUADLET_REQUEST) card->broadcast_channel = (be32_to_cpu(*data) & BROADCAST_CHANNEL_VALID) | BROADCAST_CHANNEL_INITIAL; else rcode = RCODE_TYPE_ERROR; break;
case CSR_BUS_MANAGER_ID: case CSR_BANDWIDTH_AVAILABLE: case CSR_CHANNELS_AVAILABLE_HI: case CSR_CHANNELS_AVAILABLE_LO: /* * FIXME: these are handled by the OHCI hardware and * the stack never sees these request. If we add * support for a new type of controller that doesn't * handle this in hardware we need to deal with these * transactions. */ BUG(); break;
default: rcode = RCODE_ADDRESS_ERROR; break; }
fw_send_response(card, request, rcode);}
static struct fw_address_handler registers = { .length = 0x400, .address_callback = handle_registers,};
static void handle_low_memory(struct fw_card *card, struct fw_request *request, int tcode, int destination, int source, int generation, unsigned long long offset, void *payload, size_t length, void *callback_data){ /* * This catches requests not handled by the physical DMA unit, * i.e., wrong transaction types or unauthorized source nodes. */ fw_send_response(card, request, RCODE_TYPE_ERROR);}
static struct fw_address_handler low_memory = { .length = FW_MAX_PHYSICAL_RANGE, .address_callback = handle_low_memory,};
MODULE_AUTHOR("Kristian Hoegsberg <krh@bitplanet.net>");MODULE_DESCRIPTION("Core IEEE1394 transaction logic");MODULE_LICENSE("GPL");
static const u32 vendor_textual_descriptor[] = { /* textual descriptor leaf () */ 0x00060000, 0x00000000, 0x00000000, 0x4c696e75, /* L i n u */ 0x78204669, /* x F i */ 0x72657769, /* r e w i */ 0x72650000, /* r e */};
static const u32 model_textual_descriptor[] = { /* model descriptor leaf () */ 0x00030000, 0x00000000, 0x00000000, 0x4a756a75, /* J u j u */};
static struct fw_descriptor vendor_id_descriptor = { .length = ARRAY_SIZE(vendor_textual_descriptor), .immediate = 0x03001f11, .key = 0x81000000, .data = vendor_textual_descriptor,};
static struct fw_descriptor model_id_descriptor = { .length = ARRAY_SIZE(model_textual_descriptor), .immediate = 0x17023901, .key = 0x81000000, .data = model_textual_descriptor,};
static int __init fw_core_init(void){ int ret;
fw_workqueue = alloc_workqueue("firewire", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); if (!fw_workqueue) return -ENOMEM;
ret = bus_register(&fw_bus_type); if (ret < 0) { destroy_workqueue(fw_workqueue); return ret; }
fw_cdev_major = register_chrdev(0, "firewire", &fw_device_ops); if (fw_cdev_major < 0) { bus_unregister(&fw_bus_type); destroy_workqueue(fw_workqueue); return fw_cdev_major; }
fw_core_add_address_handler(&topology_map, &topology_map_region); fw_core_add_address_handler(®isters, ®isters_region); fw_core_add_address_handler(&low_memory, &low_memory_region); fw_core_add_descriptor(&vendor_id_descriptor); fw_core_add_descriptor(&model_id_descriptor);
return 0;}
static void __exit fw_core_cleanup(void){ unregister_chrdev(fw_cdev_major, "firewire"); bus_unregister(&fw_bus_type); destroy_workqueue(fw_workqueue); xa_destroy(&fw_device_xa);}
module_init(fw_core_init);module_exit(fw_core_cleanup);