1/* Copyright (c) 2013 Coraid, Inc. See COPYING for GPL terms. */
2/*
3 * aoecmd.c
4 * Filesystem request handling methods
5 */
6
7#include <linux/ata.h>
8#include <linux/slab.h>
9#include <linux/hdreg.h>
10#include <linux/blk-mq.h>
11#include <linux/skbuff.h>
12#include <linux/netdevice.h>
13#include <linux/moduleparam.h>
14#include <linux/workqueue.h>
15#include <linux/kthread.h>
16#include <net/net_namespace.h>
17#include <linux/unaligned.h>
18#include <linux/uio.h>
19#include "aoe.h"
20
21#define MAXIOC (8192) /* default meant to avoid most soft lockups */
22
23static void ktcomplete(struct frame *, struct sk_buff *);
24static int count_targets(struct aoedev *d, int *untainted);
25
26static struct buf *nextbuf(struct aoedev *);
27
28static int aoe_deadsecs = 60 * 3;
29module_param(aoe_deadsecs, int, 0644);
30MODULE_PARM_DESC(aoe_deadsecs, "After aoe_deadsecs seconds, give up and fail dev.");
31
32static int aoe_maxout = 64;
33module_param(aoe_maxout, int, 0644);
34MODULE_PARM_DESC(aoe_maxout,
35 "Only aoe_maxout outstanding packets for every MAC on eX.Y.");
36
37/* The number of online cpus during module initialization gives us a
38 * convenient heuristic cap on the parallelism used for ktio threads
39 * doing I/O completion. It is not important that the cap equal the
40 * actual number of running CPUs at any given time, but because of CPU
41 * hotplug, we take care to use ncpus instead of using
42 * num_online_cpus() after module initialization.
43 */
44static int ncpus;
45
46/* mutex lock used for synchronization while thread spawning */
47static DEFINE_MUTEX(ktio_spawn_lock);
48
49static wait_queue_head_t *ktiowq;
50static struct ktstate *kts;
51
52/* io completion queue */
53struct iocq_ktio {
54 struct list_head head;
55 spinlock_t lock;
56};
57static struct iocq_ktio *iocq;
58
59static struct page *empty_page;
60
61static struct sk_buff *
62new_skb(ulong len)
63{
64 struct sk_buff *skb;
65
66 skb = alloc_skb(size: len + MAX_HEADER, GFP_ATOMIC);
67 if (skb) {
68 skb_reserve(skb, MAX_HEADER);
69 skb_reset_mac_header(skb);
70 skb_reset_network_header(skb);
71 skb->protocol = __constant_htons(ETH_P_AOE);
72 skb_checksum_none_assert(skb);
73 }
74 return skb;
75}
76
77static struct frame *
78getframe_deferred(struct aoedev *d, u32 tag)
79{
80 struct list_head *head, *pos, *nx;
81 struct frame *f;
82
83 head = &d->rexmitq;
84 list_for_each_safe(pos, nx, head) {
85 f = list_entry(pos, struct frame, head);
86 if (f->tag == tag) {
87 list_del(entry: pos);
88 return f;
89 }
90 }
91 return NULL;
92}
93
94static struct frame *
95getframe(struct aoedev *d, u32 tag)
96{
97 struct frame *f;
98 struct list_head *head, *pos, *nx;
99 u32 n;
100
101 n = tag % NFACTIVE;
102 head = &d->factive[n];
103 list_for_each_safe(pos, nx, head) {
104 f = list_entry(pos, struct frame, head);
105 if (f->tag == tag) {
106 list_del(entry: pos);
107 return f;
108 }
109 }
110 return NULL;
111}
112
113/*
114 * Leave the top bit clear so we have tagspace for userland.
115 * The bottom 16 bits are the xmit tick for rexmit/rttavg processing.
116 * This driver reserves tag -1 to mean "unused frame."
117 */
118static int
119newtag(struct aoedev *d)
120{
121 register ulong n;
122
123 n = jiffies & 0xffff;
124 return n | (++d->lasttag & 0x7fff) << 16;
125}
126
127static u32
128aoehdr_atainit(struct aoedev *d, struct aoetgt *t, struct aoe_hdr *h)
129{
130 u32 host_tag = newtag(d);
131
132 memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
133 memcpy(h->dst, t->addr, sizeof h->dst);
134 h->type = __constant_cpu_to_be16(ETH_P_AOE);
135 h->verfl = AOE_HVER;
136 h->major = cpu_to_be16(d->aoemajor);
137 h->minor = d->aoeminor;
138 h->cmd = AOECMD_ATA;
139 h->tag = cpu_to_be32(host_tag);
140
141 return host_tag;
142}
143
144static inline void
145put_lba(struct aoe_atahdr *ah, sector_t lba)
146{
147 ah->lba0 = lba;
148 ah->lba1 = lba >>= 8;
149 ah->lba2 = lba >>= 8;
150 ah->lba3 = lba >>= 8;
151 ah->lba4 = lba >>= 8;
152 ah->lba5 = lba >>= 8;
153}
154
155static struct aoeif *
156ifrotate(struct aoetgt *t)
157{
158 struct aoeif *ifp;
159
160 ifp = t->ifp;
161 ifp++;
162 if (ifp >= &t->ifs[NAOEIFS] || ifp->nd == NULL)
163 ifp = t->ifs;
164 if (ifp->nd == NULL)
165 return NULL;
166 return t->ifp = ifp;
167}
168
169static void
170skb_pool_put(struct aoedev *d, struct sk_buff *skb)
171{
172 __skb_queue_tail(list: &d->skbpool, newsk: skb);
173}
174
175static struct sk_buff *
176skb_pool_get(struct aoedev *d)
177{
178 struct sk_buff *skb = skb_peek(list_: &d->skbpool);
179
180 if (skb && atomic_read(v: &skb_shinfo(skb)->dataref) == 1) {
181 __skb_unlink(skb, list: &d->skbpool);
182 return skb;
183 }
184 if (skb_queue_len(list_: &d->skbpool) < NSKBPOOLMAX &&
185 (skb = new_skb(ETH_ZLEN)))
186 return skb;
187
188 return NULL;
189}
190
191void
192aoe_freetframe(struct frame *f)
193{
194 struct aoetgt *t;
195
196 t = f->t;
197 f->buf = NULL;
198 memset(&f->iter, 0, sizeof(f->iter));
199 f->r_skb = NULL;
200 f->flags = 0;
201 list_add(new: &f->head, head: &t->ffree);
202}
203
204static struct frame *
205newtframe(struct aoedev *d, struct aoetgt *t)
206{
207 struct frame *f;
208 struct sk_buff *skb;
209 struct list_head *pos;
210
211 if (list_empty(head: &t->ffree)) {
212 if (t->falloc >= NSKBPOOLMAX*2)
213 return NULL;
214 f = kcalloc(1, sizeof(*f), GFP_ATOMIC);
215 if (f == NULL)
216 return NULL;
217 t->falloc++;
218 f->t = t;
219 } else {
220 pos = t->ffree.next;
221 list_del(entry: pos);
222 f = list_entry(pos, struct frame, head);
223 }
224
225 skb = f->skb;
226 if (skb == NULL) {
227 f->skb = skb = new_skb(ETH_ZLEN);
228 if (!skb) {
229bail: aoe_freetframe(f);
230 return NULL;
231 }
232 }
233
234 if (atomic_read(v: &skb_shinfo(skb)->dataref) != 1) {
235 skb = skb_pool_get(d);
236 if (skb == NULL)
237 goto bail;
238 skb_pool_put(d, skb: f->skb);
239 f->skb = skb;
240 }
241
242 skb->truesize -= skb->data_len;
243 skb_shinfo(skb)->nr_frags = skb->data_len = 0;
244 skb_trim(skb, len: 0);
245 return f;
246}
247
248static struct frame *
249newframe(struct aoedev *d)
250{
251 struct frame *f;
252 struct aoetgt *t, **tt;
253 int totout = 0;
254 int use_tainted;
255 int has_untainted;
256
257 if (!d->targets || !d->targets[0]) {
258 printk(KERN_ERR "aoe: NULL TARGETS!\n");
259 return NULL;
260 }
261 tt = d->tgt; /* last used target */
262 for (use_tainted = 0, has_untainted = 0;;) {
263 tt++;
264 if (tt >= &d->targets[d->ntargets] || !*tt)
265 tt = d->targets;
266 t = *tt;
267 if (!t->taint) {
268 has_untainted = 1;
269 totout += t->nout;
270 }
271 if (t->nout < t->maxout
272 && (use_tainted || !t->taint)
273 && t->ifp->nd) {
274 f = newtframe(d, t);
275 if (f) {
276 ifrotate(t);
277 d->tgt = tt;
278 return f;
279 }
280 }
281 if (tt == d->tgt) { /* we've looped and found nada */
282 if (!use_tainted && !has_untainted)
283 use_tainted = 1;
284 else
285 break;
286 }
287 }
288 if (totout == 0) {
289 d->kicked++;
290 d->flags |= DEVFL_KICKME;
291 }
292 return NULL;
293}
294
295static void
296skb_fillup(struct sk_buff *skb, struct bio *bio, struct bvec_iter iter)
297{
298 int frag = 0;
299 struct bio_vec bv;
300
301 __bio_for_each_segment(bv, bio, iter, iter)
302 skb_fill_page_desc(skb, i: frag++, page: bv.bv_page,
303 off: bv.bv_offset, size: bv.bv_len);
304}
305
306static void
307fhash(struct frame *f)
308{
309 struct aoedev *d = f->t->d;
310 u32 n;
311
312 n = f->tag % NFACTIVE;
313 list_add_tail(new: &f->head, head: &d->factive[n]);
314}
315
316static void
317ata_rw_frameinit(struct frame *f)
318{
319 struct aoetgt *t;
320 struct aoe_hdr *h;
321 struct aoe_atahdr *ah;
322 struct sk_buff *skb;
323 char writebit, extbit;
324
325 skb = f->skb;
326 h = (struct aoe_hdr *) skb_mac_header(skb);
327 ah = (struct aoe_atahdr *) (h + 1);
328 skb_put(skb, len: sizeof(*h) + sizeof(*ah));
329 memset(h, 0, skb->len);
330
331 writebit = 0x10;
332 extbit = 0x4;
333
334 t = f->t;
335 f->tag = aoehdr_atainit(d: t->d, t, h);
336 fhash(f);
337 t->nout++;
338 f->waited = 0;
339 f->waited_total = 0;
340
341 /* set up ata header */
342 ah->scnt = f->iter.bi_size >> 9;
343 put_lba(ah, lba: f->iter.bi_sector);
344 if (t->d->flags & DEVFL_EXT) {
345 ah->aflags |= AOEAFL_EXT;
346 } else {
347 extbit = 0;
348 ah->lba3 &= 0x0f;
349 ah->lba3 |= 0xe0; /* LBA bit + obsolete 0xa0 */
350 }
351 if (f->buf && bio_data_dir(f->buf->bio) == WRITE) {
352 skb_fillup(skb, bio: f->buf->bio, iter: f->iter);
353 ah->aflags |= AOEAFL_WRITE;
354 skb->len += f->iter.bi_size;
355 skb->data_len = f->iter.bi_size;
356 skb->truesize += f->iter.bi_size;
357 t->wpkts++;
358 } else {
359 t->rpkts++;
360 writebit = 0;
361 }
362
363 ah->cmdstat = ATA_CMD_PIO_READ | writebit | extbit;
364 dev_hold(dev: t->ifp->nd);
365 skb->dev = t->ifp->nd;
366}
367
368static int
369aoecmd_ata_rw(struct aoedev *d)
370{
371 struct frame *f;
372 struct buf *buf;
373 struct sk_buff *skb;
374 struct sk_buff_head queue;
375
376 buf = nextbuf(d);
377 if (buf == NULL)
378 return 0;
379 f = newframe(d);
380 if (f == NULL)
381 return 0;
382
383 /* initialize the headers & frame */
384 f->buf = buf;
385 f->iter = buf->iter;
386 f->iter.bi_size = min_t(unsigned long,
387 d->maxbcnt ?: DEFAULTBCNT,
388 f->iter.bi_size);
389 bio_advance_iter(bio: buf->bio, iter: &buf->iter, bytes: f->iter.bi_size);
390
391 if (!buf->iter.bi_size)
392 d->ip.buf = NULL;
393
394 /* mark all tracking fields and load out */
395 buf->nframesout += 1;
396
397 ata_rw_frameinit(f);
398
399 skb = skb_clone(skb: f->skb, GFP_ATOMIC);
400 if (skb) {
401 f->sent = ktime_get();
402 __skb_queue_head_init(list: &queue);
403 __skb_queue_tail(list: &queue, newsk: skb);
404 aoenet_xmit(&queue);
405 } else {
406 dev_put(dev: f->t->ifp->nd);
407 }
408 return 1;
409}
410
411/* some callers cannot sleep, and they can call this function,
412 * transmitting the packets later, when interrupts are on
413 */
414static void
415aoecmd_cfg_pkts(ushort aoemajor, unsigned char aoeminor, struct sk_buff_head *queue)
416{
417 struct aoe_hdr *h;
418 struct aoe_cfghdr *ch;
419 struct sk_buff *skb;
420 struct net_device *ifp;
421
422 rcu_read_lock();
423 for_each_netdev_rcu(&init_net, ifp) {
424 dev_hold(dev: ifp);
425 if (!is_aoe_netif(ifp)) {
426 dev_put(dev: ifp);
427 continue;
428 }
429
430 skb = new_skb(len: sizeof *h + sizeof *ch);
431 if (skb == NULL) {
432 printk(KERN_INFO "aoe: skb alloc failure\n");
433 dev_put(dev: ifp);
434 continue;
435 }
436 skb_put(skb, len: sizeof *h + sizeof *ch);
437 skb->dev = ifp;
438 __skb_queue_tail(list: queue, newsk: skb);
439 h = (struct aoe_hdr *) skb_mac_header(skb);
440 memset(h, 0, sizeof *h + sizeof *ch);
441
442 memset(h->dst, 0xff, sizeof h->dst);
443 memcpy(h->src, ifp->dev_addr, sizeof h->src);
444 h->type = __constant_cpu_to_be16(ETH_P_AOE);
445 h->verfl = AOE_HVER;
446 h->major = cpu_to_be16(aoemajor);
447 h->minor = aoeminor;
448 h->cmd = AOECMD_CFG;
449 }
450 rcu_read_unlock();
451}
452
453static void
454resend(struct aoedev *d, struct frame *f)
455{
456 struct sk_buff *skb;
457 struct sk_buff_head queue;
458 struct aoe_hdr *h;
459 struct aoetgt *t;
460 char buf[128];
461 u32 n;
462
463 t = f->t;
464 n = newtag(d);
465 skb = f->skb;
466 if (ifrotate(t) == NULL) {
467 /* probably can't happen, but set it up to fail anyway */
468 pr_info("aoe: resend: no interfaces to rotate to.\n");
469 ktcomplete(f, NULL);
470 return;
471 }
472 h = (struct aoe_hdr *) skb_mac_header(skb);
473
474 if (!(f->flags & FFL_PROBE)) {
475 snprintf(buf, size: sizeof(buf),
476 fmt: "%15s e%ld.%d oldtag=%08x@%08lx newtag=%08x s=%pm d=%pm nout=%d\n",
477 "retransmit", d->aoemajor, d->aoeminor,
478 f->tag, jiffies, n,
479 h->src, h->dst, t->nout);
480 aoechr_error(buf);
481 }
482
483 f->tag = n;
484 fhash(f);
485 h->tag = cpu_to_be32(n);
486 memcpy(h->dst, t->addr, sizeof h->dst);
487 memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
488
489 dev_hold(dev: t->ifp->nd);
490 skb->dev = t->ifp->nd;
491 skb = skb_clone(skb, GFP_ATOMIC);
492 if (skb == NULL) {
493 dev_put(dev: t->ifp->nd);
494 return;
495 }
496 f->sent = ktime_get();
497 __skb_queue_head_init(list: &queue);
498 __skb_queue_tail(list: &queue, newsk: skb);
499 aoenet_xmit(&queue);
500}
501
502static int
503tsince_hr(struct frame *f)
504{
505 u64 delta = ktime_to_ns(ktime_sub(ktime_get(), f->sent));
506
507 /* delta is normally under 4.2 seconds, avoid 64-bit division */
508 if (likely(delta <= UINT_MAX))
509 return (u32)delta / NSEC_PER_USEC;
510
511 /* avoid overflow after 71 minutes */
512 if (delta > ((u64)INT_MAX * NSEC_PER_USEC))
513 return INT_MAX;
514
515 return div_u64(dividend: delta, NSEC_PER_USEC);
516}
517
518static int
519tsince(u32 tag)
520{
521 int n;
522
523 n = jiffies & 0xffff;
524 n -= tag & 0xffff;
525 if (n < 0)
526 n += 1<<16;
527 return jiffies_to_usecs(j: n + 1);
528}
529
530static struct aoeif *
531getif(struct aoetgt *t, struct net_device *nd)
532{
533 struct aoeif *p, *e;
534
535 p = t->ifs;
536 e = p + NAOEIFS;
537 for (; p < e; p++)
538 if (p->nd == nd)
539 return p;
540 return NULL;
541}
542
543static void
544ejectif(struct aoetgt *t, struct aoeif *ifp)
545{
546 struct aoeif *e;
547 struct net_device *nd;
548 ulong n;
549
550 nd = ifp->nd;
551 e = t->ifs + NAOEIFS - 1;
552 n = (e - ifp) * sizeof *ifp;
553 memmove(ifp, ifp+1, n);
554 e->nd = NULL;
555 dev_put(dev: nd);
556}
557
558static struct frame *
559reassign_frame(struct frame *f)
560{
561 struct frame *nf;
562 struct sk_buff *skb;
563
564 nf = newframe(d: f->t->d);
565 if (!nf)
566 return NULL;
567 if (nf->t == f->t) {
568 aoe_freetframe(f: nf);
569 return NULL;
570 }
571
572 skb = nf->skb;
573 nf->skb = f->skb;
574 nf->buf = f->buf;
575 nf->iter = f->iter;
576 nf->waited = 0;
577 nf->waited_total = f->waited_total;
578 nf->sent = f->sent;
579 f->skb = skb;
580
581 return nf;
582}
583
584static void
585probe(struct aoetgt *t)
586{
587 struct aoedev *d;
588 struct frame *f;
589 struct sk_buff *skb;
590 struct sk_buff_head queue;
591 size_t n, m;
592 int frag;
593
594 d = t->d;
595 f = newtframe(d, t);
596 if (!f) {
597 pr_err("%s %pm for e%ld.%d: %s\n",
598 "aoe: cannot probe remote address",
599 t->addr,
600 (long) d->aoemajor, d->aoeminor,
601 "no frame available");
602 return;
603 }
604 f->flags |= FFL_PROBE;
605 ifrotate(t);
606 f->iter.bi_size = t->d->maxbcnt ? t->d->maxbcnt : DEFAULTBCNT;
607 ata_rw_frameinit(f);
608 skb = f->skb;
609 for (frag = 0, n = f->iter.bi_size; n > 0; ++frag, n -= m) {
610 if (n < PAGE_SIZE)
611 m = n;
612 else
613 m = PAGE_SIZE;
614 skb_fill_page_desc(skb, i: frag, page: empty_page, off: 0, size: m);
615 }
616 skb->len += f->iter.bi_size;
617 skb->data_len = f->iter.bi_size;
618 skb->truesize += f->iter.bi_size;
619
620 skb = skb_clone(skb: f->skb, GFP_ATOMIC);
621 if (skb) {
622 f->sent = ktime_get();
623 __skb_queue_head_init(list: &queue);
624 __skb_queue_tail(list: &queue, newsk: skb);
625 aoenet_xmit(&queue);
626 } else {
627 dev_put(dev: f->t->ifp->nd);
628 }
629}
630
631static long
632rto(struct aoedev *d)
633{
634 long t;
635
636 t = 2 * d->rttavg >> RTTSCALE;
637 t += 8 * d->rttdev >> RTTDSCALE;
638 if (t == 0)
639 t = 1;
640
641 return t;
642}
643
644static void
645rexmit_deferred(struct aoedev *d)
646{
647 struct aoetgt *t;
648 struct frame *f;
649 struct frame *nf;
650 struct list_head *pos, *nx, *head;
651 int since;
652 int untainted;
653
654 count_targets(d, untainted: &untainted);
655
656 head = &d->rexmitq;
657 list_for_each_safe(pos, nx, head) {
658 f = list_entry(pos, struct frame, head);
659 t = f->t;
660 if (t->taint) {
661 if (!(f->flags & FFL_PROBE)) {
662 nf = reassign_frame(f);
663 if (nf) {
664 if (t->nout_probes == 0
665 && untainted > 0) {
666 probe(t);
667 t->nout_probes++;
668 }
669 list_replace(old: &f->head, new: &nf->head);
670 pos = &nf->head;
671 aoe_freetframe(f);
672 f = nf;
673 t = f->t;
674 }
675 } else if (untainted < 1) {
676 /* don't probe w/o other untainted aoetgts */
677 goto stop_probe;
678 } else if (tsince_hr(f) < t->taint * rto(d)) {
679 /* reprobe slowly when taint is high */
680 continue;
681 }
682 } else if (f->flags & FFL_PROBE) {
683stop_probe: /* don't probe untainted aoetgts */
684 list_del(entry: pos);
685 aoe_freetframe(f);
686 /* leaving d->kicked, because this is routine */
687 f->t->d->flags |= DEVFL_KICKME;
688 continue;
689 }
690 if (t->nout >= t->maxout)
691 continue;
692 list_del(entry: pos);
693 t->nout++;
694 if (f->flags & FFL_PROBE)
695 t->nout_probes++;
696 since = tsince_hr(f);
697 f->waited += since;
698 f->waited_total += since;
699 resend(d, f);
700 }
701}
702
703/* An aoetgt accumulates demerits quickly, and successful
704 * probing redeems the aoetgt slowly.
705 */
706static void
707scorn(struct aoetgt *t)
708{
709 int n;
710
711 n = t->taint++;
712 t->taint += t->taint * 2;
713 if (n > t->taint)
714 t->taint = n;
715 if (t->taint > MAX_TAINT)
716 t->taint = MAX_TAINT;
717}
718
719static int
720count_targets(struct aoedev *d, int *untainted)
721{
722 int i, good;
723
724 for (i = good = 0; i < d->ntargets && d->targets[i]; ++i)
725 if (d->targets[i]->taint == 0)
726 good++;
727
728 if (untainted)
729 *untainted = good;
730 return i;
731}
732
733static void
734rexmit_timer(struct timer_list *timer)
735{
736 struct aoedev *d;
737 struct aoetgt *t;
738 struct aoeif *ifp;
739 struct frame *f;
740 struct list_head *head, *pos, *nx;
741 LIST_HEAD(flist);
742 register long timeout;
743 ulong flags, n;
744 int i;
745 int utgts; /* number of aoetgt descriptors (not slots) */
746 int since;
747
748 d = timer_container_of(d, timer, timer);
749
750 spin_lock_irqsave(&d->lock, flags);
751
752 /* timeout based on observed timings and variations */
753 timeout = rto(d);
754
755 utgts = count_targets(d, NULL);
756
757 if (d->flags & (DEVFL_TKILL | DEVFL_DEAD)) {
758 spin_unlock_irqrestore(lock: &d->lock, flags);
759 return;
760 }
761
762 /* collect all frames to rexmit into flist */
763 for (i = 0; i < NFACTIVE; i++) {
764 head = &d->factive[i];
765 list_for_each_safe(pos, nx, head) {
766 f = list_entry(pos, struct frame, head);
767 if (tsince_hr(f) < timeout)
768 break; /* end of expired frames */
769 /* move to flist for later processing */
770 list_move_tail(list: pos, head: &flist);
771 }
772 }
773
774 /* process expired frames */
775 while (!list_empty(head: &flist)) {
776 pos = flist.next;
777 f = list_entry(pos, struct frame, head);
778 since = tsince_hr(f);
779 n = f->waited_total + since;
780 n /= USEC_PER_SEC;
781 if (aoe_deadsecs
782 && n > aoe_deadsecs
783 && !(f->flags & FFL_PROBE)) {
784 /* Waited too long. Device failure.
785 * Hang all frames on first hash bucket for downdev
786 * to clean up.
787 */
788 list_splice(list: &flist, head: &d->factive[0]);
789 d->flags |= DEVFL_DEAD;
790 queue_work(wq: aoe_wq, work: &d->work);
791 goto out;
792 }
793
794 t = f->t;
795 n = f->waited + since;
796 n /= USEC_PER_SEC;
797 if (aoe_deadsecs && utgts > 0
798 && (n > aoe_deadsecs / utgts || n > HARD_SCORN_SECS))
799 scorn(t); /* avoid this target */
800
801 if (t->maxout != 1) {
802 t->ssthresh = t->maxout / 2;
803 t->maxout = 1;
804 }
805
806 if (f->flags & FFL_PROBE) {
807 t->nout_probes--;
808 } else {
809 ifp = getif(t, nd: f->skb->dev);
810 if (ifp && ++ifp->lost > (t->nframes << 1)
811 && (ifp != t->ifs || t->ifs[1].nd)) {
812 ejectif(t, ifp);
813 ifp = NULL;
814 }
815 }
816 list_move_tail(list: pos, head: &d->rexmitq);
817 t->nout--;
818 }
819 rexmit_deferred(d);
820
821out:
822 if ((d->flags & DEVFL_KICKME) && d->blkq) {
823 d->flags &= ~DEVFL_KICKME;
824 blk_mq_run_hw_queues(q: d->blkq, async: true);
825 }
826
827 d->timer.expires = jiffies + TIMERTICK;
828 add_timer(timer: &d->timer);
829
830 spin_unlock_irqrestore(lock: &d->lock, flags);
831}
832
833static void
834bufinit(struct buf *buf, struct request *rq, struct bio *bio)
835{
836 memset(buf, 0, sizeof(*buf));
837 buf->rq = rq;
838 buf->bio = bio;
839 buf->iter = bio->bi_iter;
840}
841
842static struct buf *
843nextbuf(struct aoedev *d)
844{
845 struct request *rq;
846 struct request_queue *q;
847 struct aoe_req *req;
848 struct buf *buf;
849 struct bio *bio;
850
851 q = d->blkq;
852 if (q == NULL)
853 return NULL; /* initializing */
854 if (d->ip.buf)
855 return d->ip.buf;
856 rq = d->ip.rq;
857 if (rq == NULL) {
858 rq = list_first_entry_or_null(&d->rq_list, struct request,
859 queuelist);
860 if (rq == NULL)
861 return NULL;
862 list_del_init(entry: &rq->queuelist);
863 blk_mq_start_request(rq);
864 d->ip.rq = rq;
865 d->ip.nxbio = rq->bio;
866
867 req = blk_mq_rq_to_pdu(rq);
868 req->nr_bios = 0;
869 __rq_for_each_bio(bio, rq)
870 req->nr_bios++;
871 }
872 buf = mempool_alloc(d->bufpool, GFP_ATOMIC);
873 if (buf == NULL) {
874 pr_err("aoe: nextbuf: unable to mempool_alloc!\n");
875 return NULL;
876 }
877 bio = d->ip.nxbio;
878 bufinit(buf, rq, bio);
879 bio = bio->bi_next;
880 d->ip.nxbio = bio;
881 if (bio == NULL)
882 d->ip.rq = NULL;
883 return d->ip.buf = buf;
884}
885
886/* enters with d->lock held */
887void
888aoecmd_work(struct aoedev *d)
889{
890 rexmit_deferred(d);
891 while (aoecmd_ata_rw(d))
892 ;
893}
894
895/* this function performs work that has been deferred until sleeping is OK
896 */
897void
898aoecmd_sleepwork(struct work_struct *work)
899{
900 struct aoedev *d = container_of(work, struct aoedev, work);
901
902 if (d->flags & DEVFL_DEAD)
903 aoedev_downdev(d);
904
905 if (d->flags & DEVFL_GDALLOC)
906 aoeblk_gdalloc(d);
907
908 if (d->flags & DEVFL_NEWSIZE) {
909 set_capacity_and_notify(disk: d->gd, size: d->ssize);
910
911 spin_lock_irq(lock: &d->lock);
912 d->flags |= DEVFL_UP;
913 d->flags &= ~DEVFL_NEWSIZE;
914 spin_unlock_irq(lock: &d->lock);
915 }
916}
917
918static void
919ata_ident_fixstring(u16 *id, int ns)
920{
921 u16 s;
922
923 while (ns-- > 0) {
924 s = *id;
925 *id++ = s >> 8 | s << 8;
926 }
927}
928
929static void
930ataid_complete(struct aoedev *d, struct aoetgt *t, unsigned char *id)
931{
932 u64 ssize;
933 u16 n;
934
935 /* word 83: command set supported */
936 n = get_unaligned_le16(p: &id[83 << 1]);
937
938 /* word 86: command set/feature enabled */
939 n |= get_unaligned_le16(p: &id[86 << 1]);
940
941 if (n & (1<<10)) { /* bit 10: LBA 48 */
942 d->flags |= DEVFL_EXT;
943
944 /* word 100: number lba48 sectors */
945 ssize = get_unaligned_le64(p: &id[100 << 1]);
946
947 /* set as in ide-disk.c:init_idedisk_capacity */
948 d->geo.cylinders = ssize;
949 d->geo.cylinders /= (255 * 63);
950 d->geo.heads = 255;
951 d->geo.sectors = 63;
952 } else {
953 d->flags &= ~DEVFL_EXT;
954
955 /* number lba28 sectors */
956 ssize = get_unaligned_le32(p: &id[60 << 1]);
957
958 /* NOTE: obsolete in ATA 6 */
959 d->geo.cylinders = get_unaligned_le16(p: &id[54 << 1]);
960 d->geo.heads = get_unaligned_le16(p: &id[55 << 1]);
961 d->geo.sectors = get_unaligned_le16(p: &id[56 << 1]);
962 }
963
964 ata_ident_fixstring(id: (u16 *) &id[10<<1], ns: 10); /* serial */
965 ata_ident_fixstring(id: (u16 *) &id[23<<1], ns: 4); /* firmware */
966 ata_ident_fixstring(id: (u16 *) &id[27<<1], ns: 20); /* model */
967 memcpy(d->ident, id, sizeof(d->ident));
968
969 if (d->ssize != ssize)
970 printk(KERN_INFO
971 "aoe: %pm e%ld.%d v%04x has %llu sectors\n",
972 t->addr,
973 d->aoemajor, d->aoeminor,
974 d->fw_ver, (long long)ssize);
975 d->ssize = ssize;
976 d->geo.start = 0;
977 if (d->flags & (DEVFL_GDALLOC|DEVFL_NEWSIZE))
978 return;
979 if (d->gd != NULL)
980 d->flags |= DEVFL_NEWSIZE;
981 else
982 d->flags |= DEVFL_GDALLOC;
983 queue_work(wq: aoe_wq, work: &d->work);
984}
985
986static void
987calc_rttavg(struct aoedev *d, struct aoetgt *t, int rtt)
988{
989 register long n;
990
991 n = rtt;
992
993 /* cf. Congestion Avoidance and Control, Jacobson & Karels, 1988 */
994 n -= d->rttavg >> RTTSCALE;
995 d->rttavg += n;
996 if (n < 0)
997 n = -n;
998 n -= d->rttdev >> RTTDSCALE;
999 d->rttdev += n;
1000
1001 if (!t || t->maxout >= t->nframes)
1002 return;
1003 if (t->maxout < t->ssthresh)
1004 t->maxout += 1;
1005 else if (t->nout == t->maxout && t->next_cwnd-- == 0) {
1006 t->maxout += 1;
1007 t->next_cwnd = t->maxout;
1008 }
1009}
1010
1011static struct aoetgt *
1012gettgt(struct aoedev *d, char *addr)
1013{
1014 struct aoetgt **t, **e;
1015
1016 t = d->targets;
1017 e = t + d->ntargets;
1018 for (; t < e && *t; t++)
1019 if (memcmp(p: (*t)->addr, q: addr, size: sizeof((*t)->addr)) == 0)
1020 return *t;
1021 return NULL;
1022}
1023
1024static void
1025bvcpy(struct sk_buff *skb, struct bio *bio, struct bvec_iter iter, long cnt)
1026{
1027 int soff = 0;
1028 struct bio_vec bv;
1029
1030 iter.bi_size = cnt;
1031
1032 __bio_for_each_segment(bv, bio, iter, iter) {
1033 char *p = bvec_kmap_local(bvec: &bv);
1034 skb_copy_bits(skb, offset: soff, to: p, len: bv.bv_len);
1035 kunmap_local(p);
1036 soff += bv.bv_len;
1037 }
1038}
1039
1040void
1041aoe_end_request(struct aoedev *d, struct request *rq, int fastfail)
1042{
1043 struct bio *bio;
1044 int bok;
1045 struct request_queue *q;
1046 blk_status_t err = BLK_STS_OK;
1047
1048 q = d->blkq;
1049 if (rq == d->ip.rq)
1050 d->ip.rq = NULL;
1051 do {
1052 bio = rq->bio;
1053 bok = !fastfail && !bio->bi_status;
1054 if (!bok)
1055 err = BLK_STS_IOERR;
1056 } while (blk_update_request(rq, error: bok ? BLK_STS_OK : BLK_STS_IOERR, nr_bytes: bio->bi_iter.bi_size));
1057
1058 __blk_mq_end_request(rq, error: err);
1059
1060 /* cf. https://lore.kernel.org/lkml/20061031071040.GS14055@kernel.dk/ */
1061 if (!fastfail)
1062 blk_mq_run_hw_queues(q, async: true);
1063}
1064
1065static void
1066aoe_end_buf(struct aoedev *d, struct buf *buf)
1067{
1068 struct request *rq = buf->rq;
1069 struct aoe_req *req = blk_mq_rq_to_pdu(rq);
1070
1071 if (buf == d->ip.buf)
1072 d->ip.buf = NULL;
1073 mempool_free(element: buf, pool: d->bufpool);
1074 if (--req->nr_bios == 0)
1075 aoe_end_request(d, rq, fastfail: 0);
1076}
1077
1078static void
1079ktiocomplete(struct frame *f)
1080{
1081 struct aoe_hdr *hin, *hout;
1082 struct aoe_atahdr *ahin, *ahout;
1083 struct buf *buf;
1084 struct sk_buff *skb;
1085 struct aoetgt *t;
1086 struct aoeif *ifp;
1087 struct aoedev *d;
1088 long n;
1089 int untainted;
1090
1091 if (f == NULL)
1092 return;
1093
1094 t = f->t;
1095 d = t->d;
1096 skb = f->r_skb;
1097 buf = f->buf;
1098 if (f->flags & FFL_PROBE)
1099 goto out;
1100 if (!skb) /* just fail the buf. */
1101 goto noskb;
1102
1103 hout = (struct aoe_hdr *) skb_mac_header(skb: f->skb);
1104 ahout = (struct aoe_atahdr *) (hout+1);
1105
1106 hin = (struct aoe_hdr *) skb->data;
1107 skb_pull(skb, len: sizeof(*hin));
1108 ahin = (struct aoe_atahdr *) skb->data;
1109 skb_pull(skb, len: sizeof(*ahin));
1110 if (ahin->cmdstat & 0xa9) { /* these bits cleared on success */
1111 pr_err("aoe: ata error cmd=%2.2Xh stat=%2.2Xh from e%ld.%d\n",
1112 ahout->cmdstat, ahin->cmdstat,
1113 d->aoemajor, d->aoeminor);
1114noskb: if (buf)
1115 buf->bio->bi_status = BLK_STS_IOERR;
1116 goto out;
1117 }
1118
1119 n = ahout->scnt << 9;
1120 switch (ahout->cmdstat) {
1121 case ATA_CMD_PIO_READ:
1122 case ATA_CMD_PIO_READ_EXT:
1123 if (skb->len < n) {
1124 pr_err("%s e%ld.%d. skb->len=%d need=%ld\n",
1125 "aoe: runt data size in read from",
1126 (long) d->aoemajor, d->aoeminor,
1127 skb->len, n);
1128 buf->bio->bi_status = BLK_STS_IOERR;
1129 break;
1130 }
1131 if (n > f->iter.bi_size) {
1132 pr_err_ratelimited("%s e%ld.%d. bytes=%ld need=%u\n",
1133 "aoe: too-large data size in read from",
1134 (long) d->aoemajor, d->aoeminor,
1135 n, f->iter.bi_size);
1136 buf->bio->bi_status = BLK_STS_IOERR;
1137 break;
1138 }
1139 bvcpy(skb, bio: f->buf->bio, iter: f->iter, cnt: n);
1140 fallthrough;
1141 case ATA_CMD_PIO_WRITE:
1142 case ATA_CMD_PIO_WRITE_EXT:
1143 spin_lock_irq(lock: &d->lock);
1144 ifp = getif(t, nd: skb->dev);
1145 if (ifp)
1146 ifp->lost = 0;
1147 spin_unlock_irq(lock: &d->lock);
1148 break;
1149 case ATA_CMD_ID_ATA:
1150 if (skb->len < 512) {
1151 pr_info("%s e%ld.%d. skb->len=%d need=512\n",
1152 "aoe: runt data size in ataid from",
1153 (long) d->aoemajor, d->aoeminor,
1154 skb->len);
1155 break;
1156 }
1157 if (skb_linearize(skb))
1158 break;
1159 spin_lock_irq(lock: &d->lock);
1160 ataid_complete(d, t, id: skb->data);
1161 spin_unlock_irq(lock: &d->lock);
1162 break;
1163 default:
1164 pr_info("aoe: unrecognized ata command %2.2Xh for %d.%d\n",
1165 ahout->cmdstat,
1166 be16_to_cpu(get_unaligned(&hin->major)),
1167 hin->minor);
1168 }
1169out:
1170 spin_lock_irq(lock: &d->lock);
1171 if (t->taint > 0
1172 && --t->taint > 0
1173 && t->nout_probes == 0) {
1174 count_targets(d, untainted: &untainted);
1175 if (untainted > 0) {
1176 probe(t);
1177 t->nout_probes++;
1178 }
1179 }
1180
1181 aoe_freetframe(f);
1182
1183 if (buf && --buf->nframesout == 0 && buf->iter.bi_size == 0)
1184 aoe_end_buf(d, buf);
1185
1186 spin_unlock_irq(lock: &d->lock);
1187 aoedev_put(d);
1188 dev_kfree_skb(skb);
1189}
1190
1191/* Enters with iocq.lock held.
1192 * Returns true iff responses needing processing remain.
1193 */
1194static int
1195ktio(int id)
1196{
1197 struct frame *f;
1198 struct list_head *pos;
1199 int i;
1200 int actual_id;
1201
1202 for (i = 0; ; ++i) {
1203 if (i == MAXIOC)
1204 return 1;
1205 if (list_empty(head: &iocq[id].head))
1206 return 0;
1207 pos = iocq[id].head.next;
1208 list_del(entry: pos);
1209 f = list_entry(pos, struct frame, head);
1210 spin_unlock_irq(lock: &iocq[id].lock);
1211 ktiocomplete(f);
1212
1213 /* Figure out if extra threads are required. */
1214 actual_id = f->t->d->aoeminor % ncpus;
1215
1216 if (!kts[actual_id].active) {
1217 BUG_ON(id != 0);
1218 mutex_lock(&ktio_spawn_lock);
1219 if (!kts[actual_id].active
1220 && aoe_ktstart(k: &kts[actual_id]) == 0)
1221 kts[actual_id].active = 1;
1222 mutex_unlock(lock: &ktio_spawn_lock);
1223 }
1224 spin_lock_irq(lock: &iocq[id].lock);
1225 }
1226}
1227
1228static int
1229kthread(void *vp)
1230{
1231 struct ktstate *k;
1232 DECLARE_WAITQUEUE(wait, current);
1233 int more;
1234
1235 k = vp;
1236 current->flags |= PF_NOFREEZE;
1237 set_user_nice(current, nice: -10);
1238 complete(&k->rendez); /* tell spawner we're running */
1239 do {
1240 spin_lock_irq(lock: k->lock);
1241 more = k->fn(k->id);
1242 if (!more) {
1243 add_wait_queue(wq_head: k->waitq, wq_entry: &wait);
1244 __set_current_state(TASK_INTERRUPTIBLE);
1245 }
1246 spin_unlock_irq(lock: k->lock);
1247 if (!more) {
1248 schedule();
1249 remove_wait_queue(wq_head: k->waitq, wq_entry: &wait);
1250 } else
1251 cond_resched();
1252 } while (!kthread_should_stop());
1253 complete(&k->rendez); /* tell spawner we're stopping */
1254 return 0;
1255}
1256
1257void
1258aoe_ktstop(struct ktstate *k)
1259{
1260 kthread_stop(k: k->task);
1261 wait_for_completion(&k->rendez);
1262}
1263
1264int
1265aoe_ktstart(struct ktstate *k)
1266{
1267 struct task_struct *task;
1268
1269 init_completion(x: &k->rendez);
1270 task = kthread_run(kthread, k, "%s", k->name);
1271 if (task == NULL || IS_ERR(ptr: task))
1272 return -ENOMEM;
1273 k->task = task;
1274 wait_for_completion(&k->rendez); /* allow kthread to start */
1275 init_completion(x: &k->rendez); /* for waiting for exit later */
1276 return 0;
1277}
1278
1279/* pass it off to kthreads for processing */
1280static void
1281ktcomplete(struct frame *f, struct sk_buff *skb)
1282{
1283 int id;
1284 ulong flags;
1285
1286 f->r_skb = skb;
1287 id = f->t->d->aoeminor % ncpus;
1288 spin_lock_irqsave(&iocq[id].lock, flags);
1289 if (!kts[id].active) {
1290 spin_unlock_irqrestore(lock: &iocq[id].lock, flags);
1291 /* The thread with id has not been spawned yet,
1292 * so delegate the work to the main thread and
1293 * try spawning a new thread.
1294 */
1295 id = 0;
1296 spin_lock_irqsave(&iocq[id].lock, flags);
1297 }
1298 list_add_tail(new: &f->head, head: &iocq[id].head);
1299 spin_unlock_irqrestore(lock: &iocq[id].lock, flags);
1300 wake_up(&ktiowq[id]);
1301}
1302
1303struct sk_buff *
1304aoecmd_ata_rsp(struct sk_buff *skb)
1305{
1306 struct aoedev *d;
1307 struct aoe_hdr *h;
1308 struct frame *f;
1309 u32 n;
1310 ulong flags;
1311 char ebuf[128];
1312 u16 aoemajor;
1313
1314 h = (struct aoe_hdr *) skb->data;
1315 aoemajor = be16_to_cpu(get_unaligned(&h->major));
1316 d = aoedev_by_aoeaddr(maj: aoemajor, min: h->minor, do_alloc: 0);
1317 if (d == NULL) {
1318 snprintf(buf: ebuf, size: sizeof ebuf, fmt: "aoecmd_ata_rsp: ata response "
1319 "for unknown device %d.%d\n",
1320 aoemajor, h->minor);
1321 aoechr_error(ebuf);
1322 return skb;
1323 }
1324
1325 spin_lock_irqsave(&d->lock, flags);
1326
1327 n = be32_to_cpu(get_unaligned(&h->tag));
1328 f = getframe(d, tag: n);
1329 if (f) {
1330 calc_rttavg(d, t: f->t, rtt: tsince_hr(f));
1331 f->t->nout--;
1332 if (f->flags & FFL_PROBE)
1333 f->t->nout_probes--;
1334 } else {
1335 f = getframe_deferred(d, tag: n);
1336 if (f) {
1337 calc_rttavg(d, NULL, rtt: tsince_hr(f));
1338 } else {
1339 calc_rttavg(d, NULL, rtt: tsince(tag: n));
1340 spin_unlock_irqrestore(lock: &d->lock, flags);
1341 aoedev_put(d);
1342 snprintf(buf: ebuf, size: sizeof(ebuf),
1343 fmt: "%15s e%d.%d tag=%08x@%08lx s=%pm d=%pm\n",
1344 "unexpected rsp",
1345 get_unaligned_be16(p: &h->major),
1346 h->minor,
1347 get_unaligned_be32(p: &h->tag),
1348 jiffies,
1349 h->src,
1350 h->dst);
1351 aoechr_error(ebuf);
1352 return skb;
1353 }
1354 }
1355 aoecmd_work(d);
1356
1357 spin_unlock_irqrestore(lock: &d->lock, flags);
1358
1359 ktcomplete(f, skb);
1360
1361 /*
1362 * Note here that we do not perform an aoedev_put, as we are
1363 * leaving this reference for the ktio to release.
1364 */
1365 return NULL;
1366}
1367
1368void
1369aoecmd_cfg(ushort aoemajor, unsigned char aoeminor)
1370{
1371 struct sk_buff_head queue;
1372
1373 __skb_queue_head_init(list: &queue);
1374 aoecmd_cfg_pkts(aoemajor, aoeminor, queue: &queue);
1375 aoenet_xmit(&queue);
1376}
1377
1378struct sk_buff *
1379aoecmd_ata_id(struct aoedev *d)
1380{
1381 struct aoe_hdr *h;
1382 struct aoe_atahdr *ah;
1383 struct frame *f;
1384 struct sk_buff *skb;
1385 struct aoetgt *t;
1386
1387 f = newframe(d);
1388 if (f == NULL)
1389 return NULL;
1390
1391 t = *d->tgt;
1392
1393 /* initialize the headers & frame */
1394 skb = f->skb;
1395 h = (struct aoe_hdr *) skb_mac_header(skb);
1396 ah = (struct aoe_atahdr *) (h+1);
1397 skb_put(skb, len: sizeof *h + sizeof *ah);
1398 memset(h, 0, skb->len);
1399 f->tag = aoehdr_atainit(d, t, h);
1400 fhash(f);
1401 t->nout++;
1402 f->waited = 0;
1403 f->waited_total = 0;
1404
1405 /* set up ata header */
1406 ah->scnt = 1;
1407 ah->cmdstat = ATA_CMD_ID_ATA;
1408 ah->lba3 = 0xa0;
1409
1410 dev_hold(dev: t->ifp->nd);
1411 skb->dev = t->ifp->nd;
1412
1413 d->rttavg = RTTAVG_INIT;
1414 d->rttdev = RTTDEV_INIT;
1415 d->timer.function = rexmit_timer;
1416
1417 skb = skb_clone(skb, GFP_ATOMIC);
1418 if (skb)
1419 f->sent = ktime_get();
1420 else
1421 dev_put(dev: t->ifp->nd);
1422
1423 return skb;
1424}
1425
1426static struct aoetgt **
1427grow_targets(struct aoedev *d)
1428{
1429 ulong oldn, newn;
1430 struct aoetgt **tt;
1431
1432 oldn = d->ntargets;
1433 newn = oldn * 2;
1434 tt = kcalloc(newn, sizeof(*d->targets), GFP_ATOMIC);
1435 if (!tt)
1436 return NULL;
1437 memmove(tt, d->targets, sizeof(*d->targets) * oldn);
1438 d->tgt = tt + (d->tgt - d->targets);
1439 kfree(objp: d->targets);
1440 d->targets = tt;
1441 d->ntargets = newn;
1442
1443 return &d->targets[oldn];
1444}
1445
1446static struct aoetgt *
1447addtgt(struct aoedev *d, char *addr, ulong nframes)
1448{
1449 struct aoetgt *t, **tt, **te;
1450
1451 tt = d->targets;
1452 te = tt + d->ntargets;
1453 for (; tt < te && *tt; tt++)
1454 ;
1455
1456 if (tt == te) {
1457 tt = grow_targets(d);
1458 if (!tt)
1459 goto nomem;
1460 }
1461 t = kzalloc(sizeof(*t), GFP_ATOMIC);
1462 if (!t)
1463 goto nomem;
1464 t->nframes = nframes;
1465 t->d = d;
1466 memcpy(t->addr, addr, sizeof t->addr);
1467 t->ifp = t->ifs;
1468 aoecmd_wreset(t);
1469 t->maxout = t->nframes / 2;
1470 INIT_LIST_HEAD(list: &t->ffree);
1471 return *tt = t;
1472
1473 nomem:
1474 pr_info("aoe: cannot allocate memory to add target\n");
1475 return NULL;
1476}
1477
1478static void
1479setdbcnt(struct aoedev *d)
1480{
1481 struct aoetgt **t, **e;
1482 int bcnt = 0;
1483
1484 t = d->targets;
1485 e = t + d->ntargets;
1486 for (; t < e && *t; t++)
1487 if (bcnt == 0 || bcnt > (*t)->minbcnt)
1488 bcnt = (*t)->minbcnt;
1489 if (bcnt != d->maxbcnt) {
1490 d->maxbcnt = bcnt;
1491 pr_info("aoe: e%ld.%d: setting %d byte data frames\n",
1492 d->aoemajor, d->aoeminor, bcnt);
1493 }
1494}
1495
1496static void
1497setifbcnt(struct aoetgt *t, struct net_device *nd, int bcnt)
1498{
1499 struct aoedev *d;
1500 struct aoeif *p, *e;
1501 int minbcnt;
1502
1503 d = t->d;
1504 minbcnt = bcnt;
1505 p = t->ifs;
1506 e = p + NAOEIFS;
1507 for (; p < e; p++) {
1508 if (p->nd == NULL)
1509 break; /* end of the valid interfaces */
1510 if (p->nd == nd) {
1511 p->bcnt = bcnt; /* we're updating */
1512 nd = NULL;
1513 } else if (minbcnt > p->bcnt)
1514 minbcnt = p->bcnt; /* find the min interface */
1515 }
1516 if (nd) {
1517 if (p == e) {
1518 pr_err("aoe: device setifbcnt failure; too many interfaces.\n");
1519 return;
1520 }
1521 dev_hold(dev: nd);
1522 p->nd = nd;
1523 p->bcnt = bcnt;
1524 }
1525 t->minbcnt = minbcnt;
1526 setdbcnt(d);
1527}
1528
1529void
1530aoecmd_cfg_rsp(struct sk_buff *skb)
1531{
1532 struct aoedev *d;
1533 struct aoe_hdr *h;
1534 struct aoe_cfghdr *ch;
1535 struct aoetgt *t;
1536 ulong flags, aoemajor;
1537 struct sk_buff *sl;
1538 struct sk_buff_head queue;
1539 u16 n;
1540
1541 sl = NULL;
1542 h = (struct aoe_hdr *) skb_mac_header(skb);
1543 ch = (struct aoe_cfghdr *) (h+1);
1544
1545 /*
1546 * Enough people have their dip switches set backwards to
1547 * warrant a loud message for this special case.
1548 */
1549 aoemajor = get_unaligned_be16(p: &h->major);
1550 if (aoemajor == 0xfff) {
1551 printk(KERN_ERR "aoe: Warning: shelf address is all ones. "
1552 "Check shelf dip switches.\n");
1553 return;
1554 }
1555 if (aoemajor == 0xffff) {
1556 pr_info("aoe: e%ld.%d: broadcast shelf number invalid\n",
1557 aoemajor, (int) h->minor);
1558 return;
1559 }
1560 if (h->minor == 0xff) {
1561 pr_info("aoe: e%ld.%d: broadcast slot number invalid\n",
1562 aoemajor, (int) h->minor);
1563 return;
1564 }
1565
1566 n = be16_to_cpu(ch->bufcnt);
1567 if (n > aoe_maxout) /* keep it reasonable */
1568 n = aoe_maxout;
1569
1570 d = aoedev_by_aoeaddr(maj: aoemajor, min: h->minor, do_alloc: 1);
1571 if (d == NULL) {
1572 pr_info("aoe: device allocation failure\n");
1573 return;
1574 }
1575
1576 spin_lock_irqsave(&d->lock, flags);
1577
1578 t = gettgt(d, addr: h->src);
1579 if (t) {
1580 t->nframes = n;
1581 if (n < t->maxout)
1582 aoecmd_wreset(t);
1583 } else {
1584 t = addtgt(d, addr: h->src, nframes: n);
1585 if (!t)
1586 goto bail;
1587 }
1588 n = skb->dev->mtu;
1589 n -= sizeof(struct aoe_hdr) + sizeof(struct aoe_atahdr);
1590 n /= 512;
1591 if (n > ch->scnt)
1592 n = ch->scnt;
1593 n = n ? n * 512 : DEFAULTBCNT;
1594 setifbcnt(t, nd: skb->dev, bcnt: n);
1595
1596 /* don't change users' perspective */
1597 if (d->nopen == 0) {
1598 d->fw_ver = be16_to_cpu(ch->fwver);
1599 sl = aoecmd_ata_id(d);
1600 }
1601bail:
1602 spin_unlock_irqrestore(lock: &d->lock, flags);
1603 aoedev_put(d);
1604 if (sl) {
1605 __skb_queue_head_init(list: &queue);
1606 __skb_queue_tail(list: &queue, newsk: sl);
1607 aoenet_xmit(&queue);
1608 }
1609}
1610
1611void
1612aoecmd_wreset(struct aoetgt *t)
1613{
1614 t->maxout = 1;
1615 t->ssthresh = t->nframes / 2;
1616 t->next_cwnd = t->nframes;
1617}
1618
1619void
1620aoecmd_cleanslate(struct aoedev *d)
1621{
1622 struct aoetgt **t, **te;
1623
1624 d->rttavg = RTTAVG_INIT;
1625 d->rttdev = RTTDEV_INIT;
1626 d->maxbcnt = 0;
1627
1628 t = d->targets;
1629 te = t + d->ntargets;
1630 for (; t < te && *t; t++)
1631 aoecmd_wreset(t: *t);
1632}
1633
1634void
1635aoe_failbuf(struct aoedev *d, struct buf *buf)
1636{
1637 if (buf == NULL)
1638 return;
1639 buf->iter.bi_size = 0;
1640 buf->bio->bi_status = BLK_STS_IOERR;
1641 if (buf->nframesout == 0)
1642 aoe_end_buf(d, buf);
1643}
1644
1645void
1646aoe_flush_iocq(void)
1647{
1648 int i;
1649
1650 for (i = 0; i < ncpus; i++) {
1651 if (kts[i].active)
1652 aoe_flush_iocq_by_index(i);
1653 }
1654}
1655
1656void
1657aoe_flush_iocq_by_index(int id)
1658{
1659 struct frame *f;
1660 struct aoedev *d;
1661 LIST_HEAD(flist);
1662 struct list_head *pos;
1663 struct sk_buff *skb;
1664 ulong flags;
1665
1666 spin_lock_irqsave(&iocq[id].lock, flags);
1667 list_splice_init(list: &iocq[id].head, head: &flist);
1668 spin_unlock_irqrestore(lock: &iocq[id].lock, flags);
1669 while (!list_empty(head: &flist)) {
1670 pos = flist.next;
1671 list_del(entry: pos);
1672 f = list_entry(pos, struct frame, head);
1673 d = f->t->d;
1674 skb = f->r_skb;
1675 spin_lock_irqsave(&d->lock, flags);
1676 if (f->buf) {
1677 f->buf->nframesout--;
1678 aoe_failbuf(d, buf: f->buf);
1679 }
1680 aoe_freetframe(f);
1681 spin_unlock_irqrestore(lock: &d->lock, flags);
1682 dev_kfree_skb(skb);
1683 aoedev_put(d);
1684 }
1685}
1686
1687int __init
1688aoecmd_init(void)
1689{
1690 void *p;
1691 int i;
1692 int ret;
1693
1694 /* get_zeroed_page returns page with ref count 1 */
1695 p = (void *) get_zeroed_page(GFP_KERNEL);
1696 if (!p)
1697 return -ENOMEM;
1698 empty_page = virt_to_page(p);
1699
1700 ncpus = num_online_cpus();
1701
1702 iocq = kcalloc(ncpus, sizeof(struct iocq_ktio), GFP_KERNEL);
1703 if (!iocq)
1704 return -ENOMEM;
1705
1706 kts = kcalloc(ncpus, sizeof(struct ktstate), GFP_KERNEL);
1707 if (!kts) {
1708 ret = -ENOMEM;
1709 goto kts_fail;
1710 }
1711
1712 ktiowq = kcalloc(ncpus, sizeof(wait_queue_head_t), GFP_KERNEL);
1713 if (!ktiowq) {
1714 ret = -ENOMEM;
1715 goto ktiowq_fail;
1716 }
1717
1718 for (i = 0; i < ncpus; i++) {
1719 INIT_LIST_HEAD(list: &iocq[i].head);
1720 spin_lock_init(&iocq[i].lock);
1721 init_waitqueue_head(&ktiowq[i]);
1722 snprintf(buf: kts[i].name, size: sizeof(kts[i].name), fmt: "aoe_ktio%d", i);
1723 kts[i].fn = ktio;
1724 kts[i].waitq = &ktiowq[i];
1725 kts[i].lock = &iocq[i].lock;
1726 kts[i].id = i;
1727 kts[i].active = 0;
1728 }
1729 kts[0].active = 1;
1730 if (aoe_ktstart(k: &kts[0])) {
1731 ret = -ENOMEM;
1732 goto ktstart_fail;
1733 }
1734 return 0;
1735
1736ktstart_fail:
1737 kfree(objp: ktiowq);
1738ktiowq_fail:
1739 kfree(objp: kts);
1740kts_fail:
1741 kfree(objp: iocq);
1742
1743 return ret;
1744}
1745
1746void
1747aoecmd_exit(void)
1748{
1749 int i;
1750
1751 for (i = 0; i < ncpus; i++)
1752 if (kts[i].active)
1753 aoe_ktstop(k: &kts[i]);
1754
1755 aoe_flush_iocq();
1756
1757 /* Free up the iocq and thread speicific configuration
1758 * allocated during startup.
1759 */
1760 kfree(objp: iocq);
1761 kfree(objp: kts);
1762 kfree(objp: ktiowq);
1763
1764 __free_page(empty_page);
1765 empty_page = NULL;
1766}
1767

source code of linux/drivers/block/aoe/aoecmd.c