[S390] qdio: remove enhanced SIGA
[linux-3.10.git] / drivers / s390 / cio / qdio_main.c
1 /*
2  * linux/drivers/s390/cio/qdio_main.c
3  *
4  * Linux for s390 qdio support, buffer handling, qdio API and module support.
5  *
6  * Copyright 2000,2008 IBM Corp.
7  * Author(s): Utz Bacher <utz.bacher@de.ibm.com>
8  *            Jan Glauber <jang@linux.vnet.ibm.com>
9  * 2.6 cio integration by Cornelia Huck <cornelia.huck@de.ibm.com>
10  */
11 #include <linux/module.h>
12 #include <linux/init.h>
13 #include <linux/kernel.h>
14 #include <linux/timer.h>
15 #include <linux/delay.h>
16 #include <linux/gfp.h>
17 #include <linux/kernel_stat.h>
18 #include <asm/atomic.h>
19 #include <asm/debug.h>
20 #include <asm/qdio.h>
21
22 #include "cio.h"
23 #include "css.h"
24 #include "device.h"
25 #include "qdio.h"
26 #include "qdio_debug.h"
27
28 MODULE_AUTHOR("Utz Bacher <utz.bacher@de.ibm.com>,"\
29         "Jan Glauber <jang@linux.vnet.ibm.com>");
30 MODULE_DESCRIPTION("QDIO base support");
31 MODULE_LICENSE("GPL");
32
33 static inline int do_siga_sync(unsigned long schid,
34                                unsigned int out_mask, unsigned int in_mask,
35                                unsigned int fc)
36 {
37         register unsigned long __fc asm ("0") = fc;
38         register unsigned long __schid asm ("1") = schid;
39         register unsigned long out asm ("2") = out_mask;
40         register unsigned long in asm ("3") = in_mask;
41         int cc;
42
43         asm volatile(
44                 "       siga    0\n"
45                 "       ipm     %0\n"
46                 "       srl     %0,28\n"
47                 : "=d" (cc)
48                 : "d" (__fc), "d" (__schid), "d" (out), "d" (in) : "cc");
49         return cc;
50 }
51
52 static inline int do_siga_input(unsigned long schid, unsigned int mask,
53                                 unsigned int fc)
54 {
55         register unsigned long __fc asm ("0") = fc;
56         register unsigned long __schid asm ("1") = schid;
57         register unsigned long __mask asm ("2") = mask;
58         int cc;
59
60         asm volatile(
61                 "       siga    0\n"
62                 "       ipm     %0\n"
63                 "       srl     %0,28\n"
64                 : "=d" (cc)
65                 : "d" (__fc), "d" (__schid), "d" (__mask) : "cc", "memory");
66         return cc;
67 }
68
69 /**
70  * do_siga_output - perform SIGA-w/wt function
71  * @schid: subchannel id or in case of QEBSM the subchannel token
72  * @mask: which output queues to process
73  * @bb: busy bit indicator, set only if SIGA-w/wt could not access a buffer
74  * @fc: function code to perform
75  *
76  * Returns cc or QDIO_ERROR_SIGA_ACCESS_EXCEPTION.
77  * Note: For IQDC unicast queues only the highest priority queue is processed.
78  */
79 static inline int do_siga_output(unsigned long schid, unsigned long mask,
80                                  unsigned int *bb, unsigned int fc)
81 {
82         register unsigned long __fc asm("0") = fc;
83         register unsigned long __schid asm("1") = schid;
84         register unsigned long __mask asm("2") = mask;
85         int cc = QDIO_ERROR_SIGA_ACCESS_EXCEPTION;
86
87         asm volatile(
88                 "       siga    0\n"
89                 "0:     ipm     %0\n"
90                 "       srl     %0,28\n"
91                 "1:\n"
92                 EX_TABLE(0b, 1b)
93                 : "+d" (cc), "+d" (__fc), "+d" (__schid), "+d" (__mask)
94                 : : "cc", "memory");
95         *bb = ((unsigned int) __fc) >> 31;
96         return cc;
97 }
98
99 static inline int qdio_check_ccq(struct qdio_q *q, unsigned int ccq)
100 {
101         /* all done or next buffer state different */
102         if (ccq == 0 || ccq == 32)
103                 return 0;
104         /* not all buffers processed */
105         if (ccq == 96 || ccq == 97)
106                 return 1;
107         /* notify devices immediately */
108         DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);
109         return -EIO;
110 }
111
112 /**
113  * qdio_do_eqbs - extract buffer states for QEBSM
114  * @q: queue to manipulate
115  * @state: state of the extracted buffers
116  * @start: buffer number to start at
117  * @count: count of buffers to examine
118  * @auto_ack: automatically acknowledge buffers
119  *
120  * Returns the number of successfully extracted equal buffer states.
121  * Stops processing if a state is different from the last buffers state.
122  */
123 static int qdio_do_eqbs(struct qdio_q *q, unsigned char *state,
124                         int start, int count, int auto_ack)
125 {
126         unsigned int ccq = 0;
127         int tmp_count = count, tmp_start = start;
128         int nr = q->nr;
129         int rc;
130
131         BUG_ON(!q->irq_ptr->sch_token);
132         qperf_inc(q, eqbs);
133
134         if (!q->is_input_q)
135                 nr += q->irq_ptr->nr_input_qs;
136 again:
137         ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count,
138                       auto_ack);
139         rc = qdio_check_ccq(q, ccq);
140
141         /* At least one buffer was processed, return and extract the remaining
142          * buffers later.
143          */
144         if ((ccq == 96) && (count != tmp_count)) {
145                 qperf_inc(q, eqbs_partial);
146                 return (count - tmp_count);
147         }
148
149         if (rc == 1) {
150                 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq);
151                 goto again;
152         }
153
154         if (rc < 0) {
155                 DBF_ERROR("%4x EQBS ERROR", SCH_NO(q));
156                 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
157                 q->handler(q->irq_ptr->cdev,
158                            QDIO_ERROR_ACTIVATE_CHECK_CONDITION,
159                            0, -1, -1, q->irq_ptr->int_parm);
160                 return 0;
161         }
162         return count - tmp_count;
163 }
164
165 /**
166  * qdio_do_sqbs - set buffer states for QEBSM
167  * @q: queue to manipulate
168  * @state: new state of the buffers
169  * @start: first buffer number to change
170  * @count: how many buffers to change
171  *
172  * Returns the number of successfully changed buffers.
173  * Does retrying until the specified count of buffer states is set or an
174  * error occurs.
175  */
176 static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start,
177                         int count)
178 {
179         unsigned int ccq = 0;
180         int tmp_count = count, tmp_start = start;
181         int nr = q->nr;
182         int rc;
183
184         if (!count)
185                 return 0;
186
187         BUG_ON(!q->irq_ptr->sch_token);
188         qperf_inc(q, sqbs);
189
190         if (!q->is_input_q)
191                 nr += q->irq_ptr->nr_input_qs;
192 again:
193         ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count);
194         rc = qdio_check_ccq(q, ccq);
195         if (rc == 1) {
196                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq);
197                 qperf_inc(q, sqbs_partial);
198                 goto again;
199         }
200         if (rc < 0) {
201                 DBF_ERROR("%4x SQBS ERROR", SCH_NO(q));
202                 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
203                 q->handler(q->irq_ptr->cdev,
204                            QDIO_ERROR_ACTIVATE_CHECK_CONDITION,
205                            0, -1, -1, q->irq_ptr->int_parm);
206                 return 0;
207         }
208         WARN_ON(tmp_count);
209         return count - tmp_count;
210 }
211
212 /* returns number of examined buffers and their common state in *state */
213 static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr,
214                                  unsigned char *state, unsigned int count,
215                                  int auto_ack)
216 {
217         unsigned char __state = 0;
218         int i;
219
220         BUG_ON(bufnr > QDIO_MAX_BUFFERS_MASK);
221         BUG_ON(count > QDIO_MAX_BUFFERS_PER_Q);
222
223         if (is_qebsm(q))
224                 return qdio_do_eqbs(q, state, bufnr, count, auto_ack);
225
226         for (i = 0; i < count; i++) {
227                 if (!__state)
228                         __state = q->slsb.val[bufnr];
229                 else if (q->slsb.val[bufnr] != __state)
230                         break;
231                 bufnr = next_buf(bufnr);
232         }
233         *state = __state;
234         return i;
235 }
236
237 static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr,
238                                 unsigned char *state, int auto_ack)
239 {
240         return get_buf_states(q, bufnr, state, 1, auto_ack);
241 }
242
243 /* wrap-around safe setting of slsb states, returns number of changed buffers */
244 static inline int set_buf_states(struct qdio_q *q, int bufnr,
245                                  unsigned char state, int count)
246 {
247         int i;
248
249         BUG_ON(bufnr > QDIO_MAX_BUFFERS_MASK);
250         BUG_ON(count > QDIO_MAX_BUFFERS_PER_Q);
251
252         if (is_qebsm(q))
253                 return qdio_do_sqbs(q, state, bufnr, count);
254
255         for (i = 0; i < count; i++) {
256                 xchg(&q->slsb.val[bufnr], state);
257                 bufnr = next_buf(bufnr);
258         }
259         return count;
260 }
261
262 static inline int set_buf_state(struct qdio_q *q, int bufnr,
263                                 unsigned char state)
264 {
265         return set_buf_states(q, bufnr, state, 1);
266 }
267
268 /* set slsb states to initial state */
269 void qdio_init_buf_states(struct qdio_irq *irq_ptr)
270 {
271         struct qdio_q *q;
272         int i;
273
274         for_each_input_queue(irq_ptr, q, i)
275                 set_buf_states(q, 0, SLSB_P_INPUT_NOT_INIT,
276                                QDIO_MAX_BUFFERS_PER_Q);
277         for_each_output_queue(irq_ptr, q, i)
278                 set_buf_states(q, 0, SLSB_P_OUTPUT_NOT_INIT,
279                                QDIO_MAX_BUFFERS_PER_Q);
280 }
281
282 static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output,
283                           unsigned int input)
284 {
285         unsigned long schid = *((u32 *) &q->irq_ptr->schid);
286         unsigned int fc = QDIO_SIGA_SYNC;
287         int cc;
288
289         if (!need_siga_sync(q))
290                 return 0;
291
292         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr);
293         qperf_inc(q, siga_sync);
294
295         if (is_qebsm(q)) {
296                 schid = q->irq_ptr->sch_token;
297                 fc |= QDIO_SIGA_QEBSM_FLAG;
298         }
299
300         cc = do_siga_sync(schid, output, input, fc);
301         if (unlikely(cc))
302                 DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc);
303         return cc;
304 }
305
306 static inline int qdio_siga_sync_q(struct qdio_q *q)
307 {
308         if (q->is_input_q)
309                 return qdio_siga_sync(q, 0, q->mask);
310         else
311                 return qdio_siga_sync(q, q->mask, 0);
312 }
313
314 static inline int qdio_siga_sync_out(struct qdio_q *q)
315 {
316         return qdio_siga_sync(q, ~0U, 0);
317 }
318
319 static inline int qdio_siga_sync_all(struct qdio_q *q)
320 {
321         return qdio_siga_sync(q, ~0U, ~0U);
322 }
323
324 static int qdio_siga_output(struct qdio_q *q, unsigned int *busy_bit)
325 {
326         unsigned long schid = *((u32 *) &q->irq_ptr->schid);
327         unsigned int fc = QDIO_SIGA_WRITE;
328         u64 start_time = 0;
329         int cc;
330
331         if (is_qebsm(q)) {
332                 schid = q->irq_ptr->sch_token;
333                 fc |= QDIO_SIGA_QEBSM_FLAG;
334         }
335 again:
336         cc = do_siga_output(schid, q->mask, busy_bit, fc);
337
338         /* hipersocket busy condition */
339         if (unlikely(*busy_bit)) {
340                 WARN_ON(queue_type(q) != QDIO_IQDIO_QFMT || cc != 2);
341
342                 if (!start_time) {
343                         start_time = get_clock();
344                         goto again;
345                 }
346                 if ((get_clock() - start_time) < QDIO_BUSY_BIT_PATIENCE)
347                         goto again;
348         }
349         return cc;
350 }
351
352 static inline int qdio_siga_input(struct qdio_q *q)
353 {
354         unsigned long schid = *((u32 *) &q->irq_ptr->schid);
355         unsigned int fc = QDIO_SIGA_READ;
356         int cc;
357
358         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr);
359         qperf_inc(q, siga_read);
360
361         if (is_qebsm(q)) {
362                 schid = q->irq_ptr->sch_token;
363                 fc |= QDIO_SIGA_QEBSM_FLAG;
364         }
365
366         cc = do_siga_input(schid, q->mask, fc);
367         if (unlikely(cc))
368                 DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc);
369         return cc;
370 }
371
372 static inline void qdio_sync_after_thinint(struct qdio_q *q)
373 {
374         if (pci_out_supported(q)) {
375                 if (need_siga_sync_thinint(q))
376                         qdio_siga_sync_all(q);
377                 else if (need_siga_sync_out_thinint(q))
378                         qdio_siga_sync_out(q);
379         } else
380                 qdio_siga_sync_q(q);
381 }
382
383 int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr,
384                         unsigned char *state)
385 {
386         qdio_siga_sync_q(q);
387         return get_buf_states(q, bufnr, state, 1, 0);
388 }
389
390 static inline void qdio_stop_polling(struct qdio_q *q)
391 {
392         if (!q->u.in.polling)
393                 return;
394
395         q->u.in.polling = 0;
396         qperf_inc(q, stop_polling);
397
398         /* show the card that we are not polling anymore */
399         if (is_qebsm(q)) {
400                 set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,
401                                q->u.in.ack_count);
402                 q->u.in.ack_count = 0;
403         } else
404                 set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);
405 }
406
407 static inline void account_sbals(struct qdio_q *q, int count)
408 {
409         int pos = 0;
410
411         q->q_stats.nr_sbal_total += count;
412         if (count == QDIO_MAX_BUFFERS_MASK) {
413                 q->q_stats.nr_sbals[7]++;
414                 return;
415         }
416         while (count >>= 1)
417                 pos++;
418         q->q_stats.nr_sbals[pos]++;
419 }
420
421 static void announce_buffer_error(struct qdio_q *q, int count)
422 {
423         q->qdio_error |= QDIO_ERROR_SLSB_STATE;
424
425         /* special handling for no target buffer empty */
426         if ((!q->is_input_q &&
427             (q->sbal[q->first_to_check]->element[15].flags & 0xff) == 0x10)) {
428                 qperf_inc(q, target_full);
429                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x",
430                               q->first_to_check);
431                 return;
432         }
433
434         DBF_ERROR("%4x BUF ERROR", SCH_NO(q));
435         DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr);
436         DBF_ERROR("FTC:%3d C:%3d", q->first_to_check, count);
437         DBF_ERROR("F14:%2x F15:%2x",
438                   q->sbal[q->first_to_check]->element[14].flags & 0xff,
439                   q->sbal[q->first_to_check]->element[15].flags & 0xff);
440 }
441
442 static inline void inbound_primed(struct qdio_q *q, int count)
443 {
444         int new;
445
446         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim: %02x", count);
447
448         /* for QEBSM the ACK was already set by EQBS */
449         if (is_qebsm(q)) {
450                 if (!q->u.in.polling) {
451                         q->u.in.polling = 1;
452                         q->u.in.ack_count = count;
453                         q->u.in.ack_start = q->first_to_check;
454                         return;
455                 }
456
457                 /* delete the previous ACK's */
458                 set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,
459                                q->u.in.ack_count);
460                 q->u.in.ack_count = count;
461                 q->u.in.ack_start = q->first_to_check;
462                 return;
463         }
464
465         /*
466          * ACK the newest buffer. The ACK will be removed in qdio_stop_polling
467          * or by the next inbound run.
468          */
469         new = add_buf(q->first_to_check, count - 1);
470         if (q->u.in.polling) {
471                 /* reset the previous ACK but first set the new one */
472                 set_buf_state(q, new, SLSB_P_INPUT_ACK);
473                 set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);
474         } else {
475                 q->u.in.polling = 1;
476                 set_buf_state(q, new, SLSB_P_INPUT_ACK);
477         }
478
479         q->u.in.ack_start = new;
480         count--;
481         if (!count)
482                 return;
483         /* need to change ALL buffers to get more interrupts */
484         set_buf_states(q, q->first_to_check, SLSB_P_INPUT_NOT_INIT, count);
485 }
486
487 static int get_inbound_buffer_frontier(struct qdio_q *q)
488 {
489         int count, stop;
490         unsigned char state;
491
492         /*
493          * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
494          * would return 0.
495          */
496         count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);
497         stop = add_buf(q->first_to_check, count);
498
499         if (q->first_to_check == stop)
500                 goto out;
501
502         /*
503          * No siga sync here, as a PCI or we after a thin interrupt
504          * already sync'ed the queues.
505          */
506         count = get_buf_states(q, q->first_to_check, &state, count, 1);
507         if (!count)
508                 goto out;
509
510         switch (state) {
511         case SLSB_P_INPUT_PRIMED:
512                 inbound_primed(q, count);
513                 q->first_to_check = add_buf(q->first_to_check, count);
514                 if (atomic_sub(count, &q->nr_buf_used) == 0)
515                         qperf_inc(q, inbound_queue_full);
516                 if (q->irq_ptr->perf_stat_enabled)
517                         account_sbals(q, count);
518                 break;
519         case SLSB_P_INPUT_ERROR:
520                 announce_buffer_error(q, count);
521                 /* process the buffer, the upper layer will take care of it */
522                 q->first_to_check = add_buf(q->first_to_check, count);
523                 atomic_sub(count, &q->nr_buf_used);
524                 if (q->irq_ptr->perf_stat_enabled)
525                         account_sbals_error(q, count);
526                 break;
527         case SLSB_CU_INPUT_EMPTY:
528         case SLSB_P_INPUT_NOT_INIT:
529         case SLSB_P_INPUT_ACK:
530                 if (q->irq_ptr->perf_stat_enabled)
531                         q->q_stats.nr_sbal_nop++;
532                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop");
533                 break;
534         default:
535                 BUG();
536         }
537 out:
538         return q->first_to_check;
539 }
540
541 static int qdio_inbound_q_moved(struct qdio_q *q)
542 {
543         int bufnr;
544
545         bufnr = get_inbound_buffer_frontier(q);
546
547         if ((bufnr != q->last_move) || q->qdio_error) {
548                 q->last_move = bufnr;
549                 if (!is_thinint_irq(q->irq_ptr) && MACHINE_IS_LPAR)
550                         q->u.in.timestamp = get_clock();
551                 return 1;
552         } else
553                 return 0;
554 }
555
556 static inline int qdio_inbound_q_done(struct qdio_q *q)
557 {
558         unsigned char state = 0;
559
560         if (!atomic_read(&q->nr_buf_used))
561                 return 1;
562
563         qdio_siga_sync_q(q);
564         get_buf_state(q, q->first_to_check, &state, 0);
565
566         if (state == SLSB_P_INPUT_PRIMED || state == SLSB_P_INPUT_ERROR)
567                 /* more work coming */
568                 return 0;
569
570         if (is_thinint_irq(q->irq_ptr))
571                 return 1;
572
573         /* don't poll under z/VM */
574         if (MACHINE_IS_VM)
575                 return 1;
576
577         /*
578          * At this point we know, that inbound first_to_check
579          * has (probably) not moved (see qdio_inbound_processing).
580          */
581         if (get_clock() > q->u.in.timestamp + QDIO_INPUT_THRESHOLD) {
582                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in done:%02x",
583                               q->first_to_check);
584                 return 1;
585         } else
586                 return 0;
587 }
588
589 static void qdio_kick_handler(struct qdio_q *q)
590 {
591         int start = q->first_to_kick;
592         int end = q->first_to_check;
593         int count;
594
595         if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
596                 return;
597
598         count = sub_buf(end, start);
599
600         if (q->is_input_q) {
601                 qperf_inc(q, inbound_handler);
602                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "kih s:%02x c:%02x", start, count);
603         } else {
604                 qperf_inc(q, outbound_handler);
605                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "koh: s:%02x c:%02x",
606                               start, count);
607         }
608
609         q->handler(q->irq_ptr->cdev, q->qdio_error, q->nr, start, count,
610                    q->irq_ptr->int_parm);
611
612         /* for the next time */
613         q->first_to_kick = end;
614         q->qdio_error = 0;
615 }
616
617 static void __qdio_inbound_processing(struct qdio_q *q)
618 {
619         qperf_inc(q, tasklet_inbound);
620
621         if (!qdio_inbound_q_moved(q))
622                 return;
623
624         qdio_kick_handler(q);
625
626         if (!qdio_inbound_q_done(q)) {
627                 /* means poll time is not yet over */
628                 qperf_inc(q, tasklet_inbound_resched);
629                 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) {
630                         tasklet_schedule(&q->tasklet);
631                         return;
632                 }
633         }
634
635         qdio_stop_polling(q);
636         /*
637          * We need to check again to not lose initiative after
638          * resetting the ACK state.
639          */
640         if (!qdio_inbound_q_done(q)) {
641                 qperf_inc(q, tasklet_inbound_resched2);
642                 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED))
643                         tasklet_schedule(&q->tasklet);
644         }
645 }
646
647 void qdio_inbound_processing(unsigned long data)
648 {
649         struct qdio_q *q = (struct qdio_q *)data;
650         __qdio_inbound_processing(q);
651 }
652
653 static int get_outbound_buffer_frontier(struct qdio_q *q)
654 {
655         int count, stop;
656         unsigned char state;
657
658         if (((queue_type(q) != QDIO_IQDIO_QFMT) && !pci_out_supported(q)) ||
659             (queue_type(q) == QDIO_IQDIO_QFMT && multicast_outbound(q)))
660                 qdio_siga_sync_q(q);
661
662         /*
663          * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
664          * would return 0.
665          */
666         count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);
667         stop = add_buf(q->first_to_check, count);
668
669         if (q->first_to_check == stop)
670                 return q->first_to_check;
671
672         count = get_buf_states(q, q->first_to_check, &state, count, 0);
673         if (!count)
674                 return q->first_to_check;
675
676         switch (state) {
677         case SLSB_P_OUTPUT_EMPTY:
678                 /* the adapter got it */
679                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out empty:%1d %02x", q->nr, count);
680
681                 atomic_sub(count, &q->nr_buf_used);
682                 q->first_to_check = add_buf(q->first_to_check, count);
683                 if (q->irq_ptr->perf_stat_enabled)
684                         account_sbals(q, count);
685                 break;
686         case SLSB_P_OUTPUT_ERROR:
687                 announce_buffer_error(q, count);
688                 /* process the buffer, the upper layer will take care of it */
689                 q->first_to_check = add_buf(q->first_to_check, count);
690                 atomic_sub(count, &q->nr_buf_used);
691                 if (q->irq_ptr->perf_stat_enabled)
692                         account_sbals_error(q, count);
693                 break;
694         case SLSB_CU_OUTPUT_PRIMED:
695                 /* the adapter has not fetched the output yet */
696                 if (q->irq_ptr->perf_stat_enabled)
697                         q->q_stats.nr_sbal_nop++;
698                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d", q->nr);
699                 break;
700         case SLSB_P_OUTPUT_NOT_INIT:
701         case SLSB_P_OUTPUT_HALTED:
702                 break;
703         default:
704                 BUG();
705         }
706         return q->first_to_check;
707 }
708
709 /* all buffers processed? */
710 static inline int qdio_outbound_q_done(struct qdio_q *q)
711 {
712         return atomic_read(&q->nr_buf_used) == 0;
713 }
714
715 static inline int qdio_outbound_q_moved(struct qdio_q *q)
716 {
717         int bufnr;
718
719         bufnr = get_outbound_buffer_frontier(q);
720
721         if ((bufnr != q->last_move) || q->qdio_error) {
722                 q->last_move = bufnr;
723                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out moved:%1d", q->nr);
724                 return 1;
725         } else
726                 return 0;
727 }
728
729 static int qdio_kick_outbound_q(struct qdio_q *q)
730 {
731         unsigned int busy_bit;
732         int cc;
733
734         if (!need_siga_out(q))
735                 return 0;
736
737         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr);
738         qperf_inc(q, siga_write);
739
740         cc = qdio_siga_output(q, &busy_bit);
741         switch (cc) {
742         case 0:
743                 break;
744         case 2:
745                 if (busy_bit) {
746                         DBF_ERROR("%4x cc2 REP:%1d", SCH_NO(q), q->nr);
747                         cc |= QDIO_ERROR_SIGA_BUSY;
748                 } else
749                         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr);
750                 break;
751         case 1:
752         case 3:
753                 DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q), cc);
754                 break;
755         }
756         return cc;
757 }
758
759 static void __qdio_outbound_processing(struct qdio_q *q)
760 {
761         qperf_inc(q, tasklet_outbound);
762         BUG_ON(atomic_read(&q->nr_buf_used) < 0);
763
764         if (qdio_outbound_q_moved(q))
765                 qdio_kick_handler(q);
766
767         if (queue_type(q) == QDIO_ZFCP_QFMT)
768                 if (!pci_out_supported(q) && !qdio_outbound_q_done(q))
769                         goto sched;
770
771         /* bail out for HiperSockets unicast queues */
772         if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q))
773                 return;
774
775         if ((queue_type(q) == QDIO_IQDIO_QFMT) &&
776             (atomic_read(&q->nr_buf_used)) > QDIO_IQDIO_POLL_LVL)
777                 goto sched;
778
779         if (q->u.out.pci_out_enabled)
780                 return;
781
782         /*
783          * Now we know that queue type is either qeth without pci enabled
784          * or HiperSockets multicast. Make sure buffer switch from PRIMED to
785          * EMPTY is noticed and outbound_handler is called after some time.
786          */
787         if (qdio_outbound_q_done(q))
788                 del_timer(&q->u.out.timer);
789         else
790                 if (!timer_pending(&q->u.out.timer))
791                         mod_timer(&q->u.out.timer, jiffies + 10 * HZ);
792         return;
793
794 sched:
795         if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
796                 return;
797         tasklet_schedule(&q->tasklet);
798 }
799
800 /* outbound tasklet */
801 void qdio_outbound_processing(unsigned long data)
802 {
803         struct qdio_q *q = (struct qdio_q *)data;
804         __qdio_outbound_processing(q);
805 }
806
807 void qdio_outbound_timer(unsigned long data)
808 {
809         struct qdio_q *q = (struct qdio_q *)data;
810
811         if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
812                 return;
813         tasklet_schedule(&q->tasklet);
814 }
815
816 static inline void qdio_check_outbound_after_thinint(struct qdio_q *q)
817 {
818         struct qdio_q *out;
819         int i;
820
821         if (!pci_out_supported(q))
822                 return;
823
824         for_each_output_queue(q->irq_ptr, out, i)
825                 if (!qdio_outbound_q_done(out))
826                         tasklet_schedule(&out->tasklet);
827 }
828
829 static void __tiqdio_inbound_processing(struct qdio_q *q)
830 {
831         qperf_inc(q, tasklet_inbound);
832         qdio_sync_after_thinint(q);
833
834         /*
835          * The interrupt could be caused by a PCI request. Check the
836          * PCI capable outbound queues.
837          */
838         qdio_check_outbound_after_thinint(q);
839
840         if (!qdio_inbound_q_moved(q))
841                 return;
842
843         qdio_kick_handler(q);
844
845         if (!qdio_inbound_q_done(q)) {
846                 qperf_inc(q, tasklet_inbound_resched);
847                 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) {
848                         tasklet_schedule(&q->tasklet);
849                         return;
850                 }
851         }
852
853         qdio_stop_polling(q);
854         /*
855          * We need to check again to not lose initiative after
856          * resetting the ACK state.
857          */
858         if (!qdio_inbound_q_done(q)) {
859                 qperf_inc(q, tasklet_inbound_resched2);
860                 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED))
861                         tasklet_schedule(&q->tasklet);
862         }
863 }
864
865 void tiqdio_inbound_processing(unsigned long data)
866 {
867         struct qdio_q *q = (struct qdio_q *)data;
868         __tiqdio_inbound_processing(q);
869 }
870
871 static inline void qdio_set_state(struct qdio_irq *irq_ptr,
872                                   enum qdio_irq_states state)
873 {
874         DBF_DEV_EVENT(DBF_INFO, irq_ptr, "newstate: %1d", state);
875
876         irq_ptr->state = state;
877         mb();
878 }
879
880 static void qdio_irq_check_sense(struct qdio_irq *irq_ptr, struct irb *irb)
881 {
882         if (irb->esw.esw0.erw.cons) {
883                 DBF_ERROR("%4x sense:", irq_ptr->schid.sch_no);
884                 DBF_ERROR_HEX(irb, 64);
885                 DBF_ERROR_HEX(irb->ecw, 64);
886         }
887 }
888
889 /* PCI interrupt handler */
890 static void qdio_int_handler_pci(struct qdio_irq *irq_ptr)
891 {
892         int i;
893         struct qdio_q *q;
894
895         if (unlikely(irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
896                 return;
897
898         for_each_input_queue(irq_ptr, q, i) {
899                 if (q->u.in.queue_start_poll) {
900                         /* skip if polling is enabled or already in work */
901                         if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
902                                      &q->u.in.queue_irq_state)) {
903                                 qperf_inc(q, int_discarded);
904                                 continue;
905                         }
906                         q->u.in.queue_start_poll(q->irq_ptr->cdev, q->nr,
907                                                  q->irq_ptr->int_parm);
908                 } else
909                         tasklet_schedule(&q->tasklet);
910         }
911
912         if (!(irq_ptr->qib.ac & QIB_AC_OUTBOUND_PCI_SUPPORTED))
913                 return;
914
915         for_each_output_queue(irq_ptr, q, i) {
916                 if (qdio_outbound_q_done(q))
917                         continue;
918
919                 if (!siga_syncs_out_pci(q))
920                         qdio_siga_sync_q(q);
921
922                 tasklet_schedule(&q->tasklet);
923         }
924 }
925
926 static void qdio_handle_activate_check(struct ccw_device *cdev,
927                                 unsigned long intparm, int cstat, int dstat)
928 {
929         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
930         struct qdio_q *q;
931
932         DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no);
933         DBF_ERROR("intp :%lx", intparm);
934         DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
935
936         if (irq_ptr->nr_input_qs) {
937                 q = irq_ptr->input_qs[0];
938         } else if (irq_ptr->nr_output_qs) {
939                 q = irq_ptr->output_qs[0];
940         } else {
941                 dump_stack();
942                 goto no_handler;
943         }
944         q->handler(q->irq_ptr->cdev, QDIO_ERROR_ACTIVATE_CHECK_CONDITION,
945                    0, -1, -1, irq_ptr->int_parm);
946 no_handler:
947         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
948 }
949
950 static void qdio_establish_handle_irq(struct ccw_device *cdev, int cstat,
951                                       int dstat)
952 {
953         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
954
955         DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq");
956
957         if (cstat)
958                 goto error;
959         if (dstat & ~(DEV_STAT_DEV_END | DEV_STAT_CHN_END))
960                 goto error;
961         if (!(dstat & DEV_STAT_DEV_END))
962                 goto error;
963         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ESTABLISHED);
964         return;
965
966 error:
967         DBF_ERROR("%4x EQ:error", irq_ptr->schid.sch_no);
968         DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
969         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
970 }
971
972 /* qdio interrupt handler */
973 void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm,
974                       struct irb *irb)
975 {
976         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
977         int cstat, dstat;
978
979         if (!intparm || !irq_ptr) {
980                 DBF_ERROR("qint:%4x", cdev->private->schid.sch_no);
981                 return;
982         }
983
984         kstat_cpu(smp_processor_id()).irqs[IOINT_QDI]++;
985         if (irq_ptr->perf_stat_enabled)
986                 irq_ptr->perf_stat.qdio_int++;
987
988         if (IS_ERR(irb)) {
989                 switch (PTR_ERR(irb)) {
990                 case -EIO:
991                         DBF_ERROR("%4x IO error", irq_ptr->schid.sch_no);
992                         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
993                         wake_up(&cdev->private->wait_q);
994                         return;
995                 default:
996                         WARN_ON(1);
997                         return;
998                 }
999         }
1000         qdio_irq_check_sense(irq_ptr, irb);
1001         cstat = irb->scsw.cmd.cstat;
1002         dstat = irb->scsw.cmd.dstat;
1003
1004         switch (irq_ptr->state) {
1005         case QDIO_IRQ_STATE_INACTIVE:
1006                 qdio_establish_handle_irq(cdev, cstat, dstat);
1007                 break;
1008         case QDIO_IRQ_STATE_CLEANUP:
1009                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1010                 break;
1011         case QDIO_IRQ_STATE_ESTABLISHED:
1012         case QDIO_IRQ_STATE_ACTIVE:
1013                 if (cstat & SCHN_STAT_PCI) {
1014                         qdio_int_handler_pci(irq_ptr);
1015                         return;
1016                 }
1017                 if (cstat || dstat)
1018                         qdio_handle_activate_check(cdev, intparm, cstat,
1019                                                    dstat);
1020                 break;
1021         case QDIO_IRQ_STATE_STOPPED:
1022                 break;
1023         default:
1024                 WARN_ON(1);
1025         }
1026         wake_up(&cdev->private->wait_q);
1027 }
1028
1029 /**
1030  * qdio_get_ssqd_desc - get qdio subchannel description
1031  * @cdev: ccw device to get description for
1032  * @data: where to store the ssqd
1033  *
1034  * Returns 0 or an error code. The results of the chsc are stored in the
1035  * specified structure.
1036  */
1037 int qdio_get_ssqd_desc(struct ccw_device *cdev,
1038                        struct qdio_ssqd_desc *data)
1039 {
1040
1041         if (!cdev || !cdev->private)
1042                 return -EINVAL;
1043
1044         DBF_EVENT("get ssqd:%4x", cdev->private->schid.sch_no);
1045         return qdio_setup_get_ssqd(NULL, &cdev->private->schid, data);
1046 }
1047 EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc);
1048
1049 static void qdio_shutdown_queues(struct ccw_device *cdev)
1050 {
1051         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1052         struct qdio_q *q;
1053         int i;
1054
1055         for_each_input_queue(irq_ptr, q, i)
1056                 tasklet_kill(&q->tasklet);
1057
1058         for_each_output_queue(irq_ptr, q, i) {
1059                 del_timer(&q->u.out.timer);
1060                 tasklet_kill(&q->tasklet);
1061         }
1062 }
1063
1064 /**
1065  * qdio_shutdown - shut down a qdio subchannel
1066  * @cdev: associated ccw device
1067  * @how: use halt or clear to shutdown
1068  */
1069 int qdio_shutdown(struct ccw_device *cdev, int how)
1070 {
1071         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1072         int rc;
1073         unsigned long flags;
1074
1075         if (!irq_ptr)
1076                 return -ENODEV;
1077
1078         BUG_ON(irqs_disabled());
1079         DBF_EVENT("qshutdown:%4x", cdev->private->schid.sch_no);
1080
1081         mutex_lock(&irq_ptr->setup_mutex);
1082         /*
1083          * Subchannel was already shot down. We cannot prevent being called
1084          * twice since cio may trigger a shutdown asynchronously.
1085          */
1086         if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {
1087                 mutex_unlock(&irq_ptr->setup_mutex);
1088                 return 0;
1089         }
1090
1091         /*
1092          * Indicate that the device is going down. Scheduling the queue
1093          * tasklets is forbidden from here on.
1094          */
1095         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
1096
1097         tiqdio_remove_input_queues(irq_ptr);
1098         qdio_shutdown_queues(cdev);
1099         qdio_shutdown_debug_entries(irq_ptr, cdev);
1100
1101         /* cleanup subchannel */
1102         spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
1103
1104         if (how & QDIO_FLAG_CLEANUP_USING_CLEAR)
1105                 rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP);
1106         else
1107                 /* default behaviour is halt */
1108                 rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP);
1109         if (rc) {
1110                 DBF_ERROR("%4x SHUTD ERR", irq_ptr->schid.sch_no);
1111                 DBF_ERROR("rc:%4d", rc);
1112                 goto no_cleanup;
1113         }
1114
1115         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_CLEANUP);
1116         spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
1117         wait_event_interruptible_timeout(cdev->private->wait_q,
1118                 irq_ptr->state == QDIO_IRQ_STATE_INACTIVE ||
1119                 irq_ptr->state == QDIO_IRQ_STATE_ERR,
1120                 10 * HZ);
1121         spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
1122
1123 no_cleanup:
1124         qdio_shutdown_thinint(irq_ptr);
1125
1126         /* restore interrupt handler */
1127         if ((void *)cdev->handler == (void *)qdio_int_handler)
1128                 cdev->handler = irq_ptr->orig_handler;
1129         spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
1130
1131         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1132         mutex_unlock(&irq_ptr->setup_mutex);
1133         if (rc)
1134                 return rc;
1135         return 0;
1136 }
1137 EXPORT_SYMBOL_GPL(qdio_shutdown);
1138
1139 /**
1140  * qdio_free - free data structures for a qdio subchannel
1141  * @cdev: associated ccw device
1142  */
1143 int qdio_free(struct ccw_device *cdev)
1144 {
1145         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1146
1147         if (!irq_ptr)
1148                 return -ENODEV;
1149
1150         DBF_EVENT("qfree:%4x", cdev->private->schid.sch_no);
1151         mutex_lock(&irq_ptr->setup_mutex);
1152
1153         if (irq_ptr->debug_area != NULL) {
1154                 debug_unregister(irq_ptr->debug_area);
1155                 irq_ptr->debug_area = NULL;
1156         }
1157         cdev->private->qdio_data = NULL;
1158         mutex_unlock(&irq_ptr->setup_mutex);
1159
1160         qdio_release_memory(irq_ptr);
1161         return 0;
1162 }
1163 EXPORT_SYMBOL_GPL(qdio_free);
1164
1165 /**
1166  * qdio_allocate - allocate qdio queues and associated data
1167  * @init_data: initialization data
1168  */
1169 int qdio_allocate(struct qdio_initialize *init_data)
1170 {
1171         struct qdio_irq *irq_ptr;
1172
1173         DBF_EVENT("qallocate:%4x", init_data->cdev->private->schid.sch_no);
1174
1175         if ((init_data->no_input_qs && !init_data->input_handler) ||
1176             (init_data->no_output_qs && !init_data->output_handler))
1177                 return -EINVAL;
1178
1179         if ((init_data->no_input_qs > QDIO_MAX_QUEUES_PER_IRQ) ||
1180             (init_data->no_output_qs > QDIO_MAX_QUEUES_PER_IRQ))
1181                 return -EINVAL;
1182
1183         if ((!init_data->input_sbal_addr_array) ||
1184             (!init_data->output_sbal_addr_array))
1185                 return -EINVAL;
1186
1187         /* irq_ptr must be in GFP_DMA since it contains ccw1.cda */
1188         irq_ptr = (void *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1189         if (!irq_ptr)
1190                 goto out_err;
1191
1192         mutex_init(&irq_ptr->setup_mutex);
1193         qdio_allocate_dbf(init_data, irq_ptr);
1194
1195         /*
1196          * Allocate a page for the chsc calls in qdio_establish.
1197          * Must be pre-allocated since a zfcp recovery will call
1198          * qdio_establish. In case of low memory and swap on a zfcp disk
1199          * we may not be able to allocate memory otherwise.
1200          */
1201         irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL);
1202         if (!irq_ptr->chsc_page)
1203                 goto out_rel;
1204
1205         /* qdr is used in ccw1.cda which is u32 */
1206         irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1207         if (!irq_ptr->qdr)
1208                 goto out_rel;
1209         WARN_ON((unsigned long)irq_ptr->qdr & 0xfff);
1210
1211         if (qdio_allocate_qs(irq_ptr, init_data->no_input_qs,
1212                              init_data->no_output_qs))
1213                 goto out_rel;
1214
1215         init_data->cdev->private->qdio_data = irq_ptr;
1216         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1217         return 0;
1218 out_rel:
1219         qdio_release_memory(irq_ptr);
1220 out_err:
1221         return -ENOMEM;
1222 }
1223 EXPORT_SYMBOL_GPL(qdio_allocate);
1224
1225 /**
1226  * qdio_establish - establish queues on a qdio subchannel
1227  * @init_data: initialization data
1228  */
1229 int qdio_establish(struct qdio_initialize *init_data)
1230 {
1231         struct qdio_irq *irq_ptr;
1232         struct ccw_device *cdev = init_data->cdev;
1233         unsigned long saveflags;
1234         int rc;
1235
1236         DBF_EVENT("qestablish:%4x", cdev->private->schid.sch_no);
1237
1238         irq_ptr = cdev->private->qdio_data;
1239         if (!irq_ptr)
1240                 return -ENODEV;
1241
1242         if (cdev->private->state != DEV_STATE_ONLINE)
1243                 return -EINVAL;
1244
1245         mutex_lock(&irq_ptr->setup_mutex);
1246         qdio_setup_irq(init_data);
1247
1248         rc = qdio_establish_thinint(irq_ptr);
1249         if (rc) {
1250                 mutex_unlock(&irq_ptr->setup_mutex);
1251                 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
1252                 return rc;
1253         }
1254
1255         /* establish q */
1256         irq_ptr->ccw.cmd_code = irq_ptr->equeue.cmd;
1257         irq_ptr->ccw.flags = CCW_FLAG_SLI;
1258         irq_ptr->ccw.count = irq_ptr->equeue.count;
1259         irq_ptr->ccw.cda = (u32)((addr_t)irq_ptr->qdr);
1260
1261         spin_lock_irqsave(get_ccwdev_lock(cdev), saveflags);
1262         ccw_device_set_options_mask(cdev, 0);
1263
1264         rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ESTABLISH, 0, 0);
1265         if (rc) {
1266                 DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no);
1267                 DBF_ERROR("rc:%4x", rc);
1268         }
1269         spin_unlock_irqrestore(get_ccwdev_lock(cdev), saveflags);
1270
1271         if (rc) {
1272                 mutex_unlock(&irq_ptr->setup_mutex);
1273                 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
1274                 return rc;
1275         }
1276
1277         wait_event_interruptible_timeout(cdev->private->wait_q,
1278                 irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED ||
1279                 irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ);
1280
1281         if (irq_ptr->state != QDIO_IRQ_STATE_ESTABLISHED) {
1282                 mutex_unlock(&irq_ptr->setup_mutex);
1283                 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
1284                 return -EIO;
1285         }
1286
1287         qdio_setup_ssqd_info(irq_ptr);
1288         DBF_EVENT("qib ac:%4x", irq_ptr->qib.ac);
1289
1290         /* qebsm is now setup if available, initialize buffer states */
1291         qdio_init_buf_states(irq_ptr);
1292
1293         mutex_unlock(&irq_ptr->setup_mutex);
1294         qdio_print_subchannel_info(irq_ptr, cdev);
1295         qdio_setup_debug_entries(irq_ptr, cdev);
1296         return 0;
1297 }
1298 EXPORT_SYMBOL_GPL(qdio_establish);
1299
1300 /**
1301  * qdio_activate - activate queues on a qdio subchannel
1302  * @cdev: associated cdev
1303  */
1304 int qdio_activate(struct ccw_device *cdev)
1305 {
1306         struct qdio_irq *irq_ptr;
1307         int rc;
1308         unsigned long saveflags;
1309
1310         DBF_EVENT("qactivate:%4x", cdev->private->schid.sch_no);
1311
1312         irq_ptr = cdev->private->qdio_data;
1313         if (!irq_ptr)
1314                 return -ENODEV;
1315
1316         if (cdev->private->state != DEV_STATE_ONLINE)
1317                 return -EINVAL;
1318
1319         mutex_lock(&irq_ptr->setup_mutex);
1320         if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {
1321                 rc = -EBUSY;
1322                 goto out;
1323         }
1324
1325         irq_ptr->ccw.cmd_code = irq_ptr->aqueue.cmd;
1326         irq_ptr->ccw.flags = CCW_FLAG_SLI;
1327         irq_ptr->ccw.count = irq_ptr->aqueue.count;
1328         irq_ptr->ccw.cda = 0;
1329
1330         spin_lock_irqsave(get_ccwdev_lock(cdev), saveflags);
1331         ccw_device_set_options(cdev, CCWDEV_REPORT_ALL);
1332
1333         rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ACTIVATE,
1334                               0, DOIO_DENY_PREFETCH);
1335         if (rc) {
1336                 DBF_ERROR("%4x act IO ERR", irq_ptr->schid.sch_no);
1337                 DBF_ERROR("rc:%4x", rc);
1338         }
1339         spin_unlock_irqrestore(get_ccwdev_lock(cdev), saveflags);
1340
1341         if (rc)
1342                 goto out;
1343
1344         if (is_thinint_irq(irq_ptr))
1345                 tiqdio_add_input_queues(irq_ptr);
1346
1347         /* wait for subchannel to become active */
1348         msleep(5);
1349
1350         switch (irq_ptr->state) {
1351         case QDIO_IRQ_STATE_STOPPED:
1352         case QDIO_IRQ_STATE_ERR:
1353                 rc = -EIO;
1354                 break;
1355         default:
1356                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ACTIVE);
1357                 rc = 0;
1358         }
1359 out:
1360         mutex_unlock(&irq_ptr->setup_mutex);
1361         return rc;
1362 }
1363 EXPORT_SYMBOL_GPL(qdio_activate);
1364
1365 static inline int buf_in_between(int bufnr, int start, int count)
1366 {
1367         int end = add_buf(start, count);
1368
1369         if (end > start) {
1370                 if (bufnr >= start && bufnr < end)
1371                         return 1;
1372                 else
1373                         return 0;
1374         }
1375
1376         /* wrap-around case */
1377         if ((bufnr >= start && bufnr <= QDIO_MAX_BUFFERS_PER_Q) ||
1378             (bufnr < end))
1379                 return 1;
1380         else
1381                 return 0;
1382 }
1383
1384 /**
1385  * handle_inbound - reset processed input buffers
1386  * @q: queue containing the buffers
1387  * @callflags: flags
1388  * @bufnr: first buffer to process
1389  * @count: how many buffers are emptied
1390  */
1391 static int handle_inbound(struct qdio_q *q, unsigned int callflags,
1392                           int bufnr, int count)
1393 {
1394         int used, diff;
1395
1396         qperf_inc(q, inbound_call);
1397
1398         if (!q->u.in.polling)
1399                 goto set;
1400
1401         /* protect against stop polling setting an ACK for an emptied slsb */
1402         if (count == QDIO_MAX_BUFFERS_PER_Q) {
1403                 /* overwriting everything, just delete polling status */
1404                 q->u.in.polling = 0;
1405                 q->u.in.ack_count = 0;
1406                 goto set;
1407         } else if (buf_in_between(q->u.in.ack_start, bufnr, count)) {
1408                 if (is_qebsm(q)) {
1409                         /* partial overwrite, just update ack_start */
1410                         diff = add_buf(bufnr, count);
1411                         diff = sub_buf(diff, q->u.in.ack_start);
1412                         q->u.in.ack_count -= diff;
1413                         if (q->u.in.ack_count <= 0) {
1414                                 q->u.in.polling = 0;
1415                                 q->u.in.ack_count = 0;
1416                                 goto set;
1417                         }
1418                         q->u.in.ack_start = add_buf(q->u.in.ack_start, diff);
1419                 }
1420                 else
1421                         /* the only ACK will be deleted, so stop polling */
1422                         q->u.in.polling = 0;
1423         }
1424
1425 set:
1426         count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count);
1427
1428         used = atomic_add_return(count, &q->nr_buf_used) - count;
1429         BUG_ON(used + count > QDIO_MAX_BUFFERS_PER_Q);
1430
1431         /* no need to signal as long as the adapter had free buffers */
1432         if (used)
1433                 return 0;
1434
1435         if (need_siga_in(q))
1436                 return qdio_siga_input(q);
1437         return 0;
1438 }
1439
1440 /**
1441  * handle_outbound - process filled outbound buffers
1442  * @q: queue containing the buffers
1443  * @callflags: flags
1444  * @bufnr: first buffer to process
1445  * @count: how many buffers are filled
1446  */
1447 static int handle_outbound(struct qdio_q *q, unsigned int callflags,
1448                            int bufnr, int count)
1449 {
1450         unsigned char state;
1451         int used, rc = 0;
1452
1453         qperf_inc(q, outbound_call);
1454
1455         count = set_buf_states(q, bufnr, SLSB_CU_OUTPUT_PRIMED, count);
1456         used = atomic_add_return(count, &q->nr_buf_used);
1457         BUG_ON(used > QDIO_MAX_BUFFERS_PER_Q);
1458
1459         if (used == QDIO_MAX_BUFFERS_PER_Q)
1460                 qperf_inc(q, outbound_queue_full);
1461
1462         if (callflags & QDIO_FLAG_PCI_OUT) {
1463                 q->u.out.pci_out_enabled = 1;
1464                 qperf_inc(q, pci_request_int);
1465         } else
1466                 q->u.out.pci_out_enabled = 0;
1467
1468         if (queue_type(q) == QDIO_IQDIO_QFMT) {
1469                 /* One SIGA-W per buffer required for unicast HiperSockets. */
1470                 WARN_ON_ONCE(count > 1 && !multicast_outbound(q));
1471
1472                 rc = qdio_kick_outbound_q(q);
1473         } else if (unlikely(need_siga_sync(q))) {
1474                 rc = qdio_siga_sync_q(q);
1475         } else {
1476                 /* try to fast requeue buffers */
1477                 get_buf_state(q, prev_buf(bufnr), &state, 0);
1478                 if (state != SLSB_CU_OUTPUT_PRIMED)
1479                         rc = qdio_kick_outbound_q(q);
1480                 else
1481                         qperf_inc(q, fast_requeue);
1482         }
1483
1484         /* in case of SIGA errors we must process the error immediately */
1485         if (used >= q->u.out.scan_threshold || rc)
1486                 tasklet_schedule(&q->tasklet);
1487         else
1488                 /* free the SBALs in case of no further traffic */
1489                 if (!timer_pending(&q->u.out.timer))
1490                         mod_timer(&q->u.out.timer, jiffies + HZ);
1491         return rc;
1492 }
1493
1494 /**
1495  * do_QDIO - process input or output buffers
1496  * @cdev: associated ccw_device for the qdio subchannel
1497  * @callflags: input or output and special flags from the program
1498  * @q_nr: queue number
1499  * @bufnr: buffer number
1500  * @count: how many buffers to process
1501  */
1502 int do_QDIO(struct ccw_device *cdev, unsigned int callflags,
1503             int q_nr, unsigned int bufnr, unsigned int count)
1504 {
1505         struct qdio_irq *irq_ptr;
1506
1507         if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q)
1508                 return -EINVAL;
1509
1510         irq_ptr = cdev->private->qdio_data;
1511         if (!irq_ptr)
1512                 return -ENODEV;
1513
1514         DBF_DEV_EVENT(DBF_INFO, irq_ptr,
1515                       "do%02x b:%02x c:%02x", callflags, bufnr, count);
1516
1517         if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)
1518                 return -EBUSY;
1519
1520         if (callflags & QDIO_FLAG_SYNC_INPUT)
1521                 return handle_inbound(irq_ptr->input_qs[q_nr],
1522                                       callflags, bufnr, count);
1523         else if (callflags & QDIO_FLAG_SYNC_OUTPUT)
1524                 return handle_outbound(irq_ptr->output_qs[q_nr],
1525                                        callflags, bufnr, count);
1526         return -EINVAL;
1527 }
1528 EXPORT_SYMBOL_GPL(do_QDIO);
1529
1530 /**
1531  * qdio_start_irq - process input buffers
1532  * @cdev: associated ccw_device for the qdio subchannel
1533  * @nr: input queue number
1534  *
1535  * Return codes
1536  *   0 - success
1537  *   1 - irqs not started since new data is available
1538  */
1539 int qdio_start_irq(struct ccw_device *cdev, int nr)
1540 {
1541         struct qdio_q *q;
1542         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1543
1544         if (!irq_ptr)
1545                 return -ENODEV;
1546         q = irq_ptr->input_qs[nr];
1547
1548         WARN_ON(queue_irqs_enabled(q));
1549
1550         if (!shared_ind(q->irq_ptr->dsci))
1551                 xchg(q->irq_ptr->dsci, 0);
1552
1553         qdio_stop_polling(q);
1554         clear_bit(QDIO_QUEUE_IRQS_DISABLED, &q->u.in.queue_irq_state);
1555
1556         /*
1557          * We need to check again to not lose initiative after
1558          * resetting the ACK state.
1559          */
1560         if (!shared_ind(q->irq_ptr->dsci) && *q->irq_ptr->dsci)
1561                 goto rescan;
1562         if (!qdio_inbound_q_done(q))
1563                 goto rescan;
1564         return 0;
1565
1566 rescan:
1567         if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
1568                              &q->u.in.queue_irq_state))
1569                 return 0;
1570         else
1571                 return 1;
1572
1573 }
1574 EXPORT_SYMBOL(qdio_start_irq);
1575
1576 /**
1577  * qdio_get_next_buffers - process input buffers
1578  * @cdev: associated ccw_device for the qdio subchannel
1579  * @nr: input queue number
1580  * @bufnr: first filled buffer number
1581  * @error: buffers are in error state
1582  *
1583  * Return codes
1584  *   < 0 - error
1585  *   = 0 - no new buffers found
1586  *   > 0 - number of processed buffers
1587  */
1588 int qdio_get_next_buffers(struct ccw_device *cdev, int nr, int *bufnr,
1589                           int *error)
1590 {
1591         struct qdio_q *q;
1592         int start, end;
1593         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1594
1595         if (!irq_ptr)
1596                 return -ENODEV;
1597         q = irq_ptr->input_qs[nr];
1598         WARN_ON(queue_irqs_enabled(q));
1599
1600         qdio_sync_after_thinint(q);
1601
1602         /*
1603          * The interrupt could be caused by a PCI request. Check the
1604          * PCI capable outbound queues.
1605          */
1606         qdio_check_outbound_after_thinint(q);
1607
1608         if (!qdio_inbound_q_moved(q))
1609                 return 0;
1610
1611         /* Note: upper-layer MUST stop processing immediately here ... */
1612         if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
1613                 return -EIO;
1614
1615         start = q->first_to_kick;
1616         end = q->first_to_check;
1617         *bufnr = start;
1618         *error = q->qdio_error;
1619
1620         /* for the next time */
1621         q->first_to_kick = end;
1622         q->qdio_error = 0;
1623         return sub_buf(end, start);
1624 }
1625 EXPORT_SYMBOL(qdio_get_next_buffers);
1626
1627 /**
1628  * qdio_stop_irq - disable interrupt processing for the device
1629  * @cdev: associated ccw_device for the qdio subchannel
1630  * @nr: input queue number
1631  *
1632  * Return codes
1633  *   0 - interrupts were already disabled
1634  *   1 - interrupts successfully disabled
1635  */
1636 int qdio_stop_irq(struct ccw_device *cdev, int nr)
1637 {
1638         struct qdio_q *q;
1639         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1640
1641         if (!irq_ptr)
1642                 return -ENODEV;
1643         q = irq_ptr->input_qs[nr];
1644
1645         if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
1646                              &q->u.in.queue_irq_state))
1647                 return 0;
1648         else
1649                 return 1;
1650 }
1651 EXPORT_SYMBOL(qdio_stop_irq);
1652
1653 static int __init init_QDIO(void)
1654 {
1655         int rc;
1656
1657         rc = qdio_setup_init();
1658         if (rc)
1659                 return rc;
1660         rc = tiqdio_allocate_memory();
1661         if (rc)
1662                 goto out_cache;
1663         rc = qdio_debug_init();
1664         if (rc)
1665                 goto out_ti;
1666         rc = tiqdio_register_thinints();
1667         if (rc)
1668                 goto out_debug;
1669         return 0;
1670
1671 out_debug:
1672         qdio_debug_exit();
1673 out_ti:
1674         tiqdio_free_memory();
1675 out_cache:
1676         qdio_setup_exit();
1677         return rc;
1678 }
1679
1680 static void __exit exit_QDIO(void)
1681 {
1682         tiqdio_unregister_thinints();
1683         tiqdio_free_memory();
1684         qdio_debug_exit();
1685         qdio_setup_exit();
1686 }
1687
1688 module_init(init_QDIO);
1689 module_exit(exit_QDIO);