[S390] zcrypt: special command support for cex3 exploitation
[linux-2.6.git] / drivers / s390 / crypto / ap_bus.c
1 /*
2  * linux/drivers/s390/crypto/ap_bus.c
3  *
4  * Copyright (C) 2006 IBM Corporation
5  * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
6  *            Martin Schwidefsky <schwidefsky@de.ibm.com>
7  *            Ralph Wuerthner <rwuerthn@de.ibm.com>
8  *            Felix Beck <felix.beck@de.ibm.com>
9  *
10  * Adjunct processor bus.
11  *
12  * This program is free software; you can redistribute it and/or modify
13  * it under the terms of the GNU General Public License as published by
14  * the Free Software Foundation; either version 2, or (at your option)
15  * any later version.
16  *
17  * This program is distributed in the hope that it will be useful,
18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20  * GNU General Public License for more details.
21  *
22  * You should have received a copy of the GNU General Public License
23  * along with this program; if not, write to the Free Software
24  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
25  */
26
27 #define KMSG_COMPONENT "ap"
28 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
29
30 #include <linux/module.h>
31 #include <linux/init.h>
32 #include <linux/delay.h>
33 #include <linux/err.h>
34 #include <linux/interrupt.h>
35 #include <linux/workqueue.h>
36 #include <linux/notifier.h>
37 #include <linux/kthread.h>
38 #include <linux/mutex.h>
39 #include <asm/reset.h>
40 #include <asm/airq.h>
41 #include <asm/atomic.h>
42 #include <asm/system.h>
43 #include <asm/isc.h>
44 #include <linux/hrtimer.h>
45 #include <linux/ktime.h>
46
47 #include "ap_bus.h"
48
49 /* Some prototypes. */
50 static void ap_scan_bus(struct work_struct *);
51 static void ap_poll_all(unsigned long);
52 static enum hrtimer_restart ap_poll_timeout(struct hrtimer *);
53 static int ap_poll_thread_start(void);
54 static void ap_poll_thread_stop(void);
55 static void ap_request_timeout(unsigned long);
56 static inline void ap_schedule_poll_timer(void);
57 static int __ap_poll_device(struct ap_device *ap_dev, unsigned long *flags);
58 static int ap_device_remove(struct device *dev);
59 static int ap_device_probe(struct device *dev);
60 static void ap_interrupt_handler(void *unused1, void *unused2);
61 static void ap_reset(struct ap_device *ap_dev);
62 static void ap_config_timeout(unsigned long ptr);
63 static int ap_select_domain(void);
64
65 /*
66  * Module description.
67  */
68 MODULE_AUTHOR("IBM Corporation");
69 MODULE_DESCRIPTION("Adjunct Processor Bus driver, "
70                    "Copyright 2006 IBM Corporation");
71 MODULE_LICENSE("GPL");
72
73 /*
74  * Module parameter
75  */
76 int ap_domain_index = -1;       /* Adjunct Processor Domain Index */
77 module_param_named(domain, ap_domain_index, int, 0000);
78 MODULE_PARM_DESC(domain, "domain index for ap devices");
79 EXPORT_SYMBOL(ap_domain_index);
80
81 static int ap_thread_flag = 0;
82 module_param_named(poll_thread, ap_thread_flag, int, 0000);
83 MODULE_PARM_DESC(poll_thread, "Turn on/off poll thread, default is 0 (off).");
84
85 static struct device *ap_root_device = NULL;
86 static DEFINE_SPINLOCK(ap_device_list_lock);
87 static LIST_HEAD(ap_device_list);
88
89 /*
90  * Workqueue & timer for bus rescan.
91  */
92 static struct workqueue_struct *ap_work_queue;
93 static struct timer_list ap_config_timer;
94 static int ap_config_time = AP_CONFIG_TIME;
95 static DECLARE_WORK(ap_config_work, ap_scan_bus);
96
97 /*
98  * Tasklet & timer for AP request polling and interrupts
99  */
100 static DECLARE_TASKLET(ap_tasklet, ap_poll_all, 0);
101 static atomic_t ap_poll_requests = ATOMIC_INIT(0);
102 static DECLARE_WAIT_QUEUE_HEAD(ap_poll_wait);
103 static struct task_struct *ap_poll_kthread = NULL;
104 static DEFINE_MUTEX(ap_poll_thread_mutex);
105 static void *ap_interrupt_indicator;
106 static struct hrtimer ap_poll_timer;
107 /* In LPAR poll with 4kHz frequency. Poll every 250000 nanoseconds.
108  * If z/VM change to 1500000 nanoseconds to adjust to z/VM polling.*/
109 static unsigned long long poll_timeout = 250000;
110
111 /* Suspend flag */
112 static int ap_suspend_flag;
113 /* Flag to check if domain was set through module parameter domain=. This is
114  * important when supsend and resume is done in a z/VM environment where the
115  * domain might change. */
116 static int user_set_domain = 0;
117 static struct bus_type ap_bus_type;
118
119 /**
120  * ap_using_interrupts() - Returns non-zero if interrupt support is
121  * available.
122  */
123 static inline int ap_using_interrupts(void)
124 {
125         return ap_interrupt_indicator != NULL;
126 }
127
128 /**
129  * ap_intructions_available() - Test if AP instructions are available.
130  *
131  * Returns 0 if the AP instructions are installed.
132  */
133 static inline int ap_instructions_available(void)
134 {
135         register unsigned long reg0 asm ("0") = AP_MKQID(0,0);
136         register unsigned long reg1 asm ("1") = -ENODEV;
137         register unsigned long reg2 asm ("2") = 0UL;
138
139         asm volatile(
140                 "   .long 0xb2af0000\n"         /* PQAP(TAPQ) */
141                 "0: la    %1,0\n"
142                 "1:\n"
143                 EX_TABLE(0b, 1b)
144                 : "+d" (reg0), "+d" (reg1), "+d" (reg2) : : "cc" );
145         return reg1;
146 }
147
148 /**
149  * ap_interrupts_available(): Test if AP interrupts are available.
150  *
151  * Returns 1 if AP interrupts are available.
152  */
153 static int ap_interrupts_available(void)
154 {
155         unsigned long long facility_bits[2];
156
157         if (stfle(facility_bits, 2) <= 1)
158                 return 0;
159         if (!(facility_bits[0] & (1ULL << 61)) ||
160             !(facility_bits[1] & (1ULL << 62)))
161                 return 0;
162         return 1;
163 }
164
165 /**
166  * ap_test_queue(): Test adjunct processor queue.
167  * @qid: The AP queue number
168  * @queue_depth: Pointer to queue depth value
169  * @device_type: Pointer to device type value
170  *
171  * Returns AP queue status structure.
172  */
173 static inline struct ap_queue_status
174 ap_test_queue(ap_qid_t qid, int *queue_depth, int *device_type)
175 {
176         register unsigned long reg0 asm ("0") = qid;
177         register struct ap_queue_status reg1 asm ("1");
178         register unsigned long reg2 asm ("2") = 0UL;
179
180         asm volatile(".long 0xb2af0000"         /* PQAP(TAPQ) */
181                      : "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
182         *device_type = (int) (reg2 >> 24);
183         *queue_depth = (int) (reg2 & 0xff);
184         return reg1;
185 }
186
187 /**
188  * ap_reset_queue(): Reset adjunct processor queue.
189  * @qid: The AP queue number
190  *
191  * Returns AP queue status structure.
192  */
193 static inline struct ap_queue_status ap_reset_queue(ap_qid_t qid)
194 {
195         register unsigned long reg0 asm ("0") = qid | 0x01000000UL;
196         register struct ap_queue_status reg1 asm ("1");
197         register unsigned long reg2 asm ("2") = 0UL;
198
199         asm volatile(
200                 ".long 0xb2af0000"              /* PQAP(RAPQ) */
201                 : "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
202         return reg1;
203 }
204
205 #ifdef CONFIG_64BIT
206 /**
207  * ap_queue_interruption_control(): Enable interruption for a specific AP.
208  * @qid: The AP queue number
209  * @ind: The notification indicator byte
210  *
211  * Returns AP queue status.
212  */
213 static inline struct ap_queue_status
214 ap_queue_interruption_control(ap_qid_t qid, void *ind)
215 {
216         register unsigned long reg0 asm ("0") = qid | 0x03000000UL;
217         register unsigned long reg1_in asm ("1") = 0x0000800000000000UL | AP_ISC;
218         register struct ap_queue_status reg1_out asm ("1");
219         register void *reg2 asm ("2") = ind;
220         asm volatile(
221                 ".long 0xb2af0000"              /* PQAP(RAPQ) */
222                 : "+d" (reg0), "+d" (reg1_in), "=d" (reg1_out), "+d" (reg2)
223                 :
224                 : "cc" );
225         return reg1_out;
226 }
227 #endif
228
229 /**
230  * ap_queue_enable_interruption(): Enable interruption on an AP.
231  * @qid: The AP queue number
232  * @ind: the notification indicator byte
233  *
234  * Enables interruption on AP queue via ap_queue_interruption_control(). Based
235  * on the return value it waits a while and tests the AP queue if interrupts
236  * have been switched on using ap_test_queue().
237  */
238 static int ap_queue_enable_interruption(ap_qid_t qid, void *ind)
239 {
240 #ifdef CONFIG_64BIT
241         struct ap_queue_status status;
242         int t_depth, t_device_type, rc, i;
243
244         rc = -EBUSY;
245         status = ap_queue_interruption_control(qid, ind);
246
247         for (i = 0; i < AP_MAX_RESET; i++) {
248                 switch (status.response_code) {
249                 case AP_RESPONSE_NORMAL:
250                         if (status.int_enabled)
251                                 return 0;
252                         break;
253                 case AP_RESPONSE_RESET_IN_PROGRESS:
254                 case AP_RESPONSE_BUSY:
255                         break;
256                 case AP_RESPONSE_Q_NOT_AVAIL:
257                 case AP_RESPONSE_DECONFIGURED:
258                 case AP_RESPONSE_CHECKSTOPPED:
259                 case AP_RESPONSE_INVALID_ADDRESS:
260                         return -ENODEV;
261                 case AP_RESPONSE_OTHERWISE_CHANGED:
262                         if (status.int_enabled)
263                                 return 0;
264                         break;
265                 default:
266                         break;
267                 }
268                 if (i < AP_MAX_RESET - 1) {
269                         udelay(5);
270                         status = ap_test_queue(qid, &t_depth, &t_device_type);
271                 }
272         }
273         return rc;
274 #else
275         return -EINVAL;
276 #endif
277 }
278
279 /**
280  * __ap_send(): Send message to adjunct processor queue.
281  * @qid: The AP queue number
282  * @psmid: The program supplied message identifier
283  * @msg: The message text
284  * @length: The message length
285  * @special: Special Bit
286  *
287  * Returns AP queue status structure.
288  * Condition code 1 on NQAP can't happen because the L bit is 1.
289  * Condition code 2 on NQAP also means the send is incomplete,
290  * because a segment boundary was reached. The NQAP is repeated.
291  */
292 static inline struct ap_queue_status
293 __ap_send(ap_qid_t qid, unsigned long long psmid, void *msg, size_t length,
294           unsigned int special)
295 {
296         typedef struct { char _[length]; } msgblock;
297         register unsigned long reg0 asm ("0") = qid | 0x40000000UL;
298         register struct ap_queue_status reg1 asm ("1");
299         register unsigned long reg2 asm ("2") = (unsigned long) msg;
300         register unsigned long reg3 asm ("3") = (unsigned long) length;
301         register unsigned long reg4 asm ("4") = (unsigned int) (psmid >> 32);
302         register unsigned long reg5 asm ("5") = (unsigned int) psmid;
303
304         if (special == 1)
305                 reg0 |= 0x400000UL;
306
307         asm volatile (
308                 "0: .long 0xb2ad0042\n"         /* DQAP */
309                 "   brc   2,0b"
310                 : "+d" (reg0), "=d" (reg1), "+d" (reg2), "+d" (reg3)
311                 : "d" (reg4), "d" (reg5), "m" (*(msgblock *) msg)
312                 : "cc" );
313         return reg1;
314 }
315
316 int ap_send(ap_qid_t qid, unsigned long long psmid, void *msg, size_t length)
317 {
318         struct ap_queue_status status;
319
320         status = __ap_send(qid, psmid, msg, length, 0);
321         switch (status.response_code) {
322         case AP_RESPONSE_NORMAL:
323                 return 0;
324         case AP_RESPONSE_Q_FULL:
325         case AP_RESPONSE_RESET_IN_PROGRESS:
326                 return -EBUSY;
327         case AP_RESPONSE_REQ_FAC_NOT_INST:
328                 return -EINVAL;
329         default:        /* Device is gone. */
330                 return -ENODEV;
331         }
332 }
333 EXPORT_SYMBOL(ap_send);
334
335 /**
336  * __ap_recv(): Receive message from adjunct processor queue.
337  * @qid: The AP queue number
338  * @psmid: Pointer to program supplied message identifier
339  * @msg: The message text
340  * @length: The message length
341  *
342  * Returns AP queue status structure.
343  * Condition code 1 on DQAP means the receive has taken place
344  * but only partially.  The response is incomplete, hence the
345  * DQAP is repeated.
346  * Condition code 2 on DQAP also means the receive is incomplete,
347  * this time because a segment boundary was reached. Again, the
348  * DQAP is repeated.
349  * Note that gpr2 is used by the DQAP instruction to keep track of
350  * any 'residual' length, in case the instruction gets interrupted.
351  * Hence it gets zeroed before the instruction.
352  */
353 static inline struct ap_queue_status
354 __ap_recv(ap_qid_t qid, unsigned long long *psmid, void *msg, size_t length)
355 {
356         typedef struct { char _[length]; } msgblock;
357         register unsigned long reg0 asm("0") = qid | 0x80000000UL;
358         register struct ap_queue_status reg1 asm ("1");
359         register unsigned long reg2 asm("2") = 0UL;
360         register unsigned long reg4 asm("4") = (unsigned long) msg;
361         register unsigned long reg5 asm("5") = (unsigned long) length;
362         register unsigned long reg6 asm("6") = 0UL;
363         register unsigned long reg7 asm("7") = 0UL;
364
365
366         asm volatile(
367                 "0: .long 0xb2ae0064\n"
368                 "   brc   6,0b\n"
369                 : "+d" (reg0), "=d" (reg1), "+d" (reg2),
370                 "+d" (reg4), "+d" (reg5), "+d" (reg6), "+d" (reg7),
371                 "=m" (*(msgblock *) msg) : : "cc" );
372         *psmid = (((unsigned long long) reg6) << 32) + reg7;
373         return reg1;
374 }
375
376 int ap_recv(ap_qid_t qid, unsigned long long *psmid, void *msg, size_t length)
377 {
378         struct ap_queue_status status;
379
380         status = __ap_recv(qid, psmid, msg, length);
381         switch (status.response_code) {
382         case AP_RESPONSE_NORMAL:
383                 return 0;
384         case AP_RESPONSE_NO_PENDING_REPLY:
385                 if (status.queue_empty)
386                         return -ENOENT;
387                 return -EBUSY;
388         case AP_RESPONSE_RESET_IN_PROGRESS:
389                 return -EBUSY;
390         default:
391                 return -ENODEV;
392         }
393 }
394 EXPORT_SYMBOL(ap_recv);
395
396 /**
397  * ap_query_queue(): Check if an AP queue is available.
398  * @qid: The AP queue number
399  * @queue_depth: Pointer to queue depth value
400  * @device_type: Pointer to device type value
401  *
402  * The test is repeated for AP_MAX_RESET times.
403  */
404 static int ap_query_queue(ap_qid_t qid, int *queue_depth, int *device_type)
405 {
406         struct ap_queue_status status;
407         int t_depth, t_device_type, rc, i;
408
409         rc = -EBUSY;
410         for (i = 0; i < AP_MAX_RESET; i++) {
411                 status = ap_test_queue(qid, &t_depth, &t_device_type);
412                 switch (status.response_code) {
413                 case AP_RESPONSE_NORMAL:
414                         *queue_depth = t_depth + 1;
415                         *device_type = t_device_type;
416                         rc = 0;
417                         break;
418                 case AP_RESPONSE_Q_NOT_AVAIL:
419                         rc = -ENODEV;
420                         break;
421                 case AP_RESPONSE_RESET_IN_PROGRESS:
422                         break;
423                 case AP_RESPONSE_DECONFIGURED:
424                         rc = -ENODEV;
425                         break;
426                 case AP_RESPONSE_CHECKSTOPPED:
427                         rc = -ENODEV;
428                         break;
429                 case AP_RESPONSE_INVALID_ADDRESS:
430                         rc = -ENODEV;
431                         break;
432                 case AP_RESPONSE_OTHERWISE_CHANGED:
433                         break;
434                 case AP_RESPONSE_BUSY:
435                         break;
436                 default:
437                         BUG();
438                 }
439                 if (rc != -EBUSY)
440                         break;
441                 if (i < AP_MAX_RESET - 1)
442                         udelay(5);
443         }
444         return rc;
445 }
446
447 /**
448  * ap_init_queue(): Reset an AP queue.
449  * @qid: The AP queue number
450  *
451  * Reset an AP queue and wait for it to become available again.
452  */
453 static int ap_init_queue(ap_qid_t qid)
454 {
455         struct ap_queue_status status;
456         int rc, dummy, i;
457
458         rc = -ENODEV;
459         status = ap_reset_queue(qid);
460         for (i = 0; i < AP_MAX_RESET; i++) {
461                 switch (status.response_code) {
462                 case AP_RESPONSE_NORMAL:
463                         if (status.queue_empty)
464                                 rc = 0;
465                         break;
466                 case AP_RESPONSE_Q_NOT_AVAIL:
467                 case AP_RESPONSE_DECONFIGURED:
468                 case AP_RESPONSE_CHECKSTOPPED:
469                         i = AP_MAX_RESET;       /* return with -ENODEV */
470                         break;
471                 case AP_RESPONSE_RESET_IN_PROGRESS:
472                         rc = -EBUSY;
473                 case AP_RESPONSE_BUSY:
474                 default:
475                         break;
476                 }
477                 if (rc != -ENODEV && rc != -EBUSY)
478                         break;
479                 if (i < AP_MAX_RESET - 1) {
480                         udelay(5);
481                         status = ap_test_queue(qid, &dummy, &dummy);
482                 }
483         }
484         if (rc == 0 && ap_using_interrupts()) {
485                 rc = ap_queue_enable_interruption(qid, ap_interrupt_indicator);
486                 /* If interruption mode is supported by the machine,
487                 * but an AP can not be enabled for interruption then
488                 * the AP will be discarded.    */
489                 if (rc)
490                         pr_err("Registering adapter interrupts for "
491                                "AP %d failed\n", AP_QID_DEVICE(qid));
492         }
493         return rc;
494 }
495
496 /**
497  * ap_increase_queue_count(): Arm request timeout.
498  * @ap_dev: Pointer to an AP device.
499  *
500  * Arm request timeout if an AP device was idle and a new request is submitted.
501  */
502 static void ap_increase_queue_count(struct ap_device *ap_dev)
503 {
504         int timeout = ap_dev->drv->request_timeout;
505
506         ap_dev->queue_count++;
507         if (ap_dev->queue_count == 1) {
508                 mod_timer(&ap_dev->timeout, jiffies + timeout);
509                 ap_dev->reset = AP_RESET_ARMED;
510         }
511 }
512
513 /**
514  * ap_decrease_queue_count(): Decrease queue count.
515  * @ap_dev: Pointer to an AP device.
516  *
517  * If AP device is still alive, re-schedule request timeout if there are still
518  * pending requests.
519  */
520 static void ap_decrease_queue_count(struct ap_device *ap_dev)
521 {
522         int timeout = ap_dev->drv->request_timeout;
523
524         ap_dev->queue_count--;
525         if (ap_dev->queue_count > 0)
526                 mod_timer(&ap_dev->timeout, jiffies + timeout);
527         else
528                 /*
529                  * The timeout timer should to be disabled now - since
530                  * del_timer_sync() is very expensive, we just tell via the
531                  * reset flag to ignore the pending timeout timer.
532                  */
533                 ap_dev->reset = AP_RESET_IGNORE;
534 }
535
536 /*
537  * AP device related attributes.
538  */
539 static ssize_t ap_hwtype_show(struct device *dev,
540                               struct device_attribute *attr, char *buf)
541 {
542         struct ap_device *ap_dev = to_ap_dev(dev);
543         return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->device_type);
544 }
545
546 static DEVICE_ATTR(hwtype, 0444, ap_hwtype_show, NULL);
547 static ssize_t ap_depth_show(struct device *dev, struct device_attribute *attr,
548                              char *buf)
549 {
550         struct ap_device *ap_dev = to_ap_dev(dev);
551         return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->queue_depth);
552 }
553
554 static DEVICE_ATTR(depth, 0444, ap_depth_show, NULL);
555 static ssize_t ap_request_count_show(struct device *dev,
556                                      struct device_attribute *attr,
557                                      char *buf)
558 {
559         struct ap_device *ap_dev = to_ap_dev(dev);
560         int rc;
561
562         spin_lock_bh(&ap_dev->lock);
563         rc = snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->total_request_count);
564         spin_unlock_bh(&ap_dev->lock);
565         return rc;
566 }
567
568 static DEVICE_ATTR(request_count, 0444, ap_request_count_show, NULL);
569
570 static ssize_t ap_modalias_show(struct device *dev,
571                                 struct device_attribute *attr, char *buf)
572 {
573         return sprintf(buf, "ap:t%02X", to_ap_dev(dev)->device_type);
574 }
575
576 static DEVICE_ATTR(modalias, 0444, ap_modalias_show, NULL);
577
578 static struct attribute *ap_dev_attrs[] = {
579         &dev_attr_hwtype.attr,
580         &dev_attr_depth.attr,
581         &dev_attr_request_count.attr,
582         &dev_attr_modalias.attr,
583         NULL
584 };
585 static struct attribute_group ap_dev_attr_group = {
586         .attrs = ap_dev_attrs
587 };
588
589 /**
590  * ap_bus_match()
591  * @dev: Pointer to device
592  * @drv: Pointer to device_driver
593  *
594  * AP bus driver registration/unregistration.
595  */
596 static int ap_bus_match(struct device *dev, struct device_driver *drv)
597 {
598         struct ap_device *ap_dev = to_ap_dev(dev);
599         struct ap_driver *ap_drv = to_ap_drv(drv);
600         struct ap_device_id *id;
601
602         /*
603          * Compare device type of the device with the list of
604          * supported types of the device_driver.
605          */
606         for (id = ap_drv->ids; id->match_flags; id++) {
607                 if ((id->match_flags & AP_DEVICE_ID_MATCH_DEVICE_TYPE) &&
608                     (id->dev_type != ap_dev->device_type))
609                         continue;
610                 return 1;
611         }
612         return 0;
613 }
614
615 /**
616  * ap_uevent(): Uevent function for AP devices.
617  * @dev: Pointer to device
618  * @env: Pointer to kobj_uevent_env
619  *
620  * It sets up a single environment variable DEV_TYPE which contains the
621  * hardware device type.
622  */
623 static int ap_uevent (struct device *dev, struct kobj_uevent_env *env)
624 {
625         struct ap_device *ap_dev = to_ap_dev(dev);
626         int retval = 0;
627
628         if (!ap_dev)
629                 return -ENODEV;
630
631         /* Set up DEV_TYPE environment variable. */
632         retval = add_uevent_var(env, "DEV_TYPE=%04X", ap_dev->device_type);
633         if (retval)
634                 return retval;
635
636         /* Add MODALIAS= */
637         retval = add_uevent_var(env, "MODALIAS=ap:t%02X", ap_dev->device_type);
638
639         return retval;
640 }
641
642 static int ap_bus_suspend(struct device *dev, pm_message_t state)
643 {
644         struct ap_device *ap_dev = to_ap_dev(dev);
645         unsigned long flags;
646
647         if (!ap_suspend_flag) {
648                 ap_suspend_flag = 1;
649
650                 /* Disable scanning for devices, thus we do not want to scan
651                  * for them after removing.
652                  */
653                 del_timer_sync(&ap_config_timer);
654                 if (ap_work_queue != NULL) {
655                         destroy_workqueue(ap_work_queue);
656                         ap_work_queue = NULL;
657                 }
658
659                 tasklet_disable(&ap_tasklet);
660         }
661         /* Poll on the device until all requests are finished. */
662         do {
663                 flags = 0;
664                 spin_lock_bh(&ap_dev->lock);
665                 __ap_poll_device(ap_dev, &flags);
666                 spin_unlock_bh(&ap_dev->lock);
667         } while ((flags & 1) || (flags & 2));
668
669         spin_lock_bh(&ap_dev->lock);
670         ap_dev->unregistered = 1;
671         spin_unlock_bh(&ap_dev->lock);
672
673         return 0;
674 }
675
676 static int ap_bus_resume(struct device *dev)
677 {
678         int rc = 0;
679         struct ap_device *ap_dev = to_ap_dev(dev);
680
681         if (ap_suspend_flag) {
682                 ap_suspend_flag = 0;
683                 if (!ap_interrupts_available())
684                         ap_interrupt_indicator = NULL;
685                 if (!user_set_domain) {
686                         ap_domain_index = -1;
687                         ap_select_domain();
688                 }
689                 init_timer(&ap_config_timer);
690                 ap_config_timer.function = ap_config_timeout;
691                 ap_config_timer.data = 0;
692                 ap_config_timer.expires = jiffies + ap_config_time * HZ;
693                 add_timer(&ap_config_timer);
694                 ap_work_queue = create_singlethread_workqueue("kapwork");
695                 if (!ap_work_queue)
696                         return -ENOMEM;
697                 tasklet_enable(&ap_tasklet);
698                 if (!ap_using_interrupts())
699                         ap_schedule_poll_timer();
700                 else
701                         tasklet_schedule(&ap_tasklet);
702                 if (ap_thread_flag)
703                         rc = ap_poll_thread_start();
704         }
705         if (AP_QID_QUEUE(ap_dev->qid) != ap_domain_index) {
706                 spin_lock_bh(&ap_dev->lock);
707                 ap_dev->qid = AP_MKQID(AP_QID_DEVICE(ap_dev->qid),
708                                        ap_domain_index);
709                 spin_unlock_bh(&ap_dev->lock);
710         }
711         queue_work(ap_work_queue, &ap_config_work);
712
713         return rc;
714 }
715
716 static struct bus_type ap_bus_type = {
717         .name = "ap",
718         .match = &ap_bus_match,
719         .uevent = &ap_uevent,
720         .suspend = ap_bus_suspend,
721         .resume = ap_bus_resume
722 };
723
724 static int ap_device_probe(struct device *dev)
725 {
726         struct ap_device *ap_dev = to_ap_dev(dev);
727         struct ap_driver *ap_drv = to_ap_drv(dev->driver);
728         int rc;
729
730         ap_dev->drv = ap_drv;
731         rc = ap_drv->probe ? ap_drv->probe(ap_dev) : -ENODEV;
732         if (!rc) {
733                 spin_lock_bh(&ap_device_list_lock);
734                 list_add(&ap_dev->list, &ap_device_list);
735                 spin_unlock_bh(&ap_device_list_lock);
736         }
737         return rc;
738 }
739
740 /**
741  * __ap_flush_queue(): Flush requests.
742  * @ap_dev: Pointer to the AP device
743  *
744  * Flush all requests from the request/pending queue of an AP device.
745  */
746 static void __ap_flush_queue(struct ap_device *ap_dev)
747 {
748         struct ap_message *ap_msg, *next;
749
750         list_for_each_entry_safe(ap_msg, next, &ap_dev->pendingq, list) {
751                 list_del_init(&ap_msg->list);
752                 ap_dev->pendingq_count--;
753                 ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
754         }
755         list_for_each_entry_safe(ap_msg, next, &ap_dev->requestq, list) {
756                 list_del_init(&ap_msg->list);
757                 ap_dev->requestq_count--;
758                 ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
759         }
760 }
761
762 void ap_flush_queue(struct ap_device *ap_dev)
763 {
764         spin_lock_bh(&ap_dev->lock);
765         __ap_flush_queue(ap_dev);
766         spin_unlock_bh(&ap_dev->lock);
767 }
768 EXPORT_SYMBOL(ap_flush_queue);
769
770 static int ap_device_remove(struct device *dev)
771 {
772         struct ap_device *ap_dev = to_ap_dev(dev);
773         struct ap_driver *ap_drv = ap_dev->drv;
774
775         ap_flush_queue(ap_dev);
776         del_timer_sync(&ap_dev->timeout);
777         spin_lock_bh(&ap_device_list_lock);
778         list_del_init(&ap_dev->list);
779         spin_unlock_bh(&ap_device_list_lock);
780         if (ap_drv->remove)
781                 ap_drv->remove(ap_dev);
782         spin_lock_bh(&ap_dev->lock);
783         atomic_sub(ap_dev->queue_count, &ap_poll_requests);
784         spin_unlock_bh(&ap_dev->lock);
785         return 0;
786 }
787
788 int ap_driver_register(struct ap_driver *ap_drv, struct module *owner,
789                        char *name)
790 {
791         struct device_driver *drv = &ap_drv->driver;
792
793         drv->bus = &ap_bus_type;
794         drv->probe = ap_device_probe;
795         drv->remove = ap_device_remove;
796         drv->owner = owner;
797         drv->name = name;
798         return driver_register(drv);
799 }
800 EXPORT_SYMBOL(ap_driver_register);
801
802 void ap_driver_unregister(struct ap_driver *ap_drv)
803 {
804         driver_unregister(&ap_drv->driver);
805 }
806 EXPORT_SYMBOL(ap_driver_unregister);
807
808 /*
809  * AP bus attributes.
810  */
811 static ssize_t ap_domain_show(struct bus_type *bus, char *buf)
812 {
813         return snprintf(buf, PAGE_SIZE, "%d\n", ap_domain_index);
814 }
815
816 static BUS_ATTR(ap_domain, 0444, ap_domain_show, NULL);
817
818 static ssize_t ap_config_time_show(struct bus_type *bus, char *buf)
819 {
820         return snprintf(buf, PAGE_SIZE, "%d\n", ap_config_time);
821 }
822
823 static ssize_t ap_interrupts_show(struct bus_type *bus, char *buf)
824 {
825         return snprintf(buf, PAGE_SIZE, "%d\n",
826                         ap_using_interrupts() ? 1 : 0);
827 }
828
829 static BUS_ATTR(ap_interrupts, 0444, ap_interrupts_show, NULL);
830
831 static ssize_t ap_config_time_store(struct bus_type *bus,
832                                     const char *buf, size_t count)
833 {
834         int time;
835
836         if (sscanf(buf, "%d\n", &time) != 1 || time < 5 || time > 120)
837                 return -EINVAL;
838         ap_config_time = time;
839         if (!timer_pending(&ap_config_timer) ||
840             !mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ)) {
841                 ap_config_timer.expires = jiffies + ap_config_time * HZ;
842                 add_timer(&ap_config_timer);
843         }
844         return count;
845 }
846
847 static BUS_ATTR(config_time, 0644, ap_config_time_show, ap_config_time_store);
848
849 static ssize_t ap_poll_thread_show(struct bus_type *bus, char *buf)
850 {
851         return snprintf(buf, PAGE_SIZE, "%d\n", ap_poll_kthread ? 1 : 0);
852 }
853
854 static ssize_t ap_poll_thread_store(struct bus_type *bus,
855                                     const char *buf, size_t count)
856 {
857         int flag, rc;
858
859         if (sscanf(buf, "%d\n", &flag) != 1)
860                 return -EINVAL;
861         if (flag) {
862                 rc = ap_poll_thread_start();
863                 if (rc)
864                         return rc;
865         }
866         else
867                 ap_poll_thread_stop();
868         return count;
869 }
870
871 static BUS_ATTR(poll_thread, 0644, ap_poll_thread_show, ap_poll_thread_store);
872
873 static ssize_t poll_timeout_show(struct bus_type *bus, char *buf)
874 {
875         return snprintf(buf, PAGE_SIZE, "%llu\n", poll_timeout);
876 }
877
878 static ssize_t poll_timeout_store(struct bus_type *bus, const char *buf,
879                                   size_t count)
880 {
881         unsigned long long time;
882         ktime_t hr_time;
883
884         /* 120 seconds = maximum poll interval */
885         if (sscanf(buf, "%llu\n", &time) != 1 || time < 1 ||
886             time > 120000000000ULL)
887                 return -EINVAL;
888         poll_timeout = time;
889         hr_time = ktime_set(0, poll_timeout);
890
891         if (!hrtimer_is_queued(&ap_poll_timer) ||
892             !hrtimer_forward(&ap_poll_timer, hrtimer_get_expires(&ap_poll_timer), hr_time)) {
893                 hrtimer_set_expires(&ap_poll_timer, hr_time);
894                 hrtimer_start_expires(&ap_poll_timer, HRTIMER_MODE_ABS);
895         }
896         return count;
897 }
898
899 static BUS_ATTR(poll_timeout, 0644, poll_timeout_show, poll_timeout_store);
900
901 static struct bus_attribute *const ap_bus_attrs[] = {
902         &bus_attr_ap_domain,
903         &bus_attr_config_time,
904         &bus_attr_poll_thread,
905         &bus_attr_ap_interrupts,
906         &bus_attr_poll_timeout,
907         NULL,
908 };
909
910 /**
911  * ap_select_domain(): Select an AP domain.
912  *
913  * Pick one of the 16 AP domains.
914  */
915 static int ap_select_domain(void)
916 {
917         int queue_depth, device_type, count, max_count, best_domain;
918         int rc, i, j;
919
920         /*
921          * We want to use a single domain. Either the one specified with
922          * the "domain=" parameter or the domain with the maximum number
923          * of devices.
924          */
925         if (ap_domain_index >= 0 && ap_domain_index < AP_DOMAINS)
926                 /* Domain has already been selected. */
927                 return 0;
928         best_domain = -1;
929         max_count = 0;
930         for (i = 0; i < AP_DOMAINS; i++) {
931                 count = 0;
932                 for (j = 0; j < AP_DEVICES; j++) {
933                         ap_qid_t qid = AP_MKQID(j, i);
934                         rc = ap_query_queue(qid, &queue_depth, &device_type);
935                         if (rc)
936                                 continue;
937                         count++;
938                 }
939                 if (count > max_count) {
940                         max_count = count;
941                         best_domain = i;
942                 }
943         }
944         if (best_domain >= 0){
945                 ap_domain_index = best_domain;
946                 return 0;
947         }
948         return -ENODEV;
949 }
950
951 /**
952  * ap_probe_device_type(): Find the device type of an AP.
953  * @ap_dev: pointer to the AP device.
954  *
955  * Find the device type if query queue returned a device type of 0.
956  */
957 static int ap_probe_device_type(struct ap_device *ap_dev)
958 {
959         static unsigned char msg[] = {
960                 0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,
961                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
962                 0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,
963                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
964                 0x01,0x00,0x43,0x43,0x41,0x2d,0x41,0x50,
965                 0x50,0x4c,0x20,0x20,0x20,0x01,0x01,0x01,
966                 0x00,0x00,0x00,0x00,0x50,0x4b,0x00,0x00,
967                 0x00,0x00,0x01,0x1c,0x00,0x00,0x00,0x00,
968                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
969                 0x00,0x00,0x05,0xb8,0x00,0x00,0x00,0x00,
970                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
971                 0x70,0x00,0x41,0x00,0x00,0x00,0x00,0x00,
972                 0x00,0x00,0x54,0x32,0x01,0x00,0xa0,0x00,
973                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
974                 0x00,0x00,0x00,0x00,0xb8,0x05,0x00,0x00,
975                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
976                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
977                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
978                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
979                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
980                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
981                 0x00,0x00,0x0a,0x00,0x00,0x00,0x00,0x00,
982                 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
983                 0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,
984                 0x49,0x43,0x53,0x46,0x20,0x20,0x20,0x20,
985                 0x50,0x4b,0x0a,0x00,0x50,0x4b,0x43,0x53,
986                 0x2d,0x31,0x2e,0x32,0x37,0x00,0x11,0x22,
987                 0x33,0x44,0x55,0x66,0x77,0x88,0x99,0x00,
988                 0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,
989                 0x99,0x00,0x11,0x22,0x33,0x44,0x55,0x66,
990                 0x77,0x88,0x99,0x00,0x11,0x22,0x33,0x44,
991                 0x55,0x66,0x77,0x88,0x99,0x00,0x11,0x22,
992                 0x33,0x44,0x55,0x66,0x77,0x88,0x99,0x00,
993                 0x11,0x22,0x33,0x5d,0x00,0x5b,0x00,0x77,
994                 0x88,0x1e,0x00,0x00,0x57,0x00,0x00,0x00,
995                 0x00,0x04,0x00,0x00,0x4f,0x00,0x00,0x00,
996                 0x03,0x02,0x00,0x00,0x40,0x01,0x00,0x01,
997                 0xce,0x02,0x68,0x2d,0x5f,0xa9,0xde,0x0c,
998                 0xf6,0xd2,0x7b,0x58,0x4b,0xf9,0x28,0x68,
999                 0x3d,0xb4,0xf4,0xef,0x78,0xd5,0xbe,0x66,
1000                 0x63,0x42,0xef,0xf8,0xfd,0xa4,0xf8,0xb0,
1001                 0x8e,0x29,0xc2,0xc9,0x2e,0xd8,0x45,0xb8,
1002                 0x53,0x8c,0x6f,0x4e,0x72,0x8f,0x6c,0x04,
1003                 0x9c,0x88,0xfc,0x1e,0xc5,0x83,0x55,0x57,
1004                 0xf7,0xdd,0xfd,0x4f,0x11,0x36,0x95,0x5d,
1005         };
1006         struct ap_queue_status status;
1007         unsigned long long psmid;
1008         char *reply;
1009         int rc, i;
1010
1011         reply = (void *) get_zeroed_page(GFP_KERNEL);
1012         if (!reply) {
1013                 rc = -ENOMEM;
1014                 goto out;
1015         }
1016
1017         status = __ap_send(ap_dev->qid, 0x0102030405060708ULL,
1018                            msg, sizeof(msg), 0);
1019         if (status.response_code != AP_RESPONSE_NORMAL) {
1020                 rc = -ENODEV;
1021                 goto out_free;
1022         }
1023
1024         /* Wait for the test message to complete. */
1025         for (i = 0; i < 6; i++) {
1026                 mdelay(300);
1027                 status = __ap_recv(ap_dev->qid, &psmid, reply, 4096);
1028                 if (status.response_code == AP_RESPONSE_NORMAL &&
1029                     psmid == 0x0102030405060708ULL)
1030                         break;
1031         }
1032         if (i < 6) {
1033                 /* Got an answer. */
1034                 if (reply[0] == 0x00 && reply[1] == 0x86)
1035                         ap_dev->device_type = AP_DEVICE_TYPE_PCICC;
1036                 else
1037                         ap_dev->device_type = AP_DEVICE_TYPE_PCICA;
1038                 rc = 0;
1039         } else
1040                 rc = -ENODEV;
1041
1042 out_free:
1043         free_page((unsigned long) reply);
1044 out:
1045         return rc;
1046 }
1047
1048 static void ap_interrupt_handler(void *unused1, void *unused2)
1049 {
1050         tasklet_schedule(&ap_tasklet);
1051 }
1052
1053 /**
1054  * __ap_scan_bus(): Scan the AP bus.
1055  * @dev: Pointer to device
1056  * @data: Pointer to data
1057  *
1058  * Scan the AP bus for new devices.
1059  */
1060 static int __ap_scan_bus(struct device *dev, void *data)
1061 {
1062         return to_ap_dev(dev)->qid == (ap_qid_t)(unsigned long) data;
1063 }
1064
1065 static void ap_device_release(struct device *dev)
1066 {
1067         struct ap_device *ap_dev = to_ap_dev(dev);
1068
1069         kfree(ap_dev);
1070 }
1071
1072 static void ap_scan_bus(struct work_struct *unused)
1073 {
1074         struct ap_device *ap_dev;
1075         struct device *dev;
1076         ap_qid_t qid;
1077         int queue_depth, device_type;
1078         int rc, i;
1079
1080         if (ap_select_domain() != 0)
1081                 return;
1082         for (i = 0; i < AP_DEVICES; i++) {
1083                 qid = AP_MKQID(i, ap_domain_index);
1084                 dev = bus_find_device(&ap_bus_type, NULL,
1085                                       (void *)(unsigned long)qid,
1086                                       __ap_scan_bus);
1087                 rc = ap_query_queue(qid, &queue_depth, &device_type);
1088                 if (dev) {
1089                         if (rc == -EBUSY) {
1090                                 set_current_state(TASK_UNINTERRUPTIBLE);
1091                                 schedule_timeout(AP_RESET_TIMEOUT);
1092                                 rc = ap_query_queue(qid, &queue_depth,
1093                                                     &device_type);
1094                         }
1095                         ap_dev = to_ap_dev(dev);
1096                         spin_lock_bh(&ap_dev->lock);
1097                         if (rc || ap_dev->unregistered) {
1098                                 spin_unlock_bh(&ap_dev->lock);
1099                                 if (ap_dev->unregistered)
1100                                         i--;
1101                                 device_unregister(dev);
1102                                 put_device(dev);
1103                                 continue;
1104                         }
1105                         spin_unlock_bh(&ap_dev->lock);
1106                         put_device(dev);
1107                         continue;
1108                 }
1109                 if (rc)
1110                         continue;
1111                 rc = ap_init_queue(qid);
1112                 if (rc)
1113                         continue;
1114                 ap_dev = kzalloc(sizeof(*ap_dev), GFP_KERNEL);
1115                 if (!ap_dev)
1116                         break;
1117                 ap_dev->qid = qid;
1118                 ap_dev->queue_depth = queue_depth;
1119                 ap_dev->unregistered = 1;
1120                 spin_lock_init(&ap_dev->lock);
1121                 INIT_LIST_HEAD(&ap_dev->pendingq);
1122                 INIT_LIST_HEAD(&ap_dev->requestq);
1123                 INIT_LIST_HEAD(&ap_dev->list);
1124                 setup_timer(&ap_dev->timeout, ap_request_timeout,
1125                             (unsigned long) ap_dev);
1126                 if (device_type == 0)
1127                         ap_probe_device_type(ap_dev);
1128                 else
1129                         ap_dev->device_type = device_type;
1130
1131                 ap_dev->device.bus = &ap_bus_type;
1132                 ap_dev->device.parent = ap_root_device;
1133                 if (dev_set_name(&ap_dev->device, "card%02x",
1134                                  AP_QID_DEVICE(ap_dev->qid))) {
1135                         kfree(ap_dev);
1136                         continue;
1137                 }
1138                 ap_dev->device.release = ap_device_release;
1139                 rc = device_register(&ap_dev->device);
1140                 if (rc) {
1141                         put_device(&ap_dev->device);
1142                         continue;
1143                 }
1144                 /* Add device attributes. */
1145                 rc = sysfs_create_group(&ap_dev->device.kobj,
1146                                         &ap_dev_attr_group);
1147                 if (!rc) {
1148                         spin_lock_bh(&ap_dev->lock);
1149                         ap_dev->unregistered = 0;
1150                         spin_unlock_bh(&ap_dev->lock);
1151                 }
1152                 else
1153                         device_unregister(&ap_dev->device);
1154         }
1155 }
1156
1157 static void
1158 ap_config_timeout(unsigned long ptr)
1159 {
1160         queue_work(ap_work_queue, &ap_config_work);
1161         ap_config_timer.expires = jiffies + ap_config_time * HZ;
1162         add_timer(&ap_config_timer);
1163 }
1164
1165 /**
1166  * ap_schedule_poll_timer(): Schedule poll timer.
1167  *
1168  * Set up the timer to run the poll tasklet
1169  */
1170 static inline void ap_schedule_poll_timer(void)
1171 {
1172         ktime_t hr_time;
1173         if (ap_using_interrupts() || ap_suspend_flag)
1174                 return;
1175         if (hrtimer_is_queued(&ap_poll_timer))
1176                 return;
1177         if (ktime_to_ns(hrtimer_expires_remaining(&ap_poll_timer)) <= 0) {
1178                 hr_time = ktime_set(0, poll_timeout);
1179                 hrtimer_forward_now(&ap_poll_timer, hr_time);
1180                 hrtimer_restart(&ap_poll_timer);
1181         }
1182         return;
1183 }
1184
1185 /**
1186  * ap_poll_read(): Receive pending reply messages from an AP device.
1187  * @ap_dev: pointer to the AP device
1188  * @flags: pointer to control flags, bit 2^0 is set if another poll is
1189  *         required, bit 2^1 is set if the poll timer needs to get armed
1190  *
1191  * Returns 0 if the device is still present, -ENODEV if not.
1192  */
1193 static int ap_poll_read(struct ap_device *ap_dev, unsigned long *flags)
1194 {
1195         struct ap_queue_status status;
1196         struct ap_message *ap_msg;
1197
1198         if (ap_dev->queue_count <= 0)
1199                 return 0;
1200         status = __ap_recv(ap_dev->qid, &ap_dev->reply->psmid,
1201                            ap_dev->reply->message, ap_dev->reply->length);
1202         switch (status.response_code) {
1203         case AP_RESPONSE_NORMAL:
1204                 atomic_dec(&ap_poll_requests);
1205                 ap_decrease_queue_count(ap_dev);
1206                 list_for_each_entry(ap_msg, &ap_dev->pendingq, list) {
1207                         if (ap_msg->psmid != ap_dev->reply->psmid)
1208                                 continue;
1209                         list_del_init(&ap_msg->list);
1210                         ap_dev->pendingq_count--;
1211                         ap_dev->drv->receive(ap_dev, ap_msg, ap_dev->reply);
1212                         break;
1213                 }
1214                 if (ap_dev->queue_count > 0)
1215                         *flags |= 1;
1216                 break;
1217         case AP_RESPONSE_NO_PENDING_REPLY:
1218                 if (status.queue_empty) {
1219                         /* The card shouldn't forget requests but who knows. */
1220                         atomic_sub(ap_dev->queue_count, &ap_poll_requests);
1221                         ap_dev->queue_count = 0;
1222                         list_splice_init(&ap_dev->pendingq, &ap_dev->requestq);
1223                         ap_dev->requestq_count += ap_dev->pendingq_count;
1224                         ap_dev->pendingq_count = 0;
1225                 } else
1226                         *flags |= 2;
1227                 break;
1228         default:
1229                 return -ENODEV;
1230         }
1231         return 0;
1232 }
1233
1234 /**
1235  * ap_poll_write(): Send messages from the request queue to an AP device.
1236  * @ap_dev: pointer to the AP device
1237  * @flags: pointer to control flags, bit 2^0 is set if another poll is
1238  *         required, bit 2^1 is set if the poll timer needs to get armed
1239  *
1240  * Returns 0 if the device is still present, -ENODEV if not.
1241  */
1242 static int ap_poll_write(struct ap_device *ap_dev, unsigned long *flags)
1243 {
1244         struct ap_queue_status status;
1245         struct ap_message *ap_msg;
1246
1247         if (ap_dev->requestq_count <= 0 ||
1248             ap_dev->queue_count >= ap_dev->queue_depth)
1249                 return 0;
1250         /* Start the next request on the queue. */
1251         ap_msg = list_entry(ap_dev->requestq.next, struct ap_message, list);
1252         status = __ap_send(ap_dev->qid, ap_msg->psmid,
1253                            ap_msg->message, ap_msg->length, ap_msg->special);
1254         switch (status.response_code) {
1255         case AP_RESPONSE_NORMAL:
1256                 atomic_inc(&ap_poll_requests);
1257                 ap_increase_queue_count(ap_dev);
1258                 list_move_tail(&ap_msg->list, &ap_dev->pendingq);
1259                 ap_dev->requestq_count--;
1260                 ap_dev->pendingq_count++;
1261                 if (ap_dev->queue_count < ap_dev->queue_depth &&
1262                     ap_dev->requestq_count > 0)
1263                         *flags |= 1;
1264                 *flags |= 2;
1265                 break;
1266         case AP_RESPONSE_Q_FULL:
1267         case AP_RESPONSE_RESET_IN_PROGRESS:
1268                 *flags |= 2;
1269                 break;
1270         case AP_RESPONSE_MESSAGE_TOO_BIG:
1271         case AP_RESPONSE_REQ_FAC_NOT_INST:
1272                 return -EINVAL;
1273         default:
1274                 return -ENODEV;
1275         }
1276         return 0;
1277 }
1278
1279 /**
1280  * ap_poll_queue(): Poll AP device for pending replies and send new messages.
1281  * @ap_dev: pointer to the bus device
1282  * @flags: pointer to control flags, bit 2^0 is set if another poll is
1283  *         required, bit 2^1 is set if the poll timer needs to get armed
1284  *
1285  * Poll AP device for pending replies and send new messages. If either
1286  * ap_poll_read or ap_poll_write returns -ENODEV unregister the device.
1287  * Returns 0.
1288  */
1289 static inline int ap_poll_queue(struct ap_device *ap_dev, unsigned long *flags)
1290 {
1291         int rc;
1292
1293         rc = ap_poll_read(ap_dev, flags);
1294         if (rc)
1295                 return rc;
1296         return ap_poll_write(ap_dev, flags);
1297 }
1298
1299 /**
1300  * __ap_queue_message(): Queue a message to a device.
1301  * @ap_dev: pointer to the AP device
1302  * @ap_msg: the message to be queued
1303  *
1304  * Queue a message to a device. Returns 0 if successful.
1305  */
1306 static int __ap_queue_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
1307 {
1308         struct ap_queue_status status;
1309
1310         if (list_empty(&ap_dev->requestq) &&
1311             ap_dev->queue_count < ap_dev->queue_depth) {
1312                 status = __ap_send(ap_dev->qid, ap_msg->psmid,
1313                                    ap_msg->message, ap_msg->length,
1314                                    ap_msg->special);
1315                 switch (status.response_code) {
1316                 case AP_RESPONSE_NORMAL:
1317                         list_add_tail(&ap_msg->list, &ap_dev->pendingq);
1318                         atomic_inc(&ap_poll_requests);
1319                         ap_dev->pendingq_count++;
1320                         ap_increase_queue_count(ap_dev);
1321                         ap_dev->total_request_count++;
1322                         break;
1323                 case AP_RESPONSE_Q_FULL:
1324                 case AP_RESPONSE_RESET_IN_PROGRESS:
1325                         list_add_tail(&ap_msg->list, &ap_dev->requestq);
1326                         ap_dev->requestq_count++;
1327                         ap_dev->total_request_count++;
1328                         return -EBUSY;
1329                 case AP_RESPONSE_REQ_FAC_NOT_INST:
1330                 case AP_RESPONSE_MESSAGE_TOO_BIG:
1331                         ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-EINVAL));
1332                         return -EINVAL;
1333                 default:        /* Device is gone. */
1334                         ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
1335                         return -ENODEV;
1336                 }
1337         } else {
1338                 list_add_tail(&ap_msg->list, &ap_dev->requestq);
1339                 ap_dev->requestq_count++;
1340                 ap_dev->total_request_count++;
1341                 return -EBUSY;
1342         }
1343         ap_schedule_poll_timer();
1344         return 0;
1345 }
1346
1347 void ap_queue_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
1348 {
1349         unsigned long flags;
1350         int rc;
1351
1352         spin_lock_bh(&ap_dev->lock);
1353         if (!ap_dev->unregistered) {
1354                 /* Make room on the queue by polling for finished requests. */
1355                 rc = ap_poll_queue(ap_dev, &flags);
1356                 if (!rc)
1357                         rc = __ap_queue_message(ap_dev, ap_msg);
1358                 if (!rc)
1359                         wake_up(&ap_poll_wait);
1360                 if (rc == -ENODEV)
1361                         ap_dev->unregistered = 1;
1362         } else {
1363                 ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
1364                 rc = -ENODEV;
1365         }
1366         spin_unlock_bh(&ap_dev->lock);
1367         if (rc == -ENODEV)
1368                 device_unregister(&ap_dev->device);
1369 }
1370 EXPORT_SYMBOL(ap_queue_message);
1371
1372 /**
1373  * ap_cancel_message(): Cancel a crypto request.
1374  * @ap_dev: The AP device that has the message queued
1375  * @ap_msg: The message that is to be removed
1376  *
1377  * Cancel a crypto request. This is done by removing the request
1378  * from the device pending or request queue. Note that the
1379  * request stays on the AP queue. When it finishes the message
1380  * reply will be discarded because the psmid can't be found.
1381  */
1382 void ap_cancel_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
1383 {
1384         struct ap_message *tmp;
1385
1386         spin_lock_bh(&ap_dev->lock);
1387         if (!list_empty(&ap_msg->list)) {
1388                 list_for_each_entry(tmp, &ap_dev->pendingq, list)
1389                         if (tmp->psmid == ap_msg->psmid) {
1390                                 ap_dev->pendingq_count--;
1391                                 goto found;
1392                         }
1393                 ap_dev->requestq_count--;
1394         found:
1395                 list_del_init(&ap_msg->list);
1396         }
1397         spin_unlock_bh(&ap_dev->lock);
1398 }
1399 EXPORT_SYMBOL(ap_cancel_message);
1400
1401 /**
1402  * ap_poll_timeout(): AP receive polling for finished AP requests.
1403  * @unused: Unused pointer.
1404  *
1405  * Schedules the AP tasklet using a high resolution timer.
1406  */
1407 static enum hrtimer_restart ap_poll_timeout(struct hrtimer *unused)
1408 {
1409         tasklet_schedule(&ap_tasklet);
1410         return HRTIMER_NORESTART;
1411 }
1412
1413 /**
1414  * ap_reset(): Reset a not responding AP device.
1415  * @ap_dev: Pointer to the AP device
1416  *
1417  * Reset a not responding AP device and move all requests from the
1418  * pending queue to the request queue.
1419  */
1420 static void ap_reset(struct ap_device *ap_dev)
1421 {
1422         int rc;
1423
1424         ap_dev->reset = AP_RESET_IGNORE;
1425         atomic_sub(ap_dev->queue_count, &ap_poll_requests);
1426         ap_dev->queue_count = 0;
1427         list_splice_init(&ap_dev->pendingq, &ap_dev->requestq);
1428         ap_dev->requestq_count += ap_dev->pendingq_count;
1429         ap_dev->pendingq_count = 0;
1430         rc = ap_init_queue(ap_dev->qid);
1431         if (rc == -ENODEV)
1432                 ap_dev->unregistered = 1;
1433 }
1434
1435 static int __ap_poll_device(struct ap_device *ap_dev, unsigned long *flags)
1436 {
1437         if (!ap_dev->unregistered) {
1438                 if (ap_poll_queue(ap_dev, flags))
1439                         ap_dev->unregistered = 1;
1440                 if (ap_dev->reset == AP_RESET_DO)
1441                         ap_reset(ap_dev);
1442         }
1443         return 0;
1444 }
1445
1446 /**
1447  * ap_poll_all(): Poll all AP devices.
1448  * @dummy: Unused variable
1449  *
1450  * Poll all AP devices on the bus in a round robin fashion. Continue
1451  * polling until bit 2^0 of the control flags is not set. If bit 2^1
1452  * of the control flags has been set arm the poll timer.
1453  */
1454 static void ap_poll_all(unsigned long dummy)
1455 {
1456         unsigned long flags;
1457         struct ap_device *ap_dev;
1458
1459         /* Reset the indicator if interrupts are used. Thus new interrupts can
1460          * be received. Doing it in the beginning of the tasklet is therefor
1461          * important that no requests on any AP get lost.
1462          */
1463         if (ap_using_interrupts())
1464                 xchg((u8 *)ap_interrupt_indicator, 0);
1465         do {
1466                 flags = 0;
1467                 spin_lock(&ap_device_list_lock);
1468                 list_for_each_entry(ap_dev, &ap_device_list, list) {
1469                         spin_lock(&ap_dev->lock);
1470                         __ap_poll_device(ap_dev, &flags);
1471                         spin_unlock(&ap_dev->lock);
1472                 }
1473                 spin_unlock(&ap_device_list_lock);
1474         } while (flags & 1);
1475         if (flags & 2)
1476                 ap_schedule_poll_timer();
1477 }
1478
1479 /**
1480  * ap_poll_thread(): Thread that polls for finished requests.
1481  * @data: Unused pointer
1482  *
1483  * AP bus poll thread. The purpose of this thread is to poll for
1484  * finished requests in a loop if there is a "free" cpu - that is
1485  * a cpu that doesn't have anything better to do. The polling stops
1486  * as soon as there is another task or if all messages have been
1487  * delivered.
1488  */
1489 static int ap_poll_thread(void *data)
1490 {
1491         DECLARE_WAITQUEUE(wait, current);
1492         unsigned long flags;
1493         int requests;
1494         struct ap_device *ap_dev;
1495
1496         set_user_nice(current, 19);
1497         while (1) {
1498                 if (ap_suspend_flag)
1499                         return 0;
1500                 if (need_resched()) {
1501                         schedule();
1502                         continue;
1503                 }
1504                 add_wait_queue(&ap_poll_wait, &wait);
1505                 set_current_state(TASK_INTERRUPTIBLE);
1506                 if (kthread_should_stop())
1507                         break;
1508                 requests = atomic_read(&ap_poll_requests);
1509                 if (requests <= 0)
1510                         schedule();
1511                 set_current_state(TASK_RUNNING);
1512                 remove_wait_queue(&ap_poll_wait, &wait);
1513
1514                 flags = 0;
1515                 spin_lock_bh(&ap_device_list_lock);
1516                 list_for_each_entry(ap_dev, &ap_device_list, list) {
1517                         spin_lock(&ap_dev->lock);
1518                         __ap_poll_device(ap_dev, &flags);
1519                         spin_unlock(&ap_dev->lock);
1520                 }
1521                 spin_unlock_bh(&ap_device_list_lock);
1522         }
1523         set_current_state(TASK_RUNNING);
1524         remove_wait_queue(&ap_poll_wait, &wait);
1525         return 0;
1526 }
1527
1528 static int ap_poll_thread_start(void)
1529 {
1530         int rc;
1531
1532         if (ap_using_interrupts() || ap_suspend_flag)
1533                 return 0;
1534         mutex_lock(&ap_poll_thread_mutex);
1535         if (!ap_poll_kthread) {
1536                 ap_poll_kthread = kthread_run(ap_poll_thread, NULL, "appoll");
1537                 rc = IS_ERR(ap_poll_kthread) ? PTR_ERR(ap_poll_kthread) : 0;
1538                 if (rc)
1539                         ap_poll_kthread = NULL;
1540         }
1541         else
1542                 rc = 0;
1543         mutex_unlock(&ap_poll_thread_mutex);
1544         return rc;
1545 }
1546
1547 static void ap_poll_thread_stop(void)
1548 {
1549         mutex_lock(&ap_poll_thread_mutex);
1550         if (ap_poll_kthread) {
1551                 kthread_stop(ap_poll_kthread);
1552                 ap_poll_kthread = NULL;
1553         }
1554         mutex_unlock(&ap_poll_thread_mutex);
1555 }
1556
1557 /**
1558  * ap_request_timeout(): Handling of request timeouts
1559  * @data: Holds the AP device.
1560  *
1561  * Handles request timeouts.
1562  */
1563 static void ap_request_timeout(unsigned long data)
1564 {
1565         struct ap_device *ap_dev = (struct ap_device *) data;
1566
1567         if (ap_dev->reset == AP_RESET_ARMED) {
1568                 ap_dev->reset = AP_RESET_DO;
1569
1570                 if (ap_using_interrupts())
1571                         tasklet_schedule(&ap_tasklet);
1572         }
1573 }
1574
1575 static void ap_reset_domain(void)
1576 {
1577         int i;
1578
1579         if (ap_domain_index != -1)
1580                 for (i = 0; i < AP_DEVICES; i++)
1581                         ap_reset_queue(AP_MKQID(i, ap_domain_index));
1582 }
1583
1584 static void ap_reset_all(void)
1585 {
1586         int i, j;
1587
1588         for (i = 0; i < AP_DOMAINS; i++)
1589                 for (j = 0; j < AP_DEVICES; j++)
1590                         ap_reset_queue(AP_MKQID(j, i));
1591 }
1592
1593 static struct reset_call ap_reset_call = {
1594         .fn = ap_reset_all,
1595 };
1596
1597 /**
1598  * ap_module_init(): The module initialization code.
1599  *
1600  * Initializes the module.
1601  */
1602 int __init ap_module_init(void)
1603 {
1604         int rc, i;
1605
1606         if (ap_domain_index < -1 || ap_domain_index >= AP_DOMAINS) {
1607                 pr_warning("%d is not a valid cryptographic domain\n",
1608                            ap_domain_index);
1609                 return -EINVAL;
1610         }
1611         /* In resume callback we need to know if the user had set the domain.
1612          * If so, we can not just reset it.
1613          */
1614         if (ap_domain_index >= 0)
1615                 user_set_domain = 1;
1616
1617         if (ap_instructions_available() != 0) {
1618                 pr_warning("The hardware system does not support "
1619                            "AP instructions\n");
1620                 return -ENODEV;
1621         }
1622         if (ap_interrupts_available()) {
1623                 isc_register(AP_ISC);
1624                 ap_interrupt_indicator = s390_register_adapter_interrupt(
1625                         &ap_interrupt_handler, NULL, AP_ISC);
1626                 if (IS_ERR(ap_interrupt_indicator)) {
1627                         ap_interrupt_indicator = NULL;
1628                         isc_unregister(AP_ISC);
1629                 }
1630         }
1631
1632         register_reset_call(&ap_reset_call);
1633
1634         /* Create /sys/bus/ap. */
1635         rc = bus_register(&ap_bus_type);
1636         if (rc)
1637                 goto out;
1638         for (i = 0; ap_bus_attrs[i]; i++) {
1639                 rc = bus_create_file(&ap_bus_type, ap_bus_attrs[i]);
1640                 if (rc)
1641                         goto out_bus;
1642         }
1643
1644         /* Create /sys/devices/ap. */
1645         ap_root_device = root_device_register("ap");
1646         rc = IS_ERR(ap_root_device) ? PTR_ERR(ap_root_device) : 0;
1647         if (rc)
1648                 goto out_bus;
1649
1650         ap_work_queue = create_singlethread_workqueue("kapwork");
1651         if (!ap_work_queue) {
1652                 rc = -ENOMEM;
1653                 goto out_root;
1654         }
1655
1656         if (ap_select_domain() == 0)
1657                 ap_scan_bus(NULL);
1658
1659         /* Setup the AP bus rescan timer. */
1660         init_timer(&ap_config_timer);
1661         ap_config_timer.function = ap_config_timeout;
1662         ap_config_timer.data = 0;
1663         ap_config_timer.expires = jiffies + ap_config_time * HZ;
1664         add_timer(&ap_config_timer);
1665
1666         /* Setup the high resultion poll timer.
1667          * If we are running under z/VM adjust polling to z/VM polling rate.
1668          */
1669         if (MACHINE_IS_VM)
1670                 poll_timeout = 1500000;
1671         hrtimer_init(&ap_poll_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1672         ap_poll_timer.function = ap_poll_timeout;
1673
1674         /* Start the low priority AP bus poll thread. */
1675         if (ap_thread_flag) {
1676                 rc = ap_poll_thread_start();
1677                 if (rc)
1678                         goto out_work;
1679         }
1680
1681         return 0;
1682
1683 out_work:
1684         del_timer_sync(&ap_config_timer);
1685         hrtimer_cancel(&ap_poll_timer);
1686         destroy_workqueue(ap_work_queue);
1687 out_root:
1688         root_device_unregister(ap_root_device);
1689 out_bus:
1690         while (i--)
1691                 bus_remove_file(&ap_bus_type, ap_bus_attrs[i]);
1692         bus_unregister(&ap_bus_type);
1693 out:
1694         unregister_reset_call(&ap_reset_call);
1695         if (ap_using_interrupts()) {
1696                 s390_unregister_adapter_interrupt(ap_interrupt_indicator, AP_ISC);
1697                 isc_unregister(AP_ISC);
1698         }
1699         return rc;
1700 }
1701
1702 static int __ap_match_all(struct device *dev, void *data)
1703 {
1704         return 1;
1705 }
1706
1707 /**
1708  * ap_modules_exit(): The module termination code
1709  *
1710  * Terminates the module.
1711  */
1712 void ap_module_exit(void)
1713 {
1714         int i;
1715         struct device *dev;
1716
1717         ap_reset_domain();
1718         ap_poll_thread_stop();
1719         del_timer_sync(&ap_config_timer);
1720         hrtimer_cancel(&ap_poll_timer);
1721         destroy_workqueue(ap_work_queue);
1722         tasklet_kill(&ap_tasklet);
1723         root_device_unregister(ap_root_device);
1724         while ((dev = bus_find_device(&ap_bus_type, NULL, NULL,
1725                     __ap_match_all)))
1726         {
1727                 device_unregister(dev);
1728                 put_device(dev);
1729         }
1730         for (i = 0; ap_bus_attrs[i]; i++)
1731                 bus_remove_file(&ap_bus_type, ap_bus_attrs[i]);
1732         bus_unregister(&ap_bus_type);
1733         unregister_reset_call(&ap_reset_call);
1734         if (ap_using_interrupts()) {
1735                 s390_unregister_adapter_interrupt(ap_interrupt_indicator, AP_ISC);
1736                 isc_unregister(AP_ISC);
1737         }
1738 }
1739
1740 #ifndef CONFIG_ZCRYPT_MONOLITHIC
1741 module_init(ap_module_init);
1742 module_exit(ap_module_exit);
1743 #endif