isci: preallocate requests
[linux-2.6.git] / drivers / scsi / isci / request.c
1 /*
2  * This file is provided under a dual BSD/GPLv2 license.  When using or
3  * redistributing this file, you may do so under either license.
4  *
5  * GPL LICENSE SUMMARY
6  *
7  * Copyright(c) 2008 - 2011 Intel Corporation. All rights reserved.
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of version 2 of the GNU General Public License as
11  * published by the Free Software Foundation.
12  *
13  * This program is distributed in the hope that it will be useful, but
14  * WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16  * General Public License for more details.
17  *
18  * You should have received a copy of the GNU General Public License
19  * along with this program; if not, write to the Free Software
20  * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
21  * The full GNU General Public License is included in this distribution
22  * in the file called LICENSE.GPL.
23  *
24  * BSD LICENSE
25  *
26  * Copyright(c) 2008 - 2011 Intel Corporation. All rights reserved.
27  * All rights reserved.
28  *
29  * Redistribution and use in source and binary forms, with or without
30  * modification, are permitted provided that the following conditions
31  * are met:
32  *
33  *   * Redistributions of source code must retain the above copyright
34  *     notice, this list of conditions and the following disclaimer.
35  *   * Redistributions in binary form must reproduce the above copyright
36  *     notice, this list of conditions and the following disclaimer in
37  *     the documentation and/or other materials provided with the
38  *     distribution.
39  *   * Neither the name of Intel Corporation nor the names of its
40  *     contributors may be used to endorse or promote products derived
41  *     from this software without specific prior written permission.
42  *
43  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
44  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
45  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
46  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
47  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
48  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
49  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
50  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
51  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
52  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
53  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
54  */
55
56 #include "isci.h"
57 #include "task.h"
58 #include "request.h"
59 #include "sata.h"
60 #include "scu_completion_codes.h"
61 #include "scu_event_codes.h"
62 #include "sas.h"
63
64 static struct scu_sgl_element_pair *to_sgl_element_pair(struct scic_sds_request *sci_req,
65                                                         int idx)
66 {
67         if (idx == 0)
68                 return &sci_req->tc->sgl_pair_ab;
69         else if (idx == 1)
70                 return &sci_req->tc->sgl_pair_cd;
71         else if (idx < 0)
72                 return NULL;
73         else
74                 return &sci_req->sg_table[idx - 2];
75 }
76
77 static dma_addr_t to_sgl_element_pair_dma(struct scic_sds_controller *scic,
78                                           struct scic_sds_request *sci_req, u32 idx)
79 {
80         u32 offset;
81
82         if (idx == 0) {
83                 offset = (void *) &sci_req->tc->sgl_pair_ab -
84                          (void *) &scic->task_context_table[0];
85                 return scic->task_context_dma + offset;
86         } else if (idx == 1) {
87                 offset = (void *) &sci_req->tc->sgl_pair_cd -
88                          (void *) &scic->task_context_table[0];
89                 return scic->task_context_dma + offset;
90         }
91
92         return scic_io_request_get_dma_addr(sci_req, &sci_req->sg_table[idx - 2]);
93 }
94
95 static void init_sgl_element(struct scu_sgl_element *e, struct scatterlist *sg)
96 {
97         e->length = sg_dma_len(sg);
98         e->address_upper = upper_32_bits(sg_dma_address(sg));
99         e->address_lower = lower_32_bits(sg_dma_address(sg));
100         e->address_modifier = 0;
101 }
102
103 static void scic_sds_request_build_sgl(struct scic_sds_request *sds_request)
104 {
105         struct isci_request *isci_request = sci_req_to_ireq(sds_request);
106         struct isci_host *isci_host = isci_request->isci_host;
107         struct scic_sds_controller *scic = &isci_host->sci;
108         struct sas_task *task = isci_request_access_task(isci_request);
109         struct scatterlist *sg = NULL;
110         dma_addr_t dma_addr;
111         u32 sg_idx = 0;
112         struct scu_sgl_element_pair *scu_sg   = NULL;
113         struct scu_sgl_element_pair *prev_sg  = NULL;
114
115         if (task->num_scatter > 0) {
116                 sg = task->scatter;
117
118                 while (sg) {
119                         scu_sg = to_sgl_element_pair(sds_request, sg_idx);
120                         init_sgl_element(&scu_sg->A, sg);
121                         sg = sg_next(sg);
122                         if (sg) {
123                                 init_sgl_element(&scu_sg->B, sg);
124                                 sg = sg_next(sg);
125                         } else
126                                 memset(&scu_sg->B, 0, sizeof(scu_sg->B));
127
128                         if (prev_sg) {
129                                 dma_addr = to_sgl_element_pair_dma(scic,
130                                                                    sds_request,
131                                                                    sg_idx);
132
133                                 prev_sg->next_pair_upper =
134                                         upper_32_bits(dma_addr);
135                                 prev_sg->next_pair_lower =
136                                         lower_32_bits(dma_addr);
137                         }
138
139                         prev_sg = scu_sg;
140                         sg_idx++;
141                 }
142         } else {        /* handle when no sg */
143                 scu_sg = to_sgl_element_pair(sds_request, sg_idx);
144
145                 dma_addr = dma_map_single(&isci_host->pdev->dev,
146                                           task->scatter,
147                                           task->total_xfer_len,
148                                           task->data_dir);
149
150                 isci_request->zero_scatter_daddr = dma_addr;
151
152                 scu_sg->A.length = task->total_xfer_len;
153                 scu_sg->A.address_upper = upper_32_bits(dma_addr);
154                 scu_sg->A.address_lower = lower_32_bits(dma_addr);
155         }
156
157         if (scu_sg) {
158                 scu_sg->next_pair_upper = 0;
159                 scu_sg->next_pair_lower = 0;
160         }
161 }
162
163 static void scic_sds_io_request_build_ssp_command_iu(struct scic_sds_request *sci_req)
164 {
165         struct ssp_cmd_iu *cmd_iu;
166         struct isci_request *ireq = sci_req_to_ireq(sci_req);
167         struct sas_task *task = isci_request_access_task(ireq);
168
169         cmd_iu = &sci_req->ssp.cmd;
170
171         memcpy(cmd_iu->LUN, task->ssp_task.LUN, 8);
172         cmd_iu->add_cdb_len = 0;
173         cmd_iu->_r_a = 0;
174         cmd_iu->_r_b = 0;
175         cmd_iu->en_fburst = 0; /* unsupported */
176         cmd_iu->task_prio = task->ssp_task.task_prio;
177         cmd_iu->task_attr = task->ssp_task.task_attr;
178         cmd_iu->_r_c = 0;
179
180         sci_swab32_cpy(&cmd_iu->cdb, task->ssp_task.cdb,
181                        sizeof(task->ssp_task.cdb) / sizeof(u32));
182 }
183
184 static void scic_sds_task_request_build_ssp_task_iu(struct scic_sds_request *sci_req)
185 {
186         struct ssp_task_iu *task_iu;
187         struct isci_request *ireq = sci_req_to_ireq(sci_req);
188         struct sas_task *task = isci_request_access_task(ireq);
189         struct isci_tmf *isci_tmf = isci_request_access_tmf(ireq);
190
191         task_iu = &sci_req->ssp.tmf;
192
193         memset(task_iu, 0, sizeof(struct ssp_task_iu));
194
195         memcpy(task_iu->LUN, task->ssp_task.LUN, 8);
196
197         task_iu->task_func = isci_tmf->tmf_code;
198         task_iu->task_tag =
199                 (ireq->ttype == tmf_task) ?
200                 isci_tmf->io_tag :
201                 SCI_CONTROLLER_INVALID_IO_TAG;
202 }
203
204 /**
205  * This method is will fill in the SCU Task Context for any type of SSP request.
206  * @sci_req:
207  * @task_context:
208  *
209  */
210 static void scu_ssp_reqeust_construct_task_context(
211         struct scic_sds_request *sds_request,
212         struct scu_task_context *task_context)
213 {
214         dma_addr_t dma_addr;
215         struct scic_sds_remote_device *target_device;
216         struct scic_sds_port *target_port;
217
218         target_device = scic_sds_request_get_device(sds_request);
219         target_port = scic_sds_request_get_port(sds_request);
220
221         /* Fill in the TC with the its required data */
222         task_context->abort = 0;
223         task_context->priority = 0;
224         task_context->initiator_request = 1;
225         task_context->connection_rate = target_device->connection_rate;
226         task_context->protocol_engine_index =
227                 scic_sds_controller_get_protocol_engine_group(controller);
228         task_context->logical_port_index =
229                 scic_sds_port_get_index(target_port);
230         task_context->protocol_type = SCU_TASK_CONTEXT_PROTOCOL_SSP;
231         task_context->valid = SCU_TASK_CONTEXT_VALID;
232         task_context->context_type = SCU_TASK_CONTEXT_TYPE;
233
234         task_context->remote_node_index =
235                 scic_sds_remote_device_get_index(sds_request->target_device);
236         task_context->command_code = 0;
237
238         task_context->link_layer_control = 0;
239         task_context->do_not_dma_ssp_good_response = 1;
240         task_context->strict_ordering = 0;
241         task_context->control_frame = 0;
242         task_context->timeout_enable = 0;
243         task_context->block_guard_enable = 0;
244
245         task_context->address_modifier = 0;
246
247         /* task_context->type.ssp.tag = sci_req->io_tag; */
248         task_context->task_phase = 0x01;
249
250         sds_request->post_context = (SCU_CONTEXT_COMMAND_REQUEST_TYPE_POST_TC |
251                                      (scic_sds_controller_get_protocol_engine_group(controller) <<
252                                       SCU_CONTEXT_COMMAND_PROTOCOL_ENGINE_GROUP_SHIFT) |
253                                      (scic_sds_port_get_index(target_port) <<
254                                       SCU_CONTEXT_COMMAND_LOGICAL_PORT_SHIFT) |
255                                      ISCI_TAG_TCI(sds_request->io_tag));
256
257         /*
258          * Copy the physical address for the command buffer to the
259          * SCU Task Context
260          */
261         dma_addr = scic_io_request_get_dma_addr(sds_request,
262                                                 &sds_request->ssp.cmd);
263
264         task_context->command_iu_upper = upper_32_bits(dma_addr);
265         task_context->command_iu_lower = lower_32_bits(dma_addr);
266
267         /*
268          * Copy the physical address for the response buffer to the
269          * SCU Task Context
270          */
271         dma_addr = scic_io_request_get_dma_addr(sds_request,
272                                                 &sds_request->ssp.rsp);
273
274         task_context->response_iu_upper = upper_32_bits(dma_addr);
275         task_context->response_iu_lower = lower_32_bits(dma_addr);
276 }
277
278 /**
279  * This method is will fill in the SCU Task Context for a SSP IO request.
280  * @sci_req:
281  *
282  */
283 static void scu_ssp_io_request_construct_task_context(struct scic_sds_request *sci_req,
284                                                       enum dma_data_direction dir,
285                                                       u32 len)
286 {
287         struct scu_task_context *task_context = sci_req->tc;
288
289         scu_ssp_reqeust_construct_task_context(sci_req, task_context);
290
291         task_context->ssp_command_iu_length =
292                 sizeof(struct ssp_cmd_iu) / sizeof(u32);
293         task_context->type.ssp.frame_type = SSP_COMMAND;
294
295         switch (dir) {
296         case DMA_FROM_DEVICE:
297         case DMA_NONE:
298         default:
299                 task_context->task_type = SCU_TASK_TYPE_IOREAD;
300                 break;
301         case DMA_TO_DEVICE:
302                 task_context->task_type = SCU_TASK_TYPE_IOWRITE;
303                 break;
304         }
305
306         task_context->transfer_length_bytes = len;
307
308         if (task_context->transfer_length_bytes > 0)
309                 scic_sds_request_build_sgl(sci_req);
310 }
311
312 /**
313  * This method will fill in the SCU Task Context for a SSP Task request.  The
314  *    following important settings are utilized: -# priority ==
315  *    SCU_TASK_PRIORITY_HIGH.  This ensures that the task request is issued
316  *    ahead of other task destined for the same Remote Node. -# task_type ==
317  *    SCU_TASK_TYPE_IOREAD.  This simply indicates that a normal request type
318  *    (i.e. non-raw frame) is being utilized to perform task management. -#
319  *    control_frame == 1.  This ensures that the proper endianess is set so
320  *    that the bytes are transmitted in the right order for a task frame.
321  * @sci_req: This parameter specifies the task request object being
322  *    constructed.
323  *
324  */
325 static void scu_ssp_task_request_construct_task_context(struct scic_sds_request *sci_req)
326 {
327         struct scu_task_context *task_context = sci_req->tc;
328
329         scu_ssp_reqeust_construct_task_context(sci_req, task_context);
330
331         task_context->control_frame                = 1;
332         task_context->priority                     = SCU_TASK_PRIORITY_HIGH;
333         task_context->task_type                    = SCU_TASK_TYPE_RAW_FRAME;
334         task_context->transfer_length_bytes        = 0;
335         task_context->type.ssp.frame_type          = SSP_TASK;
336         task_context->ssp_command_iu_length =
337                 sizeof(struct ssp_task_iu) / sizeof(u32);
338 }
339
340 /**
341  * This method is will fill in the SCU Task Context for any type of SATA
342  *    request.  This is called from the various SATA constructors.
343  * @sci_req: The general IO request object which is to be used in
344  *    constructing the SCU task context.
345  * @task_context: The buffer pointer for the SCU task context which is being
346  *    constructed.
347  *
348  * The general io request construction is complete. The buffer assignment for
349  * the command buffer is complete. none Revisit task context construction to
350  * determine what is common for SSP/SMP/STP task context structures.
351  */
352 static void scu_sata_reqeust_construct_task_context(
353         struct scic_sds_request *sci_req,
354         struct scu_task_context *task_context)
355 {
356         dma_addr_t dma_addr;
357         struct scic_sds_remote_device *target_device;
358         struct scic_sds_port *target_port;
359
360         target_device = scic_sds_request_get_device(sci_req);
361         target_port = scic_sds_request_get_port(sci_req);
362
363         /* Fill in the TC with the its required data */
364         task_context->abort = 0;
365         task_context->priority = SCU_TASK_PRIORITY_NORMAL;
366         task_context->initiator_request = 1;
367         task_context->connection_rate = target_device->connection_rate;
368         task_context->protocol_engine_index =
369                 scic_sds_controller_get_protocol_engine_group(controller);
370         task_context->logical_port_index =
371                 scic_sds_port_get_index(target_port);
372         task_context->protocol_type = SCU_TASK_CONTEXT_PROTOCOL_STP;
373         task_context->valid = SCU_TASK_CONTEXT_VALID;
374         task_context->context_type = SCU_TASK_CONTEXT_TYPE;
375
376         task_context->remote_node_index =
377                 scic_sds_remote_device_get_index(sci_req->target_device);
378         task_context->command_code = 0;
379
380         task_context->link_layer_control = 0;
381         task_context->do_not_dma_ssp_good_response = 1;
382         task_context->strict_ordering = 0;
383         task_context->control_frame = 0;
384         task_context->timeout_enable = 0;
385         task_context->block_guard_enable = 0;
386
387         task_context->address_modifier = 0;
388         task_context->task_phase = 0x01;
389
390         task_context->ssp_command_iu_length =
391                 (sizeof(struct host_to_dev_fis) - sizeof(u32)) / sizeof(u32);
392
393         /* Set the first word of the H2D REG FIS */
394         task_context->type.words[0] = *(u32 *)&sci_req->stp.cmd;
395
396         sci_req->post_context = (SCU_CONTEXT_COMMAND_REQUEST_TYPE_POST_TC |
397                                  (scic_sds_controller_get_protocol_engine_group(controller) <<
398                                   SCU_CONTEXT_COMMAND_PROTOCOL_ENGINE_GROUP_SHIFT) |
399                                  (scic_sds_port_get_index(target_port) <<
400                                   SCU_CONTEXT_COMMAND_LOGICAL_PORT_SHIFT) |
401                                  ISCI_TAG_TCI(sci_req->io_tag));
402         /*
403          * Copy the physical address for the command buffer to the SCU Task
404          * Context. We must offset the command buffer by 4 bytes because the
405          * first 4 bytes are transfered in the body of the TC.
406          */
407         dma_addr = scic_io_request_get_dma_addr(sci_req,
408                                                 ((char *) &sci_req->stp.cmd) +
409                                                 sizeof(u32));
410
411         task_context->command_iu_upper = upper_32_bits(dma_addr);
412         task_context->command_iu_lower = lower_32_bits(dma_addr);
413
414         /* SATA Requests do not have a response buffer */
415         task_context->response_iu_upper = 0;
416         task_context->response_iu_lower = 0;
417 }
418
419 static void scu_stp_raw_request_construct_task_context(struct scic_sds_request *sci_req)
420 {
421         struct scu_task_context *task_context = sci_req->tc;
422
423         scu_sata_reqeust_construct_task_context(sci_req, task_context);
424
425         task_context->control_frame         = 0;
426         task_context->priority              = SCU_TASK_PRIORITY_NORMAL;
427         task_context->task_type             = SCU_TASK_TYPE_SATA_RAW_FRAME;
428         task_context->type.stp.fis_type     = FIS_REGH2D;
429         task_context->transfer_length_bytes = sizeof(struct host_to_dev_fis) - sizeof(u32);
430 }
431
432 static enum sci_status
433 scic_sds_stp_pio_request_construct(struct scic_sds_request *sci_req,
434                                    bool copy_rx_frame)
435 {
436         struct scic_sds_stp_request *stp_req = &sci_req->stp.req;
437         struct scic_sds_stp_pio_request *pio = &stp_req->type.pio;
438
439         scu_stp_raw_request_construct_task_context(sci_req);
440
441         pio->current_transfer_bytes = 0;
442         pio->ending_error = 0;
443         pio->ending_status = 0;
444
445         pio->request_current.sgl_offset = 0;
446         pio->request_current.sgl_set = SCU_SGL_ELEMENT_PAIR_A;
447
448         if (copy_rx_frame) {
449                 scic_sds_request_build_sgl(sci_req);
450                 pio->request_current.sgl_index = 0;
451         } else {
452                 /* The user does not want the data copied to the SGL buffer location */
453                 pio->request_current.sgl_index = -1;
454         }
455
456         return SCI_SUCCESS;
457 }
458
459 /**
460  *
461  * @sci_req: This parameter specifies the request to be constructed as an
462  *    optimized request.
463  * @optimized_task_type: This parameter specifies whether the request is to be
464  *    an UDMA request or a NCQ request. - A value of 0 indicates UDMA. - A
465  *    value of 1 indicates NCQ.
466  *
467  * This method will perform request construction common to all types of STP
468  * requests that are optimized by the silicon (i.e. UDMA, NCQ). This method
469  * returns an indication as to whether the construction was successful.
470  */
471 static void scic_sds_stp_optimized_request_construct(struct scic_sds_request *sci_req,
472                                                      u8 optimized_task_type,
473                                                      u32 len,
474                                                      enum dma_data_direction dir)
475 {
476         struct scu_task_context *task_context = sci_req->tc;
477
478         /* Build the STP task context structure */
479         scu_sata_reqeust_construct_task_context(sci_req, task_context);
480
481         /* Copy over the SGL elements */
482         scic_sds_request_build_sgl(sci_req);
483
484         /* Copy over the number of bytes to be transfered */
485         task_context->transfer_length_bytes = len;
486
487         if (dir == DMA_TO_DEVICE) {
488                 /*
489                  * The difference between the DMA IN and DMA OUT request task type
490                  * values are consistent with the difference between FPDMA READ
491                  * and FPDMA WRITE values.  Add the supplied task type parameter
492                  * to this difference to set the task type properly for this
493                  * DATA OUT (WRITE) case. */
494                 task_context->task_type = optimized_task_type + (SCU_TASK_TYPE_DMA_OUT
495                                                                  - SCU_TASK_TYPE_DMA_IN);
496         } else {
497                 /*
498                  * For the DATA IN (READ) case, simply save the supplied
499                  * optimized task type. */
500                 task_context->task_type = optimized_task_type;
501         }
502 }
503
504
505
506 static enum sci_status
507 scic_io_request_construct_sata(struct scic_sds_request *sci_req,
508                                u32 len,
509                                enum dma_data_direction dir,
510                                bool copy)
511 {
512         enum sci_status status = SCI_SUCCESS;
513         struct isci_request *ireq = sci_req_to_ireq(sci_req);
514         struct sas_task *task = isci_request_access_task(ireq);
515
516         /* check for management protocols */
517         if (ireq->ttype == tmf_task) {
518                 struct isci_tmf *tmf = isci_request_access_tmf(ireq);
519
520                 if (tmf->tmf_code == isci_tmf_sata_srst_high ||
521                     tmf->tmf_code == isci_tmf_sata_srst_low) {
522                         scu_stp_raw_request_construct_task_context(sci_req);
523                         return SCI_SUCCESS;
524                 } else {
525                         dev_err(scic_to_dev(sci_req->owning_controller),
526                                 "%s: Request 0x%p received un-handled SAT "
527                                 "management protocol 0x%x.\n",
528                                 __func__, sci_req, tmf->tmf_code);
529
530                         return SCI_FAILURE;
531                 }
532         }
533
534         if (!sas_protocol_ata(task->task_proto)) {
535                 dev_err(scic_to_dev(sci_req->owning_controller),
536                         "%s: Non-ATA protocol in SATA path: 0x%x\n",
537                         __func__,
538                         task->task_proto);
539                 return SCI_FAILURE;
540
541         }
542
543         /* non data */
544         if (task->data_dir == DMA_NONE) {
545                 scu_stp_raw_request_construct_task_context(sci_req);
546                 return SCI_SUCCESS;
547         }
548
549         /* NCQ */
550         if (task->ata_task.use_ncq) {
551                 scic_sds_stp_optimized_request_construct(sci_req,
552                                                          SCU_TASK_TYPE_FPDMAQ_READ,
553                                                          len, dir);
554                 return SCI_SUCCESS;
555         }
556
557         /* DMA */
558         if (task->ata_task.dma_xfer) {
559                 scic_sds_stp_optimized_request_construct(sci_req,
560                                                          SCU_TASK_TYPE_DMA_IN,
561                                                          len, dir);
562                 return SCI_SUCCESS;
563         } else /* PIO */
564                 return scic_sds_stp_pio_request_construct(sci_req, copy);
565
566         return status;
567 }
568
569 static enum sci_status scic_io_request_construct_basic_ssp(struct scic_sds_request *sci_req)
570 {
571         struct isci_request *ireq = sci_req_to_ireq(sci_req);
572         struct sas_task *task = isci_request_access_task(ireq);
573
574         sci_req->protocol = SCIC_SSP_PROTOCOL;
575
576         scu_ssp_io_request_construct_task_context(sci_req,
577                                                   task->data_dir,
578                                                   task->total_xfer_len);
579
580         scic_sds_io_request_build_ssp_command_iu(sci_req);
581
582         sci_change_state(&sci_req->sm, SCI_REQ_CONSTRUCTED);
583
584         return SCI_SUCCESS;
585 }
586
587 enum sci_status scic_task_request_construct_ssp(
588         struct scic_sds_request *sci_req)
589 {
590         /* Construct the SSP Task SCU Task Context */
591         scu_ssp_task_request_construct_task_context(sci_req);
592
593         /* Fill in the SSP Task IU */
594         scic_sds_task_request_build_ssp_task_iu(sci_req);
595
596         sci_change_state(&sci_req->sm, SCI_REQ_CONSTRUCTED);
597
598         return SCI_SUCCESS;
599 }
600
601 static enum sci_status scic_io_request_construct_basic_sata(struct scic_sds_request *sci_req)
602 {
603         enum sci_status status;
604         bool copy = false;
605         struct isci_request *isci_request = sci_req_to_ireq(sci_req);
606         struct sas_task *task = isci_request_access_task(isci_request);
607
608         sci_req->protocol = SCIC_STP_PROTOCOL;
609
610         copy = (task->data_dir == DMA_NONE) ? false : true;
611
612         status = scic_io_request_construct_sata(sci_req,
613                                                 task->total_xfer_len,
614                                                 task->data_dir,
615                                                 copy);
616
617         if (status == SCI_SUCCESS)
618                 sci_change_state(&sci_req->sm, SCI_REQ_CONSTRUCTED);
619
620         return status;
621 }
622
623 enum sci_status scic_task_request_construct_sata(struct scic_sds_request *sci_req)
624 {
625         enum sci_status status = SCI_SUCCESS;
626         struct isci_request *ireq = sci_req_to_ireq(sci_req);
627
628         /* check for management protocols */
629         if (ireq->ttype == tmf_task) {
630                 struct isci_tmf *tmf = isci_request_access_tmf(ireq);
631
632                 if (tmf->tmf_code == isci_tmf_sata_srst_high ||
633                     tmf->tmf_code == isci_tmf_sata_srst_low) {
634                         scu_stp_raw_request_construct_task_context(sci_req);
635                 } else {
636                         dev_err(scic_to_dev(sci_req->owning_controller),
637                                 "%s: Request 0x%p received un-handled SAT "
638                                 "Protocol 0x%x.\n",
639                                 __func__, sci_req, tmf->tmf_code);
640
641                         return SCI_FAILURE;
642                 }
643         }
644
645         if (status != SCI_SUCCESS)
646                 return status;
647         sci_change_state(&sci_req->sm, SCI_REQ_CONSTRUCTED);
648
649         return status;
650 }
651
652 /**
653  * sci_req_tx_bytes - bytes transferred when reply underruns request
654  * @sci_req: request that was terminated early
655  */
656 #define SCU_TASK_CONTEXT_SRAM 0x200000
657 static u32 sci_req_tx_bytes(struct scic_sds_request *sci_req)
658 {
659         struct scic_sds_controller *scic = sci_req->owning_controller;
660         u32 ret_val = 0;
661
662         if (readl(&scic->smu_registers->address_modifier) == 0) {
663                 void __iomem *scu_reg_base = scic->scu_registers;
664
665                 /* get the bytes of data from the Address == BAR1 + 20002Ch + (256*TCi) where
666                  *   BAR1 is the scu_registers
667                  *   0x20002C = 0x200000 + 0x2c
668                  *            = start of task context SRAM + offset of (type.ssp.data_offset)
669                  *   TCi is the io_tag of struct scic_sds_request
670                  */
671                 ret_val = readl(scu_reg_base +
672                                 (SCU_TASK_CONTEXT_SRAM + offsetof(struct scu_task_context, type.ssp.data_offset)) +
673                                 ((sizeof(struct scu_task_context)) * ISCI_TAG_TCI(sci_req->io_tag)));
674         }
675
676         return ret_val;
677 }
678
679 enum sci_status scic_sds_request_start(struct scic_sds_request *sci_req)
680 {
681         enum sci_base_request_states state;
682         struct scu_task_context *tc = sci_req->tc;
683         struct scic_sds_controller *scic = sci_req->owning_controller;
684
685         state = sci_req->sm.current_state_id;
686         if (state != SCI_REQ_CONSTRUCTED) {
687                 dev_warn(scic_to_dev(scic),
688                         "%s: SCIC IO Request requested to start while in wrong "
689                          "state %d\n", __func__, state);
690                 return SCI_FAILURE_INVALID_STATE;
691         }
692
693         tc->task_index = ISCI_TAG_TCI(sci_req->io_tag);
694
695         switch (tc->protocol_type) {
696         case SCU_TASK_CONTEXT_PROTOCOL_SMP:
697         case SCU_TASK_CONTEXT_PROTOCOL_SSP:
698                 /* SSP/SMP Frame */
699                 tc->type.ssp.tag = sci_req->io_tag;
700                 tc->type.ssp.target_port_transfer_tag = 0xFFFF;
701                 break;
702
703         case SCU_TASK_CONTEXT_PROTOCOL_STP:
704                 /* STP/SATA Frame
705                  * tc->type.stp.ncq_tag = sci_req->ncq_tag;
706                  */
707                 break;
708
709         case SCU_TASK_CONTEXT_PROTOCOL_NONE:
710                 /* / @todo When do we set no protocol type? */
711                 break;
712
713         default:
714                 /* This should never happen since we build the IO
715                  * requests */
716                 break;
717         }
718
719         /* Add to the post_context the io tag value */
720         sci_req->post_context |= ISCI_TAG_TCI(sci_req->io_tag);
721
722         /* Everything is good go ahead and change state */
723         sci_change_state(&sci_req->sm, SCI_REQ_STARTED);
724
725         return SCI_SUCCESS;
726 }
727
728 enum sci_status
729 scic_sds_io_request_terminate(struct scic_sds_request *sci_req)
730 {
731         enum sci_base_request_states state;
732
733         state = sci_req->sm.current_state_id;
734
735         switch (state) {
736         case SCI_REQ_CONSTRUCTED:
737                 scic_sds_request_set_status(sci_req,
738                         SCU_TASK_DONE_TASK_ABORT,
739                         SCI_FAILURE_IO_TERMINATED);
740
741                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
742                 return SCI_SUCCESS;
743         case SCI_REQ_STARTED:
744         case SCI_REQ_TASK_WAIT_TC_COMP:
745         case SCI_REQ_SMP_WAIT_RESP:
746         case SCI_REQ_SMP_WAIT_TC_COMP:
747         case SCI_REQ_STP_UDMA_WAIT_TC_COMP:
748         case SCI_REQ_STP_UDMA_WAIT_D2H:
749         case SCI_REQ_STP_NON_DATA_WAIT_H2D:
750         case SCI_REQ_STP_NON_DATA_WAIT_D2H:
751         case SCI_REQ_STP_PIO_WAIT_H2D:
752         case SCI_REQ_STP_PIO_WAIT_FRAME:
753         case SCI_REQ_STP_PIO_DATA_IN:
754         case SCI_REQ_STP_PIO_DATA_OUT:
755         case SCI_REQ_STP_SOFT_RESET_WAIT_H2D_ASSERTED:
756         case SCI_REQ_STP_SOFT_RESET_WAIT_H2D_DIAG:
757         case SCI_REQ_STP_SOFT_RESET_WAIT_D2H:
758                 sci_change_state(&sci_req->sm, SCI_REQ_ABORTING);
759                 return SCI_SUCCESS;
760         case SCI_REQ_TASK_WAIT_TC_RESP:
761                 sci_change_state(&sci_req->sm, SCI_REQ_ABORTING);
762                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
763                 return SCI_SUCCESS;
764         case SCI_REQ_ABORTING:
765                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
766                 return SCI_SUCCESS;
767         case SCI_REQ_COMPLETED:
768         default:
769                 dev_warn(scic_to_dev(sci_req->owning_controller),
770                          "%s: SCIC IO Request requested to abort while in wrong "
771                          "state %d\n",
772                          __func__,
773                          sci_req->sm.current_state_id);
774                 break;
775         }
776
777         return SCI_FAILURE_INVALID_STATE;
778 }
779
780 enum sci_status scic_sds_request_complete(struct scic_sds_request *sci_req)
781 {
782         enum sci_base_request_states state;
783         struct scic_sds_controller *scic = sci_req->owning_controller;
784
785         state = sci_req->sm.current_state_id;
786         if (WARN_ONCE(state != SCI_REQ_COMPLETED,
787                       "isci: request completion from wrong state (%d)\n", state))
788                 return SCI_FAILURE_INVALID_STATE;
789
790         if (sci_req->saved_rx_frame_index != SCU_INVALID_FRAME_INDEX)
791                 scic_sds_controller_release_frame(scic,
792                                                   sci_req->saved_rx_frame_index);
793
794         /* XXX can we just stop the machine and remove the 'final' state? */
795         sci_change_state(&sci_req->sm, SCI_REQ_FINAL);
796         return SCI_SUCCESS;
797 }
798
799 enum sci_status scic_sds_io_request_event_handler(struct scic_sds_request *sci_req,
800                                                   u32 event_code)
801 {
802         enum sci_base_request_states state;
803         struct scic_sds_controller *scic = sci_req->owning_controller;
804
805         state = sci_req->sm.current_state_id;
806
807         if (state != SCI_REQ_STP_PIO_DATA_IN) {
808                 dev_warn(scic_to_dev(scic), "%s: (%x) in wrong state %d\n",
809                          __func__, event_code, state);
810
811                 return SCI_FAILURE_INVALID_STATE;
812         }
813
814         switch (scu_get_event_specifier(event_code)) {
815         case SCU_TASK_DONE_CRC_ERR << SCU_EVENT_SPECIFIC_CODE_SHIFT:
816                 /* We are waiting for data and the SCU has R_ERR the data frame.
817                  * Go back to waiting for the D2H Register FIS
818                  */
819                 sci_change_state(&sci_req->sm, SCI_REQ_STP_PIO_WAIT_FRAME);
820                 return SCI_SUCCESS;
821         default:
822                 dev_err(scic_to_dev(scic),
823                         "%s: pio request unexpected event %#x\n",
824                         __func__, event_code);
825
826                 /* TODO Should we fail the PIO request when we get an
827                  * unexpected event?
828                  */
829                 return SCI_FAILURE;
830         }
831 }
832
833 /*
834  * This function copies response data for requests returning response data
835  *    instead of sense data.
836  * @sci_req: This parameter specifies the request object for which to copy
837  *    the response data.
838  */
839 static void scic_sds_io_request_copy_response(struct scic_sds_request *sci_req)
840 {
841         void *resp_buf;
842         u32 len;
843         struct ssp_response_iu *ssp_response;
844         struct isci_request *ireq = sci_req_to_ireq(sci_req);
845         struct isci_tmf *isci_tmf = isci_request_access_tmf(ireq);
846
847         ssp_response = &sci_req->ssp.rsp;
848
849         resp_buf = &isci_tmf->resp.resp_iu;
850
851         len = min_t(u32,
852                     SSP_RESP_IU_MAX_SIZE,
853                     be32_to_cpu(ssp_response->response_data_len));
854
855         memcpy(resp_buf, ssp_response->resp_data, len);
856 }
857
858 static enum sci_status
859 request_started_state_tc_event(struct scic_sds_request *sci_req,
860                                u32 completion_code)
861 {
862         struct ssp_response_iu *resp_iu;
863         u8 datapres;
864
865         /* TODO: Any SDMA return code of other than 0 is bad decode 0x003C0000
866          * to determine SDMA status
867          */
868         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
869         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
870                 scic_sds_request_set_status(sci_req,
871                                             SCU_TASK_DONE_GOOD,
872                                             SCI_SUCCESS);
873                 break;
874         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_EARLY_RESP): {
875                 /* There are times when the SCU hardware will return an early
876                  * response because the io request specified more data than is
877                  * returned by the target device (mode pages, inquiry data,
878                  * etc.).  We must check the response stats to see if this is
879                  * truly a failed request or a good request that just got
880                  * completed early.
881                  */
882                 struct ssp_response_iu *resp = &sci_req->ssp.rsp;
883                 ssize_t word_cnt = SSP_RESP_IU_MAX_SIZE / sizeof(u32);
884
885                 sci_swab32_cpy(&sci_req->ssp.rsp,
886                                &sci_req->ssp.rsp,
887                                word_cnt);
888
889                 if (resp->status == 0) {
890                         scic_sds_request_set_status(sci_req,
891                                                     SCU_TASK_DONE_GOOD,
892                                                     SCI_SUCCESS_IO_DONE_EARLY);
893                 } else {
894                         scic_sds_request_set_status(sci_req,
895                                                     SCU_TASK_DONE_CHECK_RESPONSE,
896                                                     SCI_FAILURE_IO_RESPONSE_VALID);
897                 }
898                 break;
899         }
900         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_CHECK_RESPONSE): {
901                 ssize_t word_cnt = SSP_RESP_IU_MAX_SIZE / sizeof(u32);
902
903                 sci_swab32_cpy(&sci_req->ssp.rsp,
904                                &sci_req->ssp.rsp,
905                                word_cnt);
906
907                 scic_sds_request_set_status(sci_req,
908                                             SCU_TASK_DONE_CHECK_RESPONSE,
909                                             SCI_FAILURE_IO_RESPONSE_VALID);
910                 break;
911         }
912
913         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_RESP_LEN_ERR):
914                 /* TODO With TASK_DONE_RESP_LEN_ERR is the response frame
915                  * guaranteed to be received before this completion status is
916                  * posted?
917                  */
918                 resp_iu = &sci_req->ssp.rsp;
919                 datapres = resp_iu->datapres;
920
921                 if (datapres == 1 || datapres == 2) {
922                         scic_sds_request_set_status(sci_req,
923                                                     SCU_TASK_DONE_CHECK_RESPONSE,
924                                                     SCI_FAILURE_IO_RESPONSE_VALID);
925                 } else
926                         scic_sds_request_set_status(sci_req,
927                                                     SCU_TASK_DONE_GOOD,
928                                                     SCI_SUCCESS);
929                 break;
930         /* only stp device gets suspended. */
931         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_ACK_NAK_TO):
932         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_LL_PERR):
933         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_NAK_ERR):
934         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_DATA_LEN_ERR):
935         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_LL_ABORT_ERR):
936         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_XR_WD_LEN):
937         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_MAX_PLD_ERR):
938         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_UNEXP_RESP):
939         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_UNEXP_SDBFIS):
940         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_REG_ERR):
941         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SDB_ERR):
942                 if (sci_req->protocol == SCIC_STP_PROTOCOL) {
943                         scic_sds_request_set_status(sci_req,
944                                 SCU_GET_COMPLETION_TL_STATUS(completion_code) >>
945                                 SCU_COMPLETION_TL_STATUS_SHIFT,
946                                 SCI_FAILURE_REMOTE_DEVICE_RESET_REQUIRED);
947                 } else {
948                         scic_sds_request_set_status(sci_req,
949                                 SCU_GET_COMPLETION_TL_STATUS(completion_code) >>
950                                 SCU_COMPLETION_TL_STATUS_SHIFT,
951                                 SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR);
952                 }
953                 break;
954
955         /* both stp/ssp device gets suspended */
956         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_LF_ERR):
957         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_WRONG_DESTINATION):
958         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_1):
959         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_2):
960         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_3):
961         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_BAD_DESTINATION):
962         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_ZONE_VIOLATION):
963         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_STP_RESOURCES_BUSY):
964         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_PROTOCOL_NOT_SUPPORTED):
965         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_CONNECTION_RATE_NOT_SUPPORTED):
966                 scic_sds_request_set_status(sci_req,
967                                             SCU_GET_COMPLETION_TL_STATUS(completion_code) >>
968                                             SCU_COMPLETION_TL_STATUS_SHIFT,
969                                             SCI_FAILURE_REMOTE_DEVICE_RESET_REQUIRED);
970                 break;
971
972         /* neither ssp nor stp gets suspended. */
973         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_NAK_CMD_ERR):
974         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_UNEXP_XR):
975         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_XR_IU_LEN_ERR):
976         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SDMA_ERR):
977         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_OFFSET_ERR):
978         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_EXCESS_DATA):
979         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_RESP_TO_ERR):
980         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_UFI_ERR):
981         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_FRM_TYPE_ERR):
982         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_LL_RX_ERR):
983         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_UNEXP_DATA):
984         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_OPEN_FAIL):
985         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_VIIT_ENTRY_NV):
986         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_IIT_ENTRY_NV):
987         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_RNCNV_OUTBOUND):
988         default:
989                 scic_sds_request_set_status(
990                         sci_req,
991                         SCU_GET_COMPLETION_TL_STATUS(completion_code) >>
992                         SCU_COMPLETION_TL_STATUS_SHIFT,
993                         SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR);
994                 break;
995         }
996
997         /*
998          * TODO: This is probably wrong for ACK/NAK timeout conditions
999          */
1000
1001         /* In all cases we will treat this as the completion of the IO req. */
1002         sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1003         return SCI_SUCCESS;
1004 }
1005
1006 static enum sci_status
1007 request_aborting_state_tc_event(struct scic_sds_request *sci_req,
1008                                 u32 completion_code)
1009 {
1010         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1011         case (SCU_TASK_DONE_GOOD << SCU_COMPLETION_TL_STATUS_SHIFT):
1012         case (SCU_TASK_DONE_TASK_ABORT << SCU_COMPLETION_TL_STATUS_SHIFT):
1013                 scic_sds_request_set_status(sci_req, SCU_TASK_DONE_TASK_ABORT,
1014                                             SCI_FAILURE_IO_TERMINATED);
1015
1016                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1017                 break;
1018
1019         default:
1020                 /* Unless we get some strange error wait for the task abort to complete
1021                  * TODO: Should there be a state change for this completion?
1022                  */
1023                 break;
1024         }
1025
1026         return SCI_SUCCESS;
1027 }
1028
1029 static enum sci_status ssp_task_request_await_tc_event(struct scic_sds_request *sci_req,
1030                                                        u32 completion_code)
1031 {
1032         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1033         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
1034                 scic_sds_request_set_status(sci_req, SCU_TASK_DONE_GOOD,
1035                                             SCI_SUCCESS);
1036
1037                 sci_change_state(&sci_req->sm, SCI_REQ_TASK_WAIT_TC_RESP);
1038                 break;
1039         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_ACK_NAK_TO):
1040                 /* Currently, the decision is to simply allow the task request
1041                  * to timeout if the task IU wasn't received successfully.
1042                  * There is a potential for receiving multiple task responses if
1043                  * we decide to send the task IU again.
1044                  */
1045                 dev_warn(scic_to_dev(sci_req->owning_controller),
1046                          "%s: TaskRequest:0x%p CompletionCode:%x - "
1047                          "ACK/NAK timeout\n", __func__, sci_req,
1048                          completion_code);
1049
1050                 sci_change_state(&sci_req->sm, SCI_REQ_TASK_WAIT_TC_RESP);
1051                 break;
1052         default:
1053                 /*
1054                  * All other completion status cause the IO to be complete.
1055                  * If a NAK was received, then it is up to the user to retry
1056                  * the request.
1057                  */
1058                 scic_sds_request_set_status(sci_req,
1059                         SCU_NORMALIZE_COMPLETION_STATUS(completion_code),
1060                         SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR);
1061
1062                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1063                 break;
1064         }
1065
1066         return SCI_SUCCESS;
1067 }
1068
1069 static enum sci_status
1070 smp_request_await_response_tc_event(struct scic_sds_request *sci_req,
1071                                     u32 completion_code)
1072 {
1073         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1074         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
1075                 /* In the AWAIT RESPONSE state, any TC completion is
1076                  * unexpected.  but if the TC has success status, we
1077                  * complete the IO anyway.
1078                  */
1079                 scic_sds_request_set_status(sci_req, SCU_TASK_DONE_GOOD,
1080                                             SCI_SUCCESS);
1081
1082                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1083                 break;
1084
1085         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_RESP_TO_ERR):
1086         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_UFI_ERR):
1087         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_FRM_TYPE_ERR):
1088         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_LL_RX_ERR):
1089                 /* These status has been seen in a specific LSI
1090                  * expander, which sometimes is not able to send smp
1091                  * response within 2 ms. This causes our hardware break
1092                  * the connection and set TC completion with one of
1093                  * these SMP_XXX_XX_ERR status. For these type of error,
1094                  * we ask scic user to retry the request.
1095                  */
1096                 scic_sds_request_set_status(sci_req, SCU_TASK_DONE_SMP_RESP_TO_ERR,
1097                                             SCI_FAILURE_RETRY_REQUIRED);
1098
1099                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1100                 break;
1101
1102         default:
1103                 /* All other completion status cause the IO to be complete.  If a NAK
1104                  * was received, then it is up to the user to retry the request
1105                  */
1106                 scic_sds_request_set_status(sci_req,
1107                                             SCU_NORMALIZE_COMPLETION_STATUS(completion_code),
1108                                             SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR);
1109
1110                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1111                 break;
1112         }
1113
1114         return SCI_SUCCESS;
1115 }
1116
1117 static enum sci_status
1118 smp_request_await_tc_event(struct scic_sds_request *sci_req,
1119                            u32 completion_code)
1120 {
1121         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1122         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
1123                 scic_sds_request_set_status(sci_req, SCU_TASK_DONE_GOOD,
1124                                             SCI_SUCCESS);
1125
1126                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1127                 break;
1128         default:
1129                 /* All other completion status cause the IO to be
1130                  * complete.  If a NAK was received, then it is up to
1131                  * the user to retry the request.
1132                  */
1133                 scic_sds_request_set_status(sci_req,
1134                                             SCU_NORMALIZE_COMPLETION_STATUS(completion_code),
1135                                             SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR);
1136
1137                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1138                 break;
1139         }
1140
1141         return SCI_SUCCESS;
1142 }
1143
1144 void scic_stp_io_request_set_ncq_tag(struct scic_sds_request *req,
1145                                      u16 ncq_tag)
1146 {
1147         /**
1148          * @note This could be made to return an error to the user if the user
1149          *       attempts to set the NCQ tag in the wrong state.
1150          */
1151         req->tc->type.stp.ncq_tag = ncq_tag;
1152 }
1153
1154 static struct scu_sgl_element *pio_sgl_next(struct scic_sds_stp_request *stp_req)
1155 {
1156         struct scu_sgl_element *sgl;
1157         struct scu_sgl_element_pair *sgl_pair;
1158         struct scic_sds_request *sci_req = to_sci_req(stp_req);
1159         struct scic_sds_request_pio_sgl *pio_sgl = &stp_req->type.pio.request_current;
1160
1161         sgl_pair = to_sgl_element_pair(sci_req, pio_sgl->sgl_index);
1162         if (!sgl_pair)
1163                 sgl = NULL;
1164         else if (pio_sgl->sgl_set == SCU_SGL_ELEMENT_PAIR_A) {
1165                 if (sgl_pair->B.address_lower == 0 &&
1166                     sgl_pair->B.address_upper == 0) {
1167                         sgl = NULL;
1168                 } else {
1169                         pio_sgl->sgl_set = SCU_SGL_ELEMENT_PAIR_B;
1170                         sgl = &sgl_pair->B;
1171                 }
1172         } else {
1173                 if (sgl_pair->next_pair_lower == 0 &&
1174                     sgl_pair->next_pair_upper == 0) {
1175                         sgl = NULL;
1176                 } else {
1177                         pio_sgl->sgl_index++;
1178                         pio_sgl->sgl_set = SCU_SGL_ELEMENT_PAIR_A;
1179                         sgl_pair = to_sgl_element_pair(sci_req, pio_sgl->sgl_index);
1180                         sgl = &sgl_pair->A;
1181                 }
1182         }
1183
1184         return sgl;
1185 }
1186
1187 static enum sci_status
1188 stp_request_non_data_await_h2d_tc_event(struct scic_sds_request *sci_req,
1189                                         u32 completion_code)
1190 {
1191         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1192         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
1193                 scic_sds_request_set_status(sci_req, SCU_TASK_DONE_GOOD,
1194                                             SCI_SUCCESS);
1195
1196                 sci_change_state(&sci_req->sm, SCI_REQ_STP_NON_DATA_WAIT_D2H);
1197                 break;
1198
1199         default:
1200                 /* All other completion status cause the IO to be
1201                  * complete.  If a NAK was received, then it is up to
1202                  * the user to retry the request.
1203                  */
1204                 scic_sds_request_set_status(sci_req,
1205                                             SCU_NORMALIZE_COMPLETION_STATUS(completion_code),
1206                                             SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR);
1207
1208                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1209                 break;
1210         }
1211
1212         return SCI_SUCCESS;
1213 }
1214
1215 #define SCU_MAX_FRAME_BUFFER_SIZE  0x400  /* 1K is the maximum SCU frame data payload */
1216
1217 /* transmit DATA_FIS from (current sgl + offset) for input
1218  * parameter length. current sgl and offset is alreay stored in the IO request
1219  */
1220 static enum sci_status scic_sds_stp_request_pio_data_out_trasmit_data_frame(
1221         struct scic_sds_request *sci_req,
1222         u32 length)
1223 {
1224         struct scic_sds_stp_request *stp_req = &sci_req->stp.req;
1225         struct scu_task_context *task_context = sci_req->tc;
1226         struct scu_sgl_element_pair *sgl_pair;
1227         struct scu_sgl_element *current_sgl;
1228
1229         /* Recycle the TC and reconstruct it for sending out DATA FIS containing
1230          * for the data from current_sgl+offset for the input length
1231          */
1232         sgl_pair = to_sgl_element_pair(sci_req, stp_req->type.pio.request_current.sgl_index);
1233         if (stp_req->type.pio.request_current.sgl_set == SCU_SGL_ELEMENT_PAIR_A)
1234                 current_sgl = &sgl_pair->A;
1235         else
1236                 current_sgl = &sgl_pair->B;
1237
1238         /* update the TC */
1239         task_context->command_iu_upper = current_sgl->address_upper;
1240         task_context->command_iu_lower = current_sgl->address_lower;
1241         task_context->transfer_length_bytes = length;
1242         task_context->type.stp.fis_type = FIS_DATA;
1243
1244         /* send the new TC out. */
1245         return scic_controller_continue_io(sci_req);
1246 }
1247
1248 static enum sci_status scic_sds_stp_request_pio_data_out_transmit_data(struct scic_sds_request *sci_req)
1249 {
1250
1251         struct scu_sgl_element *current_sgl;
1252         u32 sgl_offset;
1253         u32 remaining_bytes_in_current_sgl = 0;
1254         enum sci_status status = SCI_SUCCESS;
1255         struct scic_sds_stp_request *stp_req = &sci_req->stp.req;
1256         struct scu_sgl_element_pair *sgl_pair;
1257
1258         sgl_offset = stp_req->type.pio.request_current.sgl_offset;
1259         sgl_pair = to_sgl_element_pair(sci_req, stp_req->type.pio.request_current.sgl_index);
1260         if (WARN_ONCE(!sgl_pair, "%s: null sgl element", __func__))
1261                 return SCI_FAILURE;
1262
1263         if (stp_req->type.pio.request_current.sgl_set == SCU_SGL_ELEMENT_PAIR_A) {
1264                 current_sgl = &sgl_pair->A;
1265                 remaining_bytes_in_current_sgl = sgl_pair->A.length - sgl_offset;
1266         } else {
1267                 current_sgl = &sgl_pair->B;
1268                 remaining_bytes_in_current_sgl = sgl_pair->B.length - sgl_offset;
1269         }
1270
1271         if (stp_req->type.pio.pio_transfer_bytes > 0) {
1272                 if (stp_req->type.pio.pio_transfer_bytes >= remaining_bytes_in_current_sgl) {
1273                         /* recycle the TC and send the H2D Data FIS from (current sgl + sgl_offset) and length = remaining_bytes_in_current_sgl */
1274                         status = scic_sds_stp_request_pio_data_out_trasmit_data_frame(sci_req, remaining_bytes_in_current_sgl);
1275                         if (status == SCI_SUCCESS) {
1276                                 stp_req->type.pio.pio_transfer_bytes -= remaining_bytes_in_current_sgl;
1277
1278                                 /* update the current sgl, sgl_offset and save for future */
1279                                 current_sgl = pio_sgl_next(stp_req);
1280                                 sgl_offset = 0;
1281                         }
1282                 } else if (stp_req->type.pio.pio_transfer_bytes < remaining_bytes_in_current_sgl) {
1283                         /* recycle the TC and send the H2D Data FIS from (current sgl + sgl_offset) and length = type.pio.pio_transfer_bytes */
1284                         scic_sds_stp_request_pio_data_out_trasmit_data_frame(sci_req, stp_req->type.pio.pio_transfer_bytes);
1285
1286                         if (status == SCI_SUCCESS) {
1287                                 /* Sgl offset will be adjusted and saved for future */
1288                                 sgl_offset += stp_req->type.pio.pio_transfer_bytes;
1289                                 current_sgl->address_lower += stp_req->type.pio.pio_transfer_bytes;
1290                                 stp_req->type.pio.pio_transfer_bytes = 0;
1291                         }
1292                 }
1293         }
1294
1295         if (status == SCI_SUCCESS) {
1296                 stp_req->type.pio.request_current.sgl_offset = sgl_offset;
1297         }
1298
1299         return status;
1300 }
1301
1302 /**
1303  *
1304  * @stp_request: The request that is used for the SGL processing.
1305  * @data_buffer: The buffer of data to be copied.
1306  * @length: The length of the data transfer.
1307  *
1308  * Copy the data from the buffer for the length specified to the IO reqeust SGL
1309  * specified data region. enum sci_status
1310  */
1311 static enum sci_status
1312 scic_sds_stp_request_pio_data_in_copy_data_buffer(struct scic_sds_stp_request *stp_req,
1313                                                   u8 *data_buf, u32 len)
1314 {
1315         struct scic_sds_request *sci_req;
1316         struct isci_request *ireq;
1317         u8 *src_addr;
1318         int copy_len;
1319         struct sas_task *task;
1320         struct scatterlist *sg;
1321         void *kaddr;
1322         int total_len = len;
1323
1324         sci_req = to_sci_req(stp_req);
1325         ireq = sci_req_to_ireq(sci_req);
1326         task = isci_request_access_task(ireq);
1327         src_addr = data_buf;
1328
1329         if (task->num_scatter > 0) {
1330                 sg = task->scatter;
1331
1332                 while (total_len > 0) {
1333                         struct page *page = sg_page(sg);
1334
1335                         copy_len = min_t(int, total_len, sg_dma_len(sg));
1336                         kaddr = kmap_atomic(page, KM_IRQ0);
1337                         memcpy(kaddr + sg->offset, src_addr, copy_len);
1338                         kunmap_atomic(kaddr, KM_IRQ0);
1339                         total_len -= copy_len;
1340                         src_addr += copy_len;
1341                         sg = sg_next(sg);
1342                 }
1343         } else {
1344                 BUG_ON(task->total_xfer_len < total_len);
1345                 memcpy(task->scatter, src_addr, total_len);
1346         }
1347
1348         return SCI_SUCCESS;
1349 }
1350
1351 /**
1352  *
1353  * @sci_req: The PIO DATA IN request that is to receive the data.
1354  * @data_buffer: The buffer to copy from.
1355  *
1356  * Copy the data buffer to the io request data region. enum sci_status
1357  */
1358 static enum sci_status scic_sds_stp_request_pio_data_in_copy_data(
1359         struct scic_sds_stp_request *sci_req,
1360         u8 *data_buffer)
1361 {
1362         enum sci_status status;
1363
1364         /*
1365          * If there is less than 1K remaining in the transfer request
1366          * copy just the data for the transfer */
1367         if (sci_req->type.pio.pio_transfer_bytes < SCU_MAX_FRAME_BUFFER_SIZE) {
1368                 status = scic_sds_stp_request_pio_data_in_copy_data_buffer(
1369                         sci_req, data_buffer, sci_req->type.pio.pio_transfer_bytes);
1370
1371                 if (status == SCI_SUCCESS)
1372                         sci_req->type.pio.pio_transfer_bytes = 0;
1373         } else {
1374                 /* We are transfering the whole frame so copy */
1375                 status = scic_sds_stp_request_pio_data_in_copy_data_buffer(
1376                         sci_req, data_buffer, SCU_MAX_FRAME_BUFFER_SIZE);
1377
1378                 if (status == SCI_SUCCESS)
1379                         sci_req->type.pio.pio_transfer_bytes -= SCU_MAX_FRAME_BUFFER_SIZE;
1380         }
1381
1382         return status;
1383 }
1384
1385 static enum sci_status
1386 stp_request_pio_await_h2d_completion_tc_event(struct scic_sds_request *sci_req,
1387                                               u32 completion_code)
1388 {
1389         enum sci_status status = SCI_SUCCESS;
1390
1391         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1392         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
1393                 scic_sds_request_set_status(sci_req,
1394                                             SCU_TASK_DONE_GOOD,
1395                                             SCI_SUCCESS);
1396
1397                 sci_change_state(&sci_req->sm, SCI_REQ_STP_PIO_WAIT_FRAME);
1398                 break;
1399
1400         default:
1401                 /* All other completion status cause the IO to be
1402                  * complete.  If a NAK was received, then it is up to
1403                  * the user to retry the request.
1404                  */
1405                 scic_sds_request_set_status(sci_req,
1406                                             SCU_NORMALIZE_COMPLETION_STATUS(completion_code),
1407                                             SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR);
1408
1409                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1410                 break;
1411         }
1412
1413         return status;
1414 }
1415
1416 static enum sci_status
1417 pio_data_out_tx_done_tc_event(struct scic_sds_request *sci_req,
1418                               u32 completion_code)
1419 {
1420         enum sci_status status = SCI_SUCCESS;
1421         bool all_frames_transferred = false;
1422         struct scic_sds_stp_request *stp_req = &sci_req->stp.req;
1423
1424         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1425         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
1426                 /* Transmit data */
1427                 if (stp_req->type.pio.pio_transfer_bytes != 0) {
1428                         status = scic_sds_stp_request_pio_data_out_transmit_data(sci_req);
1429                         if (status == SCI_SUCCESS) {
1430                                 if (stp_req->type.pio.pio_transfer_bytes == 0)
1431                                         all_frames_transferred = true;
1432                         }
1433                 } else if (stp_req->type.pio.pio_transfer_bytes == 0) {
1434                         /*
1435                          * this will happen if the all data is written at the
1436                          * first time after the pio setup fis is received
1437                          */
1438                         all_frames_transferred  = true;
1439                 }
1440
1441                 /* all data transferred. */
1442                 if (all_frames_transferred) {
1443                         /*
1444                          * Change the state to SCI_REQ_STP_PIO_DATA_IN
1445                          * and wait for PIO_SETUP fis / or D2H REg fis. */
1446                         sci_change_state(&sci_req->sm, SCI_REQ_STP_PIO_WAIT_FRAME);
1447                 }
1448                 break;
1449
1450         default:
1451                 /*
1452                  * All other completion status cause the IO to be complete.
1453                  * If a NAK was received, then it is up to the user to retry
1454                  * the request.
1455                  */
1456                 scic_sds_request_set_status(
1457                         sci_req,
1458                         SCU_NORMALIZE_COMPLETION_STATUS(completion_code),
1459                         SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR);
1460
1461                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1462                 break;
1463         }
1464
1465         return status;
1466 }
1467
1468 static void scic_sds_stp_request_udma_complete_request(
1469         struct scic_sds_request *request,
1470         u32 scu_status,
1471         enum sci_status sci_status)
1472 {
1473         scic_sds_request_set_status(request, scu_status, sci_status);
1474         sci_change_state(&request->sm, SCI_REQ_COMPLETED);
1475 }
1476
1477 static enum sci_status scic_sds_stp_request_udma_general_frame_handler(struct scic_sds_request *sci_req,
1478                                                                        u32 frame_index)
1479 {
1480         struct scic_sds_controller *scic = sci_req->owning_controller;
1481         struct dev_to_host_fis *frame_header;
1482         enum sci_status status;
1483         u32 *frame_buffer;
1484
1485         status = scic_sds_unsolicited_frame_control_get_header(&scic->uf_control,
1486                                                                frame_index,
1487                                                                (void **)&frame_header);
1488
1489         if ((status == SCI_SUCCESS) &&
1490             (frame_header->fis_type == FIS_REGD2H)) {
1491                 scic_sds_unsolicited_frame_control_get_buffer(&scic->uf_control,
1492                                                               frame_index,
1493                                                               (void **)&frame_buffer);
1494
1495                 scic_sds_controller_copy_sata_response(&sci_req->stp.rsp,
1496                                                        frame_header,
1497                                                        frame_buffer);
1498         }
1499
1500         scic_sds_controller_release_frame(scic, frame_index);
1501
1502         return status;
1503 }
1504
1505 enum sci_status
1506 scic_sds_io_request_frame_handler(struct scic_sds_request *sci_req,
1507                                   u32 frame_index)
1508 {
1509         struct scic_sds_controller *scic = sci_req->owning_controller;
1510         struct scic_sds_stp_request *stp_req = &sci_req->stp.req;
1511         enum sci_base_request_states state;
1512         enum sci_status status;
1513         ssize_t word_cnt;
1514
1515         state = sci_req->sm.current_state_id;
1516         switch (state)  {
1517         case SCI_REQ_STARTED: {
1518                 struct ssp_frame_hdr ssp_hdr;
1519                 void *frame_header;
1520
1521                 scic_sds_unsolicited_frame_control_get_header(&scic->uf_control,
1522                                                               frame_index,
1523                                                               &frame_header);
1524
1525                 word_cnt = sizeof(struct ssp_frame_hdr) / sizeof(u32);
1526                 sci_swab32_cpy(&ssp_hdr, frame_header, word_cnt);
1527
1528                 if (ssp_hdr.frame_type == SSP_RESPONSE) {
1529                         struct ssp_response_iu *resp_iu;
1530                         ssize_t word_cnt = SSP_RESP_IU_MAX_SIZE / sizeof(u32);
1531
1532                         scic_sds_unsolicited_frame_control_get_buffer(&scic->uf_control,
1533                                                                       frame_index,
1534                                                                       (void **)&resp_iu);
1535
1536                         sci_swab32_cpy(&sci_req->ssp.rsp, resp_iu, word_cnt);
1537
1538                         resp_iu = &sci_req->ssp.rsp;
1539
1540                         if (resp_iu->datapres == 0x01 ||
1541                             resp_iu->datapres == 0x02) {
1542                                 scic_sds_request_set_status(sci_req,
1543                                                             SCU_TASK_DONE_CHECK_RESPONSE,
1544                                                             SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR);
1545                         } else
1546                                 scic_sds_request_set_status(sci_req,
1547                                                             SCU_TASK_DONE_GOOD,
1548                                                             SCI_SUCCESS);
1549                 } else {
1550                         /* not a response frame, why did it get forwarded? */
1551                         dev_err(scic_to_dev(scic),
1552                                 "%s: SCIC IO Request 0x%p received unexpected "
1553                                 "frame %d type 0x%02x\n", __func__, sci_req,
1554                                 frame_index, ssp_hdr.frame_type);
1555                 }
1556
1557                 /*
1558                  * In any case we are done with this frame buffer return it to
1559                  * the controller
1560                  */
1561                 scic_sds_controller_release_frame(scic, frame_index);
1562
1563                 return SCI_SUCCESS;
1564         }
1565
1566         case SCI_REQ_TASK_WAIT_TC_RESP:
1567                 scic_sds_io_request_copy_response(sci_req);
1568                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1569                 scic_sds_controller_release_frame(scic,frame_index);
1570                 return SCI_SUCCESS;
1571
1572         case SCI_REQ_SMP_WAIT_RESP: {
1573                 struct smp_resp *rsp_hdr = &sci_req->smp.rsp;
1574                 void *frame_header;
1575
1576                 scic_sds_unsolicited_frame_control_get_header(&scic->uf_control,
1577                                                               frame_index,
1578                                                               &frame_header);
1579
1580                 /* byte swap the header. */
1581                 word_cnt = SMP_RESP_HDR_SZ / sizeof(u32);
1582                 sci_swab32_cpy(rsp_hdr, frame_header, word_cnt);
1583
1584                 if (rsp_hdr->frame_type == SMP_RESPONSE) {
1585                         void *smp_resp;
1586
1587                         scic_sds_unsolicited_frame_control_get_buffer(&scic->uf_control,
1588                                                                       frame_index,
1589                                                                       &smp_resp);
1590
1591                         word_cnt = (sizeof(struct smp_resp) - SMP_RESP_HDR_SZ) /
1592                                 sizeof(u32);
1593
1594                         sci_swab32_cpy(((u8 *) rsp_hdr) + SMP_RESP_HDR_SZ,
1595                                        smp_resp, word_cnt);
1596
1597                         scic_sds_request_set_status(sci_req, SCU_TASK_DONE_GOOD,
1598                                                     SCI_SUCCESS);
1599
1600                         sci_change_state(&sci_req->sm, SCI_REQ_SMP_WAIT_TC_COMP);
1601                 } else {
1602                         /*
1603                          * This was not a response frame why did it get
1604                          * forwarded?
1605                          */
1606                         dev_err(scic_to_dev(scic),
1607                                 "%s: SCIC SMP Request 0x%p received unexpected "
1608                                 "frame %d type 0x%02x\n",
1609                                 __func__,
1610                                 sci_req,
1611                                 frame_index,
1612                                 rsp_hdr->frame_type);
1613
1614                         scic_sds_request_set_status(sci_req,
1615                                                     SCU_TASK_DONE_SMP_FRM_TYPE_ERR,
1616                                                     SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR);
1617
1618                         sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1619                 }
1620
1621                 scic_sds_controller_release_frame(scic, frame_index);
1622
1623                 return SCI_SUCCESS;
1624         }
1625
1626         case SCI_REQ_STP_UDMA_WAIT_TC_COMP:
1627                 return scic_sds_stp_request_udma_general_frame_handler(sci_req,
1628                                                                        frame_index);
1629
1630         case SCI_REQ_STP_UDMA_WAIT_D2H:
1631                 /* Use the general frame handler to copy the resposne data */
1632                 status = scic_sds_stp_request_udma_general_frame_handler(sci_req,
1633                                                                          frame_index);
1634
1635                 if (status != SCI_SUCCESS)
1636                         return status;
1637
1638                 scic_sds_stp_request_udma_complete_request(sci_req,
1639                                                            SCU_TASK_DONE_CHECK_RESPONSE,
1640                                                            SCI_FAILURE_IO_RESPONSE_VALID);
1641
1642                 return SCI_SUCCESS;
1643
1644         case SCI_REQ_STP_NON_DATA_WAIT_D2H: {
1645                 struct dev_to_host_fis *frame_header;
1646                 u32 *frame_buffer;
1647
1648                 status = scic_sds_unsolicited_frame_control_get_header(&scic->uf_control,
1649                                                                        frame_index,
1650                                                                        (void **)&frame_header);
1651
1652                 if (status != SCI_SUCCESS) {
1653                         dev_err(scic_to_dev(scic),
1654                                 "%s: SCIC IO Request 0x%p could not get frame "
1655                                 "header for frame index %d, status %x\n",
1656                                 __func__,
1657                                 stp_req,
1658                                 frame_index,
1659                                 status);
1660
1661                         return status;
1662                 }
1663
1664                 switch (frame_header->fis_type) {
1665                 case FIS_REGD2H:
1666                         scic_sds_unsolicited_frame_control_get_buffer(&scic->uf_control,
1667                                                                       frame_index,
1668                                                                       (void **)&frame_buffer);
1669
1670                         scic_sds_controller_copy_sata_response(&sci_req->stp.rsp,
1671                                                                frame_header,
1672                                                                frame_buffer);
1673
1674                         /* The command has completed with error */
1675                         scic_sds_request_set_status(sci_req, SCU_TASK_DONE_CHECK_RESPONSE,
1676                                                     SCI_FAILURE_IO_RESPONSE_VALID);
1677                         break;
1678
1679                 default:
1680                         dev_warn(scic_to_dev(scic),
1681                                  "%s: IO Request:0x%p Frame Id:%d protocol "
1682                                   "violation occurred\n", __func__, stp_req,
1683                                   frame_index);
1684
1685                         scic_sds_request_set_status(sci_req, SCU_TASK_DONE_UNEXP_FIS,
1686                                                     SCI_FAILURE_PROTOCOL_VIOLATION);
1687                         break;
1688                 }
1689
1690                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1691
1692                 /* Frame has been decoded return it to the controller */
1693                 scic_sds_controller_release_frame(scic, frame_index);
1694
1695                 return status;
1696         }
1697
1698         case SCI_REQ_STP_PIO_WAIT_FRAME: {
1699                 struct isci_request *ireq = sci_req_to_ireq(sci_req);
1700                 struct sas_task *task = isci_request_access_task(ireq);
1701                 struct dev_to_host_fis *frame_header;
1702                 u32 *frame_buffer;
1703
1704                 status = scic_sds_unsolicited_frame_control_get_header(&scic->uf_control,
1705                                                                        frame_index,
1706                                                                        (void **)&frame_header);
1707
1708                 if (status != SCI_SUCCESS) {
1709                         dev_err(scic_to_dev(scic),
1710                                 "%s: SCIC IO Request 0x%p could not get frame "
1711                                 "header for frame index %d, status %x\n",
1712                                 __func__, stp_req, frame_index, status);
1713                         return status;
1714                 }
1715
1716                 switch (frame_header->fis_type) {
1717                 case FIS_PIO_SETUP:
1718                         /* Get from the frame buffer the PIO Setup Data */
1719                         scic_sds_unsolicited_frame_control_get_buffer(&scic->uf_control,
1720                                                                       frame_index,
1721                                                                       (void **)&frame_buffer);
1722
1723                         /* Get the data from the PIO Setup The SCU Hardware
1724                          * returns first word in the frame_header and the rest
1725                          * of the data is in the frame buffer so we need to
1726                          * back up one dword
1727                          */
1728
1729                         /* transfer_count: first 16bits in the 4th dword */
1730                         stp_req->type.pio.pio_transfer_bytes = frame_buffer[3] & 0xffff;
1731
1732                         /* ending_status: 4th byte in the 3rd dword */
1733                         stp_req->type.pio.ending_status = (frame_buffer[2] >> 24) & 0xff;
1734
1735                         scic_sds_controller_copy_sata_response(&sci_req->stp.rsp,
1736                                                                frame_header,
1737                                                                frame_buffer);
1738
1739                         sci_req->stp.rsp.status = stp_req->type.pio.ending_status;
1740
1741                         /* The next state is dependent on whether the
1742                          * request was PIO Data-in or Data out
1743                          */
1744                         if (task->data_dir == DMA_FROM_DEVICE) {
1745                                 sci_change_state(&sci_req->sm, SCI_REQ_STP_PIO_DATA_IN);
1746                         } else if (task->data_dir == DMA_TO_DEVICE) {
1747                                 /* Transmit data */
1748                                 status = scic_sds_stp_request_pio_data_out_transmit_data(sci_req);
1749                                 if (status != SCI_SUCCESS)
1750                                         break;
1751                                 sci_change_state(&sci_req->sm, SCI_REQ_STP_PIO_DATA_OUT);
1752                         }
1753                         break;
1754
1755                 case FIS_SETDEVBITS:
1756                         sci_change_state(&sci_req->sm, SCI_REQ_STP_PIO_WAIT_FRAME);
1757                         break;
1758
1759                 case FIS_REGD2H:
1760                         if (frame_header->status & ATA_BUSY) {
1761                                 /*
1762                                  * Now why is the drive sending a D2H Register
1763                                  * FIS when it is still busy?  Do nothing since
1764                                  * we are still in the right state.
1765                                  */
1766                                 dev_dbg(scic_to_dev(scic),
1767                                         "%s: SCIC PIO Request 0x%p received "
1768                                         "D2H Register FIS with BSY status "
1769                                         "0x%x\n",
1770                                         __func__,
1771                                         stp_req,
1772                                         frame_header->status);
1773                                 break;
1774                         }
1775
1776                         scic_sds_unsolicited_frame_control_get_buffer(&scic->uf_control,
1777                                                                       frame_index,
1778                                                                       (void **)&frame_buffer);
1779
1780                         scic_sds_controller_copy_sata_response(&sci_req->stp.req,
1781                                                                frame_header,
1782                                                                frame_buffer);
1783
1784                         scic_sds_request_set_status(sci_req,
1785                                                     SCU_TASK_DONE_CHECK_RESPONSE,
1786                                                     SCI_FAILURE_IO_RESPONSE_VALID);
1787
1788                         sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1789                         break;
1790
1791                 default:
1792                         /* FIXME: what do we do here? */
1793                         break;
1794                 }
1795
1796                 /* Frame is decoded return it to the controller */
1797                 scic_sds_controller_release_frame(scic, frame_index);
1798
1799                 return status;
1800         }
1801
1802         case SCI_REQ_STP_PIO_DATA_IN: {
1803                 struct dev_to_host_fis *frame_header;
1804                 struct sata_fis_data *frame_buffer;
1805
1806                 status = scic_sds_unsolicited_frame_control_get_header(&scic->uf_control,
1807                                                                        frame_index,
1808                                                                        (void **)&frame_header);
1809
1810                 if (status != SCI_SUCCESS) {
1811                         dev_err(scic_to_dev(scic),
1812                                 "%s: SCIC IO Request 0x%p could not get frame "
1813                                 "header for frame index %d, status %x\n",
1814                                 __func__,
1815                                 stp_req,
1816                                 frame_index,
1817                                 status);
1818                         return status;
1819                 }
1820
1821                 if (frame_header->fis_type != FIS_DATA) {
1822                         dev_err(scic_to_dev(scic),
1823                                 "%s: SCIC PIO Request 0x%p received frame %d "
1824                                 "with fis type 0x%02x when expecting a data "
1825                                 "fis.\n",
1826                                 __func__,
1827                                 stp_req,
1828                                 frame_index,
1829                                 frame_header->fis_type);
1830
1831                         scic_sds_request_set_status(sci_req,
1832                                                     SCU_TASK_DONE_GOOD,
1833                                                     SCI_FAILURE_IO_REQUIRES_SCSI_ABORT);
1834
1835                         sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1836
1837                         /* Frame is decoded return it to the controller */
1838                         scic_sds_controller_release_frame(scic, frame_index);
1839                         return status;
1840                 }
1841
1842                 if (stp_req->type.pio.request_current.sgl_index < 0) {
1843                         sci_req->saved_rx_frame_index = frame_index;
1844                         stp_req->type.pio.pio_transfer_bytes = 0;
1845                 } else {
1846                         scic_sds_unsolicited_frame_control_get_buffer(&scic->uf_control,
1847                                                                       frame_index,
1848                                                                       (void **)&frame_buffer);
1849
1850                         status = scic_sds_stp_request_pio_data_in_copy_data(stp_req,
1851                                                                             (u8 *)frame_buffer);
1852
1853                         /* Frame is decoded return it to the controller */
1854                         scic_sds_controller_release_frame(scic, frame_index);
1855                 }
1856
1857                 /* Check for the end of the transfer, are there more
1858                  * bytes remaining for this data transfer
1859                  */
1860                 if (status != SCI_SUCCESS ||
1861                     stp_req->type.pio.pio_transfer_bytes != 0)
1862                         return status;
1863
1864                 if ((stp_req->type.pio.ending_status & ATA_BUSY) == 0) {
1865                         scic_sds_request_set_status(sci_req,
1866                                                     SCU_TASK_DONE_CHECK_RESPONSE,
1867                                                     SCI_FAILURE_IO_RESPONSE_VALID);
1868
1869                         sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1870                 } else {
1871                         sci_change_state(&sci_req->sm, SCI_REQ_STP_PIO_WAIT_FRAME);
1872                 }
1873                 return status;
1874         }
1875
1876         case SCI_REQ_STP_SOFT_RESET_WAIT_D2H: {
1877                 struct dev_to_host_fis *frame_header;
1878                 u32 *frame_buffer;
1879
1880                 status = scic_sds_unsolicited_frame_control_get_header(&scic->uf_control,
1881                                                                        frame_index,
1882                                                                        (void **)&frame_header);
1883                 if (status != SCI_SUCCESS) {
1884                         dev_err(scic_to_dev(scic),
1885                                 "%s: SCIC IO Request 0x%p could not get frame "
1886                                 "header for frame index %d, status %x\n",
1887                                 __func__,
1888                                 stp_req,
1889                                 frame_index,
1890                                 status);
1891                         return status;
1892                 }
1893
1894                 switch (frame_header->fis_type) {
1895                 case FIS_REGD2H:
1896                         scic_sds_unsolicited_frame_control_get_buffer(&scic->uf_control,
1897                                                                       frame_index,
1898                                                                       (void **)&frame_buffer);
1899
1900                         scic_sds_controller_copy_sata_response(&sci_req->stp.rsp,
1901                                                                frame_header,
1902                                                                frame_buffer);
1903
1904                         /* The command has completed with error */
1905                         scic_sds_request_set_status(sci_req,
1906                                                     SCU_TASK_DONE_CHECK_RESPONSE,
1907                                                     SCI_FAILURE_IO_RESPONSE_VALID);
1908                         break;
1909
1910                 default:
1911                         dev_warn(scic_to_dev(scic),
1912                                  "%s: IO Request:0x%p Frame Id:%d protocol "
1913                                  "violation occurred\n",
1914                                  __func__,
1915                                  stp_req,
1916                                  frame_index);
1917
1918                         scic_sds_request_set_status(sci_req,
1919                                                     SCU_TASK_DONE_UNEXP_FIS,
1920                                                     SCI_FAILURE_PROTOCOL_VIOLATION);
1921                         break;
1922                 }
1923
1924                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
1925
1926                 /* Frame has been decoded return it to the controller */
1927                 scic_sds_controller_release_frame(scic, frame_index);
1928
1929                 return status;
1930         }
1931         case SCI_REQ_ABORTING:
1932                 /*
1933                  * TODO: Is it even possible to get an unsolicited frame in the
1934                  * aborting state?
1935                  */
1936                 scic_sds_controller_release_frame(scic, frame_index);
1937                 return SCI_SUCCESS;
1938
1939         default:
1940                 dev_warn(scic_to_dev(scic),
1941                          "%s: SCIC IO Request given unexpected frame %x while "
1942                          "in state %d\n",
1943                          __func__,
1944                          frame_index,
1945                          state);
1946
1947                 scic_sds_controller_release_frame(scic, frame_index);
1948                 return SCI_FAILURE_INVALID_STATE;
1949         }
1950 }
1951
1952 static enum sci_status stp_request_udma_await_tc_event(struct scic_sds_request *sci_req,
1953                                                        u32 completion_code)
1954 {
1955         enum sci_status status = SCI_SUCCESS;
1956
1957         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1958         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
1959                 scic_sds_stp_request_udma_complete_request(sci_req,
1960                                                            SCU_TASK_DONE_GOOD,
1961                                                            SCI_SUCCESS);
1962                 break;
1963         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_UNEXP_FIS):
1964         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_REG_ERR):
1965                 /* We must check ther response buffer to see if the D2H
1966                  * Register FIS was received before we got the TC
1967                  * completion.
1968                  */
1969                 if (sci_req->stp.rsp.fis_type == FIS_REGD2H) {
1970                         scic_sds_remote_device_suspend(sci_req->target_device,
1971                                 SCU_EVENT_SPECIFIC(SCU_NORMALIZE_COMPLETION_STATUS(completion_code)));
1972
1973                         scic_sds_stp_request_udma_complete_request(sci_req,
1974                                                                    SCU_TASK_DONE_CHECK_RESPONSE,
1975                                                                    SCI_FAILURE_IO_RESPONSE_VALID);
1976                 } else {
1977                         /* If we have an error completion status for the
1978                          * TC then we can expect a D2H register FIS from
1979                          * the device so we must change state to wait
1980                          * for it
1981                          */
1982                         sci_change_state(&sci_req->sm, SCI_REQ_STP_UDMA_WAIT_D2H);
1983                 }
1984                 break;
1985
1986         /* TODO Check to see if any of these completion status need to
1987          * wait for the device to host register fis.
1988          */
1989         /* TODO We can retry the command for SCU_TASK_DONE_CMD_LL_R_ERR
1990          * - this comes only for B0
1991          */
1992         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_INV_FIS_LEN):
1993         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_MAX_PLD_ERR):
1994         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_LL_R_ERR):
1995         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_CMD_LL_R_ERR):
1996         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_CRC_ERR):
1997                 scic_sds_remote_device_suspend(sci_req->target_device,
1998                         SCU_EVENT_SPECIFIC(SCU_NORMALIZE_COMPLETION_STATUS(completion_code)));
1999         /* Fall through to the default case */
2000         default:
2001                 /* All other completion status cause the IO to be complete. */
2002                 scic_sds_stp_request_udma_complete_request(sci_req,
2003                                         SCU_NORMALIZE_COMPLETION_STATUS(completion_code),
2004                                         SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR);
2005                 break;
2006         }
2007
2008         return status;
2009 }
2010
2011 static enum sci_status
2012 stp_request_soft_reset_await_h2d_asserted_tc_event(struct scic_sds_request *sci_req,
2013                                                    u32 completion_code)
2014 {
2015         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
2016         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
2017                 scic_sds_request_set_status(sci_req, SCU_TASK_DONE_GOOD,
2018                                             SCI_SUCCESS);
2019
2020                 sci_change_state(&sci_req->sm, SCI_REQ_STP_SOFT_RESET_WAIT_H2D_DIAG);
2021                 break;
2022
2023         default:
2024                 /*
2025                  * All other completion status cause the IO to be complete.
2026                  * If a NAK was received, then it is up to the user to retry
2027                  * the request.
2028                  */
2029                 scic_sds_request_set_status(sci_req,
2030                                             SCU_NORMALIZE_COMPLETION_STATUS(completion_code),
2031                                             SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR);
2032
2033                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
2034                 break;
2035         }
2036
2037         return SCI_SUCCESS;
2038 }
2039
2040 static enum sci_status
2041 stp_request_soft_reset_await_h2d_diagnostic_tc_event(struct scic_sds_request *sci_req,
2042                                                      u32 completion_code)
2043 {
2044         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
2045         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
2046                 scic_sds_request_set_status(sci_req, SCU_TASK_DONE_GOOD,
2047                                             SCI_SUCCESS);
2048
2049                 sci_change_state(&sci_req->sm, SCI_REQ_STP_SOFT_RESET_WAIT_D2H);
2050                 break;
2051
2052         default:
2053                 /* All other completion status cause the IO to be complete.  If
2054                  * a NAK was received, then it is up to the user to retry the
2055                  * request.
2056                  */
2057                 scic_sds_request_set_status(sci_req,
2058                         SCU_NORMALIZE_COMPLETION_STATUS(completion_code),
2059                         SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR);
2060
2061                 sci_change_state(&sci_req->sm, SCI_REQ_COMPLETED);
2062                 break;
2063         }
2064
2065         return SCI_SUCCESS;
2066 }
2067
2068 enum sci_status
2069 scic_sds_io_request_tc_completion(struct scic_sds_request *sci_req,
2070                                   u32 completion_code)
2071 {
2072         enum sci_base_request_states state;
2073         struct scic_sds_controller *scic = sci_req->owning_controller;
2074
2075         state = sci_req->sm.current_state_id;
2076
2077         switch (state) {
2078         case SCI_REQ_STARTED:
2079                 return request_started_state_tc_event(sci_req, completion_code);
2080
2081         case SCI_REQ_TASK_WAIT_TC_COMP:
2082                 return ssp_task_request_await_tc_event(sci_req,
2083                                                        completion_code);
2084
2085         case SCI_REQ_SMP_WAIT_RESP:
2086                 return smp_request_await_response_tc_event(sci_req,
2087                                                            completion_code);
2088
2089         case SCI_REQ_SMP_WAIT_TC_COMP:
2090                 return smp_request_await_tc_event(sci_req, completion_code);
2091
2092         case SCI_REQ_STP_UDMA_WAIT_TC_COMP:
2093                 return stp_request_udma_await_tc_event(sci_req,
2094                                                        completion_code);
2095
2096         case SCI_REQ_STP_NON_DATA_WAIT_H2D:
2097                 return stp_request_non_data_await_h2d_tc_event(sci_req,
2098                                                                completion_code);
2099
2100         case SCI_REQ_STP_PIO_WAIT_H2D:
2101                 return stp_request_pio_await_h2d_completion_tc_event(sci_req,
2102                                                                      completion_code);
2103
2104         case SCI_REQ_STP_PIO_DATA_OUT:
2105                 return pio_data_out_tx_done_tc_event(sci_req, completion_code);
2106
2107         case SCI_REQ_STP_SOFT_RESET_WAIT_H2D_ASSERTED:
2108                 return stp_request_soft_reset_await_h2d_asserted_tc_event(sci_req,
2109                                                                           completion_code);
2110
2111         case SCI_REQ_STP_SOFT_RESET_WAIT_H2D_DIAG:
2112                 return stp_request_soft_reset_await_h2d_diagnostic_tc_event(sci_req,
2113                                                                             completion_code);
2114
2115         case SCI_REQ_ABORTING:
2116                 return request_aborting_state_tc_event(sci_req,
2117                                                        completion_code);
2118
2119         default:
2120                 dev_warn(scic_to_dev(scic),
2121                          "%s: SCIC IO Request given task completion "
2122                          "notification %x while in wrong state %d\n",
2123                          __func__,
2124                          completion_code,
2125                          state);
2126                 return SCI_FAILURE_INVALID_STATE;
2127         }
2128 }
2129
2130 /**
2131  * isci_request_process_response_iu() - This function sets the status and
2132  *    response iu, in the task struct, from the request object for the upper
2133  *    layer driver.
2134  * @sas_task: This parameter is the task struct from the upper layer driver.
2135  * @resp_iu: This parameter points to the response iu of the completed request.
2136  * @dev: This parameter specifies the linux device struct.
2137  *
2138  * none.
2139  */
2140 static void isci_request_process_response_iu(
2141         struct sas_task *task,
2142         struct ssp_response_iu *resp_iu,
2143         struct device *dev)
2144 {
2145         dev_dbg(dev,
2146                 "%s: resp_iu = %p "
2147                 "resp_iu->status = 0x%x,\nresp_iu->datapres = %d "
2148                 "resp_iu->response_data_len = %x, "
2149                 "resp_iu->sense_data_len = %x\nrepsonse data: ",
2150                 __func__,
2151                 resp_iu,
2152                 resp_iu->status,
2153                 resp_iu->datapres,
2154                 resp_iu->response_data_len,
2155                 resp_iu->sense_data_len);
2156
2157         task->task_status.stat = resp_iu->status;
2158
2159         /* libsas updates the task status fields based on the response iu. */
2160         sas_ssp_task_response(dev, task, resp_iu);
2161 }
2162
2163 /**
2164  * isci_request_set_open_reject_status() - This function prepares the I/O
2165  *    completion for OPEN_REJECT conditions.
2166  * @request: This parameter is the completed isci_request object.
2167  * @response_ptr: This parameter specifies the service response for the I/O.
2168  * @status_ptr: This parameter specifies the exec status for the I/O.
2169  * @complete_to_host_ptr: This parameter specifies the action to be taken by
2170  *    the LLDD with respect to completing this request or forcing an abort
2171  *    condition on the I/O.
2172  * @open_rej_reason: This parameter specifies the encoded reason for the
2173  *    abandon-class reject.
2174  *
2175  * none.
2176  */
2177 static void isci_request_set_open_reject_status(
2178         struct isci_request *request,
2179         struct sas_task *task,
2180         enum service_response *response_ptr,
2181         enum exec_status *status_ptr,
2182         enum isci_completion_selection *complete_to_host_ptr,
2183         enum sas_open_rej_reason open_rej_reason)
2184 {
2185         /* Task in the target is done. */
2186         set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2187         *response_ptr                     = SAS_TASK_UNDELIVERED;
2188         *status_ptr                       = SAS_OPEN_REJECT;
2189         *complete_to_host_ptr             = isci_perform_normal_io_completion;
2190         task->task_status.open_rej_reason = open_rej_reason;
2191 }
2192
2193 /**
2194  * isci_request_handle_controller_specific_errors() - This function decodes
2195  *    controller-specific I/O completion error conditions.
2196  * @request: This parameter is the completed isci_request object.
2197  * @response_ptr: This parameter specifies the service response for the I/O.
2198  * @status_ptr: This parameter specifies the exec status for the I/O.
2199  * @complete_to_host_ptr: This parameter specifies the action to be taken by
2200  *    the LLDD with respect to completing this request or forcing an abort
2201  *    condition on the I/O.
2202  *
2203  * none.
2204  */
2205 static void isci_request_handle_controller_specific_errors(
2206         struct isci_remote_device *idev,
2207         struct isci_request *request,
2208         struct sas_task *task,
2209         enum service_response *response_ptr,
2210         enum exec_status *status_ptr,
2211         enum isci_completion_selection *complete_to_host_ptr)
2212 {
2213         unsigned int cstatus;
2214
2215         cstatus = request->sci.scu_status;
2216
2217         dev_dbg(&request->isci_host->pdev->dev,
2218                 "%s: %p SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR "
2219                 "- controller status = 0x%x\n",
2220                 __func__, request, cstatus);
2221
2222         /* Decode the controller-specific errors; most
2223          * important is to recognize those conditions in which
2224          * the target may still have a task outstanding that
2225          * must be aborted.
2226          *
2227          * Note that there are SCU completion codes being
2228          * named in the decode below for which SCIC has already
2229          * done work to handle them in a way other than as
2230          * a controller-specific completion code; these are left
2231          * in the decode below for completeness sake.
2232          */
2233         switch (cstatus) {
2234         case SCU_TASK_DONE_DMASETUP_DIRERR:
2235         /* Also SCU_TASK_DONE_SMP_FRM_TYPE_ERR: */
2236         case SCU_TASK_DONE_XFERCNT_ERR:
2237                 /* Also SCU_TASK_DONE_SMP_UFI_ERR: */
2238                 if (task->task_proto == SAS_PROTOCOL_SMP) {
2239                         /* SCU_TASK_DONE_SMP_UFI_ERR == Task Done. */
2240                         *response_ptr = SAS_TASK_COMPLETE;
2241
2242                         /* See if the device has been/is being stopped. Note
2243                          * that we ignore the quiesce state, since we are
2244                          * concerned about the actual device state.
2245                          */
2246                         if (!idev)
2247                                 *status_ptr = SAS_DEVICE_UNKNOWN;
2248                         else
2249                                 *status_ptr = SAS_ABORTED_TASK;
2250
2251                         set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2252
2253                         *complete_to_host_ptr =
2254                                 isci_perform_normal_io_completion;
2255                 } else {
2256                         /* Task in the target is not done. */
2257                         *response_ptr = SAS_TASK_UNDELIVERED;
2258
2259                         if (!idev)
2260                                 *status_ptr = SAS_DEVICE_UNKNOWN;
2261                         else
2262                                 *status_ptr = SAM_STAT_TASK_ABORTED;
2263
2264                         clear_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2265
2266                         *complete_to_host_ptr =
2267                                 isci_perform_error_io_completion;
2268                 }
2269
2270                 break;
2271
2272         case SCU_TASK_DONE_CRC_ERR:
2273         case SCU_TASK_DONE_NAK_CMD_ERR:
2274         case SCU_TASK_DONE_EXCESS_DATA:
2275         case SCU_TASK_DONE_UNEXP_FIS:
2276         /* Also SCU_TASK_DONE_UNEXP_RESP: */
2277         case SCU_TASK_DONE_VIIT_ENTRY_NV:       /* TODO - conditions? */
2278         case SCU_TASK_DONE_IIT_ENTRY_NV:        /* TODO - conditions? */
2279         case SCU_TASK_DONE_RNCNV_OUTBOUND:      /* TODO - conditions? */
2280                 /* These are conditions in which the target
2281                  * has completed the task, so that no cleanup
2282                  * is necessary.
2283                  */
2284                 *response_ptr = SAS_TASK_COMPLETE;
2285
2286                 /* See if the device has been/is being stopped. Note
2287                  * that we ignore the quiesce state, since we are
2288                  * concerned about the actual device state.
2289                  */
2290                 if (!idev)
2291                         *status_ptr = SAS_DEVICE_UNKNOWN;
2292                 else
2293                         *status_ptr = SAS_ABORTED_TASK;
2294
2295                 set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2296
2297                 *complete_to_host_ptr = isci_perform_normal_io_completion;
2298                 break;
2299
2300
2301         /* Note that the only open reject completion codes seen here will be
2302          * abandon-class codes; all others are automatically retried in the SCU.
2303          */
2304         case SCU_TASK_OPEN_REJECT_WRONG_DESTINATION:
2305
2306                 isci_request_set_open_reject_status(
2307                         request, task, response_ptr, status_ptr,
2308                         complete_to_host_ptr, SAS_OREJ_WRONG_DEST);
2309                 break;
2310
2311         case SCU_TASK_OPEN_REJECT_ZONE_VIOLATION:
2312
2313                 /* Note - the return of AB0 will change when
2314                  * libsas implements detection of zone violations.
2315                  */
2316                 isci_request_set_open_reject_status(
2317                         request, task, response_ptr, status_ptr,
2318                         complete_to_host_ptr, SAS_OREJ_RESV_AB0);
2319                 break;
2320
2321         case SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_1:
2322
2323                 isci_request_set_open_reject_status(
2324                         request, task, response_ptr, status_ptr,
2325                         complete_to_host_ptr, SAS_OREJ_RESV_AB1);
2326                 break;
2327
2328         case SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_2:
2329
2330                 isci_request_set_open_reject_status(
2331                         request, task, response_ptr, status_ptr,
2332                         complete_to_host_ptr, SAS_OREJ_RESV_AB2);
2333                 break;
2334
2335         case SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_3:
2336
2337                 isci_request_set_open_reject_status(
2338                         request, task, response_ptr, status_ptr,
2339                         complete_to_host_ptr, SAS_OREJ_RESV_AB3);
2340                 break;
2341
2342         case SCU_TASK_OPEN_REJECT_BAD_DESTINATION:
2343
2344                 isci_request_set_open_reject_status(
2345                         request, task, response_ptr, status_ptr,
2346                         complete_to_host_ptr, SAS_OREJ_BAD_DEST);
2347                 break;
2348
2349         case SCU_TASK_OPEN_REJECT_STP_RESOURCES_BUSY:
2350
2351                 isci_request_set_open_reject_status(
2352                         request, task, response_ptr, status_ptr,
2353                         complete_to_host_ptr, SAS_OREJ_STP_NORES);
2354                 break;
2355
2356         case SCU_TASK_OPEN_REJECT_PROTOCOL_NOT_SUPPORTED:
2357
2358                 isci_request_set_open_reject_status(
2359                         request, task, response_ptr, status_ptr,
2360                         complete_to_host_ptr, SAS_OREJ_EPROTO);
2361                 break;
2362
2363         case SCU_TASK_OPEN_REJECT_CONNECTION_RATE_NOT_SUPPORTED:
2364
2365                 isci_request_set_open_reject_status(
2366                         request, task, response_ptr, status_ptr,
2367                         complete_to_host_ptr, SAS_OREJ_CONN_RATE);
2368                 break;
2369
2370         case SCU_TASK_DONE_LL_R_ERR:
2371         /* Also SCU_TASK_DONE_ACK_NAK_TO: */
2372         case SCU_TASK_DONE_LL_PERR:
2373         case SCU_TASK_DONE_LL_SY_TERM:
2374         /* Also SCU_TASK_DONE_NAK_ERR:*/
2375         case SCU_TASK_DONE_LL_LF_TERM:
2376         /* Also SCU_TASK_DONE_DATA_LEN_ERR: */
2377         case SCU_TASK_DONE_LL_ABORT_ERR:
2378         case SCU_TASK_DONE_SEQ_INV_TYPE:
2379         /* Also SCU_TASK_DONE_UNEXP_XR: */
2380         case SCU_TASK_DONE_XR_IU_LEN_ERR:
2381         case SCU_TASK_DONE_INV_FIS_LEN:
2382         /* Also SCU_TASK_DONE_XR_WD_LEN: */
2383         case SCU_TASK_DONE_SDMA_ERR:
2384         case SCU_TASK_DONE_OFFSET_ERR:
2385         case SCU_TASK_DONE_MAX_PLD_ERR:
2386         case SCU_TASK_DONE_LF_ERR:
2387         case SCU_TASK_DONE_SMP_RESP_TO_ERR:  /* Escalate to dev reset? */
2388         case SCU_TASK_DONE_SMP_LL_RX_ERR:
2389         case SCU_TASK_DONE_UNEXP_DATA:
2390         case SCU_TASK_DONE_UNEXP_SDBFIS:
2391         case SCU_TASK_DONE_REG_ERR:
2392         case SCU_TASK_DONE_SDB_ERR:
2393         case SCU_TASK_DONE_TASK_ABORT:
2394         default:
2395                 /* Task in the target is not done. */
2396                 *response_ptr = SAS_TASK_UNDELIVERED;
2397                 *status_ptr = SAM_STAT_TASK_ABORTED;
2398
2399                 if (task->task_proto == SAS_PROTOCOL_SMP) {
2400                         set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2401
2402                         *complete_to_host_ptr = isci_perform_normal_io_completion;
2403                 } else {
2404                         clear_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2405
2406                         *complete_to_host_ptr = isci_perform_error_io_completion;
2407                 }
2408                 break;
2409         }
2410 }
2411
2412 /**
2413  * isci_task_save_for_upper_layer_completion() - This function saves the
2414  *    request for later completion to the upper layer driver.
2415  * @host: This parameter is a pointer to the host on which the the request
2416  *    should be queued (either as an error or success).
2417  * @request: This parameter is the completed request.
2418  * @response: This parameter is the response code for the completed task.
2419  * @status: This parameter is the status code for the completed task.
2420  *
2421  * none.
2422  */
2423 static void isci_task_save_for_upper_layer_completion(
2424         struct isci_host *host,
2425         struct isci_request *request,
2426         enum service_response response,
2427         enum exec_status status,
2428         enum isci_completion_selection task_notification_selection)
2429 {
2430         struct sas_task *task = isci_request_access_task(request);
2431
2432         task_notification_selection
2433                 = isci_task_set_completion_status(task, response, status,
2434                                                   task_notification_selection);
2435
2436         /* Tasks aborted specifically by a call to the lldd_abort_task
2437          * function should not be completed to the host in the regular path.
2438          */
2439         switch (task_notification_selection) {
2440
2441         case isci_perform_normal_io_completion:
2442
2443                 /* Normal notification (task_done) */
2444                 dev_dbg(&host->pdev->dev,
2445                         "%s: Normal - task = %p, response=%d (%d), status=%d (%d)\n",
2446                         __func__,
2447                         task,
2448                         task->task_status.resp, response,
2449                         task->task_status.stat, status);
2450                 /* Add to the completed list. */
2451                 list_add(&request->completed_node,
2452                          &host->requests_to_complete);
2453
2454                 /* Take the request off the device's pending request list. */
2455                 list_del_init(&request->dev_node);
2456                 break;
2457
2458         case isci_perform_aborted_io_completion:
2459                 /* No notification to libsas because this request is
2460                  * already in the abort path.
2461                  */
2462                 dev_warn(&host->pdev->dev,
2463                          "%s: Aborted - task = %p, response=%d (%d), status=%d (%d)\n",
2464                          __func__,
2465                          task,
2466                          task->task_status.resp, response,
2467                          task->task_status.stat, status);
2468
2469                 /* Wake up whatever process was waiting for this
2470                  * request to complete.
2471                  */
2472                 WARN_ON(request->io_request_completion == NULL);
2473
2474                 if (request->io_request_completion != NULL) {
2475
2476                         /* Signal whoever is waiting that this
2477                         * request is complete.
2478                         */
2479                         complete(request->io_request_completion);
2480                 }
2481                 break;
2482
2483         case isci_perform_error_io_completion:
2484                 /* Use sas_task_abort */
2485                 dev_warn(&host->pdev->dev,
2486                          "%s: Error - task = %p, response=%d (%d), status=%d (%d)\n",
2487                          __func__,
2488                          task,
2489                          task->task_status.resp, response,
2490                          task->task_status.stat, status);
2491                 /* Add to the aborted list. */
2492                 list_add(&request->completed_node,
2493                          &host->requests_to_errorback);
2494                 break;
2495
2496         default:
2497                 dev_warn(&host->pdev->dev,
2498                          "%s: Unknown - task = %p, response=%d (%d), status=%d (%d)\n",
2499                          __func__,
2500                          task,
2501                          task->task_status.resp, response,
2502                          task->task_status.stat, status);
2503
2504                 /* Add to the error to libsas list. */
2505                 list_add(&request->completed_node,
2506                          &host->requests_to_errorback);
2507                 break;
2508         }
2509 }
2510
2511 static void isci_request_io_request_complete(struct isci_host *isci_host,
2512                                              struct isci_request *request,
2513                                              enum sci_io_status completion_status)
2514 {
2515         struct sas_task *task = isci_request_access_task(request);
2516         struct ssp_response_iu *resp_iu;
2517         void *resp_buf;
2518         unsigned long task_flags;
2519         struct isci_remote_device *idev = isci_lookup_device(task->dev);
2520         enum service_response response       = SAS_TASK_UNDELIVERED;
2521         enum exec_status status         = SAS_ABORTED_TASK;
2522         enum isci_request_status request_status;
2523         enum isci_completion_selection complete_to_host
2524                 = isci_perform_normal_io_completion;
2525
2526         dev_dbg(&isci_host->pdev->dev,
2527                 "%s: request = %p, task = %p,\n"
2528                 "task->data_dir = %d completion_status = 0x%x\n",
2529                 __func__,
2530                 request,
2531                 task,
2532                 task->data_dir,
2533                 completion_status);
2534
2535         spin_lock(&request->state_lock);
2536         request_status = isci_request_get_state(request);
2537
2538         /* Decode the request status.  Note that if the request has been
2539          * aborted by a task management function, we don't care
2540          * what the status is.
2541          */
2542         switch (request_status) {
2543
2544         case aborted:
2545                 /* "aborted" indicates that the request was aborted by a task
2546                  * management function, since once a task management request is
2547                  * perfomed by the device, the request only completes because
2548                  * of the subsequent driver terminate.
2549                  *
2550                  * Aborted also means an external thread is explicitly managing
2551                  * this request, so that we do not complete it up the stack.
2552                  *
2553                  * The target is still there (since the TMF was successful).
2554                  */
2555                 set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2556                 response = SAS_TASK_COMPLETE;
2557
2558                 /* See if the device has been/is being stopped. Note
2559                  * that we ignore the quiesce state, since we are
2560                  * concerned about the actual device state.
2561                  */
2562                 if (!idev)
2563                         status = SAS_DEVICE_UNKNOWN;
2564                 else
2565                         status = SAS_ABORTED_TASK;
2566
2567                 complete_to_host = isci_perform_aborted_io_completion;
2568                 /* This was an aborted request. */
2569
2570                 spin_unlock(&request->state_lock);
2571                 break;
2572
2573         case aborting:
2574                 /* aborting means that the task management function tried and
2575                  * failed to abort the request. We need to note the request
2576                  * as SAS_TASK_UNDELIVERED, so that the scsi mid layer marks the
2577                  * target as down.
2578                  *
2579                  * Aborting also means an external thread is explicitly managing
2580                  * this request, so that we do not complete it up the stack.
2581                  */
2582                 set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2583                 response = SAS_TASK_UNDELIVERED;
2584
2585                 if (!idev)
2586                         /* The device has been /is being stopped. Note that
2587                          * we ignore the quiesce state, since we are
2588                          * concerned about the actual device state.
2589                          */
2590                         status = SAS_DEVICE_UNKNOWN;
2591                 else
2592                         status = SAS_PHY_DOWN;
2593
2594                 complete_to_host = isci_perform_aborted_io_completion;
2595
2596                 /* This was an aborted request. */
2597
2598                 spin_unlock(&request->state_lock);
2599                 break;
2600
2601         case terminating:
2602
2603                 /* This was an terminated request.  This happens when
2604                  * the I/O is being terminated because of an action on
2605                  * the device (reset, tear down, etc.), and the I/O needs
2606                  * to be completed up the stack.
2607                  */
2608                 set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2609                 response = SAS_TASK_UNDELIVERED;
2610
2611                 /* See if the device has been/is being stopped. Note
2612                  * that we ignore the quiesce state, since we are
2613                  * concerned about the actual device state.
2614                  */
2615                 if (!idev)
2616                         status = SAS_DEVICE_UNKNOWN;
2617                 else
2618                         status = SAS_ABORTED_TASK;
2619
2620                 complete_to_host = isci_perform_aborted_io_completion;
2621
2622                 /* This was a terminated request. */
2623
2624                 spin_unlock(&request->state_lock);
2625                 break;
2626
2627         case dead:
2628                 /* This was a terminated request that timed-out during the
2629                  * termination process.  There is no task to complete to
2630                  * libsas.
2631                  */
2632                 complete_to_host = isci_perform_normal_io_completion;
2633                 spin_unlock(&request->state_lock);
2634                 break;
2635
2636         default:
2637
2638                 /* The request is done from an SCU HW perspective. */
2639                 request->status = completed;
2640
2641                 spin_unlock(&request->state_lock);
2642
2643                 /* This is an active request being completed from the core. */
2644                 switch (completion_status) {
2645
2646                 case SCI_IO_FAILURE_RESPONSE_VALID:
2647                         dev_dbg(&isci_host->pdev->dev,
2648                                 "%s: SCI_IO_FAILURE_RESPONSE_VALID (%p/%p)\n",
2649                                 __func__,
2650                                 request,
2651                                 task);
2652
2653                         if (sas_protocol_ata(task->task_proto)) {
2654                                 resp_buf = &request->sci.stp.rsp;
2655                                 isci_request_process_stp_response(task,
2656                                                                   resp_buf);
2657                         } else if (SAS_PROTOCOL_SSP == task->task_proto) {
2658
2659                                 /* crack the iu response buffer. */
2660                                 resp_iu = &request->sci.ssp.rsp;
2661                                 isci_request_process_response_iu(task, resp_iu,
2662                                                                  &isci_host->pdev->dev);
2663
2664                         } else if (SAS_PROTOCOL_SMP == task->task_proto) {
2665
2666                                 dev_err(&isci_host->pdev->dev,
2667                                         "%s: SCI_IO_FAILURE_RESPONSE_VALID: "
2668                                         "SAS_PROTOCOL_SMP protocol\n",
2669                                         __func__);
2670
2671                         } else
2672                                 dev_err(&isci_host->pdev->dev,
2673                                         "%s: unknown protocol\n", __func__);
2674
2675                         /* use the task status set in the task struct by the
2676                          * isci_request_process_response_iu call.
2677                          */
2678                         set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2679                         response = task->task_status.resp;
2680                         status = task->task_status.stat;
2681                         break;
2682
2683                 case SCI_IO_SUCCESS:
2684                 case SCI_IO_SUCCESS_IO_DONE_EARLY:
2685
2686                         response = SAS_TASK_COMPLETE;
2687                         status   = SAM_STAT_GOOD;
2688                         set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2689
2690                         if (task->task_proto == SAS_PROTOCOL_SMP) {
2691                                 void *rsp = &request->sci.smp.rsp;
2692
2693                                 dev_dbg(&isci_host->pdev->dev,
2694                                         "%s: SMP protocol completion\n",
2695                                         __func__);
2696
2697                                 sg_copy_from_buffer(
2698                                         &task->smp_task.smp_resp, 1,
2699                                         rsp, sizeof(struct smp_resp));
2700                         } else if (completion_status
2701                                    == SCI_IO_SUCCESS_IO_DONE_EARLY) {
2702
2703                                 /* This was an SSP / STP / SATA transfer.
2704                                  * There is a possibility that less data than
2705                                  * the maximum was transferred.
2706                                  */
2707                                 u32 transferred_length = sci_req_tx_bytes(&request->sci);
2708
2709                                 task->task_status.residual
2710                                         = task->total_xfer_len - transferred_length;
2711
2712                                 /* If there were residual bytes, call this an
2713                                  * underrun.
2714                                  */
2715                                 if (task->task_status.residual != 0)
2716                                         status = SAS_DATA_UNDERRUN;
2717
2718                                 dev_dbg(&isci_host->pdev->dev,
2719                                         "%s: SCI_IO_SUCCESS_IO_DONE_EARLY %d\n",
2720                                         __func__,
2721                                         status);
2722
2723                         } else
2724                                 dev_dbg(&isci_host->pdev->dev,
2725                                         "%s: SCI_IO_SUCCESS\n",
2726                                         __func__);
2727
2728                         break;
2729
2730                 case SCI_IO_FAILURE_TERMINATED:
2731                         dev_dbg(&isci_host->pdev->dev,
2732                                 "%s: SCI_IO_FAILURE_TERMINATED (%p/%p)\n",
2733                                 __func__,
2734                                 request,
2735                                 task);
2736
2737                         /* The request was terminated explicitly.  No handling
2738                          * is needed in the SCSI error handler path.
2739                          */
2740                         set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2741                         response = SAS_TASK_UNDELIVERED;
2742
2743                         /* See if the device has been/is being stopped. Note
2744                          * that we ignore the quiesce state, since we are
2745                          * concerned about the actual device state.
2746                          */
2747                         if (!idev)
2748                                 status = SAS_DEVICE_UNKNOWN;
2749                         else
2750                                 status = SAS_ABORTED_TASK;
2751
2752                         complete_to_host = isci_perform_normal_io_completion;
2753                         break;
2754
2755                 case SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR:
2756
2757                         isci_request_handle_controller_specific_errors(
2758                                 idev, request, task, &response, &status,
2759                                 &complete_to_host);
2760
2761                         break;
2762
2763                 case SCI_IO_FAILURE_REMOTE_DEVICE_RESET_REQUIRED:
2764                         /* This is a special case, in that the I/O completion
2765                          * is telling us that the device needs a reset.
2766                          * In order for the device reset condition to be
2767                          * noticed, the I/O has to be handled in the error
2768                          * handler.  Set the reset flag and cause the
2769                          * SCSI error thread to be scheduled.
2770                          */
2771                         spin_lock_irqsave(&task->task_state_lock, task_flags);
2772                         task->task_state_flags |= SAS_TASK_NEED_DEV_RESET;
2773                         spin_unlock_irqrestore(&task->task_state_lock, task_flags);
2774
2775                         /* Fail the I/O. */
2776                         response = SAS_TASK_UNDELIVERED;
2777                         status = SAM_STAT_TASK_ABORTED;
2778
2779                         complete_to_host = isci_perform_error_io_completion;
2780                         clear_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2781                         break;
2782
2783                 case SCI_FAILURE_RETRY_REQUIRED:
2784
2785                         /* Fail the I/O so it can be retried. */
2786                         response = SAS_TASK_UNDELIVERED;
2787                         if (!idev)
2788                                 status = SAS_DEVICE_UNKNOWN;
2789                         else
2790                                 status = SAS_ABORTED_TASK;
2791
2792                         complete_to_host = isci_perform_normal_io_completion;
2793                         set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2794                         break;
2795
2796
2797                 default:
2798                         /* Catch any otherwise unhandled error codes here. */
2799                         dev_warn(&isci_host->pdev->dev,
2800                                  "%s: invalid completion code: 0x%x - "
2801                                  "isci_request = %p\n",
2802                                  __func__, completion_status, request);
2803
2804                         response = SAS_TASK_UNDELIVERED;
2805
2806                         /* See if the device has been/is being stopped. Note
2807                          * that we ignore the quiesce state, since we are
2808                          * concerned about the actual device state.
2809                          */
2810                         if (!idev)
2811                                 status = SAS_DEVICE_UNKNOWN;
2812                         else
2813                                 status = SAS_ABORTED_TASK;
2814
2815                         if (SAS_PROTOCOL_SMP == task->task_proto) {
2816                                 set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2817                                 complete_to_host = isci_perform_normal_io_completion;
2818                         } else {
2819                                 clear_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2820                                 complete_to_host = isci_perform_error_io_completion;
2821                         }
2822                         break;
2823                 }
2824                 break;
2825         }
2826
2827         switch (task->task_proto) {
2828         case SAS_PROTOCOL_SSP:
2829                 if (task->data_dir == DMA_NONE)
2830                         break;
2831                 if (task->num_scatter == 0)
2832                         /* 0 indicates a single dma address */
2833                         dma_unmap_single(&isci_host->pdev->dev,
2834                                          request->zero_scatter_daddr,
2835                                          task->total_xfer_len, task->data_dir);
2836                 else  /* unmap the sgl dma addresses */
2837                         dma_unmap_sg(&isci_host->pdev->dev, task->scatter,
2838                                      request->num_sg_entries, task->data_dir);
2839                 break;
2840         case SAS_PROTOCOL_SMP: {
2841                 struct scatterlist *sg = &task->smp_task.smp_req;
2842                 struct smp_req *smp_req;
2843                 void *kaddr;
2844
2845                 dma_unmap_sg(&isci_host->pdev->dev, sg, 1, DMA_TO_DEVICE);
2846
2847                 /* need to swab it back in case the command buffer is re-used */
2848                 kaddr = kmap_atomic(sg_page(sg), KM_IRQ0);
2849                 smp_req = kaddr + sg->offset;
2850                 sci_swab32_cpy(smp_req, smp_req, sg->length / sizeof(u32));
2851                 kunmap_atomic(kaddr, KM_IRQ0);
2852                 break;
2853         }
2854         default:
2855                 break;
2856         }
2857
2858         /* Put the completed request on the correct list */
2859         isci_task_save_for_upper_layer_completion(isci_host, request, response,
2860                                                   status, complete_to_host
2861                                                   );
2862
2863         /* complete the io request to the core. */
2864         scic_controller_complete_io(&isci_host->sci,
2865                                     request->sci.target_device,
2866                                     &request->sci);
2867         isci_put_device(idev);
2868
2869         /* set terminated handle so it cannot be completed or
2870          * terminated again, and to cause any calls into abort
2871          * task to recognize the already completed case.
2872          */
2873         set_bit(IREQ_TERMINATED, &request->flags);
2874 }
2875
2876 static void scic_sds_request_started_state_enter(struct sci_base_state_machine *sm)
2877 {
2878         struct scic_sds_request *sci_req = container_of(sm, typeof(*sci_req), sm);
2879         struct isci_request *ireq = sci_req_to_ireq(sci_req);
2880         struct domain_device *dev = sci_dev_to_domain(sci_req->target_device);
2881         struct sas_task *task;
2882
2883         /* XXX as hch said always creating an internal sas_task for tmf
2884          * requests would simplify the driver
2885          */
2886         task = ireq->ttype == io_task ? isci_request_access_task(ireq) : NULL;
2887
2888         /* all unaccelerated request types (non ssp or ncq) handled with
2889          * substates
2890          */
2891         if (!task && dev->dev_type == SAS_END_DEV) {
2892                 sci_change_state(sm, SCI_REQ_TASK_WAIT_TC_COMP);
2893         } else if (!task &&
2894                    (isci_request_access_tmf(ireq)->tmf_code == isci_tmf_sata_srst_high ||
2895                     isci_request_access_tmf(ireq)->tmf_code == isci_tmf_sata_srst_low)) {
2896                 sci_change_state(sm, SCI_REQ_STP_SOFT_RESET_WAIT_H2D_ASSERTED);
2897         } else if (task && task->task_proto == SAS_PROTOCOL_SMP) {
2898                 sci_change_state(sm, SCI_REQ_SMP_WAIT_RESP);
2899         } else if (task && sas_protocol_ata(task->task_proto) &&
2900                    !task->ata_task.use_ncq) {
2901                 u32 state;
2902
2903                 if (task->data_dir == DMA_NONE)
2904                         state = SCI_REQ_STP_NON_DATA_WAIT_H2D;
2905                 else if (task->ata_task.dma_xfer)
2906                         state = SCI_REQ_STP_UDMA_WAIT_TC_COMP;
2907                 else /* PIO */
2908                         state = SCI_REQ_STP_PIO_WAIT_H2D;
2909
2910                 sci_change_state(sm, state);
2911         }
2912 }
2913
2914 static void scic_sds_request_completed_state_enter(struct sci_base_state_machine *sm)
2915 {
2916         struct scic_sds_request *sci_req = container_of(sm, typeof(*sci_req), sm);
2917         struct scic_sds_controller *scic = sci_req->owning_controller;
2918         struct isci_host *ihost = scic_to_ihost(scic);
2919         struct isci_request *ireq = sci_req_to_ireq(sci_req);
2920
2921         /* Tell the SCI_USER that the IO request is complete */
2922         if (!test_bit(IREQ_TMF, &ireq->flags))
2923                 isci_request_io_request_complete(ihost, ireq,
2924                                                  sci_req->sci_status);
2925         else
2926                 isci_task_request_complete(ihost, ireq, sci_req->sci_status);
2927 }
2928
2929 static void scic_sds_request_aborting_state_enter(struct sci_base_state_machine *sm)
2930 {
2931         struct scic_sds_request *sci_req = container_of(sm, typeof(*sci_req), sm);
2932
2933         /* Setting the abort bit in the Task Context is required by the silicon. */
2934         sci_req->tc->abort = 1;
2935 }
2936
2937 static void scic_sds_stp_request_started_non_data_await_h2d_completion_enter(struct sci_base_state_machine *sm)
2938 {
2939         struct scic_sds_request *sci_req = container_of(sm, typeof(*sci_req), sm);
2940
2941         scic_sds_remote_device_set_working_request(sci_req->target_device,
2942                                                    sci_req);
2943 }
2944
2945 static void scic_sds_stp_request_started_pio_await_h2d_completion_enter(struct sci_base_state_machine *sm)
2946 {
2947         struct scic_sds_request *sci_req = container_of(sm, typeof(*sci_req), sm);
2948
2949         scic_sds_remote_device_set_working_request(sci_req->target_device,
2950                                                    sci_req);
2951 }
2952
2953 static void scic_sds_stp_request_started_soft_reset_await_h2d_asserted_completion_enter(struct sci_base_state_machine *sm)
2954 {
2955         struct scic_sds_request *sci_req = container_of(sm, typeof(*sci_req), sm);
2956
2957         scic_sds_remote_device_set_working_request(sci_req->target_device,
2958                                                    sci_req);
2959 }
2960
2961 static void scic_sds_stp_request_started_soft_reset_await_h2d_diagnostic_completion_enter(struct sci_base_state_machine *sm)
2962 {
2963         struct scic_sds_request *sci_req = container_of(sm, typeof(*sci_req), sm);
2964         struct scu_task_context *tc = sci_req->tc;
2965         struct host_to_dev_fis *h2d_fis;
2966         enum sci_status status;
2967
2968         /* Clear the SRST bit */
2969         h2d_fis = &sci_req->stp.cmd;
2970         h2d_fis->control = 0;
2971
2972         /* Clear the TC control bit */
2973         tc->control_frame = 0;
2974
2975         status = scic_controller_continue_io(sci_req);
2976         WARN_ONCE(status != SCI_SUCCESS, "isci: continue io failure\n");
2977 }
2978
2979 static const struct sci_base_state scic_sds_request_state_table[] = {
2980         [SCI_REQ_INIT] = { },
2981         [SCI_REQ_CONSTRUCTED] = { },
2982         [SCI_REQ_STARTED] = {
2983                 .enter_state = scic_sds_request_started_state_enter,
2984         },
2985         [SCI_REQ_STP_NON_DATA_WAIT_H2D] = {
2986                 .enter_state = scic_sds_stp_request_started_non_data_await_h2d_completion_enter,
2987         },
2988         [SCI_REQ_STP_NON_DATA_WAIT_D2H] = { },
2989         [SCI_REQ_STP_PIO_WAIT_H2D] = {
2990                 .enter_state = scic_sds_stp_request_started_pio_await_h2d_completion_enter,
2991         },
2992         [SCI_REQ_STP_PIO_WAIT_FRAME] = { },
2993         [SCI_REQ_STP_PIO_DATA_IN] = { },
2994         [SCI_REQ_STP_PIO_DATA_OUT] = { },
2995         [SCI_REQ_STP_UDMA_WAIT_TC_COMP] = { },
2996         [SCI_REQ_STP_UDMA_WAIT_D2H] = { },
2997         [SCI_REQ_STP_SOFT_RESET_WAIT_H2D_ASSERTED] = {
2998                 .enter_state = scic_sds_stp_request_started_soft_reset_await_h2d_asserted_completion_enter,
2999         },
3000         [SCI_REQ_STP_SOFT_RESET_WAIT_H2D_DIAG] = {
3001                 .enter_state = scic_sds_stp_request_started_soft_reset_await_h2d_diagnostic_completion_enter,
3002         },
3003         [SCI_REQ_STP_SOFT_RESET_WAIT_D2H] = { },
3004         [SCI_REQ_TASK_WAIT_TC_COMP] = { },
3005         [SCI_REQ_TASK_WAIT_TC_RESP] = { },
3006         [SCI_REQ_SMP_WAIT_RESP] = { },
3007         [SCI_REQ_SMP_WAIT_TC_COMP] = { },
3008         [SCI_REQ_COMPLETED] = {
3009                 .enter_state = scic_sds_request_completed_state_enter,
3010         },
3011         [SCI_REQ_ABORTING] = {
3012                 .enter_state = scic_sds_request_aborting_state_enter,
3013         },
3014         [SCI_REQ_FINAL] = { },
3015 };
3016
3017 static void
3018 scic_sds_general_request_construct(struct scic_sds_controller *scic,
3019                                    struct scic_sds_remote_device *sci_dev,
3020                                    struct scic_sds_request *sci_req)
3021 {
3022         sci_init_sm(&sci_req->sm, scic_sds_request_state_table, SCI_REQ_INIT);
3023
3024         sci_req->target_device = sci_dev;
3025         sci_req->protocol = SCIC_NO_PROTOCOL;
3026         sci_req->saved_rx_frame_index = SCU_INVALID_FRAME_INDEX;
3027
3028         sci_req->sci_status   = SCI_SUCCESS;
3029         sci_req->scu_status   = 0;
3030         sci_req->post_context = 0xFFFFFFFF;
3031 }
3032
3033 static enum sci_status
3034 scic_io_request_construct(struct scic_sds_controller *scic,
3035                           struct scic_sds_remote_device *sci_dev,
3036                           struct scic_sds_request *sci_req)
3037 {
3038         struct domain_device *dev = sci_dev_to_domain(sci_dev);
3039         enum sci_status status = SCI_SUCCESS;
3040
3041         /* Build the common part of the request */
3042         scic_sds_general_request_construct(scic, sci_dev, sci_req);
3043
3044         if (sci_dev->rnc.remote_node_index == SCIC_SDS_REMOTE_NODE_CONTEXT_INVALID_INDEX)
3045                 return SCI_FAILURE_INVALID_REMOTE_DEVICE;
3046
3047         if (dev->dev_type == SAS_END_DEV)
3048                 /* pass */;
3049         else if (dev->dev_type == SATA_DEV || (dev->tproto & SAS_PROTOCOL_STP))
3050                 memset(&sci_req->stp.cmd, 0, sizeof(sci_req->stp.cmd));
3051         else if (dev_is_expander(dev))
3052                 /* pass */;
3053         else
3054                 return SCI_FAILURE_UNSUPPORTED_PROTOCOL;
3055
3056         memset(sci_req->tc, 0, offsetof(struct scu_task_context, sgl_pair_ab));
3057
3058         return status;
3059 }
3060
3061 enum sci_status scic_task_request_construct(struct scic_sds_controller *scic,
3062                                             struct scic_sds_remote_device *sci_dev,
3063                                             u16 io_tag, struct scic_sds_request *sci_req)
3064 {
3065         struct domain_device *dev = sci_dev_to_domain(sci_dev);
3066         enum sci_status status = SCI_SUCCESS;
3067
3068         /* Build the common part of the request */
3069         scic_sds_general_request_construct(scic, sci_dev, sci_req);
3070
3071         if (dev->dev_type == SAS_END_DEV ||
3072             dev->dev_type == SATA_DEV || (dev->tproto & SAS_PROTOCOL_STP)) {
3073                 set_bit(IREQ_TMF, &sci_req_to_ireq(sci_req)->flags);
3074                 memset(sci_req->tc, 0, sizeof(struct scu_task_context));
3075         } else
3076                 status = SCI_FAILURE_UNSUPPORTED_PROTOCOL;
3077
3078         return status;
3079 }
3080
3081 static enum sci_status isci_request_ssp_request_construct(
3082         struct isci_request *request)
3083 {
3084         enum sci_status status;
3085
3086         dev_dbg(&request->isci_host->pdev->dev,
3087                 "%s: request = %p\n",
3088                 __func__,
3089                 request);
3090         status = scic_io_request_construct_basic_ssp(&request->sci);
3091         return status;
3092 }
3093
3094 static enum sci_status isci_request_stp_request_construct(
3095         struct isci_request *request)
3096 {
3097         struct sas_task *task = isci_request_access_task(request);
3098         enum sci_status status;
3099         struct host_to_dev_fis *register_fis;
3100
3101         dev_dbg(&request->isci_host->pdev->dev,
3102                 "%s: request = %p\n",
3103                 __func__,
3104                 request);
3105
3106         /* Get the host_to_dev_fis from the core and copy
3107          * the fis from the task into it.
3108          */
3109         register_fis = isci_sata_task_to_fis_copy(task);
3110
3111         status = scic_io_request_construct_basic_sata(&request->sci);
3112
3113         /* Set the ncq tag in the fis, from the queue
3114          * command in the task.
3115          */
3116         if (isci_sata_is_task_ncq(task)) {
3117
3118                 isci_sata_set_ncq_tag(
3119                         register_fis,
3120                         task
3121                         );
3122         }
3123
3124         return status;
3125 }
3126
3127 static enum sci_status
3128 scic_io_request_construct_smp(struct device *dev,
3129                               struct scic_sds_request *sci_req,
3130                               struct sas_task *task)
3131 {
3132         struct scatterlist *sg = &task->smp_task.smp_req;
3133         struct scic_sds_remote_device *sci_dev;
3134         struct scu_task_context *task_context;
3135         struct scic_sds_port *sci_port;
3136         struct smp_req *smp_req;
3137         void *kaddr;
3138         u8 req_len;
3139         u32 cmd;
3140
3141         kaddr = kmap_atomic(sg_page(sg), KM_IRQ0);
3142         smp_req = kaddr + sg->offset;
3143         /*
3144          * Look at the SMP requests' header fields; for certain SAS 1.x SMP
3145          * functions under SAS 2.0, a zero request length really indicates
3146          * a non-zero default length.
3147          */
3148         if (smp_req->req_len == 0) {
3149                 switch (smp_req->func) {
3150                 case SMP_DISCOVER:
3151                 case SMP_REPORT_PHY_ERR_LOG:
3152                 case SMP_REPORT_PHY_SATA:
3153                 case SMP_REPORT_ROUTE_INFO:
3154                         smp_req->req_len = 2;
3155                         break;
3156                 case SMP_CONF_ROUTE_INFO:
3157                 case SMP_PHY_CONTROL:
3158                 case SMP_PHY_TEST_FUNCTION:
3159                         smp_req->req_len = 9;
3160                         break;
3161                         /* Default - zero is a valid default for 2.0. */
3162                 }
3163         }
3164         req_len = smp_req->req_len;
3165         sci_swab32_cpy(smp_req, smp_req, sg->length / sizeof(u32));
3166         cmd = *(u32 *) smp_req;
3167         kunmap_atomic(kaddr, KM_IRQ0);
3168
3169         if (!dma_map_sg(dev, sg, 1, DMA_TO_DEVICE))
3170                 return SCI_FAILURE;
3171
3172         sci_req->protocol = SCIC_SMP_PROTOCOL;
3173
3174         /* byte swap the smp request. */
3175
3176         task_context = sci_req->tc;
3177
3178         sci_dev = scic_sds_request_get_device(sci_req);
3179         sci_port = scic_sds_request_get_port(sci_req);
3180
3181         /*
3182          * Fill in the TC with the its required data
3183          * 00h
3184          */
3185         task_context->priority = 0;
3186         task_context->initiator_request = 1;
3187         task_context->connection_rate = sci_dev->connection_rate;
3188         task_context->protocol_engine_index =
3189                 scic_sds_controller_get_protocol_engine_group(scic);
3190         task_context->logical_port_index = scic_sds_port_get_index(sci_port);
3191         task_context->protocol_type = SCU_TASK_CONTEXT_PROTOCOL_SMP;
3192         task_context->abort = 0;
3193         task_context->valid = SCU_TASK_CONTEXT_VALID;
3194         task_context->context_type = SCU_TASK_CONTEXT_TYPE;
3195
3196         /* 04h */
3197         task_context->remote_node_index = sci_dev->rnc.remote_node_index;
3198         task_context->command_code = 0;
3199         task_context->task_type = SCU_TASK_TYPE_SMP_REQUEST;
3200
3201         /* 08h */
3202         task_context->link_layer_control = 0;
3203         task_context->do_not_dma_ssp_good_response = 1;
3204         task_context->strict_ordering = 0;
3205         task_context->control_frame = 1;
3206         task_context->timeout_enable = 0;
3207         task_context->block_guard_enable = 0;
3208
3209         /* 0ch */
3210         task_context->address_modifier = 0;
3211
3212         /* 10h */
3213         task_context->ssp_command_iu_length = req_len;
3214
3215         /* 14h */
3216         task_context->transfer_length_bytes = 0;
3217
3218         /*
3219          * 18h ~ 30h, protocol specific
3220          * since commandIU has been build by framework at this point, we just
3221          * copy the frist DWord from command IU to this location. */
3222         memcpy(&task_context->type.smp, &cmd, sizeof(u32));
3223
3224         /*
3225          * 40h
3226          * "For SMP you could program it to zero. We would prefer that way
3227          * so that done code will be consistent." - Venki
3228          */
3229         task_context->task_phase = 0;
3230
3231         sci_req->post_context = (SCU_CONTEXT_COMMAND_REQUEST_TYPE_POST_TC |
3232                                  (scic_sds_controller_get_protocol_engine_group(scic) <<
3233                                   SCU_CONTEXT_COMMAND_PROTOCOL_ENGINE_GROUP_SHIFT) |
3234                                  (scic_sds_port_get_index(sci_port) <<
3235                                   SCU_CONTEXT_COMMAND_LOGICAL_PORT_SHIFT) |
3236                                  ISCI_TAG_TCI(sci_req->io_tag));
3237         /*
3238          * Copy the physical address for the command buffer to the SCU Task
3239          * Context command buffer should not contain command header.
3240          */
3241         task_context->command_iu_upper = upper_32_bits(sg_dma_address(sg));
3242         task_context->command_iu_lower = lower_32_bits(sg_dma_address(sg) + sizeof(u32));
3243
3244         /* SMP response comes as UF, so no need to set response IU address. */
3245         task_context->response_iu_upper = 0;
3246         task_context->response_iu_lower = 0;
3247
3248         sci_change_state(&sci_req->sm, SCI_REQ_CONSTRUCTED);
3249
3250         return SCI_SUCCESS;
3251 }
3252
3253 /*
3254  * isci_smp_request_build() - This function builds the smp request.
3255  * @ireq: This parameter points to the isci_request allocated in the
3256  *    request construct function.
3257  *
3258  * SCI_SUCCESS on successfull completion, or specific failure code.
3259  */
3260 static enum sci_status isci_smp_request_build(struct isci_request *ireq)
3261 {
3262         struct sas_task *task = isci_request_access_task(ireq);
3263         struct device *dev = &ireq->isci_host->pdev->dev;
3264         struct scic_sds_request *sci_req = &ireq->sci;
3265         enum sci_status status = SCI_FAILURE;
3266
3267         status = scic_io_request_construct_smp(dev, sci_req, task);
3268         if (status != SCI_SUCCESS)
3269                 dev_warn(&ireq->isci_host->pdev->dev,
3270                          "%s: failed with status = %d\n",
3271                          __func__,
3272                          status);
3273
3274         return status;
3275 }
3276
3277 /**
3278  * isci_io_request_build() - This function builds the io request object.
3279  * @isci_host: This parameter specifies the ISCI host object
3280  * @request: This parameter points to the isci_request object allocated in the
3281  *    request construct function.
3282  * @sci_device: This parameter is the handle for the sci core's remote device
3283  *    object that is the destination for this request.
3284  *
3285  * SCI_SUCCESS on successfull completion, or specific failure code.
3286  */
3287 static enum sci_status isci_io_request_build(struct isci_host *isci_host,
3288                                              struct isci_request *request,
3289                                              struct isci_remote_device *isci_device)
3290 {
3291         enum sci_status status = SCI_SUCCESS;
3292         struct sas_task *task = isci_request_access_task(request);
3293         struct scic_sds_remote_device *sci_device = &isci_device->sci;
3294
3295         dev_dbg(&isci_host->pdev->dev,
3296                 "%s: isci_device = 0x%p; request = %p, "
3297                 "num_scatter = %d\n",
3298                 __func__,
3299                 isci_device,
3300                 request,
3301                 task->num_scatter);
3302
3303         /* map the sgl addresses, if present.
3304          * libata does the mapping for sata devices
3305          * before we get the request.
3306          */
3307         if (task->num_scatter &&
3308             !sas_protocol_ata(task->task_proto) &&
3309             !(SAS_PROTOCOL_SMP & task->task_proto)) {
3310
3311                 request->num_sg_entries = dma_map_sg(
3312                         &isci_host->pdev->dev,
3313                         task->scatter,
3314                         task->num_scatter,
3315                         task->data_dir
3316                         );
3317
3318                 if (request->num_sg_entries == 0)
3319                         return SCI_FAILURE_INSUFFICIENT_RESOURCES;
3320         }
3321
3322         status = scic_io_request_construct(&isci_host->sci, sci_device,
3323                                            &request->sci);
3324
3325         if (status != SCI_SUCCESS) {
3326                 dev_warn(&isci_host->pdev->dev,
3327                          "%s: failed request construct\n",
3328                          __func__);
3329                 return SCI_FAILURE;
3330         }
3331
3332         switch (task->task_proto) {
3333         case SAS_PROTOCOL_SMP:
3334                 status = isci_smp_request_build(request);
3335                 break;
3336         case SAS_PROTOCOL_SSP:
3337                 status = isci_request_ssp_request_construct(request);
3338                 break;
3339         case SAS_PROTOCOL_SATA:
3340         case SAS_PROTOCOL_STP:
3341         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
3342                 status = isci_request_stp_request_construct(request);
3343                 break;
3344         default:
3345                 dev_warn(&isci_host->pdev->dev,
3346                          "%s: unknown protocol\n", __func__);
3347                 return SCI_FAILURE;
3348         }
3349
3350         return SCI_SUCCESS;
3351 }
3352
3353 static struct isci_request *isci_request_from_tag(struct isci_host *ihost, u16 tag)
3354 {
3355         struct isci_request *ireq;
3356
3357         ireq = ihost->reqs[ISCI_TAG_TCI(tag)];
3358         ireq->sci.io_tag = tag;
3359         ireq->io_request_completion = NULL;
3360         ireq->flags = 0;
3361         ireq->num_sg_entries = 0;
3362         INIT_LIST_HEAD(&ireq->completed_node);
3363         INIT_LIST_HEAD(&ireq->dev_node);
3364         isci_request_change_state(ireq, allocated);
3365
3366         return ireq;
3367 }
3368
3369 static struct isci_request *isci_io_request_from_tag(struct isci_host *ihost,
3370                                                      struct sas_task *task,
3371                                                      u16 tag)
3372 {
3373         struct isci_request *ireq;
3374
3375         ireq = isci_request_from_tag(ihost, tag);
3376         ireq->ttype_ptr.io_task_ptr = task;
3377         ireq->ttype = io_task;
3378         task->lldd_task = ireq;
3379
3380         return ireq;
3381 }
3382
3383 struct isci_request *isci_tmf_request_from_tag(struct isci_host *ihost,
3384                                                struct isci_tmf *isci_tmf,
3385                                                u16 tag)
3386 {
3387         struct isci_request *ireq;
3388
3389         ireq = isci_request_from_tag(ihost, tag);
3390         ireq->ttype_ptr.tmf_task_ptr = isci_tmf;
3391         ireq->ttype = tmf_task;
3392
3393         return ireq;
3394 }
3395
3396 int isci_request_execute(struct isci_host *ihost, struct isci_remote_device *idev,
3397                          struct sas_task *task, u16 tag)
3398 {
3399         enum sci_status status = SCI_FAILURE_UNSUPPORTED_PROTOCOL;
3400         struct isci_request *ireq;
3401         unsigned long flags;
3402         int ret = 0;
3403
3404         /* do common allocation and init of request object. */
3405         ireq = isci_io_request_from_tag(ihost, task, tag);
3406
3407         status = isci_io_request_build(ihost, ireq, idev);
3408         if (status != SCI_SUCCESS) {
3409                 dev_warn(&ihost->pdev->dev,
3410                          "%s: request_construct failed - status = 0x%x\n",
3411                          __func__,
3412                          status);
3413                 return status;
3414         }
3415
3416         spin_lock_irqsave(&ihost->scic_lock, flags);
3417
3418         if (test_bit(IDEV_IO_NCQERROR, &idev->flags)) {
3419
3420                 if (isci_task_is_ncq_recovery(task)) {
3421
3422                         /* The device is in an NCQ recovery state.  Issue the
3423                          * request on the task side.  Note that it will
3424                          * complete on the I/O request side because the
3425                          * request was built that way (ie.
3426                          * ireq->is_task_management_request is false).
3427                          */
3428                         status = scic_controller_start_task(&ihost->sci,
3429                                                             &idev->sci,
3430                                                             &ireq->sci);
3431                 } else {
3432                         status = SCI_FAILURE;
3433                 }
3434         } else {
3435                 /* send the request, let the core assign the IO TAG.    */
3436                 status = scic_controller_start_io(&ihost->sci, &idev->sci,
3437                                                   &ireq->sci);
3438         }
3439
3440         if (status != SCI_SUCCESS &&
3441             status != SCI_FAILURE_REMOTE_DEVICE_RESET_REQUIRED) {
3442                 dev_warn(&ihost->pdev->dev,
3443                          "%s: failed request start (0x%x)\n",
3444                          __func__, status);
3445                 spin_unlock_irqrestore(&ihost->scic_lock, flags);
3446                 return status;
3447         }
3448
3449         /* Either I/O started OK, or the core has signaled that
3450          * the device needs a target reset.
3451          *
3452          * In either case, hold onto the I/O for later.
3453          *
3454          * Update it's status and add it to the list in the
3455          * remote device object.
3456          */
3457         list_add(&ireq->dev_node, &idev->reqs_in_process);
3458
3459         if (status == SCI_SUCCESS) {
3460                 /* Save the tag for possible task mgmt later. */
3461                 ireq->io_tag = ireq->sci.io_tag;
3462                 isci_request_change_state(ireq, started);
3463         } else {
3464                 /* The request did not really start in the
3465                  * hardware, so clear the request handle
3466                  * here so no terminations will be done.
3467                  */
3468                 set_bit(IREQ_TERMINATED, &ireq->flags);
3469                 isci_request_change_state(ireq, completed);
3470         }
3471         spin_unlock_irqrestore(&ihost->scic_lock, flags);
3472
3473         if (status ==
3474             SCI_FAILURE_REMOTE_DEVICE_RESET_REQUIRED) {
3475                 /* Signal libsas that we need the SCSI error
3476                  * handler thread to work on this I/O and that
3477                  * we want a device reset.
3478                  */
3479                 spin_lock_irqsave(&task->task_state_lock, flags);
3480                 task->task_state_flags |= SAS_TASK_NEED_DEV_RESET;
3481                 spin_unlock_irqrestore(&task->task_state_lock, flags);
3482
3483                 /* Cause this task to be scheduled in the SCSI error
3484                  * handler thread.
3485                  */
3486                 isci_execpath_callback(ihost, task,
3487                                        sas_task_abort);
3488
3489                 /* Change the status, since we are holding
3490                  * the I/O until it is managed by the SCSI
3491                  * error handler.
3492                  */
3493                 status = SCI_SUCCESS;
3494         }
3495
3496         return ret;
3497 }