Merge branch 'drm-next' of git://people.freedesktop.org/~airlied/linux
[linux-3.10.git] / drivers / gpu / drm / radeon / evergreen_cs.c
1 /*
2  * Copyright 2010 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <drm/drmP.h>
29 #include "radeon.h"
30 #include "evergreend.h"
31 #include "evergreen_reg_safe.h"
32 #include "cayman_reg_safe.h"
33
34 #define MAX(a,b)                   (((a)>(b))?(a):(b))
35 #define MIN(a,b)                   (((a)<(b))?(a):(b))
36
37 int r600_dma_cs_next_reloc(struct radeon_cs_parser *p,
38                            struct radeon_cs_reloc **cs_reloc);
39 struct evergreen_cs_track {
40         u32                     group_size;
41         u32                     nbanks;
42         u32                     npipes;
43         u32                     row_size;
44         /* value we track */
45         u32                     nsamples;               /* unused */
46         struct radeon_bo        *cb_color_bo[12];
47         u32                     cb_color_bo_offset[12];
48         struct radeon_bo        *cb_color_fmask_bo[8];  /* unused */
49         struct radeon_bo        *cb_color_cmask_bo[8];  /* unused */
50         u32                     cb_color_info[12];
51         u32                     cb_color_view[12];
52         u32                     cb_color_pitch[12];
53         u32                     cb_color_slice[12];
54         u32                     cb_color_slice_idx[12];
55         u32                     cb_color_attrib[12];
56         u32                     cb_color_cmask_slice[8];/* unused */
57         u32                     cb_color_fmask_slice[8];/* unused */
58         u32                     cb_target_mask;
59         u32                     cb_shader_mask; /* unused */
60         u32                     vgt_strmout_config;
61         u32                     vgt_strmout_buffer_config;
62         struct radeon_bo        *vgt_strmout_bo[4];
63         u32                     vgt_strmout_bo_offset[4];
64         u32                     vgt_strmout_size[4];
65         u32                     db_depth_control;
66         u32                     db_depth_view;
67         u32                     db_depth_slice;
68         u32                     db_depth_size;
69         u32                     db_z_info;
70         u32                     db_z_read_offset;
71         u32                     db_z_write_offset;
72         struct radeon_bo        *db_z_read_bo;
73         struct radeon_bo        *db_z_write_bo;
74         u32                     db_s_info;
75         u32                     db_s_read_offset;
76         u32                     db_s_write_offset;
77         struct radeon_bo        *db_s_read_bo;
78         struct radeon_bo        *db_s_write_bo;
79         bool                    sx_misc_kill_all_prims;
80         bool                    cb_dirty;
81         bool                    db_dirty;
82         bool                    streamout_dirty;
83         u32                     htile_offset;
84         u32                     htile_surface;
85         struct radeon_bo        *htile_bo;
86 };
87
88 static u32 evergreen_cs_get_aray_mode(u32 tiling_flags)
89 {
90         if (tiling_flags & RADEON_TILING_MACRO)
91                 return ARRAY_2D_TILED_THIN1;
92         else if (tiling_flags & RADEON_TILING_MICRO)
93                 return ARRAY_1D_TILED_THIN1;
94         else
95                 return ARRAY_LINEAR_GENERAL;
96 }
97
98 static u32 evergreen_cs_get_num_banks(u32 nbanks)
99 {
100         switch (nbanks) {
101         case 2:
102                 return ADDR_SURF_2_BANK;
103         case 4:
104                 return ADDR_SURF_4_BANK;
105         case 8:
106         default:
107                 return ADDR_SURF_8_BANK;
108         case 16:
109                 return ADDR_SURF_16_BANK;
110         }
111 }
112
113 static void evergreen_cs_track_init(struct evergreen_cs_track *track)
114 {
115         int i;
116
117         for (i = 0; i < 8; i++) {
118                 track->cb_color_fmask_bo[i] = NULL;
119                 track->cb_color_cmask_bo[i] = NULL;
120                 track->cb_color_cmask_slice[i] = 0;
121                 track->cb_color_fmask_slice[i] = 0;
122         }
123
124         for (i = 0; i < 12; i++) {
125                 track->cb_color_bo[i] = NULL;
126                 track->cb_color_bo_offset[i] = 0xFFFFFFFF;
127                 track->cb_color_info[i] = 0;
128                 track->cb_color_view[i] = 0xFFFFFFFF;
129                 track->cb_color_pitch[i] = 0;
130                 track->cb_color_slice[i] = 0xfffffff;
131                 track->cb_color_slice_idx[i] = 0;
132         }
133         track->cb_target_mask = 0xFFFFFFFF;
134         track->cb_shader_mask = 0xFFFFFFFF;
135         track->cb_dirty = true;
136
137         track->db_depth_slice = 0xffffffff;
138         track->db_depth_view = 0xFFFFC000;
139         track->db_depth_size = 0xFFFFFFFF;
140         track->db_depth_control = 0xFFFFFFFF;
141         track->db_z_info = 0xFFFFFFFF;
142         track->db_z_read_offset = 0xFFFFFFFF;
143         track->db_z_write_offset = 0xFFFFFFFF;
144         track->db_z_read_bo = NULL;
145         track->db_z_write_bo = NULL;
146         track->db_s_info = 0xFFFFFFFF;
147         track->db_s_read_offset = 0xFFFFFFFF;
148         track->db_s_write_offset = 0xFFFFFFFF;
149         track->db_s_read_bo = NULL;
150         track->db_s_write_bo = NULL;
151         track->db_dirty = true;
152         track->htile_bo = NULL;
153         track->htile_offset = 0xFFFFFFFF;
154         track->htile_surface = 0;
155
156         for (i = 0; i < 4; i++) {
157                 track->vgt_strmout_size[i] = 0;
158                 track->vgt_strmout_bo[i] = NULL;
159                 track->vgt_strmout_bo_offset[i] = 0xFFFFFFFF;
160         }
161         track->streamout_dirty = true;
162         track->sx_misc_kill_all_prims = false;
163 }
164
165 struct eg_surface {
166         /* value gathered from cs */
167         unsigned        nbx;
168         unsigned        nby;
169         unsigned        format;
170         unsigned        mode;
171         unsigned        nbanks;
172         unsigned        bankw;
173         unsigned        bankh;
174         unsigned        tsplit;
175         unsigned        mtilea;
176         unsigned        nsamples;
177         /* output value */
178         unsigned        bpe;
179         unsigned        layer_size;
180         unsigned        palign;
181         unsigned        halign;
182         unsigned long   base_align;
183 };
184
185 static int evergreen_surface_check_linear(struct radeon_cs_parser *p,
186                                           struct eg_surface *surf,
187                                           const char *prefix)
188 {
189         surf->layer_size = surf->nbx * surf->nby * surf->bpe * surf->nsamples;
190         surf->base_align = surf->bpe;
191         surf->palign = 1;
192         surf->halign = 1;
193         return 0;
194 }
195
196 static int evergreen_surface_check_linear_aligned(struct radeon_cs_parser *p,
197                                                   struct eg_surface *surf,
198                                                   const char *prefix)
199 {
200         struct evergreen_cs_track *track = p->track;
201         unsigned palign;
202
203         palign = MAX(64, track->group_size / surf->bpe);
204         surf->layer_size = surf->nbx * surf->nby * surf->bpe * surf->nsamples;
205         surf->base_align = track->group_size;
206         surf->palign = palign;
207         surf->halign = 1;
208         if (surf->nbx & (palign - 1)) {
209                 if (prefix) {
210                         dev_warn(p->dev, "%s:%d %s pitch %d invalid must be aligned with %d\n",
211                                  __func__, __LINE__, prefix, surf->nbx, palign);
212                 }
213                 return -EINVAL;
214         }
215         return 0;
216 }
217
218 static int evergreen_surface_check_1d(struct radeon_cs_parser *p,
219                                       struct eg_surface *surf,
220                                       const char *prefix)
221 {
222         struct evergreen_cs_track *track = p->track;
223         unsigned palign;
224
225         palign = track->group_size / (8 * surf->bpe * surf->nsamples);
226         palign = MAX(8, palign);
227         surf->layer_size = surf->nbx * surf->nby * surf->bpe;
228         surf->base_align = track->group_size;
229         surf->palign = palign;
230         surf->halign = 8;
231         if ((surf->nbx & (palign - 1))) {
232                 if (prefix) {
233                         dev_warn(p->dev, "%s:%d %s pitch %d invalid must be aligned with %d (%d %d %d)\n",
234                                  __func__, __LINE__, prefix, surf->nbx, palign,
235                                  track->group_size, surf->bpe, surf->nsamples);
236                 }
237                 return -EINVAL;
238         }
239         if ((surf->nby & (8 - 1))) {
240                 if (prefix) {
241                         dev_warn(p->dev, "%s:%d %s height %d invalid must be aligned with 8\n",
242                                  __func__, __LINE__, prefix, surf->nby);
243                 }
244                 return -EINVAL;
245         }
246         return 0;
247 }
248
249 static int evergreen_surface_check_2d(struct radeon_cs_parser *p,
250                                       struct eg_surface *surf,
251                                       const char *prefix)
252 {
253         struct evergreen_cs_track *track = p->track;
254         unsigned palign, halign, tileb, slice_pt;
255         unsigned mtile_pr, mtile_ps, mtileb;
256
257         tileb = 64 * surf->bpe * surf->nsamples;
258         slice_pt = 1;
259         if (tileb > surf->tsplit) {
260                 slice_pt = tileb / surf->tsplit;
261         }
262         tileb = tileb / slice_pt;
263         /* macro tile width & height */
264         palign = (8 * surf->bankw * track->npipes) * surf->mtilea;
265         halign = (8 * surf->bankh * surf->nbanks) / surf->mtilea;
266         mtileb = (palign / 8) * (halign / 8) * tileb;
267         mtile_pr = surf->nbx / palign;
268         mtile_ps = (mtile_pr * surf->nby) / halign;
269         surf->layer_size = mtile_ps * mtileb * slice_pt;
270         surf->base_align = (palign / 8) * (halign / 8) * tileb;
271         surf->palign = palign;
272         surf->halign = halign;
273
274         if ((surf->nbx & (palign - 1))) {
275                 if (prefix) {
276                         dev_warn(p->dev, "%s:%d %s pitch %d invalid must be aligned with %d\n",
277                                  __func__, __LINE__, prefix, surf->nbx, palign);
278                 }
279                 return -EINVAL;
280         }
281         if ((surf->nby & (halign - 1))) {
282                 if (prefix) {
283                         dev_warn(p->dev, "%s:%d %s height %d invalid must be aligned with %d\n",
284                                  __func__, __LINE__, prefix, surf->nby, halign);
285                 }
286                 return -EINVAL;
287         }
288
289         return 0;
290 }
291
292 static int evergreen_surface_check(struct radeon_cs_parser *p,
293                                    struct eg_surface *surf,
294                                    const char *prefix)
295 {
296         /* some common value computed here */
297         surf->bpe = r600_fmt_get_blocksize(surf->format);
298
299         switch (surf->mode) {
300         case ARRAY_LINEAR_GENERAL:
301                 return evergreen_surface_check_linear(p, surf, prefix);
302         case ARRAY_LINEAR_ALIGNED:
303                 return evergreen_surface_check_linear_aligned(p, surf, prefix);
304         case ARRAY_1D_TILED_THIN1:
305                 return evergreen_surface_check_1d(p, surf, prefix);
306         case ARRAY_2D_TILED_THIN1:
307                 return evergreen_surface_check_2d(p, surf, prefix);
308         default:
309                 dev_warn(p->dev, "%s:%d %s invalid array mode %d\n",
310                                 __func__, __LINE__, prefix, surf->mode);
311                 return -EINVAL;
312         }
313         return -EINVAL;
314 }
315
316 static int evergreen_surface_value_conv_check(struct radeon_cs_parser *p,
317                                               struct eg_surface *surf,
318                                               const char *prefix)
319 {
320         switch (surf->mode) {
321         case ARRAY_2D_TILED_THIN1:
322                 break;
323         case ARRAY_LINEAR_GENERAL:
324         case ARRAY_LINEAR_ALIGNED:
325         case ARRAY_1D_TILED_THIN1:
326                 return 0;
327         default:
328                 dev_warn(p->dev, "%s:%d %s invalid array mode %d\n",
329                                 __func__, __LINE__, prefix, surf->mode);
330                 return -EINVAL;
331         }
332
333         switch (surf->nbanks) {
334         case 0: surf->nbanks = 2; break;
335         case 1: surf->nbanks = 4; break;
336         case 2: surf->nbanks = 8; break;
337         case 3: surf->nbanks = 16; break;
338         default:
339                 dev_warn(p->dev, "%s:%d %s invalid number of banks %d\n",
340                          __func__, __LINE__, prefix, surf->nbanks);
341                 return -EINVAL;
342         }
343         switch (surf->bankw) {
344         case 0: surf->bankw = 1; break;
345         case 1: surf->bankw = 2; break;
346         case 2: surf->bankw = 4; break;
347         case 3: surf->bankw = 8; break;
348         default:
349                 dev_warn(p->dev, "%s:%d %s invalid bankw %d\n",
350                          __func__, __LINE__, prefix, surf->bankw);
351                 return -EINVAL;
352         }
353         switch (surf->bankh) {
354         case 0: surf->bankh = 1; break;
355         case 1: surf->bankh = 2; break;
356         case 2: surf->bankh = 4; break;
357         case 3: surf->bankh = 8; break;
358         default:
359                 dev_warn(p->dev, "%s:%d %s invalid bankh %d\n",
360                          __func__, __LINE__, prefix, surf->bankh);
361                 return -EINVAL;
362         }
363         switch (surf->mtilea) {
364         case 0: surf->mtilea = 1; break;
365         case 1: surf->mtilea = 2; break;
366         case 2: surf->mtilea = 4; break;
367         case 3: surf->mtilea = 8; break;
368         default:
369                 dev_warn(p->dev, "%s:%d %s invalid macro tile aspect %d\n",
370                          __func__, __LINE__, prefix, surf->mtilea);
371                 return -EINVAL;
372         }
373         switch (surf->tsplit) {
374         case 0: surf->tsplit = 64; break;
375         case 1: surf->tsplit = 128; break;
376         case 2: surf->tsplit = 256; break;
377         case 3: surf->tsplit = 512; break;
378         case 4: surf->tsplit = 1024; break;
379         case 5: surf->tsplit = 2048; break;
380         case 6: surf->tsplit = 4096; break;
381         default:
382                 dev_warn(p->dev, "%s:%d %s invalid tile split %d\n",
383                          __func__, __LINE__, prefix, surf->tsplit);
384                 return -EINVAL;
385         }
386         return 0;
387 }
388
389 static int evergreen_cs_track_validate_cb(struct radeon_cs_parser *p, unsigned id)
390 {
391         struct evergreen_cs_track *track = p->track;
392         struct eg_surface surf;
393         unsigned pitch, slice, mslice;
394         unsigned long offset;
395         int r;
396
397         mslice = G_028C6C_SLICE_MAX(track->cb_color_view[id]) + 1;
398         pitch = track->cb_color_pitch[id];
399         slice = track->cb_color_slice[id];
400         surf.nbx = (pitch + 1) * 8;
401         surf.nby = ((slice + 1) * 64) / surf.nbx;
402         surf.mode = G_028C70_ARRAY_MODE(track->cb_color_info[id]);
403         surf.format = G_028C70_FORMAT(track->cb_color_info[id]);
404         surf.tsplit = G_028C74_TILE_SPLIT(track->cb_color_attrib[id]);
405         surf.nbanks = G_028C74_NUM_BANKS(track->cb_color_attrib[id]);
406         surf.bankw = G_028C74_BANK_WIDTH(track->cb_color_attrib[id]);
407         surf.bankh = G_028C74_BANK_HEIGHT(track->cb_color_attrib[id]);
408         surf.mtilea = G_028C74_MACRO_TILE_ASPECT(track->cb_color_attrib[id]);
409         surf.nsamples = 1;
410
411         if (!r600_fmt_is_valid_color(surf.format)) {
412                 dev_warn(p->dev, "%s:%d cb invalid format %d for %d (0x%08x)\n",
413                          __func__, __LINE__, surf.format,
414                         id, track->cb_color_info[id]);
415                 return -EINVAL;
416         }
417
418         r = evergreen_surface_value_conv_check(p, &surf, "cb");
419         if (r) {
420                 return r;
421         }
422
423         r = evergreen_surface_check(p, &surf, "cb");
424         if (r) {
425                 dev_warn(p->dev, "%s:%d cb[%d] invalid (0x%08x 0x%08x 0x%08x 0x%08x)\n",
426                          __func__, __LINE__, id, track->cb_color_pitch[id],
427                          track->cb_color_slice[id], track->cb_color_attrib[id],
428                          track->cb_color_info[id]);
429                 return r;
430         }
431
432         offset = track->cb_color_bo_offset[id] << 8;
433         if (offset & (surf.base_align - 1)) {
434                 dev_warn(p->dev, "%s:%d cb[%d] bo base %ld not aligned with %ld\n",
435                          __func__, __LINE__, id, offset, surf.base_align);
436                 return -EINVAL;
437         }
438
439         offset += surf.layer_size * mslice;
440         if (offset > radeon_bo_size(track->cb_color_bo[id])) {
441                 /* old ddx are broken they allocate bo with w*h*bpp but
442                  * program slice with ALIGN(h, 8), catch this and patch
443                  * command stream.
444                  */
445                 if (!surf.mode) {
446                         volatile u32 *ib = p->ib.ptr;
447                         unsigned long tmp, nby, bsize, size, min = 0;
448
449                         /* find the height the ddx wants */
450                         if (surf.nby > 8) {
451                                 min = surf.nby - 8;
452                         }
453                         bsize = radeon_bo_size(track->cb_color_bo[id]);
454                         tmp = track->cb_color_bo_offset[id] << 8;
455                         for (nby = surf.nby; nby > min; nby--) {
456                                 size = nby * surf.nbx * surf.bpe * surf.nsamples;
457                                 if ((tmp + size * mslice) <= bsize) {
458                                         break;
459                                 }
460                         }
461                         if (nby > min) {
462                                 surf.nby = nby;
463                                 slice = ((nby * surf.nbx) / 64) - 1;
464                                 if (!evergreen_surface_check(p, &surf, "cb")) {
465                                         /* check if this one works */
466                                         tmp += surf.layer_size * mslice;
467                                         if (tmp <= bsize) {
468                                                 ib[track->cb_color_slice_idx[id]] = slice;
469                                                 goto old_ddx_ok;
470                                         }
471                                 }
472                         }
473                 }
474                 dev_warn(p->dev, "%s:%d cb[%d] bo too small (layer size %d, "
475                          "offset %d, max layer %d, bo size %ld, slice %d)\n",
476                          __func__, __LINE__, id, surf.layer_size,
477                         track->cb_color_bo_offset[id] << 8, mslice,
478                         radeon_bo_size(track->cb_color_bo[id]), slice);
479                 dev_warn(p->dev, "%s:%d problematic surf: (%d %d) (%d %d %d %d %d %d %d)\n",
480                          __func__, __LINE__, surf.nbx, surf.nby,
481                         surf.mode, surf.bpe, surf.nsamples,
482                         surf.bankw, surf.bankh,
483                         surf.tsplit, surf.mtilea);
484                 return -EINVAL;
485         }
486 old_ddx_ok:
487
488         return 0;
489 }
490
491 static int evergreen_cs_track_validate_htile(struct radeon_cs_parser *p,
492                                                 unsigned nbx, unsigned nby)
493 {
494         struct evergreen_cs_track *track = p->track;
495         unsigned long size;
496
497         if (track->htile_bo == NULL) {
498                 dev_warn(p->dev, "%s:%d htile enabled without htile surface 0x%08x\n",
499                                 __func__, __LINE__, track->db_z_info);
500                 return -EINVAL;
501         }
502
503         if (G_028ABC_LINEAR(track->htile_surface)) {
504                 /* pitch must be 16 htiles aligned == 16 * 8 pixel aligned */
505                 nbx = round_up(nbx, 16 * 8);
506                 /* height is npipes htiles aligned == npipes * 8 pixel aligned */
507                 nby = round_up(nby, track->npipes * 8);
508         } else {
509                 /* always assume 8x8 htile */
510                 /* align is htile align * 8, htile align vary according to
511                  * number of pipe and tile width and nby
512                  */
513                 switch (track->npipes) {
514                 case 8:
515                         /* HTILE_WIDTH = 8 & HTILE_HEIGHT = 8*/
516                         nbx = round_up(nbx, 64 * 8);
517                         nby = round_up(nby, 64 * 8);
518                         break;
519                 case 4:
520                         /* HTILE_WIDTH = 8 & HTILE_HEIGHT = 8*/
521                         nbx = round_up(nbx, 64 * 8);
522                         nby = round_up(nby, 32 * 8);
523                         break;
524                 case 2:
525                         /* HTILE_WIDTH = 8 & HTILE_HEIGHT = 8*/
526                         nbx = round_up(nbx, 32 * 8);
527                         nby = round_up(nby, 32 * 8);
528                         break;
529                 case 1:
530                         /* HTILE_WIDTH = 8 & HTILE_HEIGHT = 8*/
531                         nbx = round_up(nbx, 32 * 8);
532                         nby = round_up(nby, 16 * 8);
533                         break;
534                 default:
535                         dev_warn(p->dev, "%s:%d invalid num pipes %d\n",
536                                         __func__, __LINE__, track->npipes);
537                         return -EINVAL;
538                 }
539         }
540         /* compute number of htile */
541         nbx = nbx >> 3;
542         nby = nby >> 3;
543         /* size must be aligned on npipes * 2K boundary */
544         size = roundup(nbx * nby * 4, track->npipes * (2 << 10));
545         size += track->htile_offset;
546
547         if (size > radeon_bo_size(track->htile_bo)) {
548                 dev_warn(p->dev, "%s:%d htile surface too small %ld for %ld (%d %d)\n",
549                                 __func__, __LINE__, radeon_bo_size(track->htile_bo),
550                                 size, nbx, nby);
551                 return -EINVAL;
552         }
553         return 0;
554 }
555
556 static int evergreen_cs_track_validate_stencil(struct radeon_cs_parser *p)
557 {
558         struct evergreen_cs_track *track = p->track;
559         struct eg_surface surf;
560         unsigned pitch, slice, mslice;
561         unsigned long offset;
562         int r;
563
564         mslice = G_028008_SLICE_MAX(track->db_depth_view) + 1;
565         pitch = G_028058_PITCH_TILE_MAX(track->db_depth_size);
566         slice = track->db_depth_slice;
567         surf.nbx = (pitch + 1) * 8;
568         surf.nby = ((slice + 1) * 64) / surf.nbx;
569         surf.mode = G_028040_ARRAY_MODE(track->db_z_info);
570         surf.format = G_028044_FORMAT(track->db_s_info);
571         surf.tsplit = G_028044_TILE_SPLIT(track->db_s_info);
572         surf.nbanks = G_028040_NUM_BANKS(track->db_z_info);
573         surf.bankw = G_028040_BANK_WIDTH(track->db_z_info);
574         surf.bankh = G_028040_BANK_HEIGHT(track->db_z_info);
575         surf.mtilea = G_028040_MACRO_TILE_ASPECT(track->db_z_info);
576         surf.nsamples = 1;
577
578         if (surf.format != 1) {
579                 dev_warn(p->dev, "%s:%d stencil invalid format %d\n",
580                          __func__, __LINE__, surf.format);
581                 return -EINVAL;
582         }
583         /* replace by color format so we can use same code */
584         surf.format = V_028C70_COLOR_8;
585
586         r = evergreen_surface_value_conv_check(p, &surf, "stencil");
587         if (r) {
588                 return r;
589         }
590
591         r = evergreen_surface_check(p, &surf, NULL);
592         if (r) {
593                 /* old userspace doesn't compute proper depth/stencil alignment
594                  * check that alignment against a bigger byte per elements and
595                  * only report if that alignment is wrong too.
596                  */
597                 surf.format = V_028C70_COLOR_8_8_8_8;
598                 r = evergreen_surface_check(p, &surf, "stencil");
599                 if (r) {
600                         dev_warn(p->dev, "%s:%d stencil invalid (0x%08x 0x%08x 0x%08x 0x%08x)\n",
601                                  __func__, __LINE__, track->db_depth_size,
602                                  track->db_depth_slice, track->db_s_info, track->db_z_info);
603                 }
604                 return r;
605         }
606
607         offset = track->db_s_read_offset << 8;
608         if (offset & (surf.base_align - 1)) {
609                 dev_warn(p->dev, "%s:%d stencil read bo base %ld not aligned with %ld\n",
610                          __func__, __LINE__, offset, surf.base_align);
611                 return -EINVAL;
612         }
613         offset += surf.layer_size * mslice;
614         if (offset > radeon_bo_size(track->db_s_read_bo)) {
615                 dev_warn(p->dev, "%s:%d stencil read bo too small (layer size %d, "
616                          "offset %ld, max layer %d, bo size %ld)\n",
617                          __func__, __LINE__, surf.layer_size,
618                         (unsigned long)track->db_s_read_offset << 8, mslice,
619                         radeon_bo_size(track->db_s_read_bo));
620                 dev_warn(p->dev, "%s:%d stencil invalid (0x%08x 0x%08x 0x%08x 0x%08x)\n",
621                          __func__, __LINE__, track->db_depth_size,
622                          track->db_depth_slice, track->db_s_info, track->db_z_info);
623                 return -EINVAL;
624         }
625
626         offset = track->db_s_write_offset << 8;
627         if (offset & (surf.base_align - 1)) {
628                 dev_warn(p->dev, "%s:%d stencil write bo base %ld not aligned with %ld\n",
629                          __func__, __LINE__, offset, surf.base_align);
630                 return -EINVAL;
631         }
632         offset += surf.layer_size * mslice;
633         if (offset > radeon_bo_size(track->db_s_write_bo)) {
634                 dev_warn(p->dev, "%s:%d stencil write bo too small (layer size %d, "
635                          "offset %ld, max layer %d, bo size %ld)\n",
636                          __func__, __LINE__, surf.layer_size,
637                         (unsigned long)track->db_s_write_offset << 8, mslice,
638                         radeon_bo_size(track->db_s_write_bo));
639                 return -EINVAL;
640         }
641
642         /* hyperz */
643         if (G_028040_TILE_SURFACE_ENABLE(track->db_z_info)) {
644                 r = evergreen_cs_track_validate_htile(p, surf.nbx, surf.nby);
645                 if (r) {
646                         return r;
647                 }
648         }
649
650         return 0;
651 }
652
653 static int evergreen_cs_track_validate_depth(struct radeon_cs_parser *p)
654 {
655         struct evergreen_cs_track *track = p->track;
656         struct eg_surface surf;
657         unsigned pitch, slice, mslice;
658         unsigned long offset;
659         int r;
660
661         mslice = G_028008_SLICE_MAX(track->db_depth_view) + 1;
662         pitch = G_028058_PITCH_TILE_MAX(track->db_depth_size);
663         slice = track->db_depth_slice;
664         surf.nbx = (pitch + 1) * 8;
665         surf.nby = ((slice + 1) * 64) / surf.nbx;
666         surf.mode = G_028040_ARRAY_MODE(track->db_z_info);
667         surf.format = G_028040_FORMAT(track->db_z_info);
668         surf.tsplit = G_028040_TILE_SPLIT(track->db_z_info);
669         surf.nbanks = G_028040_NUM_BANKS(track->db_z_info);
670         surf.bankw = G_028040_BANK_WIDTH(track->db_z_info);
671         surf.bankh = G_028040_BANK_HEIGHT(track->db_z_info);
672         surf.mtilea = G_028040_MACRO_TILE_ASPECT(track->db_z_info);
673         surf.nsamples = 1;
674
675         switch (surf.format) {
676         case V_028040_Z_16:
677                 surf.format = V_028C70_COLOR_16;
678                 break;
679         case V_028040_Z_24:
680         case V_028040_Z_32_FLOAT:
681                 surf.format = V_028C70_COLOR_8_8_8_8;
682                 break;
683         default:
684                 dev_warn(p->dev, "%s:%d depth invalid format %d\n",
685                          __func__, __LINE__, surf.format);
686                 return -EINVAL;
687         }
688
689         r = evergreen_surface_value_conv_check(p, &surf, "depth");
690         if (r) {
691                 dev_warn(p->dev, "%s:%d depth invalid (0x%08x 0x%08x 0x%08x)\n",
692                          __func__, __LINE__, track->db_depth_size,
693                          track->db_depth_slice, track->db_z_info);
694                 return r;
695         }
696
697         r = evergreen_surface_check(p, &surf, "depth");
698         if (r) {
699                 dev_warn(p->dev, "%s:%d depth invalid (0x%08x 0x%08x 0x%08x)\n",
700                          __func__, __LINE__, track->db_depth_size,
701                          track->db_depth_slice, track->db_z_info);
702                 return r;
703         }
704
705         offset = track->db_z_read_offset << 8;
706         if (offset & (surf.base_align - 1)) {
707                 dev_warn(p->dev, "%s:%d stencil read bo base %ld not aligned with %ld\n",
708                          __func__, __LINE__, offset, surf.base_align);
709                 return -EINVAL;
710         }
711         offset += surf.layer_size * mslice;
712         if (offset > radeon_bo_size(track->db_z_read_bo)) {
713                 dev_warn(p->dev, "%s:%d depth read bo too small (layer size %d, "
714                          "offset %ld, max layer %d, bo size %ld)\n",
715                          __func__, __LINE__, surf.layer_size,
716                         (unsigned long)track->db_z_read_offset << 8, mslice,
717                         radeon_bo_size(track->db_z_read_bo));
718                 return -EINVAL;
719         }
720
721         offset = track->db_z_write_offset << 8;
722         if (offset & (surf.base_align - 1)) {
723                 dev_warn(p->dev, "%s:%d stencil write bo base %ld not aligned with %ld\n",
724                          __func__, __LINE__, offset, surf.base_align);
725                 return -EINVAL;
726         }
727         offset += surf.layer_size * mslice;
728         if (offset > radeon_bo_size(track->db_z_write_bo)) {
729                 dev_warn(p->dev, "%s:%d depth write bo too small (layer size %d, "
730                          "offset %ld, max layer %d, bo size %ld)\n",
731                          __func__, __LINE__, surf.layer_size,
732                         (unsigned long)track->db_z_write_offset << 8, mslice,
733                         radeon_bo_size(track->db_z_write_bo));
734                 return -EINVAL;
735         }
736
737         /* hyperz */
738         if (G_028040_TILE_SURFACE_ENABLE(track->db_z_info)) {
739                 r = evergreen_cs_track_validate_htile(p, surf.nbx, surf.nby);
740                 if (r) {
741                         return r;
742                 }
743         }
744
745         return 0;
746 }
747
748 static int evergreen_cs_track_validate_texture(struct radeon_cs_parser *p,
749                                                struct radeon_bo *texture,
750                                                struct radeon_bo *mipmap,
751                                                unsigned idx)
752 {
753         struct eg_surface surf;
754         unsigned long toffset, moffset;
755         unsigned dim, llevel, mslice, width, height, depth, i;
756         u32 texdw[8];
757         int r;
758
759         texdw[0] = radeon_get_ib_value(p, idx + 0);
760         texdw[1] = radeon_get_ib_value(p, idx + 1);
761         texdw[2] = radeon_get_ib_value(p, idx + 2);
762         texdw[3] = radeon_get_ib_value(p, idx + 3);
763         texdw[4] = radeon_get_ib_value(p, idx + 4);
764         texdw[5] = radeon_get_ib_value(p, idx + 5);
765         texdw[6] = radeon_get_ib_value(p, idx + 6);
766         texdw[7] = radeon_get_ib_value(p, idx + 7);
767         dim = G_030000_DIM(texdw[0]);
768         llevel = G_030014_LAST_LEVEL(texdw[5]);
769         mslice = G_030014_LAST_ARRAY(texdw[5]) + 1;
770         width = G_030000_TEX_WIDTH(texdw[0]) + 1;
771         height =  G_030004_TEX_HEIGHT(texdw[1]) + 1;
772         depth = G_030004_TEX_DEPTH(texdw[1]) + 1;
773         surf.format = G_03001C_DATA_FORMAT(texdw[7]);
774         surf.nbx = (G_030000_PITCH(texdw[0]) + 1) * 8;
775         surf.nbx = r600_fmt_get_nblocksx(surf.format, surf.nbx);
776         surf.nby = r600_fmt_get_nblocksy(surf.format, height);
777         surf.mode = G_030004_ARRAY_MODE(texdw[1]);
778         surf.tsplit = G_030018_TILE_SPLIT(texdw[6]);
779         surf.nbanks = G_03001C_NUM_BANKS(texdw[7]);
780         surf.bankw = G_03001C_BANK_WIDTH(texdw[7]);
781         surf.bankh = G_03001C_BANK_HEIGHT(texdw[7]);
782         surf.mtilea = G_03001C_MACRO_TILE_ASPECT(texdw[7]);
783         surf.nsamples = 1;
784         toffset = texdw[2] << 8;
785         moffset = texdw[3] << 8;
786
787         if (!r600_fmt_is_valid_texture(surf.format, p->family)) {
788                 dev_warn(p->dev, "%s:%d texture invalid format %d\n",
789                          __func__, __LINE__, surf.format);
790                 return -EINVAL;
791         }
792         switch (dim) {
793         case V_030000_SQ_TEX_DIM_1D:
794         case V_030000_SQ_TEX_DIM_2D:
795         case V_030000_SQ_TEX_DIM_CUBEMAP:
796         case V_030000_SQ_TEX_DIM_1D_ARRAY:
797         case V_030000_SQ_TEX_DIM_2D_ARRAY:
798                 depth = 1;
799                 break;
800         case V_030000_SQ_TEX_DIM_2D_MSAA:
801         case V_030000_SQ_TEX_DIM_2D_ARRAY_MSAA:
802                 surf.nsamples = 1 << llevel;
803                 llevel = 0;
804                 depth = 1;
805                 break;
806         case V_030000_SQ_TEX_DIM_3D:
807                 break;
808         default:
809                 dev_warn(p->dev, "%s:%d texture invalid dimension %d\n",
810                          __func__, __LINE__, dim);
811                 return -EINVAL;
812         }
813
814         r = evergreen_surface_value_conv_check(p, &surf, "texture");
815         if (r) {
816                 return r;
817         }
818
819         /* align height */
820         evergreen_surface_check(p, &surf, NULL);
821         surf.nby = ALIGN(surf.nby, surf.halign);
822
823         r = evergreen_surface_check(p, &surf, "texture");
824         if (r) {
825                 dev_warn(p->dev, "%s:%d texture invalid 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
826                          __func__, __LINE__, texdw[0], texdw[1], texdw[4],
827                          texdw[5], texdw[6], texdw[7]);
828                 return r;
829         }
830
831         /* check texture size */
832         if (toffset & (surf.base_align - 1)) {
833                 dev_warn(p->dev, "%s:%d texture bo base %ld not aligned with %ld\n",
834                          __func__, __LINE__, toffset, surf.base_align);
835                 return -EINVAL;
836         }
837         if (moffset & (surf.base_align - 1)) {
838                 dev_warn(p->dev, "%s:%d mipmap bo base %ld not aligned with %ld\n",
839                          __func__, __LINE__, moffset, surf.base_align);
840                 return -EINVAL;
841         }
842         if (dim == SQ_TEX_DIM_3D) {
843                 toffset += surf.layer_size * depth;
844         } else {
845                 toffset += surf.layer_size * mslice;
846         }
847         if (toffset > radeon_bo_size(texture)) {
848                 dev_warn(p->dev, "%s:%d texture bo too small (layer size %d, "
849                          "offset %ld, max layer %d, depth %d, bo size %ld) (%d %d)\n",
850                          __func__, __LINE__, surf.layer_size,
851                         (unsigned long)texdw[2] << 8, mslice,
852                         depth, radeon_bo_size(texture),
853                         surf.nbx, surf.nby);
854                 return -EINVAL;
855         }
856
857         if (!mipmap) {
858                 if (llevel) {
859                         dev_warn(p->dev, "%s:%i got NULL MIP_ADDRESS relocation\n",
860                                  __func__, __LINE__);
861                         return -EINVAL;
862                 } else {
863                         return 0; /* everything's ok */
864                 }
865         }
866
867         /* check mipmap size */
868         for (i = 1; i <= llevel; i++) {
869                 unsigned w, h, d;
870
871                 w = r600_mip_minify(width, i);
872                 h = r600_mip_minify(height, i);
873                 d = r600_mip_minify(depth, i);
874                 surf.nbx = r600_fmt_get_nblocksx(surf.format, w);
875                 surf.nby = r600_fmt_get_nblocksy(surf.format, h);
876
877                 switch (surf.mode) {
878                 case ARRAY_2D_TILED_THIN1:
879                         if (surf.nbx < surf.palign || surf.nby < surf.halign) {
880                                 surf.mode = ARRAY_1D_TILED_THIN1;
881                         }
882                         /* recompute alignment */
883                         evergreen_surface_check(p, &surf, NULL);
884                         break;
885                 case ARRAY_LINEAR_GENERAL:
886                 case ARRAY_LINEAR_ALIGNED:
887                 case ARRAY_1D_TILED_THIN1:
888                         break;
889                 default:
890                         dev_warn(p->dev, "%s:%d invalid array mode %d\n",
891                                  __func__, __LINE__, surf.mode);
892                         return -EINVAL;
893                 }
894                 surf.nbx = ALIGN(surf.nbx, surf.palign);
895                 surf.nby = ALIGN(surf.nby, surf.halign);
896
897                 r = evergreen_surface_check(p, &surf, "mipmap");
898                 if (r) {
899                         return r;
900                 }
901
902                 if (dim == SQ_TEX_DIM_3D) {
903                         moffset += surf.layer_size * d;
904                 } else {
905                         moffset += surf.layer_size * mslice;
906                 }
907                 if (moffset > radeon_bo_size(mipmap)) {
908                         dev_warn(p->dev, "%s:%d mipmap [%d] bo too small (layer size %d, "
909                                         "offset %ld, coffset %ld, max layer %d, depth %d, "
910                                         "bo size %ld) level0 (%d %d %d)\n",
911                                         __func__, __LINE__, i, surf.layer_size,
912                                         (unsigned long)texdw[3] << 8, moffset, mslice,
913                                         d, radeon_bo_size(mipmap),
914                                         width, height, depth);
915                         dev_warn(p->dev, "%s:%d problematic surf: (%d %d) (%d %d %d %d %d %d %d)\n",
916                                  __func__, __LINE__, surf.nbx, surf.nby,
917                                 surf.mode, surf.bpe, surf.nsamples,
918                                 surf.bankw, surf.bankh,
919                                 surf.tsplit, surf.mtilea);
920                         return -EINVAL;
921                 }
922         }
923
924         return 0;
925 }
926
927 static int evergreen_cs_track_check(struct radeon_cs_parser *p)
928 {
929         struct evergreen_cs_track *track = p->track;
930         unsigned tmp, i;
931         int r;
932         unsigned buffer_mask = 0;
933
934         /* check streamout */
935         if (track->streamout_dirty && track->vgt_strmout_config) {
936                 for (i = 0; i < 4; i++) {
937                         if (track->vgt_strmout_config & (1 << i)) {
938                                 buffer_mask |= (track->vgt_strmout_buffer_config >> (i * 4)) & 0xf;
939                         }
940                 }
941
942                 for (i = 0; i < 4; i++) {
943                         if (buffer_mask & (1 << i)) {
944                                 if (track->vgt_strmout_bo[i]) {
945                                         u64 offset = (u64)track->vgt_strmout_bo_offset[i] +
946                                                         (u64)track->vgt_strmout_size[i];
947                                         if (offset > radeon_bo_size(track->vgt_strmout_bo[i])) {
948                                                 DRM_ERROR("streamout %d bo too small: 0x%llx, 0x%lx\n",
949                                                           i, offset,
950                                                           radeon_bo_size(track->vgt_strmout_bo[i]));
951                                                 return -EINVAL;
952                                         }
953                                 } else {
954                                         dev_warn(p->dev, "No buffer for streamout %d\n", i);
955                                         return -EINVAL;
956                                 }
957                         }
958                 }
959                 track->streamout_dirty = false;
960         }
961
962         if (track->sx_misc_kill_all_prims)
963                 return 0;
964
965         /* check that we have a cb for each enabled target
966          */
967         if (track->cb_dirty) {
968                 tmp = track->cb_target_mask;
969                 for (i = 0; i < 8; i++) {
970                         if ((tmp >> (i * 4)) & 0xF) {
971                                 /* at least one component is enabled */
972                                 if (track->cb_color_bo[i] == NULL) {
973                                         dev_warn(p->dev, "%s:%d mask 0x%08X | 0x%08X no cb for %d\n",
974                                                 __func__, __LINE__, track->cb_target_mask, track->cb_shader_mask, i);
975                                         return -EINVAL;
976                                 }
977                                 /* check cb */
978                                 r = evergreen_cs_track_validate_cb(p, i);
979                                 if (r) {
980                                         return r;
981                                 }
982                         }
983                 }
984                 track->cb_dirty = false;
985         }
986
987         if (track->db_dirty) {
988                 /* Check stencil buffer */
989                 if (G_028044_FORMAT(track->db_s_info) != V_028044_STENCIL_INVALID &&
990                     G_028800_STENCIL_ENABLE(track->db_depth_control)) {
991                         r = evergreen_cs_track_validate_stencil(p);
992                         if (r)
993                                 return r;
994                 }
995                 /* Check depth buffer */
996                 if (G_028040_FORMAT(track->db_z_info) != V_028040_Z_INVALID &&
997                     G_028800_Z_ENABLE(track->db_depth_control)) {
998                         r = evergreen_cs_track_validate_depth(p);
999                         if (r)
1000                                 return r;
1001                 }
1002                 track->db_dirty = false;
1003         }
1004
1005         return 0;
1006 }
1007
1008 /**
1009  * evergreen_cs_packet_parse_vline() - parse userspace VLINE packet
1010  * @parser:             parser structure holding parsing context.
1011  *
1012  * This is an Evergreen(+)-specific function for parsing VLINE packets.
1013  * Real work is done by r600_cs_common_vline_parse function.
1014  * Here we just set up ASIC-specific register table and call
1015  * the common implementation function.
1016  */
1017 static int evergreen_cs_packet_parse_vline(struct radeon_cs_parser *p)
1018 {
1019
1020         static uint32_t vline_start_end[6] = {
1021                 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC0_REGISTER_OFFSET,
1022                 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC1_REGISTER_OFFSET,
1023                 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC2_REGISTER_OFFSET,
1024                 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC3_REGISTER_OFFSET,
1025                 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC4_REGISTER_OFFSET,
1026                 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC5_REGISTER_OFFSET
1027         };
1028         static uint32_t vline_status[6] = {
1029                 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
1030                 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
1031                 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
1032                 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
1033                 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
1034                 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET
1035         };
1036
1037         return r600_cs_common_vline_parse(p, vline_start_end, vline_status);
1038 }
1039
1040 static int evergreen_packet0_check(struct radeon_cs_parser *p,
1041                                    struct radeon_cs_packet *pkt,
1042                                    unsigned idx, unsigned reg)
1043 {
1044         int r;
1045
1046         switch (reg) {
1047         case EVERGREEN_VLINE_START_END:
1048                 r = evergreen_cs_packet_parse_vline(p);
1049                 if (r) {
1050                         DRM_ERROR("No reloc for ib[%d]=0x%04X\n",
1051                                         idx, reg);
1052                         return r;
1053                 }
1054                 break;
1055         default:
1056                 printk(KERN_ERR "Forbidden register 0x%04X in cs at %d\n",
1057                        reg, idx);
1058                 return -EINVAL;
1059         }
1060         return 0;
1061 }
1062
1063 static int evergreen_cs_parse_packet0(struct radeon_cs_parser *p,
1064                                       struct radeon_cs_packet *pkt)
1065 {
1066         unsigned reg, i;
1067         unsigned idx;
1068         int r;
1069
1070         idx = pkt->idx + 1;
1071         reg = pkt->reg;
1072         for (i = 0; i <= pkt->count; i++, idx++, reg += 4) {
1073                 r = evergreen_packet0_check(p, pkt, idx, reg);
1074                 if (r) {
1075                         return r;
1076                 }
1077         }
1078         return 0;
1079 }
1080
1081 /**
1082  * evergreen_cs_check_reg() - check if register is authorized or not
1083  * @parser: parser structure holding parsing context
1084  * @reg: register we are testing
1085  * @idx: index into the cs buffer
1086  *
1087  * This function will test against evergreen_reg_safe_bm and return 0
1088  * if register is safe. If register is not flag as safe this function
1089  * will test it against a list of register needind special handling.
1090  */
1091 static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
1092 {
1093         struct evergreen_cs_track *track = (struct evergreen_cs_track *)p->track;
1094         struct radeon_cs_reloc *reloc;
1095         u32 last_reg;
1096         u32 m, i, tmp, *ib;
1097         int r;
1098
1099         if (p->rdev->family >= CHIP_CAYMAN)
1100                 last_reg = ARRAY_SIZE(cayman_reg_safe_bm);
1101         else
1102                 last_reg = ARRAY_SIZE(evergreen_reg_safe_bm);
1103
1104         i = (reg >> 7);
1105         if (i >= last_reg) {
1106                 dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
1107                 return -EINVAL;
1108         }
1109         m = 1 << ((reg >> 2) & 31);
1110         if (p->rdev->family >= CHIP_CAYMAN) {
1111                 if (!(cayman_reg_safe_bm[i] & m))
1112                         return 0;
1113         } else {
1114                 if (!(evergreen_reg_safe_bm[i] & m))
1115                         return 0;
1116         }
1117         ib = p->ib.ptr;
1118         switch (reg) {
1119         /* force following reg to 0 in an attempt to disable out buffer
1120          * which will need us to better understand how it works to perform
1121          * security check on it (Jerome)
1122          */
1123         case SQ_ESGS_RING_SIZE:
1124         case SQ_GSVS_RING_SIZE:
1125         case SQ_ESTMP_RING_SIZE:
1126         case SQ_GSTMP_RING_SIZE:
1127         case SQ_HSTMP_RING_SIZE:
1128         case SQ_LSTMP_RING_SIZE:
1129         case SQ_PSTMP_RING_SIZE:
1130         case SQ_VSTMP_RING_SIZE:
1131         case SQ_ESGS_RING_ITEMSIZE:
1132         case SQ_ESTMP_RING_ITEMSIZE:
1133         case SQ_GSTMP_RING_ITEMSIZE:
1134         case SQ_GSVS_RING_ITEMSIZE:
1135         case SQ_GS_VERT_ITEMSIZE:
1136         case SQ_GS_VERT_ITEMSIZE_1:
1137         case SQ_GS_VERT_ITEMSIZE_2:
1138         case SQ_GS_VERT_ITEMSIZE_3:
1139         case SQ_GSVS_RING_OFFSET_1:
1140         case SQ_GSVS_RING_OFFSET_2:
1141         case SQ_GSVS_RING_OFFSET_3:
1142         case SQ_HSTMP_RING_ITEMSIZE:
1143         case SQ_LSTMP_RING_ITEMSIZE:
1144         case SQ_PSTMP_RING_ITEMSIZE:
1145         case SQ_VSTMP_RING_ITEMSIZE:
1146         case VGT_TF_RING_SIZE:
1147                 /* get value to populate the IB don't remove */
1148                 /*tmp =radeon_get_ib_value(p, idx);
1149                   ib[idx] = 0;*/
1150                 break;
1151         case SQ_ESGS_RING_BASE:
1152         case SQ_GSVS_RING_BASE:
1153         case SQ_ESTMP_RING_BASE:
1154         case SQ_GSTMP_RING_BASE:
1155         case SQ_HSTMP_RING_BASE:
1156         case SQ_LSTMP_RING_BASE:
1157         case SQ_PSTMP_RING_BASE:
1158         case SQ_VSTMP_RING_BASE:
1159                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1160                 if (r) {
1161                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1162                                         "0x%04X\n", reg);
1163                         return -EINVAL;
1164                 }
1165                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1166                 break;
1167         case DB_DEPTH_CONTROL:
1168                 track->db_depth_control = radeon_get_ib_value(p, idx);
1169                 track->db_dirty = true;
1170                 break;
1171         case CAYMAN_DB_EQAA:
1172                 if (p->rdev->family < CHIP_CAYMAN) {
1173                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1174                                  "0x%04X\n", reg);
1175                         return -EINVAL;
1176                 }
1177                 break;
1178         case CAYMAN_DB_DEPTH_INFO:
1179                 if (p->rdev->family < CHIP_CAYMAN) {
1180                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1181                                  "0x%04X\n", reg);
1182                         return -EINVAL;
1183                 }
1184                 break;
1185         case DB_Z_INFO:
1186                 track->db_z_info = radeon_get_ib_value(p, idx);
1187                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
1188                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1189                         if (r) {
1190                                 dev_warn(p->dev, "bad SET_CONTEXT_REG "
1191                                                 "0x%04X\n", reg);
1192                                 return -EINVAL;
1193                         }
1194                         ib[idx] &= ~Z_ARRAY_MODE(0xf);
1195                         track->db_z_info &= ~Z_ARRAY_MODE(0xf);
1196                         ib[idx] |= Z_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
1197                         track->db_z_info |= Z_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
1198                         if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) {
1199                                 unsigned bankw, bankh, mtaspect, tile_split;
1200
1201                                 evergreen_tiling_fields(reloc->lobj.tiling_flags,
1202                                                         &bankw, &bankh, &mtaspect,
1203                                                         &tile_split);
1204                                 ib[idx] |= DB_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks));
1205                                 ib[idx] |= DB_TILE_SPLIT(tile_split) |
1206                                                 DB_BANK_WIDTH(bankw) |
1207                                                 DB_BANK_HEIGHT(bankh) |
1208                                                 DB_MACRO_TILE_ASPECT(mtaspect);
1209                         }
1210                 }
1211                 track->db_dirty = true;
1212                 break;
1213         case DB_STENCIL_INFO:
1214                 track->db_s_info = radeon_get_ib_value(p, idx);
1215                 track->db_dirty = true;
1216                 break;
1217         case DB_DEPTH_VIEW:
1218                 track->db_depth_view = radeon_get_ib_value(p, idx);
1219                 track->db_dirty = true;
1220                 break;
1221         case DB_DEPTH_SIZE:
1222                 track->db_depth_size = radeon_get_ib_value(p, idx);
1223                 track->db_dirty = true;
1224                 break;
1225         case R_02805C_DB_DEPTH_SLICE:
1226                 track->db_depth_slice = radeon_get_ib_value(p, idx);
1227                 track->db_dirty = true;
1228                 break;
1229         case DB_Z_READ_BASE:
1230                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1231                 if (r) {
1232                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1233                                         "0x%04X\n", reg);
1234                         return -EINVAL;
1235                 }
1236                 track->db_z_read_offset = radeon_get_ib_value(p, idx);
1237                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1238                 track->db_z_read_bo = reloc->robj;
1239                 track->db_dirty = true;
1240                 break;
1241         case DB_Z_WRITE_BASE:
1242                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1243                 if (r) {
1244                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1245                                         "0x%04X\n", reg);
1246                         return -EINVAL;
1247                 }
1248                 track->db_z_write_offset = radeon_get_ib_value(p, idx);
1249                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1250                 track->db_z_write_bo = reloc->robj;
1251                 track->db_dirty = true;
1252                 break;
1253         case DB_STENCIL_READ_BASE:
1254                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1255                 if (r) {
1256                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1257                                         "0x%04X\n", reg);
1258                         return -EINVAL;
1259                 }
1260                 track->db_s_read_offset = radeon_get_ib_value(p, idx);
1261                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1262                 track->db_s_read_bo = reloc->robj;
1263                 track->db_dirty = true;
1264                 break;
1265         case DB_STENCIL_WRITE_BASE:
1266                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1267                 if (r) {
1268                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1269                                         "0x%04X\n", reg);
1270                         return -EINVAL;
1271                 }
1272                 track->db_s_write_offset = radeon_get_ib_value(p, idx);
1273                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1274                 track->db_s_write_bo = reloc->robj;
1275                 track->db_dirty = true;
1276                 break;
1277         case VGT_STRMOUT_CONFIG:
1278                 track->vgt_strmout_config = radeon_get_ib_value(p, idx);
1279                 track->streamout_dirty = true;
1280                 break;
1281         case VGT_STRMOUT_BUFFER_CONFIG:
1282                 track->vgt_strmout_buffer_config = radeon_get_ib_value(p, idx);
1283                 track->streamout_dirty = true;
1284                 break;
1285         case VGT_STRMOUT_BUFFER_BASE_0:
1286         case VGT_STRMOUT_BUFFER_BASE_1:
1287         case VGT_STRMOUT_BUFFER_BASE_2:
1288         case VGT_STRMOUT_BUFFER_BASE_3:
1289                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1290                 if (r) {
1291                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1292                                         "0x%04X\n", reg);
1293                         return -EINVAL;
1294                 }
1295                 tmp = (reg - VGT_STRMOUT_BUFFER_BASE_0) / 16;
1296                 track->vgt_strmout_bo_offset[tmp] = radeon_get_ib_value(p, idx) << 8;
1297                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1298                 track->vgt_strmout_bo[tmp] = reloc->robj;
1299                 track->streamout_dirty = true;
1300                 break;
1301         case VGT_STRMOUT_BUFFER_SIZE_0:
1302         case VGT_STRMOUT_BUFFER_SIZE_1:
1303         case VGT_STRMOUT_BUFFER_SIZE_2:
1304         case VGT_STRMOUT_BUFFER_SIZE_3:
1305                 tmp = (reg - VGT_STRMOUT_BUFFER_SIZE_0) / 16;
1306                 /* size in register is DWs, convert to bytes */
1307                 track->vgt_strmout_size[tmp] = radeon_get_ib_value(p, idx) * 4;
1308                 track->streamout_dirty = true;
1309                 break;
1310         case CP_COHER_BASE:
1311                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1312                 if (r) {
1313                         dev_warn(p->dev, "missing reloc for CP_COHER_BASE "
1314                                         "0x%04X\n", reg);
1315                         return -EINVAL;
1316                 }
1317                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1318         case CB_TARGET_MASK:
1319                 track->cb_target_mask = radeon_get_ib_value(p, idx);
1320                 track->cb_dirty = true;
1321                 break;
1322         case CB_SHADER_MASK:
1323                 track->cb_shader_mask = radeon_get_ib_value(p, idx);
1324                 track->cb_dirty = true;
1325                 break;
1326         case PA_SC_AA_CONFIG:
1327                 if (p->rdev->family >= CHIP_CAYMAN) {
1328                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1329                                  "0x%04X\n", reg);
1330                         return -EINVAL;
1331                 }
1332                 tmp = radeon_get_ib_value(p, idx) & MSAA_NUM_SAMPLES_MASK;
1333                 track->nsamples = 1 << tmp;
1334                 break;
1335         case CAYMAN_PA_SC_AA_CONFIG:
1336                 if (p->rdev->family < CHIP_CAYMAN) {
1337                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1338                                  "0x%04X\n", reg);
1339                         return -EINVAL;
1340                 }
1341                 tmp = radeon_get_ib_value(p, idx) & CAYMAN_MSAA_NUM_SAMPLES_MASK;
1342                 track->nsamples = 1 << tmp;
1343                 break;
1344         case CB_COLOR0_VIEW:
1345         case CB_COLOR1_VIEW:
1346         case CB_COLOR2_VIEW:
1347         case CB_COLOR3_VIEW:
1348         case CB_COLOR4_VIEW:
1349         case CB_COLOR5_VIEW:
1350         case CB_COLOR6_VIEW:
1351         case CB_COLOR7_VIEW:
1352                 tmp = (reg - CB_COLOR0_VIEW) / 0x3c;
1353                 track->cb_color_view[tmp] = radeon_get_ib_value(p, idx);
1354                 track->cb_dirty = true;
1355                 break;
1356         case CB_COLOR8_VIEW:
1357         case CB_COLOR9_VIEW:
1358         case CB_COLOR10_VIEW:
1359         case CB_COLOR11_VIEW:
1360                 tmp = ((reg - CB_COLOR8_VIEW) / 0x1c) + 8;
1361                 track->cb_color_view[tmp] = radeon_get_ib_value(p, idx);
1362                 track->cb_dirty = true;
1363                 break;
1364         case CB_COLOR0_INFO:
1365         case CB_COLOR1_INFO:
1366         case CB_COLOR2_INFO:
1367         case CB_COLOR3_INFO:
1368         case CB_COLOR4_INFO:
1369         case CB_COLOR5_INFO:
1370         case CB_COLOR6_INFO:
1371         case CB_COLOR7_INFO:
1372                 tmp = (reg - CB_COLOR0_INFO) / 0x3c;
1373                 track->cb_color_info[tmp] = radeon_get_ib_value(p, idx);
1374                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
1375                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1376                         if (r) {
1377                                 dev_warn(p->dev, "bad SET_CONTEXT_REG "
1378                                                 "0x%04X\n", reg);
1379                                 return -EINVAL;
1380                         }
1381                         ib[idx] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
1382                         track->cb_color_info[tmp] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
1383                 }
1384                 track->cb_dirty = true;
1385                 break;
1386         case CB_COLOR8_INFO:
1387         case CB_COLOR9_INFO:
1388         case CB_COLOR10_INFO:
1389         case CB_COLOR11_INFO:
1390                 tmp = ((reg - CB_COLOR8_INFO) / 0x1c) + 8;
1391                 track->cb_color_info[tmp] = radeon_get_ib_value(p, idx);
1392                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
1393                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1394                         if (r) {
1395                                 dev_warn(p->dev, "bad SET_CONTEXT_REG "
1396                                                 "0x%04X\n", reg);
1397                                 return -EINVAL;
1398                         }
1399                         ib[idx] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
1400                         track->cb_color_info[tmp] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
1401                 }
1402                 track->cb_dirty = true;
1403                 break;
1404         case CB_COLOR0_PITCH:
1405         case CB_COLOR1_PITCH:
1406         case CB_COLOR2_PITCH:
1407         case CB_COLOR3_PITCH:
1408         case CB_COLOR4_PITCH:
1409         case CB_COLOR5_PITCH:
1410         case CB_COLOR6_PITCH:
1411         case CB_COLOR7_PITCH:
1412                 tmp = (reg - CB_COLOR0_PITCH) / 0x3c;
1413                 track->cb_color_pitch[tmp] = radeon_get_ib_value(p, idx);
1414                 track->cb_dirty = true;
1415                 break;
1416         case CB_COLOR8_PITCH:
1417         case CB_COLOR9_PITCH:
1418         case CB_COLOR10_PITCH:
1419         case CB_COLOR11_PITCH:
1420                 tmp = ((reg - CB_COLOR8_PITCH) / 0x1c) + 8;
1421                 track->cb_color_pitch[tmp] = radeon_get_ib_value(p, idx);
1422                 track->cb_dirty = true;
1423                 break;
1424         case CB_COLOR0_SLICE:
1425         case CB_COLOR1_SLICE:
1426         case CB_COLOR2_SLICE:
1427         case CB_COLOR3_SLICE:
1428         case CB_COLOR4_SLICE:
1429         case CB_COLOR5_SLICE:
1430         case CB_COLOR6_SLICE:
1431         case CB_COLOR7_SLICE:
1432                 tmp = (reg - CB_COLOR0_SLICE) / 0x3c;
1433                 track->cb_color_slice[tmp] = radeon_get_ib_value(p, idx);
1434                 track->cb_color_slice_idx[tmp] = idx;
1435                 track->cb_dirty = true;
1436                 break;
1437         case CB_COLOR8_SLICE:
1438         case CB_COLOR9_SLICE:
1439         case CB_COLOR10_SLICE:
1440         case CB_COLOR11_SLICE:
1441                 tmp = ((reg - CB_COLOR8_SLICE) / 0x1c) + 8;
1442                 track->cb_color_slice[tmp] = radeon_get_ib_value(p, idx);
1443                 track->cb_color_slice_idx[tmp] = idx;
1444                 track->cb_dirty = true;
1445                 break;
1446         case CB_COLOR0_ATTRIB:
1447         case CB_COLOR1_ATTRIB:
1448         case CB_COLOR2_ATTRIB:
1449         case CB_COLOR3_ATTRIB:
1450         case CB_COLOR4_ATTRIB:
1451         case CB_COLOR5_ATTRIB:
1452         case CB_COLOR6_ATTRIB:
1453         case CB_COLOR7_ATTRIB:
1454                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1455                 if (r) {
1456                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1457                                         "0x%04X\n", reg);
1458                         return -EINVAL;
1459                 }
1460                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
1461                         if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) {
1462                                 unsigned bankw, bankh, mtaspect, tile_split;
1463
1464                                 evergreen_tiling_fields(reloc->lobj.tiling_flags,
1465                                                         &bankw, &bankh, &mtaspect,
1466                                                         &tile_split);
1467                                 ib[idx] |= CB_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks));
1468                                 ib[idx] |= CB_TILE_SPLIT(tile_split) |
1469                                            CB_BANK_WIDTH(bankw) |
1470                                            CB_BANK_HEIGHT(bankh) |
1471                                            CB_MACRO_TILE_ASPECT(mtaspect);
1472                         }
1473                 }
1474                 tmp = ((reg - CB_COLOR0_ATTRIB) / 0x3c);
1475                 track->cb_color_attrib[tmp] = ib[idx];
1476                 track->cb_dirty = true;
1477                 break;
1478         case CB_COLOR8_ATTRIB:
1479         case CB_COLOR9_ATTRIB:
1480         case CB_COLOR10_ATTRIB:
1481         case CB_COLOR11_ATTRIB:
1482                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1483                 if (r) {
1484                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1485                                         "0x%04X\n", reg);
1486                         return -EINVAL;
1487                 }
1488                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
1489                         if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) {
1490                                 unsigned bankw, bankh, mtaspect, tile_split;
1491
1492                                 evergreen_tiling_fields(reloc->lobj.tiling_flags,
1493                                                         &bankw, &bankh, &mtaspect,
1494                                                         &tile_split);
1495                                 ib[idx] |= CB_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks));
1496                                 ib[idx] |= CB_TILE_SPLIT(tile_split) |
1497                                            CB_BANK_WIDTH(bankw) |
1498                                            CB_BANK_HEIGHT(bankh) |
1499                                            CB_MACRO_TILE_ASPECT(mtaspect);
1500                         }
1501                 }
1502                 tmp = ((reg - CB_COLOR8_ATTRIB) / 0x1c) + 8;
1503                 track->cb_color_attrib[tmp] = ib[idx];
1504                 track->cb_dirty = true;
1505                 break;
1506         case CB_COLOR0_FMASK:
1507         case CB_COLOR1_FMASK:
1508         case CB_COLOR2_FMASK:
1509         case CB_COLOR3_FMASK:
1510         case CB_COLOR4_FMASK:
1511         case CB_COLOR5_FMASK:
1512         case CB_COLOR6_FMASK:
1513         case CB_COLOR7_FMASK:
1514                 tmp = (reg - CB_COLOR0_FMASK) / 0x3c;
1515                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1516                 if (r) {
1517                         dev_err(p->dev, "bad SET_CONTEXT_REG 0x%04X\n", reg);
1518                         return -EINVAL;
1519                 }
1520                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1521                 track->cb_color_fmask_bo[tmp] = reloc->robj;
1522                 break;
1523         case CB_COLOR0_CMASK:
1524         case CB_COLOR1_CMASK:
1525         case CB_COLOR2_CMASK:
1526         case CB_COLOR3_CMASK:
1527         case CB_COLOR4_CMASK:
1528         case CB_COLOR5_CMASK:
1529         case CB_COLOR6_CMASK:
1530         case CB_COLOR7_CMASK:
1531                 tmp = (reg - CB_COLOR0_CMASK) / 0x3c;
1532                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1533                 if (r) {
1534                         dev_err(p->dev, "bad SET_CONTEXT_REG 0x%04X\n", reg);
1535                         return -EINVAL;
1536                 }
1537                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1538                 track->cb_color_cmask_bo[tmp] = reloc->robj;
1539                 break;
1540         case CB_COLOR0_FMASK_SLICE:
1541         case CB_COLOR1_FMASK_SLICE:
1542         case CB_COLOR2_FMASK_SLICE:
1543         case CB_COLOR3_FMASK_SLICE:
1544         case CB_COLOR4_FMASK_SLICE:
1545         case CB_COLOR5_FMASK_SLICE:
1546         case CB_COLOR6_FMASK_SLICE:
1547         case CB_COLOR7_FMASK_SLICE:
1548                 tmp = (reg - CB_COLOR0_FMASK_SLICE) / 0x3c;
1549                 track->cb_color_fmask_slice[tmp] = radeon_get_ib_value(p, idx);
1550                 break;
1551         case CB_COLOR0_CMASK_SLICE:
1552         case CB_COLOR1_CMASK_SLICE:
1553         case CB_COLOR2_CMASK_SLICE:
1554         case CB_COLOR3_CMASK_SLICE:
1555         case CB_COLOR4_CMASK_SLICE:
1556         case CB_COLOR5_CMASK_SLICE:
1557         case CB_COLOR6_CMASK_SLICE:
1558         case CB_COLOR7_CMASK_SLICE:
1559                 tmp = (reg - CB_COLOR0_CMASK_SLICE) / 0x3c;
1560                 track->cb_color_cmask_slice[tmp] = radeon_get_ib_value(p, idx);
1561                 break;
1562         case CB_COLOR0_BASE:
1563         case CB_COLOR1_BASE:
1564         case CB_COLOR2_BASE:
1565         case CB_COLOR3_BASE:
1566         case CB_COLOR4_BASE:
1567         case CB_COLOR5_BASE:
1568         case CB_COLOR6_BASE:
1569         case CB_COLOR7_BASE:
1570                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1571                 if (r) {
1572                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1573                                         "0x%04X\n", reg);
1574                         return -EINVAL;
1575                 }
1576                 tmp = (reg - CB_COLOR0_BASE) / 0x3c;
1577                 track->cb_color_bo_offset[tmp] = radeon_get_ib_value(p, idx);
1578                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1579                 track->cb_color_bo[tmp] = reloc->robj;
1580                 track->cb_dirty = true;
1581                 break;
1582         case CB_COLOR8_BASE:
1583         case CB_COLOR9_BASE:
1584         case CB_COLOR10_BASE:
1585         case CB_COLOR11_BASE:
1586                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1587                 if (r) {
1588                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1589                                         "0x%04X\n", reg);
1590                         return -EINVAL;
1591                 }
1592                 tmp = ((reg - CB_COLOR8_BASE) / 0x1c) + 8;
1593                 track->cb_color_bo_offset[tmp] = radeon_get_ib_value(p, idx);
1594                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1595                 track->cb_color_bo[tmp] = reloc->robj;
1596                 track->cb_dirty = true;
1597                 break;
1598         case DB_HTILE_DATA_BASE:
1599                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1600                 if (r) {
1601                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1602                                         "0x%04X\n", reg);
1603                         return -EINVAL;
1604                 }
1605                 track->htile_offset = radeon_get_ib_value(p, idx);
1606                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1607                 track->htile_bo = reloc->robj;
1608                 track->db_dirty = true;
1609                 break;
1610         case DB_HTILE_SURFACE:
1611                 /* 8x8 only */
1612                 track->htile_surface = radeon_get_ib_value(p, idx);
1613                 /* force 8x8 htile width and height */
1614                 ib[idx] |= 3;
1615                 track->db_dirty = true;
1616                 break;
1617         case CB_IMMED0_BASE:
1618         case CB_IMMED1_BASE:
1619         case CB_IMMED2_BASE:
1620         case CB_IMMED3_BASE:
1621         case CB_IMMED4_BASE:
1622         case CB_IMMED5_BASE:
1623         case CB_IMMED6_BASE:
1624         case CB_IMMED7_BASE:
1625         case CB_IMMED8_BASE:
1626         case CB_IMMED9_BASE:
1627         case CB_IMMED10_BASE:
1628         case CB_IMMED11_BASE:
1629         case SQ_PGM_START_FS:
1630         case SQ_PGM_START_ES:
1631         case SQ_PGM_START_VS:
1632         case SQ_PGM_START_GS:
1633         case SQ_PGM_START_PS:
1634         case SQ_PGM_START_HS:
1635         case SQ_PGM_START_LS:
1636         case SQ_CONST_MEM_BASE:
1637         case SQ_ALU_CONST_CACHE_GS_0:
1638         case SQ_ALU_CONST_CACHE_GS_1:
1639         case SQ_ALU_CONST_CACHE_GS_2:
1640         case SQ_ALU_CONST_CACHE_GS_3:
1641         case SQ_ALU_CONST_CACHE_GS_4:
1642         case SQ_ALU_CONST_CACHE_GS_5:
1643         case SQ_ALU_CONST_CACHE_GS_6:
1644         case SQ_ALU_CONST_CACHE_GS_7:
1645         case SQ_ALU_CONST_CACHE_GS_8:
1646         case SQ_ALU_CONST_CACHE_GS_9:
1647         case SQ_ALU_CONST_CACHE_GS_10:
1648         case SQ_ALU_CONST_CACHE_GS_11:
1649         case SQ_ALU_CONST_CACHE_GS_12:
1650         case SQ_ALU_CONST_CACHE_GS_13:
1651         case SQ_ALU_CONST_CACHE_GS_14:
1652         case SQ_ALU_CONST_CACHE_GS_15:
1653         case SQ_ALU_CONST_CACHE_PS_0:
1654         case SQ_ALU_CONST_CACHE_PS_1:
1655         case SQ_ALU_CONST_CACHE_PS_2:
1656         case SQ_ALU_CONST_CACHE_PS_3:
1657         case SQ_ALU_CONST_CACHE_PS_4:
1658         case SQ_ALU_CONST_CACHE_PS_5:
1659         case SQ_ALU_CONST_CACHE_PS_6:
1660         case SQ_ALU_CONST_CACHE_PS_7:
1661         case SQ_ALU_CONST_CACHE_PS_8:
1662         case SQ_ALU_CONST_CACHE_PS_9:
1663         case SQ_ALU_CONST_CACHE_PS_10:
1664         case SQ_ALU_CONST_CACHE_PS_11:
1665         case SQ_ALU_CONST_CACHE_PS_12:
1666         case SQ_ALU_CONST_CACHE_PS_13:
1667         case SQ_ALU_CONST_CACHE_PS_14:
1668         case SQ_ALU_CONST_CACHE_PS_15:
1669         case SQ_ALU_CONST_CACHE_VS_0:
1670         case SQ_ALU_CONST_CACHE_VS_1:
1671         case SQ_ALU_CONST_CACHE_VS_2:
1672         case SQ_ALU_CONST_CACHE_VS_3:
1673         case SQ_ALU_CONST_CACHE_VS_4:
1674         case SQ_ALU_CONST_CACHE_VS_5:
1675         case SQ_ALU_CONST_CACHE_VS_6:
1676         case SQ_ALU_CONST_CACHE_VS_7:
1677         case SQ_ALU_CONST_CACHE_VS_8:
1678         case SQ_ALU_CONST_CACHE_VS_9:
1679         case SQ_ALU_CONST_CACHE_VS_10:
1680         case SQ_ALU_CONST_CACHE_VS_11:
1681         case SQ_ALU_CONST_CACHE_VS_12:
1682         case SQ_ALU_CONST_CACHE_VS_13:
1683         case SQ_ALU_CONST_CACHE_VS_14:
1684         case SQ_ALU_CONST_CACHE_VS_15:
1685         case SQ_ALU_CONST_CACHE_HS_0:
1686         case SQ_ALU_CONST_CACHE_HS_1:
1687         case SQ_ALU_CONST_CACHE_HS_2:
1688         case SQ_ALU_CONST_CACHE_HS_3:
1689         case SQ_ALU_CONST_CACHE_HS_4:
1690         case SQ_ALU_CONST_CACHE_HS_5:
1691         case SQ_ALU_CONST_CACHE_HS_6:
1692         case SQ_ALU_CONST_CACHE_HS_7:
1693         case SQ_ALU_CONST_CACHE_HS_8:
1694         case SQ_ALU_CONST_CACHE_HS_9:
1695         case SQ_ALU_CONST_CACHE_HS_10:
1696         case SQ_ALU_CONST_CACHE_HS_11:
1697         case SQ_ALU_CONST_CACHE_HS_12:
1698         case SQ_ALU_CONST_CACHE_HS_13:
1699         case SQ_ALU_CONST_CACHE_HS_14:
1700         case SQ_ALU_CONST_CACHE_HS_15:
1701         case SQ_ALU_CONST_CACHE_LS_0:
1702         case SQ_ALU_CONST_CACHE_LS_1:
1703         case SQ_ALU_CONST_CACHE_LS_2:
1704         case SQ_ALU_CONST_CACHE_LS_3:
1705         case SQ_ALU_CONST_CACHE_LS_4:
1706         case SQ_ALU_CONST_CACHE_LS_5:
1707         case SQ_ALU_CONST_CACHE_LS_6:
1708         case SQ_ALU_CONST_CACHE_LS_7:
1709         case SQ_ALU_CONST_CACHE_LS_8:
1710         case SQ_ALU_CONST_CACHE_LS_9:
1711         case SQ_ALU_CONST_CACHE_LS_10:
1712         case SQ_ALU_CONST_CACHE_LS_11:
1713         case SQ_ALU_CONST_CACHE_LS_12:
1714         case SQ_ALU_CONST_CACHE_LS_13:
1715         case SQ_ALU_CONST_CACHE_LS_14:
1716         case SQ_ALU_CONST_CACHE_LS_15:
1717                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1718                 if (r) {
1719                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1720                                         "0x%04X\n", reg);
1721                         return -EINVAL;
1722                 }
1723                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1724                 break;
1725         case SX_MEMORY_EXPORT_BASE:
1726                 if (p->rdev->family >= CHIP_CAYMAN) {
1727                         dev_warn(p->dev, "bad SET_CONFIG_REG "
1728                                  "0x%04X\n", reg);
1729                         return -EINVAL;
1730                 }
1731                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1732                 if (r) {
1733                         dev_warn(p->dev, "bad SET_CONFIG_REG "
1734                                         "0x%04X\n", reg);
1735                         return -EINVAL;
1736                 }
1737                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1738                 break;
1739         case CAYMAN_SX_SCATTER_EXPORT_BASE:
1740                 if (p->rdev->family < CHIP_CAYMAN) {
1741                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1742                                  "0x%04X\n", reg);
1743                         return -EINVAL;
1744                 }
1745                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1746                 if (r) {
1747                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1748                                         "0x%04X\n", reg);
1749                         return -EINVAL;
1750                 }
1751                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1752                 break;
1753         case SX_MISC:
1754                 track->sx_misc_kill_all_prims = (radeon_get_ib_value(p, idx) & 0x1) != 0;
1755                 break;
1756         default:
1757                 dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
1758                 return -EINVAL;
1759         }
1760         return 0;
1761 }
1762
1763 static bool evergreen_is_safe_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
1764 {
1765         u32 last_reg, m, i;
1766
1767         if (p->rdev->family >= CHIP_CAYMAN)
1768                 last_reg = ARRAY_SIZE(cayman_reg_safe_bm);
1769         else
1770                 last_reg = ARRAY_SIZE(evergreen_reg_safe_bm);
1771
1772         i = (reg >> 7);
1773         if (i >= last_reg) {
1774                 dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
1775                 return false;
1776         }
1777         m = 1 << ((reg >> 2) & 31);
1778         if (p->rdev->family >= CHIP_CAYMAN) {
1779                 if (!(cayman_reg_safe_bm[i] & m))
1780                         return true;
1781         } else {
1782                 if (!(evergreen_reg_safe_bm[i] & m))
1783                         return true;
1784         }
1785         dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
1786         return false;
1787 }
1788
1789 static int evergreen_packet3_check(struct radeon_cs_parser *p,
1790                                    struct radeon_cs_packet *pkt)
1791 {
1792         struct radeon_cs_reloc *reloc;
1793         struct evergreen_cs_track *track;
1794         volatile u32 *ib;
1795         unsigned idx;
1796         unsigned i;
1797         unsigned start_reg, end_reg, reg;
1798         int r;
1799         u32 idx_value;
1800
1801         track = (struct evergreen_cs_track *)p->track;
1802         ib = p->ib.ptr;
1803         idx = pkt->idx + 1;
1804         idx_value = radeon_get_ib_value(p, idx);
1805
1806         switch (pkt->opcode) {
1807         case PACKET3_SET_PREDICATION:
1808         {
1809                 int pred_op;
1810                 int tmp;
1811                 uint64_t offset;
1812
1813                 if (pkt->count != 1) {
1814                         DRM_ERROR("bad SET PREDICATION\n");
1815                         return -EINVAL;
1816                 }
1817
1818                 tmp = radeon_get_ib_value(p, idx + 1);
1819                 pred_op = (tmp >> 16) & 0x7;
1820
1821                 /* for the clear predicate operation */
1822                 if (pred_op == 0)
1823                         return 0;
1824
1825                 if (pred_op > 2) {
1826                         DRM_ERROR("bad SET PREDICATION operation %d\n", pred_op);
1827                         return -EINVAL;
1828                 }
1829
1830                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1831                 if (r) {
1832                         DRM_ERROR("bad SET PREDICATION\n");
1833                         return -EINVAL;
1834                 }
1835
1836                 offset = reloc->lobj.gpu_offset +
1837                          (idx_value & 0xfffffff0) +
1838                          ((u64)(tmp & 0xff) << 32);
1839
1840                 ib[idx + 0] = offset;
1841                 ib[idx + 1] = (tmp & 0xffffff00) | (upper_32_bits(offset) & 0xff);
1842         }
1843         break;
1844         case PACKET3_CONTEXT_CONTROL:
1845                 if (pkt->count != 1) {
1846                         DRM_ERROR("bad CONTEXT_CONTROL\n");
1847                         return -EINVAL;
1848                 }
1849                 break;
1850         case PACKET3_INDEX_TYPE:
1851         case PACKET3_NUM_INSTANCES:
1852         case PACKET3_CLEAR_STATE:
1853                 if (pkt->count) {
1854                         DRM_ERROR("bad INDEX_TYPE/NUM_INSTANCES/CLEAR_STATE\n");
1855                         return -EINVAL;
1856                 }
1857                 break;
1858         case CAYMAN_PACKET3_DEALLOC_STATE:
1859                 if (p->rdev->family < CHIP_CAYMAN) {
1860                         DRM_ERROR("bad PACKET3_DEALLOC_STATE\n");
1861                         return -EINVAL;
1862                 }
1863                 if (pkt->count) {
1864                         DRM_ERROR("bad INDEX_TYPE/NUM_INSTANCES/CLEAR_STATE\n");
1865                         return -EINVAL;
1866                 }
1867                 break;
1868         case PACKET3_INDEX_BASE:
1869         {
1870                 uint64_t offset;
1871
1872                 if (pkt->count != 1) {
1873                         DRM_ERROR("bad INDEX_BASE\n");
1874                         return -EINVAL;
1875                 }
1876                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1877                 if (r) {
1878                         DRM_ERROR("bad INDEX_BASE\n");
1879                         return -EINVAL;
1880                 }
1881
1882                 offset = reloc->lobj.gpu_offset +
1883                          idx_value +
1884                          ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32);
1885
1886                 ib[idx+0] = offset;
1887                 ib[idx+1] = upper_32_bits(offset) & 0xff;
1888
1889                 r = evergreen_cs_track_check(p);
1890                 if (r) {
1891                         dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
1892                         return r;
1893                 }
1894                 break;
1895         }
1896         case PACKET3_DRAW_INDEX:
1897         {
1898                 uint64_t offset;
1899                 if (pkt->count != 3) {
1900                         DRM_ERROR("bad DRAW_INDEX\n");
1901                         return -EINVAL;
1902                 }
1903                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1904                 if (r) {
1905                         DRM_ERROR("bad DRAW_INDEX\n");
1906                         return -EINVAL;
1907                 }
1908
1909                 offset = reloc->lobj.gpu_offset +
1910                          idx_value +
1911                          ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32);
1912
1913                 ib[idx+0] = offset;
1914                 ib[idx+1] = upper_32_bits(offset) & 0xff;
1915
1916                 r = evergreen_cs_track_check(p);
1917                 if (r) {
1918                         dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
1919                         return r;
1920                 }
1921                 break;
1922         }
1923         case PACKET3_DRAW_INDEX_2:
1924         {
1925                 uint64_t offset;
1926
1927                 if (pkt->count != 4) {
1928                         DRM_ERROR("bad DRAW_INDEX_2\n");
1929                         return -EINVAL;
1930                 }
1931                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1932                 if (r) {
1933                         DRM_ERROR("bad DRAW_INDEX_2\n");
1934                         return -EINVAL;
1935                 }
1936
1937                 offset = reloc->lobj.gpu_offset +
1938                          radeon_get_ib_value(p, idx+1) +
1939                          ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
1940
1941                 ib[idx+1] = offset;
1942                 ib[idx+2] = upper_32_bits(offset) & 0xff;
1943
1944                 r = evergreen_cs_track_check(p);
1945                 if (r) {
1946                         dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
1947                         return r;
1948                 }
1949                 break;
1950         }
1951         case PACKET3_DRAW_INDEX_AUTO:
1952                 if (pkt->count != 1) {
1953                         DRM_ERROR("bad DRAW_INDEX_AUTO\n");
1954                         return -EINVAL;
1955                 }
1956                 r = evergreen_cs_track_check(p);
1957                 if (r) {
1958                         dev_warn(p->dev, "%s:%d invalid cmd stream %d\n", __func__, __LINE__, idx);
1959                         return r;
1960                 }
1961                 break;
1962         case PACKET3_DRAW_INDEX_MULTI_AUTO:
1963                 if (pkt->count != 2) {
1964                         DRM_ERROR("bad DRAW_INDEX_MULTI_AUTO\n");
1965                         return -EINVAL;
1966                 }
1967                 r = evergreen_cs_track_check(p);
1968                 if (r) {
1969                         dev_warn(p->dev, "%s:%d invalid cmd stream %d\n", __func__, __LINE__, idx);
1970                         return r;
1971                 }
1972                 break;
1973         case PACKET3_DRAW_INDEX_IMMD:
1974                 if (pkt->count < 2) {
1975                         DRM_ERROR("bad DRAW_INDEX_IMMD\n");
1976                         return -EINVAL;
1977                 }
1978                 r = evergreen_cs_track_check(p);
1979                 if (r) {
1980                         dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
1981                         return r;
1982                 }
1983                 break;
1984         case PACKET3_DRAW_INDEX_OFFSET:
1985                 if (pkt->count != 2) {
1986                         DRM_ERROR("bad DRAW_INDEX_OFFSET\n");
1987                         return -EINVAL;
1988                 }
1989                 r = evergreen_cs_track_check(p);
1990                 if (r) {
1991                         dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
1992                         return r;
1993                 }
1994                 break;
1995         case PACKET3_DRAW_INDEX_OFFSET_2:
1996                 if (pkt->count != 3) {
1997                         DRM_ERROR("bad DRAW_INDEX_OFFSET_2\n");
1998                         return -EINVAL;
1999                 }
2000                 r = evergreen_cs_track_check(p);
2001                 if (r) {
2002                         dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
2003                         return r;
2004                 }
2005                 break;
2006         case PACKET3_DISPATCH_DIRECT:
2007                 if (pkt->count != 3) {
2008                         DRM_ERROR("bad DISPATCH_DIRECT\n");
2009                         return -EINVAL;
2010                 }
2011                 r = evergreen_cs_track_check(p);
2012                 if (r) {
2013                         dev_warn(p->dev, "%s:%d invalid cmd stream %d\n", __func__, __LINE__, idx);
2014                         return r;
2015                 }
2016                 break;
2017         case PACKET3_DISPATCH_INDIRECT:
2018                 if (pkt->count != 1) {
2019                         DRM_ERROR("bad DISPATCH_INDIRECT\n");
2020                         return -EINVAL;
2021                 }
2022                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2023                 if (r) {
2024                         DRM_ERROR("bad DISPATCH_INDIRECT\n");
2025                         return -EINVAL;
2026                 }
2027                 ib[idx+0] = idx_value + (u32)(reloc->lobj.gpu_offset & 0xffffffff);
2028                 r = evergreen_cs_track_check(p);
2029                 if (r) {
2030                         dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
2031                         return r;
2032                 }
2033                 break;
2034         case PACKET3_WAIT_REG_MEM:
2035                 if (pkt->count != 5) {
2036                         DRM_ERROR("bad WAIT_REG_MEM\n");
2037                         return -EINVAL;
2038                 }
2039                 /* bit 4 is reg (0) or mem (1) */
2040                 if (idx_value & 0x10) {
2041                         uint64_t offset;
2042
2043                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2044                         if (r) {
2045                                 DRM_ERROR("bad WAIT_REG_MEM\n");
2046                                 return -EINVAL;
2047                         }
2048
2049                         offset = reloc->lobj.gpu_offset +
2050                                  (radeon_get_ib_value(p, idx+1) & 0xfffffffc) +
2051                                  ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
2052
2053                         ib[idx+1] = (ib[idx+1] & 0x3) | (offset & 0xfffffffc);
2054                         ib[idx+2] = upper_32_bits(offset) & 0xff;
2055                 } else if (idx_value & 0x100) {
2056                         DRM_ERROR("cannot use PFP on REG wait\n");
2057                         return -EINVAL;
2058                 }
2059                 break;
2060         case PACKET3_CP_DMA:
2061         {
2062                 u32 command, size, info;
2063                 u64 offset, tmp;
2064                 if (pkt->count != 4) {
2065                         DRM_ERROR("bad CP DMA\n");
2066                         return -EINVAL;
2067                 }
2068                 command = radeon_get_ib_value(p, idx+4);
2069                 size = command & 0x1fffff;
2070                 info = radeon_get_ib_value(p, idx+1);
2071                 if ((((info & 0x60000000) >> 29) != 0) || /* src = GDS or DATA */
2072                     (((info & 0x00300000) >> 20) != 0) || /* dst = GDS */
2073                     ((((info & 0x00300000) >> 20) == 0) &&
2074                      (command & PACKET3_CP_DMA_CMD_DAS)) || /* dst = register */
2075                     ((((info & 0x60000000) >> 29) == 0) &&
2076                      (command & PACKET3_CP_DMA_CMD_SAS))) { /* src = register */
2077                         /* non mem to mem copies requires dw aligned count */
2078                         if (size % 4) {
2079                                 DRM_ERROR("CP DMA command requires dw count alignment\n");
2080                                 return -EINVAL;
2081                         }
2082                 }
2083                 if (command & PACKET3_CP_DMA_CMD_SAS) {
2084                         /* src address space is register */
2085                         /* GDS is ok */
2086                         if (((info & 0x60000000) >> 29) != 1) {
2087                                 DRM_ERROR("CP DMA SAS not supported\n");
2088                                 return -EINVAL;
2089                         }
2090                 } else {
2091                         if (command & PACKET3_CP_DMA_CMD_SAIC) {
2092                                 DRM_ERROR("CP DMA SAIC only supported for registers\n");
2093                                 return -EINVAL;
2094                         }
2095                         /* src address space is memory */
2096                         if (((info & 0x60000000) >> 29) == 0) {
2097                                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2098                                 if (r) {
2099                                         DRM_ERROR("bad CP DMA SRC\n");
2100                                         return -EINVAL;
2101                                 }
2102
2103                                 tmp = radeon_get_ib_value(p, idx) +
2104                                         ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32);
2105
2106                                 offset = reloc->lobj.gpu_offset + tmp;
2107
2108                                 if ((tmp + size) > radeon_bo_size(reloc->robj)) {
2109                                         dev_warn(p->dev, "CP DMA src buffer too small (%llu %lu)\n",
2110                                                  tmp + size, radeon_bo_size(reloc->robj));
2111                                         return -EINVAL;
2112                                 }
2113
2114                                 ib[idx] = offset;
2115                                 ib[idx+1] = (ib[idx+1] & 0xffffff00) | (upper_32_bits(offset) & 0xff);
2116                         } else if (((info & 0x60000000) >> 29) != 2) {
2117                                 DRM_ERROR("bad CP DMA SRC_SEL\n");
2118                                 return -EINVAL;
2119                         }
2120                 }
2121                 if (command & PACKET3_CP_DMA_CMD_DAS) {
2122                         /* dst address space is register */
2123                         /* GDS is ok */
2124                         if (((info & 0x00300000) >> 20) != 1) {
2125                                 DRM_ERROR("CP DMA DAS not supported\n");
2126                                 return -EINVAL;
2127                         }
2128                 } else {
2129                         /* dst address space is memory */
2130                         if (command & PACKET3_CP_DMA_CMD_DAIC) {
2131                                 DRM_ERROR("CP DMA DAIC only supported for registers\n");
2132                                 return -EINVAL;
2133                         }
2134                         if (((info & 0x00300000) >> 20) == 0) {
2135                                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2136                                 if (r) {
2137                                         DRM_ERROR("bad CP DMA DST\n");
2138                                         return -EINVAL;
2139                                 }
2140
2141                                 tmp = radeon_get_ib_value(p, idx+2) +
2142                                         ((u64)(radeon_get_ib_value(p, idx+3) & 0xff) << 32);
2143
2144                                 offset = reloc->lobj.gpu_offset + tmp;
2145
2146                                 if ((tmp + size) > radeon_bo_size(reloc->robj)) {
2147                                         dev_warn(p->dev, "CP DMA dst buffer too small (%llu %lu)\n",
2148                                                  tmp + size, radeon_bo_size(reloc->robj));
2149                                         return -EINVAL;
2150                                 }
2151
2152                                 ib[idx+2] = offset;
2153                                 ib[idx+3] = upper_32_bits(offset) & 0xff;
2154                         } else {
2155                                 DRM_ERROR("bad CP DMA DST_SEL\n");
2156                                 return -EINVAL;
2157                         }
2158                 }
2159                 break;
2160         }
2161         case PACKET3_SURFACE_SYNC:
2162                 if (pkt->count != 3) {
2163                         DRM_ERROR("bad SURFACE_SYNC\n");
2164                         return -EINVAL;
2165                 }
2166                 /* 0xffffffff/0x0 is flush all cache flag */
2167                 if (radeon_get_ib_value(p, idx + 1) != 0xffffffff ||
2168                     radeon_get_ib_value(p, idx + 2) != 0) {
2169                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2170                         if (r) {
2171                                 DRM_ERROR("bad SURFACE_SYNC\n");
2172                                 return -EINVAL;
2173                         }
2174                         ib[idx+2] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
2175                 }
2176                 break;
2177         case PACKET3_EVENT_WRITE:
2178                 if (pkt->count != 2 && pkt->count != 0) {
2179                         DRM_ERROR("bad EVENT_WRITE\n");
2180                         return -EINVAL;
2181                 }
2182                 if (pkt->count) {
2183                         uint64_t offset;
2184
2185                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2186                         if (r) {
2187                                 DRM_ERROR("bad EVENT_WRITE\n");
2188                                 return -EINVAL;
2189                         }
2190                         offset = reloc->lobj.gpu_offset +
2191                                  (radeon_get_ib_value(p, idx+1) & 0xfffffff8) +
2192                                  ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
2193
2194                         ib[idx+1] = offset & 0xfffffff8;
2195                         ib[idx+2] = upper_32_bits(offset) & 0xff;
2196                 }
2197                 break;
2198         case PACKET3_EVENT_WRITE_EOP:
2199         {
2200                 uint64_t offset;
2201
2202                 if (pkt->count != 4) {
2203                         DRM_ERROR("bad EVENT_WRITE_EOP\n");
2204                         return -EINVAL;
2205                 }
2206                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2207                 if (r) {
2208                         DRM_ERROR("bad EVENT_WRITE_EOP\n");
2209                         return -EINVAL;
2210                 }
2211
2212                 offset = reloc->lobj.gpu_offset +
2213                          (radeon_get_ib_value(p, idx+1) & 0xfffffffc) +
2214                          ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
2215
2216                 ib[idx+1] = offset & 0xfffffffc;
2217                 ib[idx+2] = (ib[idx+2] & 0xffffff00) | (upper_32_bits(offset) & 0xff);
2218                 break;
2219         }
2220         case PACKET3_EVENT_WRITE_EOS:
2221         {
2222                 uint64_t offset;
2223
2224                 if (pkt->count != 3) {
2225                         DRM_ERROR("bad EVENT_WRITE_EOS\n");
2226                         return -EINVAL;
2227                 }
2228                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2229                 if (r) {
2230                         DRM_ERROR("bad EVENT_WRITE_EOS\n");
2231                         return -EINVAL;
2232                 }
2233
2234                 offset = reloc->lobj.gpu_offset +
2235                          (radeon_get_ib_value(p, idx+1) & 0xfffffffc) +
2236                          ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
2237
2238                 ib[idx+1] = offset & 0xfffffffc;
2239                 ib[idx+2] = (ib[idx+2] & 0xffffff00) | (upper_32_bits(offset) & 0xff);
2240                 break;
2241         }
2242         case PACKET3_SET_CONFIG_REG:
2243                 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
2244                 end_reg = 4 * pkt->count + start_reg - 4;
2245                 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
2246                     (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
2247                     (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
2248                         DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
2249                         return -EINVAL;
2250                 }
2251                 for (i = 0; i < pkt->count; i++) {
2252                         reg = start_reg + (4 * i);
2253                         r = evergreen_cs_check_reg(p, reg, idx+1+i);
2254                         if (r)
2255                                 return r;
2256                 }
2257                 break;
2258         case PACKET3_SET_CONTEXT_REG:
2259                 start_reg = (idx_value << 2) + PACKET3_SET_CONTEXT_REG_START;
2260                 end_reg = 4 * pkt->count + start_reg - 4;
2261                 if ((start_reg < PACKET3_SET_CONTEXT_REG_START) ||
2262                     (start_reg >= PACKET3_SET_CONTEXT_REG_END) ||
2263                     (end_reg >= PACKET3_SET_CONTEXT_REG_END)) {
2264                         DRM_ERROR("bad PACKET3_SET_CONTEXT_REG\n");
2265                         return -EINVAL;
2266                 }
2267                 for (i = 0; i < pkt->count; i++) {
2268                         reg = start_reg + (4 * i);
2269                         r = evergreen_cs_check_reg(p, reg, idx+1+i);
2270                         if (r)
2271                                 return r;
2272                 }
2273                 break;
2274         case PACKET3_SET_RESOURCE:
2275                 if (pkt->count % 8) {
2276                         DRM_ERROR("bad SET_RESOURCE\n");
2277                         return -EINVAL;
2278                 }
2279                 start_reg = (idx_value << 2) + PACKET3_SET_RESOURCE_START;
2280                 end_reg = 4 * pkt->count + start_reg - 4;
2281                 if ((start_reg < PACKET3_SET_RESOURCE_START) ||
2282                     (start_reg >= PACKET3_SET_RESOURCE_END) ||
2283                     (end_reg >= PACKET3_SET_RESOURCE_END)) {
2284                         DRM_ERROR("bad SET_RESOURCE\n");
2285                         return -EINVAL;
2286                 }
2287                 for (i = 0; i < (pkt->count / 8); i++) {
2288                         struct radeon_bo *texture, *mipmap;
2289                         u32 toffset, moffset;
2290                         u32 size, offset, mip_address, tex_dim;
2291
2292                         switch (G__SQ_CONSTANT_TYPE(radeon_get_ib_value(p, idx+1+(i*8)+7))) {
2293                         case SQ_TEX_VTX_VALID_TEXTURE:
2294                                 /* tex base */
2295                                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2296                                 if (r) {
2297                                         DRM_ERROR("bad SET_RESOURCE (tex)\n");
2298                                         return -EINVAL;
2299                                 }
2300                                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
2301                                         ib[idx+1+(i*8)+1] |=
2302                                                 TEX_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
2303                                         if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) {
2304                                                 unsigned bankw, bankh, mtaspect, tile_split;
2305
2306                                                 evergreen_tiling_fields(reloc->lobj.tiling_flags,
2307                                                                         &bankw, &bankh, &mtaspect,
2308                                                                         &tile_split);
2309                                                 ib[idx+1+(i*8)+6] |= TEX_TILE_SPLIT(tile_split);
2310                                                 ib[idx+1+(i*8)+7] |=
2311                                                         TEX_BANK_WIDTH(bankw) |
2312                                                         TEX_BANK_HEIGHT(bankh) |
2313                                                         MACRO_TILE_ASPECT(mtaspect) |
2314                                                         TEX_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks));
2315                                         }
2316                                 }
2317                                 texture = reloc->robj;
2318                                 toffset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
2319
2320                                 /* tex mip base */
2321                                 tex_dim = ib[idx+1+(i*8)+0] & 0x7;
2322                                 mip_address = ib[idx+1+(i*8)+3];
2323
2324                                 if ((tex_dim == SQ_TEX_DIM_2D_MSAA || tex_dim == SQ_TEX_DIM_2D_ARRAY_MSAA) &&
2325                                     !mip_address &&
2326                                     !radeon_cs_packet_next_is_pkt3_nop(p)) {
2327                                         /* MIP_ADDRESS should point to FMASK for an MSAA texture.
2328                                          * It should be 0 if FMASK is disabled. */
2329                                         moffset = 0;
2330                                         mipmap = NULL;
2331                                 } else {
2332                                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2333                                         if (r) {
2334                                                 DRM_ERROR("bad SET_RESOURCE (tex)\n");
2335                                                 return -EINVAL;
2336                                         }
2337                                         moffset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
2338                                         mipmap = reloc->robj;
2339                                 }
2340
2341                                 r = evergreen_cs_track_validate_texture(p, texture, mipmap, idx+1+(i*8));
2342                                 if (r)
2343                                         return r;
2344                                 ib[idx+1+(i*8)+2] += toffset;
2345                                 ib[idx+1+(i*8)+3] += moffset;
2346                                 break;
2347                         case SQ_TEX_VTX_VALID_BUFFER:
2348                         {
2349                                 uint64_t offset64;
2350                                 /* vtx base */
2351                                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2352                                 if (r) {
2353                                         DRM_ERROR("bad SET_RESOURCE (vtx)\n");
2354                                         return -EINVAL;
2355                                 }
2356                                 offset = radeon_get_ib_value(p, idx+1+(i*8)+0);
2357                                 size = radeon_get_ib_value(p, idx+1+(i*8)+1);
2358                                 if (p->rdev && (size + offset) > radeon_bo_size(reloc->robj)) {
2359                                         /* force size to size of the buffer */
2360                                         dev_warn(p->dev, "vbo resource seems too big for the bo\n");
2361                                         ib[idx+1+(i*8)+1] = radeon_bo_size(reloc->robj) - offset;
2362                                 }
2363
2364                                 offset64 = reloc->lobj.gpu_offset + offset;
2365                                 ib[idx+1+(i*8)+0] = offset64;
2366                                 ib[idx+1+(i*8)+2] = (ib[idx+1+(i*8)+2] & 0xffffff00) |
2367                                                     (upper_32_bits(offset64) & 0xff);
2368                                 break;
2369                         }
2370                         case SQ_TEX_VTX_INVALID_TEXTURE:
2371                         case SQ_TEX_VTX_INVALID_BUFFER:
2372                         default:
2373                                 DRM_ERROR("bad SET_RESOURCE\n");
2374                                 return -EINVAL;
2375                         }
2376                 }
2377                 break;
2378         case PACKET3_SET_ALU_CONST:
2379                 /* XXX fix me ALU const buffers only */
2380                 break;
2381         case PACKET3_SET_BOOL_CONST:
2382                 start_reg = (idx_value << 2) + PACKET3_SET_BOOL_CONST_START;
2383                 end_reg = 4 * pkt->count + start_reg - 4;
2384                 if ((start_reg < PACKET3_SET_BOOL_CONST_START) ||
2385                     (start_reg >= PACKET3_SET_BOOL_CONST_END) ||
2386                     (end_reg >= PACKET3_SET_BOOL_CONST_END)) {
2387                         DRM_ERROR("bad SET_BOOL_CONST\n");
2388                         return -EINVAL;
2389                 }
2390                 break;
2391         case PACKET3_SET_LOOP_CONST:
2392                 start_reg = (idx_value << 2) + PACKET3_SET_LOOP_CONST_START;
2393                 end_reg = 4 * pkt->count + start_reg - 4;
2394                 if ((start_reg < PACKET3_SET_LOOP_CONST_START) ||
2395                     (start_reg >= PACKET3_SET_LOOP_CONST_END) ||
2396                     (end_reg >= PACKET3_SET_LOOP_CONST_END)) {
2397                         DRM_ERROR("bad SET_LOOP_CONST\n");
2398                         return -EINVAL;
2399                 }
2400                 break;
2401         case PACKET3_SET_CTL_CONST:
2402                 start_reg = (idx_value << 2) + PACKET3_SET_CTL_CONST_START;
2403                 end_reg = 4 * pkt->count + start_reg - 4;
2404                 if ((start_reg < PACKET3_SET_CTL_CONST_START) ||
2405                     (start_reg >= PACKET3_SET_CTL_CONST_END) ||
2406                     (end_reg >= PACKET3_SET_CTL_CONST_END)) {
2407                         DRM_ERROR("bad SET_CTL_CONST\n");
2408                         return -EINVAL;
2409                 }
2410                 break;
2411         case PACKET3_SET_SAMPLER:
2412                 if (pkt->count % 3) {
2413                         DRM_ERROR("bad SET_SAMPLER\n");
2414                         return -EINVAL;
2415                 }
2416                 start_reg = (idx_value << 2) + PACKET3_SET_SAMPLER_START;
2417                 end_reg = 4 * pkt->count + start_reg - 4;
2418                 if ((start_reg < PACKET3_SET_SAMPLER_START) ||
2419                     (start_reg >= PACKET3_SET_SAMPLER_END) ||
2420                     (end_reg >= PACKET3_SET_SAMPLER_END)) {
2421                         DRM_ERROR("bad SET_SAMPLER\n");
2422                         return -EINVAL;
2423                 }
2424                 break;
2425         case PACKET3_STRMOUT_BUFFER_UPDATE:
2426                 if (pkt->count != 4) {
2427                         DRM_ERROR("bad STRMOUT_BUFFER_UPDATE (invalid count)\n");
2428                         return -EINVAL;
2429                 }
2430                 /* Updating memory at DST_ADDRESS. */
2431                 if (idx_value & 0x1) {
2432                         u64 offset;
2433                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2434                         if (r) {
2435                                 DRM_ERROR("bad STRMOUT_BUFFER_UPDATE (missing dst reloc)\n");
2436                                 return -EINVAL;
2437                         }
2438                         offset = radeon_get_ib_value(p, idx+1);
2439                         offset += ((u64)(radeon_get_ib_value(p, idx+2) & 0xff)) << 32;
2440                         if ((offset + 4) > radeon_bo_size(reloc->robj)) {
2441                                 DRM_ERROR("bad STRMOUT_BUFFER_UPDATE dst bo too small: 0x%llx, 0x%lx\n",
2442                                           offset + 4, radeon_bo_size(reloc->robj));
2443                                 return -EINVAL;
2444                         }
2445                         offset += reloc->lobj.gpu_offset;
2446                         ib[idx+1] = offset;
2447                         ib[idx+2] = upper_32_bits(offset) & 0xff;
2448                 }
2449                 /* Reading data from SRC_ADDRESS. */
2450                 if (((idx_value >> 1) & 0x3) == 2) {
2451                         u64 offset;
2452                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2453                         if (r) {
2454                                 DRM_ERROR("bad STRMOUT_BUFFER_UPDATE (missing src reloc)\n");
2455                                 return -EINVAL;
2456                         }
2457                         offset = radeon_get_ib_value(p, idx+3);
2458                         offset += ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32;
2459                         if ((offset + 4) > radeon_bo_size(reloc->robj)) {
2460                                 DRM_ERROR("bad STRMOUT_BUFFER_UPDATE src bo too small: 0x%llx, 0x%lx\n",
2461                                           offset + 4, radeon_bo_size(reloc->robj));
2462                                 return -EINVAL;
2463                         }
2464                         offset += reloc->lobj.gpu_offset;
2465                         ib[idx+3] = offset;
2466                         ib[idx+4] = upper_32_bits(offset) & 0xff;
2467                 }
2468                 break;
2469         case PACKET3_MEM_WRITE:
2470         {
2471                 u64 offset;
2472
2473                 if (pkt->count != 3) {
2474                         DRM_ERROR("bad MEM_WRITE (invalid count)\n");
2475                         return -EINVAL;
2476                 }
2477                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2478                 if (r) {
2479                         DRM_ERROR("bad MEM_WRITE (missing reloc)\n");
2480                         return -EINVAL;
2481                 }
2482                 offset = radeon_get_ib_value(p, idx+0);
2483                 offset += ((u64)(radeon_get_ib_value(p, idx+1) & 0xff)) << 32UL;
2484                 if (offset & 0x7) {
2485                         DRM_ERROR("bad MEM_WRITE (address not qwords aligned)\n");
2486                         return -EINVAL;
2487                 }
2488                 if ((offset + 8) > radeon_bo_size(reloc->robj)) {
2489                         DRM_ERROR("bad MEM_WRITE bo too small: 0x%llx, 0x%lx\n",
2490                                   offset + 8, radeon_bo_size(reloc->robj));
2491                         return -EINVAL;
2492                 }
2493                 offset += reloc->lobj.gpu_offset;
2494                 ib[idx+0] = offset;
2495                 ib[idx+1] = upper_32_bits(offset) & 0xff;
2496                 break;
2497         }
2498         case PACKET3_COPY_DW:
2499                 if (pkt->count != 4) {
2500                         DRM_ERROR("bad COPY_DW (invalid count)\n");
2501                         return -EINVAL;
2502                 }
2503                 if (idx_value & 0x1) {
2504                         u64 offset;
2505                         /* SRC is memory. */
2506                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2507                         if (r) {
2508                                 DRM_ERROR("bad COPY_DW (missing src reloc)\n");
2509                                 return -EINVAL;
2510                         }
2511                         offset = radeon_get_ib_value(p, idx+1);
2512                         offset += ((u64)(radeon_get_ib_value(p, idx+2) & 0xff)) << 32;
2513                         if ((offset + 4) > radeon_bo_size(reloc->robj)) {
2514                                 DRM_ERROR("bad COPY_DW src bo too small: 0x%llx, 0x%lx\n",
2515                                           offset + 4, radeon_bo_size(reloc->robj));
2516                                 return -EINVAL;
2517                         }
2518                         offset += reloc->lobj.gpu_offset;
2519                         ib[idx+1] = offset;
2520                         ib[idx+2] = upper_32_bits(offset) & 0xff;
2521                 } else {
2522                         /* SRC is a reg. */
2523                         reg = radeon_get_ib_value(p, idx+1) << 2;
2524                         if (!evergreen_is_safe_reg(p, reg, idx+1))
2525                                 return -EINVAL;
2526                 }
2527                 if (idx_value & 0x2) {
2528                         u64 offset;
2529                         /* DST is memory. */
2530                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2531                         if (r) {
2532                                 DRM_ERROR("bad COPY_DW (missing dst reloc)\n");
2533                                 return -EINVAL;
2534                         }
2535                         offset = radeon_get_ib_value(p, idx+3);
2536                         offset += ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32;
2537                         if ((offset + 4) > radeon_bo_size(reloc->robj)) {
2538                                 DRM_ERROR("bad COPY_DW dst bo too small: 0x%llx, 0x%lx\n",
2539                                           offset + 4, radeon_bo_size(reloc->robj));
2540                                 return -EINVAL;
2541                         }
2542                         offset += reloc->lobj.gpu_offset;
2543                         ib[idx+3] = offset;
2544                         ib[idx+4] = upper_32_bits(offset) & 0xff;
2545                 } else {
2546                         /* DST is a reg. */
2547                         reg = radeon_get_ib_value(p, idx+3) << 2;
2548                         if (!evergreen_is_safe_reg(p, reg, idx+3))
2549                                 return -EINVAL;
2550                 }
2551                 break;
2552         case PACKET3_NOP:
2553                 break;
2554         default:
2555                 DRM_ERROR("Packet3 opcode %x not supported\n", pkt->opcode);
2556                 return -EINVAL;
2557         }
2558         return 0;
2559 }
2560
2561 int evergreen_cs_parse(struct radeon_cs_parser *p)
2562 {
2563         struct radeon_cs_packet pkt;
2564         struct evergreen_cs_track *track;
2565         u32 tmp;
2566         int r;
2567
2568         if (p->track == NULL) {
2569                 /* initialize tracker, we are in kms */
2570                 track = kzalloc(sizeof(*track), GFP_KERNEL);
2571                 if (track == NULL)
2572                         return -ENOMEM;
2573                 evergreen_cs_track_init(track);
2574                 if (p->rdev->family >= CHIP_CAYMAN)
2575                         tmp = p->rdev->config.cayman.tile_config;
2576                 else
2577                         tmp = p->rdev->config.evergreen.tile_config;
2578
2579                 switch (tmp & 0xf) {
2580                 case 0:
2581                         track->npipes = 1;
2582                         break;
2583                 case 1:
2584                 default:
2585                         track->npipes = 2;
2586                         break;
2587                 case 2:
2588                         track->npipes = 4;
2589                         break;
2590                 case 3:
2591                         track->npipes = 8;
2592                         break;
2593                 }
2594
2595                 switch ((tmp & 0xf0) >> 4) {
2596                 case 0:
2597                         track->nbanks = 4;
2598                         break;
2599                 case 1:
2600                 default:
2601                         track->nbanks = 8;
2602                         break;
2603                 case 2:
2604                         track->nbanks = 16;
2605                         break;
2606                 }
2607
2608                 switch ((tmp & 0xf00) >> 8) {
2609                 case 0:
2610                         track->group_size = 256;
2611                         break;
2612                 case 1:
2613                 default:
2614                         track->group_size = 512;
2615                         break;
2616                 }
2617
2618                 switch ((tmp & 0xf000) >> 12) {
2619                 case 0:
2620                         track->row_size = 1;
2621                         break;
2622                 case 1:
2623                 default:
2624                         track->row_size = 2;
2625                         break;
2626                 case 2:
2627                         track->row_size = 4;
2628                         break;
2629                 }
2630
2631                 p->track = track;
2632         }
2633         do {
2634                 r = radeon_cs_packet_parse(p, &pkt, p->idx);
2635                 if (r) {
2636                         kfree(p->track);
2637                         p->track = NULL;
2638                         return r;
2639                 }
2640                 p->idx += pkt.count + 2;
2641                 switch (pkt.type) {
2642                 case RADEON_PACKET_TYPE0:
2643                         r = evergreen_cs_parse_packet0(p, &pkt);
2644                         break;
2645                 case RADEON_PACKET_TYPE2:
2646                         break;
2647                 case RADEON_PACKET_TYPE3:
2648                         r = evergreen_packet3_check(p, &pkt);
2649                         break;
2650                 default:
2651                         DRM_ERROR("Unknown packet type %d !\n", pkt.type);
2652                         kfree(p->track);
2653                         p->track = NULL;
2654                         return -EINVAL;
2655                 }
2656                 if (r) {
2657                         kfree(p->track);
2658                         p->track = NULL;
2659                         return r;
2660                 }
2661         } while (p->idx < p->chunks[p->chunk_ib_idx].length_dw);
2662 #if 0
2663         for (r = 0; r < p->ib.length_dw; r++) {
2664                 printk(KERN_INFO "%05d  0x%08X\n", r, p->ib.ptr[r]);
2665                 mdelay(1);
2666         }
2667 #endif
2668         kfree(p->track);
2669         p->track = NULL;
2670         return 0;
2671 }
2672
2673 /**
2674  * evergreen_dma_cs_parse() - parse the DMA IB
2675  * @p:          parser structure holding parsing context.
2676  *
2677  * Parses the DMA IB from the CS ioctl and updates
2678  * the GPU addresses based on the reloc information and
2679  * checks for errors. (Evergreen-Cayman)
2680  * Returns 0 for success and an error on failure.
2681  **/
2682 int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
2683 {
2684         struct radeon_cs_chunk *ib_chunk = &p->chunks[p->chunk_ib_idx];
2685         struct radeon_cs_reloc *src_reloc, *dst_reloc, *dst2_reloc;
2686         u32 header, cmd, count, sub_cmd;
2687         volatile u32 *ib = p->ib.ptr;
2688         u32 idx;
2689         u64 src_offset, dst_offset, dst2_offset;
2690         int r;
2691
2692         do {
2693                 if (p->idx >= ib_chunk->length_dw) {
2694                         DRM_ERROR("Can not parse packet at %d after CS end %d !\n",
2695                                   p->idx, ib_chunk->length_dw);
2696                         return -EINVAL;
2697                 }
2698                 idx = p->idx;
2699                 header = radeon_get_ib_value(p, idx);
2700                 cmd = GET_DMA_CMD(header);
2701                 count = GET_DMA_COUNT(header);
2702                 sub_cmd = GET_DMA_SUB_CMD(header);
2703
2704                 switch (cmd) {
2705                 case DMA_PACKET_WRITE:
2706                         r = r600_dma_cs_next_reloc(p, &dst_reloc);
2707                         if (r) {
2708                                 DRM_ERROR("bad DMA_PACKET_WRITE\n");
2709                                 return -EINVAL;
2710                         }
2711                         switch (sub_cmd) {
2712                         /* tiled */
2713                         case 8:
2714                                 dst_offset = radeon_get_ib_value(p, idx+1);
2715                                 dst_offset <<= 8;
2716
2717                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
2718                                 p->idx += count + 7;
2719                                 break;
2720                         /* linear */
2721                         case 0:
2722                                 dst_offset = radeon_get_ib_value(p, idx+1);
2723                                 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+2) & 0xff)) << 32;
2724
2725                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
2726                                 ib[idx+2] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2727                                 p->idx += count + 3;
2728                                 break;
2729                         default:
2730                                 DRM_ERROR("bad DMA_PACKET_WRITE [%6d] 0x%08x sub cmd is not 0 or 8\n", idx, header);
2731                                 return -EINVAL;
2732                         }
2733                         if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
2734                                 dev_warn(p->dev, "DMA write buffer too small (%llu %lu)\n",
2735                                          dst_offset, radeon_bo_size(dst_reloc->robj));
2736                                 return -EINVAL;
2737                         }
2738                         break;
2739                 case DMA_PACKET_COPY:
2740                         r = r600_dma_cs_next_reloc(p, &src_reloc);
2741                         if (r) {
2742                                 DRM_ERROR("bad DMA_PACKET_COPY\n");
2743                                 return -EINVAL;
2744                         }
2745                         r = r600_dma_cs_next_reloc(p, &dst_reloc);
2746                         if (r) {
2747                                 DRM_ERROR("bad DMA_PACKET_COPY\n");
2748                                 return -EINVAL;
2749                         }
2750                         switch (sub_cmd) {
2751                         /* Copy L2L, DW aligned */
2752                         case 0x00:
2753                                 /* L2L, dw */
2754                                 src_offset = radeon_get_ib_value(p, idx+2);
2755                                 src_offset |= ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32;
2756                                 dst_offset = radeon_get_ib_value(p, idx+1);
2757                                 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0xff)) << 32;
2758                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
2759                                         dev_warn(p->dev, "DMA L2L, dw src buffer too small (%llu %lu)\n",
2760                                                         src_offset + (count * 4), radeon_bo_size(src_reloc->robj));
2761                                         return -EINVAL;
2762                                 }
2763                                 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
2764                                         dev_warn(p->dev, "DMA L2L, dw dst buffer too small (%llu %lu)\n",
2765                                                         dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
2766                                         return -EINVAL;
2767                                 }
2768                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
2769                                 ib[idx+2] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
2770                                 ib[idx+3] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2771                                 ib[idx+4] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2772                                 p->idx += 5;
2773                                 break;
2774                         /* Copy L2T/T2L */
2775                         case 0x08:
2776                                 /* detile bit */
2777                                 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) {
2778                                         /* tiled src, linear dst */
2779                                         src_offset = radeon_get_ib_value(p, idx+1);
2780                                         src_offset <<= 8;
2781                                         ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8);
2782
2783                                         dst_offset = radeon_get_ib_value(p, idx + 7);
2784                                         dst_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32;
2785                                         ib[idx+7] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
2786                                         ib[idx+8] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2787                                 } else {
2788                                         /* linear src, tiled dst */
2789                                         src_offset = radeon_get_ib_value(p, idx+7);
2790                                         src_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32;
2791                                         ib[idx+7] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
2792                                         ib[idx+8] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2793
2794                                         dst_offset = radeon_get_ib_value(p, idx+1);
2795                                         dst_offset <<= 8;
2796                                         ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
2797                                 }
2798                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
2799                                         dev_warn(p->dev, "DMA L2T, src buffer too small (%llu %lu)\n",
2800                                                         src_offset + (count * 4), radeon_bo_size(src_reloc->robj));
2801                                         return -EINVAL;
2802                                 }
2803                                 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
2804                                         dev_warn(p->dev, "DMA L2T, dst buffer too small (%llu %lu)\n",
2805                                                         dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
2806                                         return -EINVAL;
2807                                 }
2808                                 p->idx += 9;
2809                                 break;
2810                         /* Copy L2L, byte aligned */
2811                         case 0x40:
2812                                 /* L2L, byte */
2813                                 src_offset = radeon_get_ib_value(p, idx+2);
2814                                 src_offset |= ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32;
2815                                 dst_offset = radeon_get_ib_value(p, idx+1);
2816                                 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0xff)) << 32;
2817                                 if ((src_offset + count) > radeon_bo_size(src_reloc->robj)) {
2818                                         dev_warn(p->dev, "DMA L2L, byte src buffer too small (%llu %lu)\n",
2819                                                         src_offset + count, radeon_bo_size(src_reloc->robj));
2820                                         return -EINVAL;
2821                                 }
2822                                 if ((dst_offset + count) > radeon_bo_size(dst_reloc->robj)) {
2823                                         dev_warn(p->dev, "DMA L2L, byte dst buffer too small (%llu %lu)\n",
2824                                                         dst_offset + count, radeon_bo_size(dst_reloc->robj));
2825                                         return -EINVAL;
2826                                 }
2827                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xffffffff);
2828                                 ib[idx+2] += (u32)(src_reloc->lobj.gpu_offset & 0xffffffff);
2829                                 ib[idx+3] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2830                                 ib[idx+4] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2831                                 p->idx += 5;
2832                                 break;
2833                         /* Copy L2L, partial */
2834                         case 0x41:
2835                                 /* L2L, partial */
2836                                 if (p->family < CHIP_CAYMAN) {
2837                                         DRM_ERROR("L2L Partial is cayman only !\n");
2838                                         return -EINVAL;
2839                                 }
2840                                 ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset & 0xffffffff);
2841                                 ib[idx+2] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2842                                 ib[idx+4] += (u32)(dst_reloc->lobj.gpu_offset & 0xffffffff);
2843                                 ib[idx+5] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2844
2845                                 p->idx += 9;
2846                                 break;
2847                         /* Copy L2L, DW aligned, broadcast */
2848                         case 0x44:
2849                                 /* L2L, dw, broadcast */
2850                                 r = r600_dma_cs_next_reloc(p, &dst2_reloc);
2851                                 if (r) {
2852                                         DRM_ERROR("bad L2L, dw, broadcast DMA_PACKET_COPY\n");
2853                                         return -EINVAL;
2854                                 }
2855                                 dst_offset = radeon_get_ib_value(p, idx+1);
2856                                 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32;
2857                                 dst2_offset = radeon_get_ib_value(p, idx+2);
2858                                 dst2_offset |= ((u64)(radeon_get_ib_value(p, idx+5) & 0xff)) << 32;
2859                                 src_offset = radeon_get_ib_value(p, idx+3);
2860                                 src_offset |= ((u64)(radeon_get_ib_value(p, idx+6) & 0xff)) << 32;
2861                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
2862                                         dev_warn(p->dev, "DMA L2L, dw, broadcast src buffer too small (%llu %lu)\n",
2863                                                         src_offset + (count * 4), radeon_bo_size(src_reloc->robj));
2864                                         return -EINVAL;
2865                                 }
2866                                 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
2867                                         dev_warn(p->dev, "DMA L2L, dw, broadcast dst buffer too small (%llu %lu)\n",
2868                                                         dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
2869                                         return -EINVAL;
2870                                 }
2871                                 if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) {
2872                                         dev_warn(p->dev, "DMA L2L, dw, broadcast dst2 buffer too small (%llu %lu)\n",
2873                                                         dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj));
2874                                         return -EINVAL;
2875                                 }
2876                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
2877                                 ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset & 0xfffffffc);
2878                                 ib[idx+3] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
2879                                 ib[idx+4] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2880                                 ib[idx+5] += upper_32_bits(dst2_reloc->lobj.gpu_offset) & 0xff;
2881                                 ib[idx+6] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2882                                 p->idx += 7;
2883                                 break;
2884                         /* Copy L2T Frame to Field */
2885                         case 0x48:
2886                                 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) {
2887                                         DRM_ERROR("bad L2T, frame to fields DMA_PACKET_COPY\n");
2888                                         return -EINVAL;
2889                                 }
2890                                 r = r600_dma_cs_next_reloc(p, &dst2_reloc);
2891                                 if (r) {
2892                                         DRM_ERROR("bad L2T, frame to fields DMA_PACKET_COPY\n");
2893                                         return -EINVAL;
2894                                 }
2895                                 dst_offset = radeon_get_ib_value(p, idx+1);
2896                                 dst_offset <<= 8;
2897                                 dst2_offset = radeon_get_ib_value(p, idx+2);
2898                                 dst2_offset <<= 8;
2899                                 src_offset = radeon_get_ib_value(p, idx+8);
2900                                 src_offset |= ((u64)(radeon_get_ib_value(p, idx+9) & 0xff)) << 32;
2901                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
2902                                         dev_warn(p->dev, "DMA L2T, frame to fields src buffer too small (%llu %lu)\n",
2903                                                         src_offset + (count * 4), radeon_bo_size(src_reloc->robj));
2904                                         return -EINVAL;
2905                                 }
2906                                 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
2907                                         dev_warn(p->dev, "DMA L2T, frame to fields buffer too small (%llu %lu)\n",
2908                                                         dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
2909                                         return -EINVAL;
2910                                 }
2911                                 if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) {
2912                                         dev_warn(p->dev, "DMA L2T, frame to fields buffer too small (%llu %lu)\n",
2913                                                         dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj));
2914                                         return -EINVAL;
2915                                 }
2916                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
2917                                 ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset >> 8);
2918                                 ib[idx+8] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
2919                                 ib[idx+9] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2920                                 p->idx += 10;
2921                                 break;
2922                         /* Copy L2T/T2L, partial */
2923                         case 0x49:
2924                                 /* L2T, T2L partial */
2925                                 if (p->family < CHIP_CAYMAN) {
2926                                         DRM_ERROR("L2T, T2L Partial is cayman only !\n");
2927                                         return -EINVAL;
2928                                 }
2929                                 /* detile bit */
2930                                 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) {
2931                                         /* tiled src, linear dst */
2932                                         ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8);
2933
2934                                         ib[idx+7] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
2935                                         ib[idx+8] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2936                                 } else {
2937                                         /* linear src, tiled dst */
2938                                         ib[idx+7] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
2939                                         ib[idx+8] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2940
2941                                         ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
2942                                 }
2943                                 p->idx += 12;
2944                                 break;
2945                         /* Copy L2T broadcast */
2946                         case 0x4b:
2947                                 /* L2T, broadcast */
2948                                 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) {
2949                                         DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n");
2950                                         return -EINVAL;
2951                                 }
2952                                 r = r600_dma_cs_next_reloc(p, &dst2_reloc);
2953                                 if (r) {
2954                                         DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n");
2955                                         return -EINVAL;
2956                                 }
2957                                 dst_offset = radeon_get_ib_value(p, idx+1);
2958                                 dst_offset <<= 8;
2959                                 dst2_offset = radeon_get_ib_value(p, idx+2);
2960                                 dst2_offset <<= 8;
2961                                 src_offset = radeon_get_ib_value(p, idx+8);
2962                                 src_offset |= ((u64)(radeon_get_ib_value(p, idx+9) & 0xff)) << 32;
2963                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
2964                                         dev_warn(p->dev, "DMA L2T, broadcast src buffer too small (%llu %lu)\n",
2965                                                         src_offset + (count * 4), radeon_bo_size(src_reloc->robj));
2966                                         return -EINVAL;
2967                                 }
2968                                 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
2969                                         dev_warn(p->dev, "DMA L2T, broadcast dst buffer too small (%llu %lu)\n",
2970                                                         dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
2971                                         return -EINVAL;
2972                                 }
2973                                 if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) {
2974                                         dev_warn(p->dev, "DMA L2T, broadcast dst2 buffer too small (%llu %lu)\n",
2975                                                         dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj));
2976                                         return -EINVAL;
2977                                 }
2978                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
2979                                 ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset >> 8);
2980                                 ib[idx+8] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
2981                                 ib[idx+9] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2982                                 p->idx += 10;
2983                                 break;
2984                         /* Copy L2T/T2L (tile units) */
2985                         case 0x4c:
2986                                 /* L2T, T2L */
2987                                 /* detile bit */
2988                                 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) {
2989                                         /* tiled src, linear dst */
2990                                         src_offset = radeon_get_ib_value(p, idx+1);
2991                                         src_offset <<= 8;
2992                                         ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8);
2993
2994                                         dst_offset = radeon_get_ib_value(p, idx+7);
2995                                         dst_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32;
2996                                         ib[idx+7] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
2997                                         ib[idx+8] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2998                                 } else {
2999                                         /* linear src, tiled dst */
3000                                         src_offset = radeon_get_ib_value(p, idx+7);
3001                                         src_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32;
3002                                         ib[idx+7] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
3003                                         ib[idx+8] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
3004
3005                                         dst_offset = radeon_get_ib_value(p, idx+1);
3006                                         dst_offset <<= 8;
3007                                         ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
3008                                 }
3009                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
3010                                         dev_warn(p->dev, "DMA L2T, T2L src buffer too small (%llu %lu)\n",
3011                                                         src_offset + (count * 4), radeon_bo_size(src_reloc->robj));
3012                                         return -EINVAL;
3013                                 }
3014                                 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
3015                                         dev_warn(p->dev, "DMA L2T, T2L dst buffer too small (%llu %lu)\n",
3016                                                         dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
3017                                         return -EINVAL;
3018                                 }
3019                                 p->idx += 9;
3020                                 break;
3021                         /* Copy T2T, partial (tile units) */
3022                         case 0x4d:
3023                                 /* T2T partial */
3024                                 if (p->family < CHIP_CAYMAN) {
3025                                         DRM_ERROR("L2T, T2L Partial is cayman only !\n");
3026                                         return -EINVAL;
3027                                 }
3028                                 ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8);
3029                                 ib[idx+4] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
3030                                 p->idx += 13;
3031                                 break;
3032                         /* Copy L2T broadcast (tile units) */
3033                         case 0x4f:
3034                                 /* L2T, broadcast */
3035                                 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) {
3036                                         DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n");
3037                                         return -EINVAL;
3038                                 }
3039                                 r = r600_dma_cs_next_reloc(p, &dst2_reloc);
3040                                 if (r) {
3041                                         DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n");
3042                                         return -EINVAL;
3043                                 }
3044                                 dst_offset = radeon_get_ib_value(p, idx+1);
3045                                 dst_offset <<= 8;
3046                                 dst2_offset = radeon_get_ib_value(p, idx+2);
3047                                 dst2_offset <<= 8;
3048                                 src_offset = radeon_get_ib_value(p, idx+8);
3049                                 src_offset |= ((u64)(radeon_get_ib_value(p, idx+9) & 0xff)) << 32;
3050                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
3051                                         dev_warn(p->dev, "DMA L2T, broadcast src buffer too small (%llu %lu)\n",
3052                                                         src_offset + (count * 4), radeon_bo_size(src_reloc->robj));
3053                                         return -EINVAL;
3054                                 }
3055                                 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
3056                                         dev_warn(p->dev, "DMA L2T, broadcast dst buffer too small (%llu %lu)\n",
3057                                                         dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
3058                                         return -EINVAL;
3059                                 }
3060                                 if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) {
3061                                         dev_warn(p->dev, "DMA L2T, broadcast dst2 buffer too small (%llu %lu)\n",
3062                                                         dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj));
3063                                         return -EINVAL;
3064                                 }
3065                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
3066                                 ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset >> 8);
3067                                 ib[idx+8] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
3068                                 ib[idx+9] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
3069                                 p->idx += 10;
3070                                 break;
3071                         default:
3072                                 DRM_ERROR("bad DMA_PACKET_COPY [%6d] 0x%08x invalid sub cmd\n", idx, header);
3073                                 return -EINVAL;
3074                         }
3075                         break;
3076                 case DMA_PACKET_CONSTANT_FILL:
3077                         r = r600_dma_cs_next_reloc(p, &dst_reloc);
3078                         if (r) {
3079                                 DRM_ERROR("bad DMA_PACKET_CONSTANT_FILL\n");
3080                                 return -EINVAL;
3081                         }
3082                         dst_offset = radeon_get_ib_value(p, idx+1);
3083                         dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0x00ff0000)) << 16;
3084                         if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
3085                                 dev_warn(p->dev, "DMA constant fill buffer too small (%llu %lu)\n",
3086                                          dst_offset, radeon_bo_size(dst_reloc->robj));
3087                                 return -EINVAL;
3088                         }
3089                         ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
3090                         ib[idx+3] += (upper_32_bits(dst_reloc->lobj.gpu_offset) << 16) & 0x00ff0000;
3091                         p->idx += 4;
3092                         break;
3093                 case DMA_PACKET_NOP:
3094                         p->idx += 1;
3095                         break;
3096                 default:
3097                         DRM_ERROR("Unknown packet type %d at %d !\n", cmd, idx);
3098                         return -EINVAL;
3099                 }
3100         } while (p->idx < p->chunks[p->chunk_ib_idx].length_dw);
3101 #if 0
3102         for (r = 0; r < p->ib->length_dw; r++) {
3103                 printk(KERN_INFO "%05d  0x%08X\n", r, p->ib.ptr[r]);
3104                 mdelay(1);
3105         }
3106 #endif
3107         return 0;
3108 }
3109
3110 /* vm parser */
3111 static bool evergreen_vm_reg_valid(u32 reg)
3112 {
3113         /* context regs are fine */
3114         if (reg >= 0x28000)
3115                 return true;
3116
3117         /* check config regs */
3118         switch (reg) {
3119         case WAIT_UNTIL:
3120         case GRBM_GFX_INDEX:
3121         case CP_STRMOUT_CNTL:
3122         case CP_COHER_CNTL:
3123         case CP_COHER_SIZE:
3124         case VGT_VTX_VECT_EJECT_REG:
3125         case VGT_CACHE_INVALIDATION:
3126         case VGT_GS_VERTEX_REUSE:
3127         case VGT_PRIMITIVE_TYPE:
3128         case VGT_INDEX_TYPE:
3129         case VGT_NUM_INDICES:
3130         case VGT_NUM_INSTANCES:
3131         case VGT_COMPUTE_DIM_X:
3132         case VGT_COMPUTE_DIM_Y:
3133         case VGT_COMPUTE_DIM_Z:
3134         case VGT_COMPUTE_START_X:
3135         case VGT_COMPUTE_START_Y:
3136         case VGT_COMPUTE_START_Z:
3137         case VGT_COMPUTE_INDEX:
3138         case VGT_COMPUTE_THREAD_GROUP_SIZE:
3139         case VGT_HS_OFFCHIP_PARAM:
3140         case PA_CL_ENHANCE:
3141         case PA_SU_LINE_STIPPLE_VALUE:
3142         case PA_SC_LINE_STIPPLE_STATE:
3143         case PA_SC_ENHANCE:
3144         case SQ_DYN_GPR_CNTL_PS_FLUSH_REQ:
3145         case SQ_DYN_GPR_SIMD_LOCK_EN:
3146         case SQ_CONFIG:
3147         case SQ_GPR_RESOURCE_MGMT_1:
3148         case SQ_GLOBAL_GPR_RESOURCE_MGMT_1:
3149         case SQ_GLOBAL_GPR_RESOURCE_MGMT_2:
3150         case SQ_CONST_MEM_BASE:
3151         case SQ_STATIC_THREAD_MGMT_1:
3152         case SQ_STATIC_THREAD_MGMT_2:
3153         case SQ_STATIC_THREAD_MGMT_3:
3154         case SPI_CONFIG_CNTL:
3155         case SPI_CONFIG_CNTL_1:
3156         case TA_CNTL_AUX:
3157         case DB_DEBUG:
3158         case DB_DEBUG2:
3159         case DB_DEBUG3:
3160         case DB_DEBUG4:
3161         case DB_WATERMARKS:
3162         case TD_PS_BORDER_COLOR_INDEX:
3163         case TD_PS_BORDER_COLOR_RED:
3164         case TD_PS_BORDER_COLOR_GREEN:
3165         case TD_PS_BORDER_COLOR_BLUE:
3166         case TD_PS_BORDER_COLOR_ALPHA:
3167         case TD_VS_BORDER_COLOR_INDEX:
3168         case TD_VS_BORDER_COLOR_RED:
3169         case TD_VS_BORDER_COLOR_GREEN:
3170         case TD_VS_BORDER_COLOR_BLUE:
3171         case TD_VS_BORDER_COLOR_ALPHA:
3172         case TD_GS_BORDER_COLOR_INDEX:
3173         case TD_GS_BORDER_COLOR_RED:
3174         case TD_GS_BORDER_COLOR_GREEN:
3175         case TD_GS_BORDER_COLOR_BLUE:
3176         case TD_GS_BORDER_COLOR_ALPHA:
3177         case TD_HS_BORDER_COLOR_INDEX:
3178         case TD_HS_BORDER_COLOR_RED:
3179         case TD_HS_BORDER_COLOR_GREEN:
3180         case TD_HS_BORDER_COLOR_BLUE:
3181         case TD_HS_BORDER_COLOR_ALPHA:
3182         case TD_LS_BORDER_COLOR_INDEX:
3183         case TD_LS_BORDER_COLOR_RED:
3184         case TD_LS_BORDER_COLOR_GREEN:
3185         case TD_LS_BORDER_COLOR_BLUE:
3186         case TD_LS_BORDER_COLOR_ALPHA:
3187         case TD_CS_BORDER_COLOR_INDEX:
3188         case TD_CS_BORDER_COLOR_RED:
3189         case TD_CS_BORDER_COLOR_GREEN:
3190         case TD_CS_BORDER_COLOR_BLUE:
3191         case TD_CS_BORDER_COLOR_ALPHA:
3192         case SQ_ESGS_RING_SIZE:
3193         case SQ_GSVS_RING_SIZE:
3194         case SQ_ESTMP_RING_SIZE:
3195         case SQ_GSTMP_RING_SIZE:
3196         case SQ_HSTMP_RING_SIZE:
3197         case SQ_LSTMP_RING_SIZE:
3198         case SQ_PSTMP_RING_SIZE:
3199         case SQ_VSTMP_RING_SIZE:
3200         case SQ_ESGS_RING_ITEMSIZE:
3201         case SQ_ESTMP_RING_ITEMSIZE:
3202         case SQ_GSTMP_RING_ITEMSIZE:
3203         case SQ_GSVS_RING_ITEMSIZE:
3204         case SQ_GS_VERT_ITEMSIZE:
3205         case SQ_GS_VERT_ITEMSIZE_1:
3206         case SQ_GS_VERT_ITEMSIZE_2:
3207         case SQ_GS_VERT_ITEMSIZE_3:
3208         case SQ_GSVS_RING_OFFSET_1:
3209         case SQ_GSVS_RING_OFFSET_2:
3210         case SQ_GSVS_RING_OFFSET_3:
3211         case SQ_HSTMP_RING_ITEMSIZE:
3212         case SQ_LSTMP_RING_ITEMSIZE:
3213         case SQ_PSTMP_RING_ITEMSIZE:
3214         case SQ_VSTMP_RING_ITEMSIZE:
3215         case VGT_TF_RING_SIZE:
3216         case SQ_ESGS_RING_BASE:
3217         case SQ_GSVS_RING_BASE:
3218         case SQ_ESTMP_RING_BASE:
3219         case SQ_GSTMP_RING_BASE:
3220         case SQ_HSTMP_RING_BASE:
3221         case SQ_LSTMP_RING_BASE:
3222         case SQ_PSTMP_RING_BASE:
3223         case SQ_VSTMP_RING_BASE:
3224         case CAYMAN_VGT_OFFCHIP_LDS_BASE:
3225         case CAYMAN_SQ_EX_ALLOC_TABLE_SLOTS:
3226                 return true;
3227         default:
3228                 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
3229                 return false;
3230         }
3231 }
3232
3233 static int evergreen_vm_packet3_check(struct radeon_device *rdev,
3234                                       u32 *ib, struct radeon_cs_packet *pkt)
3235 {
3236         u32 idx = pkt->idx + 1;
3237         u32 idx_value = ib[idx];
3238         u32 start_reg, end_reg, reg, i;
3239         u32 command, info;
3240
3241         switch (pkt->opcode) {
3242         case PACKET3_NOP:
3243         case PACKET3_SET_BASE:
3244         case PACKET3_CLEAR_STATE:
3245         case PACKET3_INDEX_BUFFER_SIZE:
3246         case PACKET3_DISPATCH_DIRECT:
3247         case PACKET3_DISPATCH_INDIRECT:
3248         case PACKET3_MODE_CONTROL:
3249         case PACKET3_SET_PREDICATION:
3250         case PACKET3_COND_EXEC:
3251         case PACKET3_PRED_EXEC:
3252         case PACKET3_DRAW_INDIRECT:
3253         case PACKET3_DRAW_INDEX_INDIRECT:
3254         case PACKET3_INDEX_BASE:
3255         case PACKET3_DRAW_INDEX_2:
3256         case PACKET3_CONTEXT_CONTROL:
3257         case PACKET3_DRAW_INDEX_OFFSET:
3258         case PACKET3_INDEX_TYPE:
3259         case PACKET3_DRAW_INDEX:
3260         case PACKET3_DRAW_INDEX_AUTO:
3261         case PACKET3_DRAW_INDEX_IMMD:
3262         case PACKET3_NUM_INSTANCES:
3263         case PACKET3_DRAW_INDEX_MULTI_AUTO:
3264         case PACKET3_STRMOUT_BUFFER_UPDATE:
3265         case PACKET3_DRAW_INDEX_OFFSET_2:
3266         case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
3267         case PACKET3_MPEG_INDEX:
3268         case PACKET3_WAIT_REG_MEM:
3269         case PACKET3_MEM_WRITE:
3270         case PACKET3_SURFACE_SYNC:
3271         case PACKET3_EVENT_WRITE:
3272         case PACKET3_EVENT_WRITE_EOP:
3273         case PACKET3_EVENT_WRITE_EOS:
3274         case PACKET3_SET_CONTEXT_REG:
3275         case PACKET3_SET_BOOL_CONST:
3276         case PACKET3_SET_LOOP_CONST:
3277         case PACKET3_SET_RESOURCE:
3278         case PACKET3_SET_SAMPLER:
3279         case PACKET3_SET_CTL_CONST:
3280         case PACKET3_SET_RESOURCE_OFFSET:
3281         case PACKET3_SET_CONTEXT_REG_INDIRECT:
3282         case PACKET3_SET_RESOURCE_INDIRECT:
3283         case CAYMAN_PACKET3_DEALLOC_STATE:
3284                 break;
3285         case PACKET3_COND_WRITE:
3286                 if (idx_value & 0x100) {
3287                         reg = ib[idx + 5] * 4;
3288                         if (!evergreen_vm_reg_valid(reg))
3289                                 return -EINVAL;
3290                 }
3291                 break;
3292         case PACKET3_COPY_DW:
3293                 if (idx_value & 0x2) {
3294                         reg = ib[idx + 3] * 4;
3295                         if (!evergreen_vm_reg_valid(reg))
3296                                 return -EINVAL;
3297                 }
3298                 break;
3299         case PACKET3_SET_CONFIG_REG:
3300                 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
3301                 end_reg = 4 * pkt->count + start_reg - 4;
3302                 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
3303                     (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
3304                     (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
3305                         DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
3306                         return -EINVAL;
3307                 }
3308                 for (i = 0; i < pkt->count; i++) {
3309                         reg = start_reg + (4 * i);
3310                         if (!evergreen_vm_reg_valid(reg))
3311                                 return -EINVAL;
3312                 }
3313                 break;
3314         case PACKET3_CP_DMA:
3315                 command = ib[idx + 4];
3316                 info = ib[idx + 1];
3317                 if ((((info & 0x60000000) >> 29) != 0) || /* src = GDS or DATA */
3318                     (((info & 0x00300000) >> 20) != 0) || /* dst = GDS */
3319                     ((((info & 0x00300000) >> 20) == 0) &&
3320                      (command & PACKET3_CP_DMA_CMD_DAS)) || /* dst = register */
3321                     ((((info & 0x60000000) >> 29) == 0) &&
3322                      (command & PACKET3_CP_DMA_CMD_SAS))) { /* src = register */
3323                         /* non mem to mem copies requires dw aligned count */
3324                         if ((command & 0x1fffff) % 4) {
3325                                 DRM_ERROR("CP DMA command requires dw count alignment\n");
3326                                 return -EINVAL;
3327                         }
3328                 }
3329                 if (command & PACKET3_CP_DMA_CMD_SAS) {
3330                         /* src address space is register */
3331                         if (((info & 0x60000000) >> 29) == 0) {
3332                                 start_reg = idx_value << 2;
3333                                 if (command & PACKET3_CP_DMA_CMD_SAIC) {
3334                                         reg = start_reg;
3335                                         if (!evergreen_vm_reg_valid(reg)) {
3336                                                 DRM_ERROR("CP DMA Bad SRC register\n");
3337                                                 return -EINVAL;
3338                                         }
3339                                 } else {
3340                                         for (i = 0; i < (command & 0x1fffff); i++) {
3341                                                 reg = start_reg + (4 * i);
3342                                                 if (!evergreen_vm_reg_valid(reg)) {
3343                                                         DRM_ERROR("CP DMA Bad SRC register\n");
3344                                                         return -EINVAL;
3345                                                 }
3346                                         }
3347                                 }
3348                         }
3349                 }
3350                 if (command & PACKET3_CP_DMA_CMD_DAS) {
3351                         /* dst address space is register */
3352                         if (((info & 0x00300000) >> 20) == 0) {
3353                                 start_reg = ib[idx + 2];
3354                                 if (command & PACKET3_CP_DMA_CMD_DAIC) {
3355                                         reg = start_reg;
3356                                         if (!evergreen_vm_reg_valid(reg)) {
3357                                                 DRM_ERROR("CP DMA Bad DST register\n");
3358                                                 return -EINVAL;
3359                                         }
3360                                 } else {
3361                                         for (i = 0; i < (command & 0x1fffff); i++) {
3362                                                 reg = start_reg + (4 * i);
3363                                                 if (!evergreen_vm_reg_valid(reg)) {
3364                                                         DRM_ERROR("CP DMA Bad DST register\n");
3365                                                         return -EINVAL;
3366                                                 }
3367                                         }
3368                                 }
3369                         }
3370                 }
3371                 break;
3372         default:
3373                 return -EINVAL;
3374         }
3375         return 0;
3376 }
3377
3378 int evergreen_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
3379 {
3380         int ret = 0;
3381         u32 idx = 0;
3382         struct radeon_cs_packet pkt;
3383
3384         do {
3385                 pkt.idx = idx;
3386                 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
3387                 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
3388                 pkt.one_reg_wr = 0;
3389                 switch (pkt.type) {
3390                 case RADEON_PACKET_TYPE0:
3391                         dev_err(rdev->dev, "Packet0 not allowed!\n");
3392                         ret = -EINVAL;
3393                         break;
3394                 case RADEON_PACKET_TYPE2:
3395                         idx += 1;
3396                         break;
3397                 case RADEON_PACKET_TYPE3:
3398                         pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
3399                         ret = evergreen_vm_packet3_check(rdev, ib->ptr, &pkt);
3400                         idx += pkt.count + 2;
3401                         break;
3402                 default:
3403                         dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
3404                         ret = -EINVAL;
3405                         break;
3406                 }
3407                 if (ret)
3408                         break;
3409         } while (idx < ib->length_dw);
3410
3411         return ret;
3412 }
3413
3414 /**
3415  * evergreen_dma_ib_parse() - parse the DMA IB for VM
3416  * @rdev: radeon_device pointer
3417  * @ib: radeon_ib pointer
3418  *
3419  * Parses the DMA IB from the VM CS ioctl
3420  * checks for errors. (Cayman-SI)
3421  * Returns 0 for success and an error on failure.
3422  **/
3423 int evergreen_dma_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
3424 {
3425         u32 idx = 0;
3426         u32 header, cmd, count, sub_cmd;
3427
3428         do {
3429                 header = ib->ptr[idx];
3430                 cmd = GET_DMA_CMD(header);
3431                 count = GET_DMA_COUNT(header);
3432                 sub_cmd = GET_DMA_SUB_CMD(header);
3433
3434                 switch (cmd) {
3435                 case DMA_PACKET_WRITE:
3436                         switch (sub_cmd) {
3437                         /* tiled */
3438                         case 8:
3439                                 idx += count + 7;
3440                                 break;
3441                         /* linear */
3442                         case 0:
3443                                 idx += count + 3;
3444                                 break;
3445                         default:
3446                                 DRM_ERROR("bad DMA_PACKET_WRITE [%6d] 0x%08x sub cmd is not 0 or 8\n", idx, ib->ptr[idx]);
3447                                 return -EINVAL;
3448                         }
3449                         break;
3450                 case DMA_PACKET_COPY:
3451                         switch (sub_cmd) {
3452                         /* Copy L2L, DW aligned */
3453                         case 0x00:
3454                                 idx += 5;
3455                                 break;
3456                         /* Copy L2T/T2L */
3457                         case 0x08:
3458                                 idx += 9;
3459                                 break;
3460                         /* Copy L2L, byte aligned */
3461                         case 0x40:
3462                                 idx += 5;
3463                                 break;
3464                         /* Copy L2L, partial */
3465                         case 0x41:
3466                                 idx += 9;
3467                                 break;
3468                         /* Copy L2L, DW aligned, broadcast */
3469                         case 0x44:
3470                                 idx += 7;
3471                                 break;
3472                         /* Copy L2T Frame to Field */
3473                         case 0x48:
3474                                 idx += 10;
3475                                 break;
3476                         /* Copy L2T/T2L, partial */
3477                         case 0x49:
3478                                 idx += 12;
3479                                 break;
3480                         /* Copy L2T broadcast */
3481                         case 0x4b:
3482                                 idx += 10;
3483                                 break;
3484                         /* Copy L2T/T2L (tile units) */
3485                         case 0x4c:
3486                                 idx += 9;
3487                                 break;
3488                         /* Copy T2T, partial (tile units) */
3489                         case 0x4d:
3490                                 idx += 13;
3491                                 break;
3492                         /* Copy L2T broadcast (tile units) */
3493                         case 0x4f:
3494                                 idx += 10;
3495                                 break;
3496                         default:
3497                                 DRM_ERROR("bad DMA_PACKET_COPY [%6d] 0x%08x invalid sub cmd\n", idx, ib->ptr[idx]);
3498                                 return -EINVAL;
3499                         }
3500                         break;
3501                 case DMA_PACKET_CONSTANT_FILL:
3502                         idx += 4;
3503                         break;
3504                 case DMA_PACKET_NOP:
3505                         idx += 1;
3506                         break;
3507                 default:
3508                         DRM_ERROR("Unknown packet type %d at %d !\n", cmd, idx);
3509                         return -EINVAL;
3510                 }
3511         } while (idx < ib->length_dw);
3512
3513         return 0;
3514 }