e7ef19a840ccd7f22169e57b14e24028186d9f16
[linux-2.6.git] / drivers / video / tegra / host / gr3d / scale3d.c
1 /*
2  * drivers/video/tegra/host/t20/scale3d.c
3  *
4  * Tegra Graphics Host 3D clock scaling
5  *
6  * Copyright (c) 2010-2011, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 /*
24  * 3d clock scaling
25  *
26  * module3d_notify_busy() is called upon submit, module3d_notify_idle() is
27  * called when all outstanding submits are completed. Idle times are measured
28  * over a fixed time period (scale3d.p_period). If the 3d module idle time
29  * percentage goes over the limit (set in scale3d.p_idle_max), 3d clocks are
30  * scaled down. If the percentage goes under the minimum limit (set in
31  * scale3d.p_idle_min), 3d clocks are scaled up. An additional test is made
32  * over the time frame given in scale3d.p_fast_response for clocking up
33  * quickly in response to load peaks.
34  *
35  * 3d.emc clock is scaled proportionately to 3d clock, with a quadratic-
36  * bezier-like factor added to pull 3d.emc rate a bit lower.
37  */
38
39 #include <linux/debugfs.h>
40 #include <linux/types.h>
41 #include <linux/clk.h>
42 #include <mach/clk.h>
43 #include <mach/hardware.h>
44 #include "scale3d.h"
45 #include "../dev.h"
46
47 static int scale3d_is_enabled(void);
48 static void scale3d_enable(int enable);
49
50 #define POW2(x) ((x) * (x))
51
52 /*
53  * debugfs parameters to control 3d clock scaling test
54  *
55  * period        - time period for clock rate evaluation
56  * fast_response - time period for evaluation of 'busy' spikes
57  * idle_min      - if less than [idle_min] percent idle over [fast_response]
58  *                 microseconds, clock up.
59  * idle_max      - if over [idle_max] percent idle over [period] microseconds,
60  *                 clock down.
61  * max_scale     - limits rate changes to no less than (100 - max_scale)% or
62  *                 (100 + 2 * max_scale)% of current clock rate
63  * verbosity     - set above 5 for debug printouts
64  */
65
66 struct scale3d_info_rec {
67         struct mutex lock; /* lock for timestamps etc */
68         int enable;
69         int init;
70         ktime_t idle_frame;
71         ktime_t fast_frame;
72         ktime_t last_idle;
73         ktime_t last_short_term_idle;
74         int is_idle;
75         ktime_t last_tweak;
76         ktime_t last_down;
77         int fast_up_count;
78         int slow_down_count;
79         int is_scaled;
80         int fast_responses;
81         unsigned long idle_total;
82         unsigned long idle_short_term_total;
83         unsigned long max_rate_3d;
84         long emc_slope;
85         long emc_offset;
86         long emc_dip_slope;
87         long emc_dip_offset;
88         long emc_xmid;
89         unsigned long min_rate_3d;
90         struct work_struct work;
91         struct delayed_work idle_timer;
92         unsigned int scale;
93         unsigned int p_period;
94         unsigned int period;
95         unsigned int p_idle_min;
96         unsigned int idle_min;
97         unsigned int p_idle_max;
98         unsigned int idle_max;
99         unsigned int p_fast_response;
100         unsigned int fast_response;
101         unsigned int p_adjust;
102         unsigned int p_scale_emc;
103         unsigned int p_emc_dip;
104         unsigned int p_verbosity;
105         struct clk *clk_3d;
106         struct clk *clk_3d2;
107         struct clk *clk_3d_emc;
108 };
109
110 static struct scale3d_info_rec scale3d;
111
112 static void scale3d_clocks(unsigned long percent)
113 {
114         unsigned long hz, curr;
115
116         if (!tegra_is_clk_enabled(scale3d.clk_3d))
117                 return;
118
119         if (tegra_get_chipid() == TEGRA_CHIPID_TEGRA3)
120                 if (!tegra_is_clk_enabled(scale3d.clk_3d2))
121                         return;
122
123         curr = clk_get_rate(scale3d.clk_3d);
124         hz = percent * (curr / 100);
125
126         if (!(hz >= scale3d.max_rate_3d && curr == scale3d.max_rate_3d)) {
127                 if (tegra_get_chipid() == TEGRA_CHIPID_TEGRA3)
128                         clk_set_rate(scale3d.clk_3d2, 0);
129                 clk_set_rate(scale3d.clk_3d, hz);
130
131                 if (scale3d.p_scale_emc) {
132                         long after = (long) clk_get_rate(scale3d.clk_3d);
133                         hz = after * scale3d.emc_slope + scale3d.emc_offset;
134                         if (scale3d.p_emc_dip)
135                                 hz -=
136                                         (scale3d.emc_dip_slope *
137                                         POW2(after / 1000 - scale3d.emc_xmid) +
138                                         scale3d.emc_dip_offset);
139                         clk_set_rate(scale3d.clk_3d_emc, hz);
140                 }
141         }
142 }
143
144 static void scale3d_clocks_handler(struct work_struct *work)
145 {
146         unsigned int scale;
147
148         mutex_lock(&scale3d.lock);
149         scale = scale3d.scale;
150         mutex_unlock(&scale3d.lock);
151
152         if (scale != 0)
153                 scale3d_clocks(scale);
154 }
155
156 void nvhost_scale3d_suspend(struct nvhost_device *dev)
157 {
158         cancel_work_sync(&scale3d.work);
159         cancel_delayed_work(&scale3d.idle_timer);
160 }
161
162 /* set 3d clocks to max */
163 static void reset_3d_clocks(void)
164 {
165         if (clk_get_rate(scale3d.clk_3d) != scale3d.max_rate_3d) {
166                 clk_set_rate(scale3d.clk_3d, scale3d.max_rate_3d);
167                 if (tegra_get_chipid() == TEGRA_CHIPID_TEGRA3)
168                         clk_set_rate(scale3d.clk_3d2, scale3d.max_rate_3d);
169                 if (scale3d.p_scale_emc)
170                         clk_set_rate(scale3d.clk_3d_emc,
171                                 clk_round_rate(scale3d.clk_3d_emc, UINT_MAX));
172         }
173 }
174
175 static int scale3d_is_enabled(void)
176 {
177         int enable;
178
179         mutex_lock(&scale3d.lock);
180         enable = scale3d.enable;
181         mutex_unlock(&scale3d.lock);
182
183         return enable;
184 }
185
186 static void scale3d_enable(int enable)
187 {
188         int disable = 0;
189
190         mutex_lock(&scale3d.lock);
191
192         if (enable) {
193                 if (scale3d.max_rate_3d != scale3d.min_rate_3d)
194                         scale3d.enable = 1;
195         } else {
196                 scale3d.enable = 0;
197                 disable = 1;
198         }
199
200         mutex_unlock(&scale3d.lock);
201
202         if (disable)
203                 reset_3d_clocks();
204 }
205
206 static void reset_scaling_counters(ktime_t time)
207 {
208         scale3d.idle_total = 0;
209         scale3d.idle_short_term_total = 0;
210         scale3d.last_idle = time;
211         scale3d.last_short_term_idle = time;
212         scale3d.idle_frame = time;
213 }
214
215 /* scaling_adjust - use scale up / scale down hint counts to adjust scaling
216  * parameters.
217  *
218  * hint_ratio is 100 x the ratio of scale up to scale down hints. Three cases
219  * are distinguished:
220  *
221  * hint_ratio < HINT_RATIO_MIN - set parameters to maximize scaling effect
222  * hint_ratio > HINT_RATIO_MAX - set parameters to minimize scaling effect
223  * hint_ratio between limits - scale parameters linearly
224  *
225  * the parameters adjusted are
226  *
227  * * fast_response time
228  * * period - time for scaling down estimate
229  * * idle_min percentage
230  * * idle_max percentage
231  */
232 #define SCALING_ADJUST_PERIOD 1000000
233 #define HINT_RATIO_MAX 400
234 #define HINT_RATIO_MIN 100
235 #define HINT_RATIO_MID ((HINT_RATIO_MAX + HINT_RATIO_MIN) / 2)
236 #define HINT_RATIO_DIFF (HINT_RATIO_MAX - HINT_RATIO_MIN)
237
238 static void scaling_adjust(ktime_t time)
239 {
240         long hint_ratio;
241         long fast_response_adjustment;
242         long period_adjustment;
243         int idle_min_adjustment;
244         int idle_max_adjustment;
245         unsigned long dt;
246
247         dt = (unsigned long) ktime_us_delta(time, scale3d.last_tweak);
248         if (dt < SCALING_ADJUST_PERIOD)
249                 return;
250
251         hint_ratio = (100 * (scale3d.fast_up_count + 1)) /
252                                  (scale3d.slow_down_count + 1);
253
254         if (hint_ratio > HINT_RATIO_MAX) {
255                 fast_response_adjustment = -((int) scale3d.p_fast_response) / 4;
256                 period_adjustment = scale3d.p_period / 2;
257                 idle_min_adjustment = scale3d.p_idle_min;
258                 idle_max_adjustment = scale3d.p_idle_max;
259         } else if (hint_ratio < HINT_RATIO_MIN) {
260                 fast_response_adjustment = scale3d.p_fast_response / 2;
261                 period_adjustment = -((int) scale3d.p_period) / 4;
262                 idle_min_adjustment = -((int) scale3d.p_idle_min) / 2;
263                 idle_max_adjustment = -((int) scale3d.p_idle_max) / 2;
264         } else {
265                 int diff;
266                 int factor;
267
268                 diff = HINT_RATIO_MID - hint_ratio;
269                 if (diff < 0)
270                         factor = -diff * 2;
271                 else {
272                         factor = -diff;
273                         diff *= 2;
274                 }
275
276                 fast_response_adjustment = diff *
277                         (scale3d.p_fast_response / (HINT_RATIO_DIFF * 2));
278                 period_adjustment =
279                         diff * (scale3d.p_period / HINT_RATIO_DIFF);
280                 idle_min_adjustment =
281                         (factor * (int) scale3d.p_idle_min) / HINT_RATIO_DIFF;
282                 idle_max_adjustment =
283                         (factor * (int) scale3d.p_idle_max) / HINT_RATIO_DIFF;
284         }
285
286         scale3d.fast_response =
287                 scale3d.p_fast_response + fast_response_adjustment;
288         scale3d.period = scale3d.p_period + period_adjustment;
289                 scale3d.idle_min = scale3d.p_idle_min + idle_min_adjustment;
290         scale3d.idle_max = scale3d.p_idle_max + idle_max_adjustment;
291
292         if (scale3d.p_verbosity >= 10)
293                 pr_info("scale3d stats: + %d - %d (/ %d) f %u p %u min %u max %u\n",
294                         scale3d.fast_up_count, scale3d.slow_down_count,
295                         scale3d.fast_responses, scale3d.fast_response,
296                         scale3d.period, scale3d.idle_min, scale3d.idle_max);
297
298         scale3d.fast_up_count = 0;
299         scale3d.slow_down_count = 0;
300         scale3d.fast_responses = 0;
301         scale3d.last_down = time;
302         scale3d.last_tweak = time;
303 }
304
305 #undef SCALING_ADJUST_PERIOD
306 #undef HINT_RATIO_MAX
307 #undef HINT_RATIO_MIN
308 #undef HINT_RATIO_MID
309 #undef HINT_RATIO_DIFF
310
311 static void scaling_state_check(ktime_t time)
312 {
313         unsigned long dt;
314
315         /* adjustment: set scale parameters (fast_response, period) +/- 25%
316          * based on ratio of scale up to scale down hints
317          */
318         if (scale3d.p_adjust)
319                 scaling_adjust(time);
320         else {
321                 scale3d.fast_response = scale3d.p_fast_response;
322                 scale3d.period = scale3d.p_period;
323                 scale3d.idle_min = scale3d.p_idle_min;
324                 scale3d.idle_max = scale3d.p_idle_max;
325         }
326
327         /* check for load peaks */
328         dt = (unsigned long) ktime_us_delta(time, scale3d.fast_frame);
329         if (dt > scale3d.fast_response) {
330                 unsigned long idleness =
331                         (scale3d.idle_short_term_total * 100) / dt;
332                 scale3d.fast_responses++;
333                 scale3d.fast_frame = time;
334                 /* if too busy, scale up */
335                 if (idleness < scale3d.idle_min) {
336                         scale3d.is_scaled = 0;
337                         scale3d.fast_up_count++;
338                         if (scale3d.p_verbosity >= 5)
339                                 pr_info("scale3d: %ld%% busy\n",
340                                         100 - idleness);
341
342                         reset_3d_clocks();
343                         reset_scaling_counters(time);
344                         return;
345                 }
346                 scale3d.idle_short_term_total = 0;
347                 scale3d.last_short_term_idle = time;
348         }
349
350         dt = (unsigned long) ktime_us_delta(time, scale3d.idle_frame);
351         if (dt > scale3d.period) {
352                 unsigned long idleness = (scale3d.idle_total * 100) / dt;
353
354                 if (scale3d.p_verbosity >= 5)
355                         pr_info("scale3d: idle %lu, ~%lu%%\n",
356                                 scale3d.idle_total, idleness);
357
358                 if (idleness > scale3d.idle_max) {
359                         if (!scale3d.is_scaled) {
360                                 scale3d.is_scaled = 1;
361                                 scale3d.last_down = time;
362                         }
363                         scale3d.slow_down_count++;
364                         /* if idle time is high, clock down */
365                         scale3d.scale = 100 - (idleness - scale3d.idle_min);
366                         schedule_work(&scale3d.work);
367                 }
368
369                 reset_scaling_counters(time);
370         }
371 }
372
373 void nvhost_scale3d_notify_idle(struct nvhost_device *dev)
374 {
375         ktime_t t;
376         unsigned long dt;
377
378         mutex_lock(&scale3d.lock);
379
380         if (!scale3d.enable)
381                 goto done;
382
383         t = ktime_get();
384
385         if (scale3d.is_idle) {
386                 dt = ktime_us_delta(t, scale3d.last_idle);
387                 scale3d.idle_total += dt;
388                 dt = ktime_us_delta(t, scale3d.last_short_term_idle);
389                 scale3d.idle_short_term_total += dt;
390         } else
391                 scale3d.is_idle = 1;
392
393         scale3d.last_idle = t;
394         scale3d.last_short_term_idle = t;
395
396         scaling_state_check(scale3d.last_idle);
397
398         /* delay idle_max % of 2 * fast_response time (given in microseconds) */
399         schedule_delayed_work(&scale3d.idle_timer,
400                 msecs_to_jiffies((scale3d.idle_max * scale3d.fast_response)
401                         / 50000));
402
403 done:
404         mutex_unlock(&scale3d.lock);
405 }
406
407 void nvhost_scale3d_notify_busy(struct nvhost_device *dev)
408 {
409         unsigned long idle;
410         unsigned long short_term_idle;
411         ktime_t t;
412
413         mutex_lock(&scale3d.lock);
414
415         if (!scale3d.enable)
416                 goto done;
417
418         cancel_delayed_work(&scale3d.idle_timer);
419
420         t = ktime_get();
421
422         if (scale3d.is_idle) {
423                 idle = (unsigned long)
424                         ktime_us_delta(t, scale3d.last_idle);
425                 scale3d.idle_total += idle;
426                 short_term_idle =
427                         ktime_us_delta(t, scale3d.last_short_term_idle);
428                 scale3d.idle_short_term_total += short_term_idle;
429                 scale3d.is_idle = 0;
430         }
431
432         scaling_state_check(t);
433
434 done:
435         mutex_unlock(&scale3d.lock);
436 }
437
438 static void scale3d_idle_handler(struct work_struct *work)
439 {
440         int notify_idle = 0;
441
442         mutex_lock(&scale3d.lock);
443
444         if (scale3d.enable && scale3d.is_idle &&
445                 tegra_is_clk_enabled(scale3d.clk_3d)) {
446                 unsigned long curr = clk_get_rate(scale3d.clk_3d);
447                 if (curr > scale3d.min_rate_3d)
448                         notify_idle = 1;
449         }
450
451         mutex_unlock(&scale3d.lock);
452
453         if (notify_idle)
454                 nvhost_scale3d_notify_idle(NULL);
455 }
456
457 void nvhost_scale3d_reset()
458 {
459         ktime_t t = ktime_get();
460         mutex_lock(&scale3d.lock);
461         reset_scaling_counters(t);
462         mutex_unlock(&scale3d.lock);
463 }
464
465 /*
466  * debugfs parameters to control 3d clock scaling
467  */
468
469 void nvhost_scale3d_debug_init(struct dentry *de)
470 {
471         struct dentry *d, *f;
472
473         d = debugfs_create_dir("scaling", de);
474         if (!d) {
475                 pr_err("scale3d: can\'t create debugfs directory\n");
476                 return;
477         }
478
479 #define CREATE_SCALE3D_FILE(fname) \
480         do {\
481                 f = debugfs_create_u32(#fname, S_IRUGO | S_IWUSR, d,\
482                         &scale3d.p_##fname);\
483                 if (NULL == f) {\
484                         pr_err("scale3d: can\'t create file " #fname "\n");\
485                         return;\
486                 } \
487         } while (0)
488
489         CREATE_SCALE3D_FILE(fast_response);
490         CREATE_SCALE3D_FILE(idle_min);
491         CREATE_SCALE3D_FILE(idle_max);
492         CREATE_SCALE3D_FILE(period);
493         CREATE_SCALE3D_FILE(adjust);
494         CREATE_SCALE3D_FILE(scale_emc);
495         CREATE_SCALE3D_FILE(emc_dip);
496         CREATE_SCALE3D_FILE(verbosity);
497 #undef CREATE_SCALE3D_FILE
498 }
499
500 static ssize_t enable_3d_scaling_show(struct device *device,
501         struct device_attribute *attr, char *buf)
502 {
503         ssize_t res;
504
505         res = snprintf(buf, PAGE_SIZE, "%d\n", scale3d_is_enabled());
506
507         return res;
508 }
509
510 static ssize_t enable_3d_scaling_store(struct device *dev,
511         struct device_attribute *attr, const char *buf, size_t count)
512 {
513         unsigned long val = 0;
514
515         if (strict_strtoul(buf, 10, &val) < 0)
516                 return -EINVAL;
517
518         scale3d_enable(val);
519
520         return count;
521 }
522
523 static DEVICE_ATTR(enable_3d_scaling, S_IRUGO | S_IWUGO,
524         enable_3d_scaling_show, enable_3d_scaling_store);
525
526 void nvhost_scale3d_init(struct nvhost_device *d)
527 {
528         if (!scale3d.init) {
529                 int error;
530                 unsigned long max_emc, min_emc;
531                 long correction;
532                 mutex_init(&scale3d.lock);
533
534                 scale3d.clk_3d = d->clk[0];
535                 if (tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) {
536                         scale3d.clk_3d2 = d->clk[1];
537                         scale3d.clk_3d_emc = d->clk[2];
538                 } else
539                         scale3d.clk_3d_emc = d->clk[1];
540
541                 scale3d.max_rate_3d = clk_round_rate(scale3d.clk_3d, UINT_MAX);
542                 scale3d.min_rate_3d = clk_round_rate(scale3d.clk_3d, 0);
543
544                 if (scale3d.max_rate_3d == scale3d.min_rate_3d) {
545                         pr_warn("scale3d: 3d max rate = min rate (%lu), "
546                                 "disabling\n", scale3d.max_rate_3d);
547                         scale3d.enable = 0;
548                         return;
549                 }
550
551                 /* emc scaling:
552                  *
553                  * Remc = S * R3d + O - (Sd * (R3d - Rm)^2 + Od)
554                  *
555                  * Remc - 3d.emc rate
556                  * R3d  - 3d.cbus rate
557                  * Rm   - 3d.cbus 'middle' rate = (max + min)/2
558                  * S    - emc_slope
559                  * O    - emc_offset
560                  * Sd   - emc_dip_slope
561                  * Od   - emc_dip_offset
562                  *
563                  * this superposes a quadratic dip centered around the middle 3d
564                  * frequency over a linear correlation of 3d.emc to 3d clock
565                  * rates.
566                  *
567                  * S, O are chosen so that the maximum 3d rate produces the
568                  * maximum 3d.emc rate exactly, and the minimum 3d rate produces
569                  * at least the minimum 3d.emc rate.
570                  *
571                  * Sd and Od are chosen to produce the largest dip that will
572                  * keep 3d.emc frequencies monotonously decreasing with 3d
573                  * frequencies. To achieve this, the first derivative of Remc
574                  * with respect to R3d should be zero for the minimal 3d rate:
575                  *
576                  *   R'emc = S - 2 * Sd * (R3d - Rm)
577                  *   R'emc(R3d-min) = 0
578                  *   S = 2 * Sd * (R3d-min - Rm)
579                  *     = 2 * Sd * (R3d-min - R3d-max) / 2
580                  *   Sd = S / (R3d-min - R3d-max)
581                  *
582                  *   +---------------------------------------------------+
583                  *   | Sd = -(emc-max - emc-min) / (R3d-min - R3d-max)^2 |
584                  *   +---------------------------------------------------+
585                  *
586                  *   dip = Sd * (R3d - Rm)^2 + Od
587                  *
588                  * requiring dip(R3d-min) = 0 and dip(R3d-max) = 0 gives
589                  *
590                  *   Sd * (R3d-min - Rm)^2 + Od = 0
591                  *   Od = -Sd * ((R3d-min - R3d-max) / 2)^2
592                  *      = -Sd * ((R3d-min - R3d-max)^2) / 4
593                  *
594                  *   +------------------------------+
595                  *   | Od = (emc-max - emc-min) / 4 |
596                  *   +------------------------------+
597                  */
598
599                 max_emc = clk_round_rate(scale3d.clk_3d_emc, UINT_MAX);
600                 min_emc = clk_round_rate(scale3d.clk_3d_emc, 0);
601
602                 scale3d.emc_slope = (max_emc - min_emc) /
603                          (scale3d.max_rate_3d - scale3d.min_rate_3d);
604                 scale3d.emc_offset = max_emc -
605                         scale3d.emc_slope * scale3d.max_rate_3d;
606                 /* guarantee max 3d rate maps to max emc rate */
607                 scale3d.emc_offset += max_emc -
608                         (scale3d.emc_slope * scale3d.max_rate_3d +
609                         scale3d.emc_offset);
610
611                 scale3d.emc_dip_offset = (max_emc - min_emc) / 4;
612                 scale3d.emc_dip_slope =
613                         -4 * (scale3d.emc_dip_offset /
614                         (POW2(scale3d.max_rate_3d - scale3d.min_rate_3d)));
615                 scale3d.emc_xmid =
616                         (scale3d.max_rate_3d + scale3d.min_rate_3d) / 2;
617                 correction =
618                         scale3d.emc_dip_offset +
619                                 scale3d.emc_dip_slope *
620                                 POW2(scale3d.max_rate_3d - scale3d.emc_xmid);
621                 scale3d.emc_dip_offset -= correction;
622
623                 INIT_WORK(&scale3d.work, scale3d_clocks_handler);
624                 INIT_DELAYED_WORK(&scale3d.idle_timer, scale3d_idle_handler);
625
626                 /* set scaling parameter defaults */
627                 scale3d.enable = 1;
628                 scale3d.period = scale3d.p_period = 100000;
629                 scale3d.idle_min = scale3d.p_idle_min = 10;
630                 scale3d.idle_max = scale3d.p_idle_max = 15;
631                 scale3d.fast_response = scale3d.p_fast_response = 7000;
632                 scale3d.p_scale_emc = 1;
633                 scale3d.p_emc_dip = 1;
634                 scale3d.p_verbosity = 0;
635                 scale3d.p_adjust = 1;
636
637                 error = device_create_file(&d->dev,
638                                 &dev_attr_enable_3d_scaling);
639                 if (error)
640                         dev_err(&d->dev, "failed to create sysfs attributes");
641
642                 scale3d.init = 1;
643         }
644
645         nvhost_scale3d_reset();
646 }
647
648 void nvhost_scale3d_deinit(struct nvhost_device *dev)
649 {
650         device_remove_file(&dev->dev, &dev_attr_enable_3d_scaling);
651         scale3d.init = 0;
652 }