Linux-2.6.12-rc2
[linux-2.6.git] / arch / i386 / kernel / cpu / cpufreq / powernow-k8.c
1 /*
2  *   (c) 2003, 2004 Advanced Micro Devices, Inc.
3  *  Your use of this code is subject to the terms and conditions of the
4  *  GNU general public license version 2. See "COPYING" or
5  *  http://www.gnu.org/licenses/gpl.html
6  *
7  *  Support : paul.devriendt@amd.com
8  *
9  *  Based on the powernow-k7.c module written by Dave Jones.
10  *  (C) 2003 Dave Jones <davej@codemonkey.org.uk> on behalf of SuSE Labs
11  *  (C) 2004 Dominik Brodowski <linux@brodo.de>
12  *  (C) 2004 Pavel Machek <pavel@suse.cz>
13  *  Licensed under the terms of the GNU GPL License version 2.
14  *  Based upon datasheets & sample CPUs kindly provided by AMD.
15  *
16  *  Valuable input gratefully received from Dave Jones, Pavel Machek,
17  *  Dominik Brodowski, and others.
18  *  Processor information obtained from Chapter 9 (Power and Thermal Management)
19  *  of the "BIOS and Kernel Developer's Guide for the AMD Athlon 64 and AMD
20  *  Opteron Processors" available for download from www.amd.com
21  *
22  *  Tables for specific CPUs can be infrerred from
23  *      http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/30430.pdf
24  */
25
26 #include <linux/kernel.h>
27 #include <linux/smp.h>
28 #include <linux/module.h>
29 #include <linux/init.h>
30 #include <linux/cpufreq.h>
31 #include <linux/slab.h>
32 #include <linux/string.h>
33
34 #include <asm/msr.h>
35 #include <asm/io.h>
36 #include <asm/delay.h>
37
38 #ifdef CONFIG_X86_POWERNOW_K8_ACPI
39 #include <linux/acpi.h>
40 #include <acpi/processor.h>
41 #endif
42
43 #define PFX "powernow-k8: "
44 #define BFX PFX "BIOS error: "
45 #define VERSION "version 1.00.09e"
46 #include "powernow-k8.h"
47
48 /* serialize freq changes  */
49 static DECLARE_MUTEX(fidvid_sem);
50
51 static struct powernow_k8_data *powernow_data[NR_CPUS];
52
53 /* Return a frequency in MHz, given an input fid */
54 static u32 find_freq_from_fid(u32 fid)
55 {
56         return 800 + (fid * 100);
57 }
58
59 /* Return a frequency in KHz, given an input fid */
60 static u32 find_khz_freq_from_fid(u32 fid)
61 {
62         return 1000 * find_freq_from_fid(fid);
63 }
64
65 /* Return a voltage in miliVolts, given an input vid */
66 static u32 find_millivolts_from_vid(struct powernow_k8_data *data, u32 vid)
67 {
68         return 1550-vid*25;
69 }
70
71 /* Return the vco fid for an input fid
72  *
73  * Each "low" fid has corresponding "high" fid, and you can get to "low" fids
74  * only from corresponding high fids. This returns "high" fid corresponding to
75  * "low" one.
76  */
77 static u32 convert_fid_to_vco_fid(u32 fid)
78 {
79         if (fid < HI_FID_TABLE_BOTTOM) {
80                 return 8 + (2 * fid);
81         } else {
82                 return fid;
83         }
84 }
85
86 /*
87  * Return 1 if the pending bit is set. Unless we just instructed the processor
88  * to transition to a new state, seeing this bit set is really bad news.
89  */
90 static int pending_bit_stuck(void)
91 {
92         u32 lo, hi;
93
94         rdmsr(MSR_FIDVID_STATUS, lo, hi);
95         return lo & MSR_S_LO_CHANGE_PENDING ? 1 : 0;
96 }
97
98 /*
99  * Update the global current fid / vid values from the status msr.
100  * Returns 1 on error.
101  */
102 static int query_current_values_with_pending_wait(struct powernow_k8_data *data)
103 {
104         u32 lo, hi;
105         u32 i = 0;
106
107         lo = MSR_S_LO_CHANGE_PENDING;
108         while (lo & MSR_S_LO_CHANGE_PENDING) {
109                 if (i++ > 0x1000000) {
110                         printk(KERN_ERR PFX "detected change pending stuck\n");
111                         return 1;
112                 }
113                 rdmsr(MSR_FIDVID_STATUS, lo, hi);
114         }
115
116         data->currvid = hi & MSR_S_HI_CURRENT_VID;
117         data->currfid = lo & MSR_S_LO_CURRENT_FID;
118
119         return 0;
120 }
121
122 /* the isochronous relief time */
123 static void count_off_irt(struct powernow_k8_data *data)
124 {
125         udelay((1 << data->irt) * 10);
126         return;
127 }
128
129 /* the voltage stabalization time */
130 static void count_off_vst(struct powernow_k8_data *data)
131 {
132         udelay(data->vstable * VST_UNITS_20US);
133         return;
134 }
135
136 /* need to init the control msr to a safe value (for each cpu) */
137 static void fidvid_msr_init(void)
138 {
139         u32 lo, hi;
140         u8 fid, vid;
141
142         rdmsr(MSR_FIDVID_STATUS, lo, hi);
143         vid = hi & MSR_S_HI_CURRENT_VID;
144         fid = lo & MSR_S_LO_CURRENT_FID;
145         lo = fid | (vid << MSR_C_LO_VID_SHIFT);
146         hi = MSR_C_HI_STP_GNT_BENIGN;
147         dprintk("cpu%d, init lo 0x%x, hi 0x%x\n", smp_processor_id(), lo, hi);
148         wrmsr(MSR_FIDVID_CTL, lo, hi);
149 }
150
151
152 /* write the new fid value along with the other control fields to the msr */
153 static int write_new_fid(struct powernow_k8_data *data, u32 fid)
154 {
155         u32 lo;
156         u32 savevid = data->currvid;
157
158         if ((fid & INVALID_FID_MASK) || (data->currvid & INVALID_VID_MASK)) {
159                 printk(KERN_ERR PFX "internal error - overflow on fid write\n");
160                 return 1;
161         }
162
163         lo = fid | (data->currvid << MSR_C_LO_VID_SHIFT) | MSR_C_LO_INIT_FID_VID;
164
165         dprintk("writing fid 0x%x, lo 0x%x, hi 0x%x\n",
166                 fid, lo, data->plllock * PLL_LOCK_CONVERSION);
167
168         wrmsr(MSR_FIDVID_CTL, lo, data->plllock * PLL_LOCK_CONVERSION);
169
170         if (query_current_values_with_pending_wait(data))
171                 return 1;
172
173         count_off_irt(data);
174
175         if (savevid != data->currvid) {
176                 printk(KERN_ERR PFX "vid change on fid trans, old 0x%x, new 0x%x\n",
177                        savevid, data->currvid);
178                 return 1;
179         }
180
181         if (fid != data->currfid) {
182                 printk(KERN_ERR PFX "fid trans failed, fid 0x%x, curr 0x%x\n", fid,
183                         data->currfid);
184                 return 1;
185         }
186
187         return 0;
188 }
189
190 /* Write a new vid to the hardware */
191 static int write_new_vid(struct powernow_k8_data *data, u32 vid)
192 {
193         u32 lo;
194         u32 savefid = data->currfid;
195
196         if ((data->currfid & INVALID_FID_MASK) || (vid & INVALID_VID_MASK)) {
197                 printk(KERN_ERR PFX "internal error - overflow on vid write\n");
198                 return 1;
199         }
200
201         lo = data->currfid | (vid << MSR_C_LO_VID_SHIFT) | MSR_C_LO_INIT_FID_VID;
202
203         dprintk("writing vid 0x%x, lo 0x%x, hi 0x%x\n",
204                 vid, lo, STOP_GRANT_5NS);
205
206         wrmsr(MSR_FIDVID_CTL, lo, STOP_GRANT_5NS);
207
208         if (query_current_values_with_pending_wait(data))
209                 return 1;
210
211         if (savefid != data->currfid) {
212                 printk(KERN_ERR PFX "fid changed on vid trans, old 0x%x new 0x%x\n",
213                        savefid, data->currfid);
214                 return 1;
215         }
216
217         if (vid != data->currvid) {
218                 printk(KERN_ERR PFX "vid trans failed, vid 0x%x, curr 0x%x\n", vid,
219                                 data->currvid);
220                 return 1;
221         }
222
223         return 0;
224 }
225
226 /*
227  * Reduce the vid by the max of step or reqvid.
228  * Decreasing vid codes represent increasing voltages:
229  * vid of 0 is 1.550V, vid of 0x1e is 0.800V, vid of 0x1f is off.
230  */
231 static int decrease_vid_code_by_step(struct powernow_k8_data *data, u32 reqvid, u32 step)
232 {
233         if ((data->currvid - reqvid) > step)
234                 reqvid = data->currvid - step;
235
236         if (write_new_vid(data, reqvid))
237                 return 1;
238
239         count_off_vst(data);
240
241         return 0;
242 }
243
244 /* Change the fid and vid, by the 3 phases. */
245 static int transition_fid_vid(struct powernow_k8_data *data, u32 reqfid, u32 reqvid)
246 {
247         if (core_voltage_pre_transition(data, reqvid))
248                 return 1;
249
250         if (core_frequency_transition(data, reqfid))
251                 return 1;
252
253         if (core_voltage_post_transition(data, reqvid))
254                 return 1;
255
256         if (query_current_values_with_pending_wait(data))
257                 return 1;
258
259         if ((reqfid != data->currfid) || (reqvid != data->currvid)) {
260                 printk(KERN_ERR PFX "failed (cpu%d): req 0x%x 0x%x, curr 0x%x 0x%x\n",
261                                 smp_processor_id(),
262                                 reqfid, reqvid, data->currfid, data->currvid);
263                 return 1;
264         }
265
266         dprintk("transitioned (cpu%d): new fid 0x%x, vid 0x%x\n",
267                 smp_processor_id(), data->currfid, data->currvid);
268
269         return 0;
270 }
271
272 /* Phase 1 - core voltage transition ... setup voltage */
273 static int core_voltage_pre_transition(struct powernow_k8_data *data, u32 reqvid)
274 {
275         u32 rvosteps = data->rvo;
276         u32 savefid = data->currfid;
277
278         dprintk("ph1 (cpu%d): start, currfid 0x%x, currvid 0x%x, reqvid 0x%x, rvo 0x%x\n",
279                 smp_processor_id(),
280                 data->currfid, data->currvid, reqvid, data->rvo);
281
282         while (data->currvid > reqvid) {
283                 dprintk("ph1: curr 0x%x, req vid 0x%x\n",
284                         data->currvid, reqvid);
285                 if (decrease_vid_code_by_step(data, reqvid, data->vidmvs))
286                         return 1;
287         }
288
289         while ((rvosteps > 0)  && ((data->rvo + data->currvid) > reqvid)) {
290                 if (data->currvid == 0) {
291                         rvosteps = 0;
292                 } else {
293                         dprintk("ph1: changing vid for rvo, req 0x%x\n",
294                                 data->currvid - 1);
295                         if (decrease_vid_code_by_step(data, data->currvid - 1, 1))
296                                 return 1;
297                         rvosteps--;
298                 }
299         }
300
301         if (query_current_values_with_pending_wait(data))
302                 return 1;
303
304         if (savefid != data->currfid) {
305                 printk(KERN_ERR PFX "ph1 err, currfid changed 0x%x\n", data->currfid);
306                 return 1;
307         }
308
309         dprintk("ph1 complete, currfid 0x%x, currvid 0x%x\n",
310                 data->currfid, data->currvid);
311
312         return 0;
313 }
314
315 /* Phase 2 - core frequency transition */
316 static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid)
317 {
318         u32 vcoreqfid, vcocurrfid, vcofiddiff, savevid = data->currvid;
319
320         if ((reqfid < HI_FID_TABLE_BOTTOM) && (data->currfid < HI_FID_TABLE_BOTTOM)) {
321                 printk(KERN_ERR PFX "ph2: illegal lo-lo transition 0x%x 0x%x\n",
322                         reqfid, data->currfid);
323                 return 1;
324         }
325
326         if (data->currfid == reqfid) {
327                 printk(KERN_ERR PFX "ph2 null fid transition 0x%x\n", data->currfid);
328                 return 0;
329         }
330
331         dprintk("ph2 (cpu%d): starting, currfid 0x%x, currvid 0x%x, reqfid 0x%x\n",
332                 smp_processor_id(),
333                 data->currfid, data->currvid, reqfid);
334
335         vcoreqfid = convert_fid_to_vco_fid(reqfid);
336         vcocurrfid = convert_fid_to_vco_fid(data->currfid);
337         vcofiddiff = vcocurrfid > vcoreqfid ? vcocurrfid - vcoreqfid
338             : vcoreqfid - vcocurrfid;
339
340         while (vcofiddiff > 2) {
341                 if (reqfid > data->currfid) {
342                         if (data->currfid > LO_FID_TABLE_TOP) {
343                                 if (write_new_fid(data, data->currfid + 2)) {
344                                         return 1;
345                                 }
346                         } else {
347                                 if (write_new_fid
348                                     (data, 2 + convert_fid_to_vco_fid(data->currfid))) {
349                                         return 1;
350                                 }
351                         }
352                 } else {
353                         if (write_new_fid(data, data->currfid - 2))
354                                 return 1;
355                 }
356
357                 vcocurrfid = convert_fid_to_vco_fid(data->currfid);
358                 vcofiddiff = vcocurrfid > vcoreqfid ? vcocurrfid - vcoreqfid
359                     : vcoreqfid - vcocurrfid;
360         }
361
362         if (write_new_fid(data, reqfid))
363                 return 1;
364
365         if (query_current_values_with_pending_wait(data))
366                 return 1;
367
368         if (data->currfid != reqfid) {
369                 printk(KERN_ERR PFX
370                         "ph2: mismatch, failed fid transition, curr 0x%x, req 0x%x\n",
371                         data->currfid, reqfid);
372                 return 1;
373         }
374
375         if (savevid != data->currvid) {
376                 printk(KERN_ERR PFX "ph2: vid changed, save 0x%x, curr 0x%x\n",
377                         savevid, data->currvid);
378                 return 1;
379         }
380
381         dprintk("ph2 complete, currfid 0x%x, currvid 0x%x\n",
382                 data->currfid, data->currvid);
383
384         return 0;
385 }
386
387 /* Phase 3 - core voltage transition flow ... jump to the final vid. */
388 static int core_voltage_post_transition(struct powernow_k8_data *data, u32 reqvid)
389 {
390         u32 savefid = data->currfid;
391         u32 savereqvid = reqvid;
392
393         dprintk("ph3 (cpu%d): starting, currfid 0x%x, currvid 0x%x\n",
394                 smp_processor_id(),
395                 data->currfid, data->currvid);
396
397         if (reqvid != data->currvid) {
398                 if (write_new_vid(data, reqvid))
399                         return 1;
400
401                 if (savefid != data->currfid) {
402                         printk(KERN_ERR PFX
403                                "ph3: bad fid change, save 0x%x, curr 0x%x\n",
404                                savefid, data->currfid);
405                         return 1;
406                 }
407
408                 if (data->currvid != reqvid) {
409                         printk(KERN_ERR PFX
410                                "ph3: failed vid transition\n, req 0x%x, curr 0x%x",
411                                reqvid, data->currvid);
412                         return 1;
413                 }
414         }
415
416         if (query_current_values_with_pending_wait(data))
417                 return 1;
418
419         if (savereqvid != data->currvid) {
420                 dprintk("ph3 failed, currvid 0x%x\n", data->currvid);
421                 return 1;
422         }
423
424         if (savefid != data->currfid) {
425                 dprintk("ph3 failed, currfid changed 0x%x\n",
426                         data->currfid);
427                 return 1;
428         }
429
430         dprintk("ph3 complete, currfid 0x%x, currvid 0x%x\n",
431                 data->currfid, data->currvid);
432
433         return 0;
434 }
435
436 static int check_supported_cpu(unsigned int cpu)
437 {
438         cpumask_t oldmask = CPU_MASK_ALL;
439         u32 eax, ebx, ecx, edx;
440         unsigned int rc = 0;
441
442         oldmask = current->cpus_allowed;
443         set_cpus_allowed(current, cpumask_of_cpu(cpu));
444         schedule();
445
446         if (smp_processor_id() != cpu) {
447                 printk(KERN_ERR "limiting to cpu %u failed\n", cpu);
448                 goto out;
449         }
450
451         if (current_cpu_data.x86_vendor != X86_VENDOR_AMD)
452                 goto out;
453
454         eax = cpuid_eax(CPUID_PROCESSOR_SIGNATURE);
455         if (((eax & CPUID_USE_XFAM_XMOD) != CPUID_USE_XFAM_XMOD) ||
456             ((eax & CPUID_XFAM) != CPUID_XFAM_K8) ||
457             ((eax & CPUID_XMOD) > CPUID_XMOD_REV_E)) {
458                 printk(KERN_INFO PFX "Processor cpuid %x not supported\n", eax);
459                 goto out;
460         }
461
462         eax = cpuid_eax(CPUID_GET_MAX_CAPABILITIES);
463         if (eax < CPUID_FREQ_VOLT_CAPABILITIES) {
464                 printk(KERN_INFO PFX
465                        "No frequency change capabilities detected\n");
466                 goto out;
467         }
468
469         cpuid(CPUID_FREQ_VOLT_CAPABILITIES, &eax, &ebx, &ecx, &edx);
470         if ((edx & P_STATE_TRANSITION_CAPABLE) != P_STATE_TRANSITION_CAPABLE) {
471                 printk(KERN_INFO PFX "Power state transitions not supported\n");
472                 goto out;
473         }
474
475         rc = 1;
476
477 out:
478         set_cpus_allowed(current, oldmask);
479         schedule();
480         return rc;
481
482 }
483
484 static int check_pst_table(struct powernow_k8_data *data, struct pst_s *pst, u8 maxvid)
485 {
486         unsigned int j;
487         u8 lastfid = 0xff;
488
489         for (j = 0; j < data->numps; j++) {
490                 if (pst[j].vid > LEAST_VID) {
491                         printk(KERN_ERR PFX "vid %d invalid : 0x%x\n", j, pst[j].vid);
492                         return -EINVAL;
493                 }
494                 if (pst[j].vid < data->rvo) {   /* vid + rvo >= 0 */
495                         printk(KERN_ERR BFX "0 vid exceeded with pstate %d\n", j);
496                         return -ENODEV;
497                 }
498                 if (pst[j].vid < maxvid + data->rvo) {  /* vid + rvo >= maxvid */
499                         printk(KERN_ERR BFX "maxvid exceeded with pstate %d\n", j);
500                         return -ENODEV;
501                 }
502                 if ((pst[j].fid > MAX_FID)
503                     || (pst[j].fid & 1)
504                     || (j && (pst[j].fid < HI_FID_TABLE_BOTTOM))) {
505                         /* Only first fid is allowed to be in "low" range */
506                         printk(KERN_ERR PFX "two low fids - %d : 0x%x\n", j, pst[j].fid);
507                         return -EINVAL;
508                 }
509                 if (pst[j].fid < lastfid)
510                         lastfid = pst[j].fid;
511         }
512         if (lastfid & 1) {
513                 printk(KERN_ERR PFX "lastfid invalid\n");
514                 return -EINVAL;
515         }
516         if (lastfid > LO_FID_TABLE_TOP)
517                 printk(KERN_INFO PFX  "first fid not from lo freq table\n");
518
519         return 0;
520 }
521
522 static void print_basics(struct powernow_k8_data *data)
523 {
524         int j;
525         for (j = 0; j < data->numps; j++) {
526                 if (data->powernow_table[j].frequency != CPUFREQ_ENTRY_INVALID)
527                         printk(KERN_INFO PFX "   %d : fid 0x%x (%d MHz), vid 0x%x (%d mV)\n", j,
528                                 data->powernow_table[j].index & 0xff,
529                                 data->powernow_table[j].frequency/1000,
530                                 data->powernow_table[j].index >> 8,
531                                 find_millivolts_from_vid(data, data->powernow_table[j].index >> 8));
532         }
533         if (data->batps)
534                 printk(KERN_INFO PFX "Only %d pstates on battery\n", data->batps);
535 }
536
537 static int fill_powernow_table(struct powernow_k8_data *data, struct pst_s *pst, u8 maxvid)
538 {
539         struct cpufreq_frequency_table *powernow_table;
540         unsigned int j;
541
542         if (data->batps) {    /* use ACPI support to get full speed on mains power */
543                 printk(KERN_WARNING PFX "Only %d pstates usable (use ACPI driver for full range\n", data->batps);
544                 data->numps = data->batps;
545         }
546
547         for ( j=1; j<data->numps; j++ ) {
548                 if (pst[j-1].fid >= pst[j].fid) {
549                         printk(KERN_ERR PFX "PST out of sequence\n");
550                         return -EINVAL;
551                 }
552         }
553
554         if (data->numps < 2) {
555                 printk(KERN_ERR PFX "no p states to transition\n");
556                 return -ENODEV;
557         }
558
559         if (check_pst_table(data, pst, maxvid))
560                 return -EINVAL;
561
562         powernow_table = kmalloc((sizeof(struct cpufreq_frequency_table)
563                 * (data->numps + 1)), GFP_KERNEL);
564         if (!powernow_table) {
565                 printk(KERN_ERR PFX "powernow_table memory alloc failure\n");
566                 return -ENOMEM;
567         }
568
569         for (j = 0; j < data->numps; j++) {
570                 powernow_table[j].index = pst[j].fid; /* lower 8 bits */
571                 powernow_table[j].index |= (pst[j].vid << 8); /* upper 8 bits */
572                 powernow_table[j].frequency = find_khz_freq_from_fid(pst[j].fid);
573         }
574         powernow_table[data->numps].frequency = CPUFREQ_TABLE_END;
575         powernow_table[data->numps].index = 0;
576
577         if (query_current_values_with_pending_wait(data)) {
578                 kfree(powernow_table);
579                 return -EIO;
580         }
581
582         dprintk("cfid 0x%x, cvid 0x%x\n", data->currfid, data->currvid);
583         data->powernow_table = powernow_table;
584         print_basics(data);
585
586         for (j = 0; j < data->numps; j++)
587                 if ((pst[j].fid==data->currfid) && (pst[j].vid==data->currvid))
588                         return 0;
589
590         dprintk("currfid/vid do not match PST, ignoring\n");
591         return 0;
592 }
593
594 /* Find and validate the PSB/PST table in BIOS. */
595 static int find_psb_table(struct powernow_k8_data *data)
596 {
597         struct psb_s *psb;
598         unsigned int i;
599         u32 mvs;
600         u8 maxvid;
601         u32 cpst = 0;
602         u32 thiscpuid;
603
604         for (i = 0xc0000; i < 0xffff0; i += 0x10) {
605                 /* Scan BIOS looking for the signature. */
606                 /* It can not be at ffff0 - it is too big. */
607
608                 psb = phys_to_virt(i);
609                 if (memcmp(psb, PSB_ID_STRING, PSB_ID_STRING_LEN) != 0)
610                         continue;
611
612                 dprintk("found PSB header at 0x%p\n", psb);
613
614                 dprintk("table vers: 0x%x\n", psb->tableversion);
615                 if (psb->tableversion != PSB_VERSION_1_4) {
616                         printk(KERN_INFO BFX "PSB table is not v1.4\n");
617                         return -ENODEV;
618                 }
619
620                 dprintk("flags: 0x%x\n", psb->flags1);
621                 if (psb->flags1) {
622                         printk(KERN_ERR BFX "unknown flags\n");
623                         return -ENODEV;
624                 }
625
626                 data->vstable = psb->vstable;
627                 dprintk("voltage stabilization time: %d(*20us)\n", data->vstable);
628
629                 dprintk("flags2: 0x%x\n", psb->flags2);
630                 data->rvo = psb->flags2 & 3;
631                 data->irt = ((psb->flags2) >> 2) & 3;
632                 mvs = ((psb->flags2) >> 4) & 3;
633                 data->vidmvs = 1 << mvs;
634                 data->batps = ((psb->flags2) >> 6) & 3;
635
636                 dprintk("ramp voltage offset: %d\n", data->rvo);
637                 dprintk("isochronous relief time: %d\n", data->irt);
638                 dprintk("maximum voltage step: %d - 0x%x\n", mvs, data->vidmvs);
639
640                 dprintk("numpst: 0x%x\n", psb->num_tables);
641                 cpst = psb->num_tables;
642                 if ((psb->cpuid == 0x00000fc0) || (psb->cpuid == 0x00000fe0) ){
643                         thiscpuid = cpuid_eax(CPUID_PROCESSOR_SIGNATURE);
644                         if ((thiscpuid == 0x00000fc0) || (thiscpuid == 0x00000fe0) ) {
645                                 cpst = 1;
646                         }
647                 }
648                 if (cpst != 1) {
649                         printk(KERN_ERR BFX "numpst must be 1\n");
650                         return -ENODEV;
651                 }
652
653                 data->plllock = psb->plllocktime;
654                 dprintk("plllocktime: 0x%x (units 1us)\n", psb->plllocktime);
655                 dprintk("maxfid: 0x%x\n", psb->maxfid);
656                 dprintk("maxvid: 0x%x\n", psb->maxvid);
657                 maxvid = psb->maxvid;
658
659                 data->numps = psb->numps;
660                 dprintk("numpstates: 0x%x\n", data->numps);
661                 return fill_powernow_table(data, (struct pst_s *)(psb+1), maxvid);
662         }
663         /*
664          * If you see this message, complain to BIOS manufacturer. If
665          * he tells you "we do not support Linux" or some similar
666          * nonsense, remember that Windows 2000 uses the same legacy
667          * mechanism that the old Linux PSB driver uses. Tell them it
668          * is broken with Windows 2000.
669          *
670          * The reference to the AMD documentation is chapter 9 in the
671          * BIOS and Kernel Developer's Guide, which is available on
672          * www.amd.com
673          */
674         printk(KERN_ERR PFX "BIOS error - no PSB\n");
675         return -ENODEV;
676 }
677
678 #ifdef CONFIG_X86_POWERNOW_K8_ACPI
679 static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data, unsigned int index)
680 {
681         if (!data->acpi_data.state_count)
682                 return;
683
684         data->irt = (data->acpi_data.states[index].control >> IRT_SHIFT) & IRT_MASK;
685         data->rvo = (data->acpi_data.states[index].control >> RVO_SHIFT) & RVO_MASK;
686         data->plllock = (data->acpi_data.states[index].control >> PLL_L_SHIFT) & PLL_L_MASK;
687         data->vidmvs = 1 << ((data->acpi_data.states[index].control >> MVS_SHIFT) & MVS_MASK);
688         data->vstable = (data->acpi_data.states[index].control >> VST_SHIFT) & VST_MASK;
689 }
690
691 static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data)
692 {
693         int i;
694         int cntlofreq = 0;
695         struct cpufreq_frequency_table *powernow_table;
696
697         if (acpi_processor_register_performance(&data->acpi_data, data->cpu)) {
698                 dprintk("register performance failed\n");
699                 return -EIO;
700         }
701
702         /* verify the data contained in the ACPI structures */
703         if (data->acpi_data.state_count <= 1) {
704                 dprintk("No ACPI P-States\n");
705                 goto err_out;
706         }
707
708         if ((data->acpi_data.control_register.space_id != ACPI_ADR_SPACE_FIXED_HARDWARE) ||
709                 (data->acpi_data.status_register.space_id != ACPI_ADR_SPACE_FIXED_HARDWARE)) {
710                 dprintk("Invalid control/status registers (%x - %x)\n",
711                         data->acpi_data.control_register.space_id,
712                         data->acpi_data.status_register.space_id);
713                 goto err_out;
714         }
715
716         /* fill in data->powernow_table */
717         powernow_table = kmalloc((sizeof(struct cpufreq_frequency_table)
718                 * (data->acpi_data.state_count + 1)), GFP_KERNEL);
719         if (!powernow_table) {
720                 dprintk("powernow_table memory alloc failure\n");
721                 goto err_out;
722         }
723
724         for (i = 0; i < data->acpi_data.state_count; i++) {
725                 u32 fid = data->acpi_data.states[i].control & FID_MASK;
726                 u32 vid = (data->acpi_data.states[i].control >> VID_SHIFT) & VID_MASK;
727
728                 dprintk("   %d : fid 0x%x, vid 0x%x\n", i, fid, vid);
729
730                 powernow_table[i].index = fid; /* lower 8 bits */
731                 powernow_table[i].index |= (vid << 8); /* upper 8 bits */
732                 powernow_table[i].frequency = find_khz_freq_from_fid(fid);
733
734                 /* verify frequency is OK */
735                 if ((powernow_table[i].frequency > (MAX_FREQ * 1000)) ||
736                         (powernow_table[i].frequency < (MIN_FREQ * 1000))) {
737                         dprintk("invalid freq %u kHz, ignoring\n", powernow_table[i].frequency);
738                         powernow_table[i].frequency = CPUFREQ_ENTRY_INVALID;
739                         continue;
740                 }
741
742                 /* verify voltage is OK - BIOSs are using "off" to indicate invalid */
743                 if (vid == 0x1f) {
744                         dprintk("invalid vid %u, ignoring\n", vid);
745                         powernow_table[i].frequency = CPUFREQ_ENTRY_INVALID;
746                         continue;
747                 }
748
749                 if (fid < HI_FID_TABLE_BOTTOM) {
750                         if (cntlofreq) {
751                                 /* if both entries are the same, ignore this
752                                  * one... 
753                                  */
754                                 if ((powernow_table[i].frequency != powernow_table[cntlofreq].frequency) ||
755                                     (powernow_table[i].index != powernow_table[cntlofreq].index)) {
756                                         printk(KERN_ERR PFX "Too many lo freq table entries\n");
757                                         goto err_out_mem;
758                                 }
759                                 
760                                 dprintk("double low frequency table entry, ignoring it.\n");
761                                 powernow_table[i].frequency = CPUFREQ_ENTRY_INVALID;
762                                 continue;
763                         } else
764                                 cntlofreq = i;
765                 }
766
767                 if (powernow_table[i].frequency != (data->acpi_data.states[i].core_frequency * 1000)) {
768                         printk(KERN_INFO PFX "invalid freq entries %u kHz vs. %u kHz\n",
769                                 powernow_table[i].frequency,
770                                 (unsigned int) (data->acpi_data.states[i].core_frequency * 1000));
771                         powernow_table[i].frequency = CPUFREQ_ENTRY_INVALID;
772                         continue;
773                 }
774         }
775
776         powernow_table[data->acpi_data.state_count].frequency = CPUFREQ_TABLE_END;
777         powernow_table[data->acpi_data.state_count].index = 0;
778         data->powernow_table = powernow_table;
779
780         /* fill in data */
781         data->numps = data->acpi_data.state_count;
782         print_basics(data);
783         powernow_k8_acpi_pst_values(data, 0);
784
785         /* notify BIOS that we exist */
786         acpi_processor_notify_smm(THIS_MODULE);
787
788         return 0;
789
790 err_out_mem:
791         kfree(powernow_table);
792
793 err_out:
794         acpi_processor_unregister_performance(&data->acpi_data, data->cpu);
795
796         /* data->acpi_data.state_count informs us at ->exit() whether ACPI was used */
797         data->acpi_data.state_count = 0;
798
799         return -ENODEV;
800 }
801
802 static void powernow_k8_cpu_exit_acpi(struct powernow_k8_data *data)
803 {
804         if (data->acpi_data.state_count)
805                 acpi_processor_unregister_performance(&data->acpi_data, data->cpu);
806 }
807
808 #else
809 static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data) { return -ENODEV; }
810 static void powernow_k8_cpu_exit_acpi(struct powernow_k8_data *data) { return; }
811 static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data, unsigned int index) { return; }
812 #endif /* CONFIG_X86_POWERNOW_K8_ACPI */
813
814 /* Take a frequency, and issue the fid/vid transition command */
815 static int transition_frequency(struct powernow_k8_data *data, unsigned int index)
816 {
817         u32 fid;
818         u32 vid;
819         int res;
820         struct cpufreq_freqs freqs;
821
822         dprintk("cpu %d transition to index %u\n", smp_processor_id(), index);
823
824         /* fid are the lower 8 bits of the index we stored into
825          * the cpufreq frequency table in find_psb_table, vid are 
826          * the upper 8 bits.
827          */
828
829         fid = data->powernow_table[index].index & 0xFF;
830         vid = (data->powernow_table[index].index & 0xFF00) >> 8;
831
832         dprintk("table matched fid 0x%x, giving vid 0x%x\n", fid, vid);
833
834         if (query_current_values_with_pending_wait(data))
835                 return 1;
836
837         if ((data->currvid == vid) && (data->currfid == fid)) {
838                 dprintk("target matches current values (fid 0x%x, vid 0x%x)\n",
839                         fid, vid);
840                 return 0;
841         }
842
843         if ((fid < HI_FID_TABLE_BOTTOM) && (data->currfid < HI_FID_TABLE_BOTTOM)) {
844                 printk("ignoring illegal change in lo freq table-%x to 0x%x\n",
845                        data->currfid, fid);
846                 return 1;
847         }
848
849         dprintk("cpu %d, changing to fid 0x%x, vid 0x%x\n",
850                 smp_processor_id(), fid, vid);
851
852         freqs.cpu = data->cpu;
853
854         freqs.old = find_khz_freq_from_fid(data->currfid);
855         freqs.new = find_khz_freq_from_fid(fid);
856         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
857
858         down(&fidvid_sem);
859         res = transition_fid_vid(data, fid, vid);
860         up(&fidvid_sem);
861
862         freqs.new = find_khz_freq_from_fid(data->currfid);
863         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
864
865         return res;
866 }
867
868 /* Driver entry point to switch to the target frequency */
869 static int powernowk8_target(struct cpufreq_policy *pol, unsigned targfreq, unsigned relation)
870 {
871         cpumask_t oldmask = CPU_MASK_ALL;
872         struct powernow_k8_data *data = powernow_data[pol->cpu];
873         u32 checkfid = data->currfid;
874         u32 checkvid = data->currvid;
875         unsigned int newstate;
876         int ret = -EIO;
877
878         /* only run on specific CPU from here on */
879         oldmask = current->cpus_allowed;
880         set_cpus_allowed(current, cpumask_of_cpu(pol->cpu));
881         schedule();
882
883         if (smp_processor_id() != pol->cpu) {
884                 printk(KERN_ERR "limiting to cpu %u failed\n", pol->cpu);
885                 goto err_out;
886         }
887
888         if (pending_bit_stuck()) {
889                 printk(KERN_ERR PFX "failing targ, change pending bit set\n");
890                 goto err_out;
891         }
892
893         dprintk("targ: cpu %d, %d kHz, min %d, max %d, relation %d\n",
894                 pol->cpu, targfreq, pol->min, pol->max, relation);
895
896         if (query_current_values_with_pending_wait(data)) {
897                 ret = -EIO;
898                 goto err_out;
899         }
900
901         dprintk("targ: curr fid 0x%x, vid 0x%x\n",
902                 data->currfid, data->currvid);
903
904         if ((checkvid != data->currvid) || (checkfid != data->currfid)) {
905                 printk(KERN_ERR PFX
906                        "error - out of sync, fid 0x%x 0x%x, vid 0x%x 0x%x\n",
907                        checkfid, data->currfid, checkvid, data->currvid);
908         }
909
910         if (cpufreq_frequency_table_target(pol, data->powernow_table, targfreq, relation, &newstate))
911                 goto err_out;
912
913         powernow_k8_acpi_pst_values(data, newstate);
914
915         if (transition_frequency(data, newstate)) {
916                 printk(KERN_ERR PFX "transition frequency failed\n");
917                 ret = 1;
918                 goto err_out;
919         }
920
921         pol->cur = find_khz_freq_from_fid(data->currfid);
922         ret = 0;
923
924 err_out:
925         set_cpus_allowed(current, oldmask);
926         schedule();
927
928         return ret;
929 }
930
931 /* Driver entry point to verify the policy and range of frequencies */
932 static int powernowk8_verify(struct cpufreq_policy *pol)
933 {
934         struct powernow_k8_data *data = powernow_data[pol->cpu];
935
936         return cpufreq_frequency_table_verify(pol, data->powernow_table);
937 }
938
939 /* per CPU init entry point to the driver */
940 static int __init powernowk8_cpu_init(struct cpufreq_policy *pol)
941 {
942         struct powernow_k8_data *data;
943         cpumask_t oldmask = CPU_MASK_ALL;
944         int rc;
945
946         if (!check_supported_cpu(pol->cpu))
947                 return -ENODEV;
948
949         data = kmalloc(sizeof(struct powernow_k8_data), GFP_KERNEL);
950         if (!data) {
951                 printk(KERN_ERR PFX "unable to alloc powernow_k8_data");
952                 return -ENOMEM;
953         }
954         memset(data,0,sizeof(struct powernow_k8_data));
955
956         data->cpu = pol->cpu;
957
958         if (powernow_k8_cpu_init_acpi(data)) {
959                 /*
960                  * Use the PSB BIOS structure. This is only availabe on
961                  * an UP version, and is deprecated by AMD.
962                  */
963
964                 if ((num_online_cpus() != 1) || (num_possible_cpus() != 1)) {
965                         printk(KERN_INFO PFX "MP systems not supported by PSB BIOS structure\n");
966                         kfree(data);
967                         return -ENODEV;
968                 }
969                 if (pol->cpu != 0) {
970                         printk(KERN_ERR PFX "init not cpu 0\n");
971                         kfree(data);
972                         return -ENODEV;
973                 }
974                 rc = find_psb_table(data);
975                 if (rc) {
976                         kfree(data);
977                         return -ENODEV;
978                 }
979         }
980
981         /* only run on specific CPU from here on */
982         oldmask = current->cpus_allowed;
983         set_cpus_allowed(current, cpumask_of_cpu(pol->cpu));
984         schedule();
985
986         if (smp_processor_id() != pol->cpu) {
987                 printk(KERN_ERR "limiting to cpu %u failed\n", pol->cpu);
988                 goto err_out;
989         }
990
991         if (pending_bit_stuck()) {
992                 printk(KERN_ERR PFX "failing init, change pending bit set\n");
993                 goto err_out;
994         }
995
996         if (query_current_values_with_pending_wait(data))
997                 goto err_out;
998
999         fidvid_msr_init();
1000
1001         /* run on any CPU again */
1002         set_cpus_allowed(current, oldmask);
1003         schedule();
1004
1005         pol->governor = CPUFREQ_DEFAULT_GOVERNOR;
1006
1007         /* Take a crude guess here. 
1008          * That guess was in microseconds, so multiply with 1000 */
1009         pol->cpuinfo.transition_latency = (((data->rvo + 8) * data->vstable * VST_UNITS_20US)
1010             + (3 * (1 << data->irt) * 10)) * 1000;
1011
1012         pol->cur = find_khz_freq_from_fid(data->currfid);
1013         dprintk("policy current frequency %d kHz\n", pol->cur);
1014
1015         /* min/max the cpu is capable of */
1016         if (cpufreq_frequency_table_cpuinfo(pol, data->powernow_table)) {
1017                 printk(KERN_ERR PFX "invalid powernow_table\n");
1018                 powernow_k8_cpu_exit_acpi(data);
1019                 kfree(data->powernow_table);
1020                 kfree(data);
1021                 return -EINVAL;
1022         }
1023
1024         cpufreq_frequency_table_get_attr(data->powernow_table, pol->cpu);
1025
1026         printk("cpu_init done, current fid 0x%x, vid 0x%x\n",
1027                data->currfid, data->currvid);
1028
1029         powernow_data[pol->cpu] = data;
1030
1031         return 0;
1032
1033 err_out:
1034         set_cpus_allowed(current, oldmask);
1035         schedule();
1036         powernow_k8_cpu_exit_acpi(data);
1037
1038         kfree(data);
1039         return -ENODEV;
1040 }
1041
1042 static int __devexit powernowk8_cpu_exit (struct cpufreq_policy *pol)
1043 {
1044         struct powernow_k8_data *data = powernow_data[pol->cpu];
1045
1046         if (!data)
1047                 return -EINVAL;
1048
1049         powernow_k8_cpu_exit_acpi(data);
1050
1051         cpufreq_frequency_table_put_attr(pol->cpu);
1052
1053         kfree(data->powernow_table);
1054         kfree(data);
1055
1056         return 0;
1057 }
1058
1059 static unsigned int powernowk8_get (unsigned int cpu)
1060 {
1061         struct powernow_k8_data *data = powernow_data[cpu];
1062         cpumask_t oldmask = current->cpus_allowed;
1063         unsigned int khz = 0;
1064
1065         set_cpus_allowed(current, cpumask_of_cpu(cpu));
1066         if (smp_processor_id() != cpu) {
1067                 printk(KERN_ERR PFX "limiting to CPU %d failed in powernowk8_get\n", cpu);
1068                 set_cpus_allowed(current, oldmask);
1069                 return 0;
1070         }
1071         preempt_disable();
1072
1073         if (query_current_values_with_pending_wait(data))
1074                 goto out;
1075
1076         khz = find_khz_freq_from_fid(data->currfid);
1077
1078  out:
1079         preempt_enable_no_resched();
1080         set_cpus_allowed(current, oldmask);
1081
1082         return khz;
1083 }
1084
1085 static struct freq_attr* powernow_k8_attr[] = {
1086         &cpufreq_freq_attr_scaling_available_freqs,
1087         NULL,
1088 };
1089
1090 static struct cpufreq_driver cpufreq_amd64_driver = {
1091         .verify = powernowk8_verify,
1092         .target = powernowk8_target,
1093         .init = powernowk8_cpu_init,
1094         .exit = __devexit_p(powernowk8_cpu_exit),
1095         .get = powernowk8_get,
1096         .name = "powernow-k8",
1097         .owner = THIS_MODULE,
1098         .attr = powernow_k8_attr,
1099 };
1100
1101 /* driver entry point for init */
1102 static int __init powernowk8_init(void)
1103 {
1104         unsigned int i, supported_cpus = 0;
1105
1106         for (i=0; i<NR_CPUS; i++) {
1107                 if (!cpu_online(i))
1108                         continue;
1109                 if (check_supported_cpu(i))
1110                         supported_cpus++;
1111         }
1112
1113         if (supported_cpus == num_online_cpus()) {
1114                 printk(KERN_INFO PFX "Found %d AMD Athlon 64 / Opteron processors (" VERSION ")\n",
1115                         supported_cpus);
1116                 return cpufreq_register_driver(&cpufreq_amd64_driver);
1117         }
1118
1119         return -ENODEV;
1120 }
1121
1122 /* driver entry point for term */
1123 static void __exit powernowk8_exit(void)
1124 {
1125         dprintk("exit\n");
1126
1127         cpufreq_unregister_driver(&cpufreq_amd64_driver);
1128 }
1129
1130 MODULE_AUTHOR("Paul Devriendt <paul.devriendt@amd.com>");
1131 MODULE_DESCRIPTION("AMD Athlon 64 and Opteron processor frequency driver.");
1132 MODULE_LICENSE("GPL");
1133
1134 late_initcall(powernowk8_init);
1135 module_exit(powernowk8_exit);