blob: 8c46f555d82a0d94b0d76577f1ea08d55ba670d4 [file] [log] [blame]
Stefano Stabellini4ccefbe2015-11-05 15:15:07 +00001/*
2 * Xen stolen ticks accounting.
3 */
4#include <linux/kernel.h>
5#include <linux/kernel_stat.h>
6#include <linux/math64.h>
7#include <linux/gfp.h>
Dongli Zhang5e25f5d2017-11-01 09:46:33 +08008#include <linux/slab.h>
Stefano Stabellini4ccefbe2015-11-05 15:15:07 +00009
Juergen Grossecb23dc2016-05-20 09:26:48 +020010#include <asm/paravirt.h>
Stefano Stabellini4ccefbe2015-11-05 15:15:07 +000011#include <asm/xen/hypervisor.h>
12#include <asm/xen/hypercall.h>
13
14#include <xen/events.h>
15#include <xen/features.h>
16#include <xen/interface/xen.h>
17#include <xen/interface/vcpu.h>
18#include <xen/xen-ops.h>
19
20/* runstate info updated by Xen */
21static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate);
22
Dongli Zhang5e25f5d2017-11-01 09:46:33 +080023static DEFINE_PER_CPU(u64[4], old_runstate_time);
24
Stefano Stabellini4ccefbe2015-11-05 15:15:07 +000025/* return an consistent snapshot of 64-bit time/counter value */
26static u64 get64(const u64 *p)
27{
28 u64 ret;
29
30 if (BITS_PER_LONG < 64) {
31 u32 *p32 = (u32 *)p;
Stefano Stabellini2dd887e2015-11-20 15:02:44 +000032 u32 h, l, h2;
Stefano Stabellini4ccefbe2015-11-05 15:15:07 +000033
34 /*
35 * Read high then low, and then make sure high is
36 * still the same; this will only loop if low wraps
37 * and carries into high.
38 * XXX some clean way to make this endian-proof?
39 */
40 do {
Stefano Stabellini2dd887e2015-11-20 15:02:44 +000041 h = READ_ONCE(p32[1]);
42 l = READ_ONCE(p32[0]);
43 h2 = READ_ONCE(p32[1]);
44 } while(h2 != h);
Stefano Stabellini4ccefbe2015-11-05 15:15:07 +000045
46 ret = (((u64)h) << 32) | l;
47 } else
Stefano Stabellini2dd887e2015-11-20 15:02:44 +000048 ret = READ_ONCE(*p);
Stefano Stabellini4ccefbe2015-11-05 15:15:07 +000049
50 return ret;
51}
52
Dongli Zhang5e25f5d2017-11-01 09:46:33 +080053static void xen_get_runstate_snapshot_cpu_delta(
54 struct vcpu_runstate_info *res, unsigned int cpu)
Stefano Stabellini4ccefbe2015-11-05 15:15:07 +000055{
56 u64 state_time;
57 struct vcpu_runstate_info *state;
58
59 BUG_ON(preemptible());
60
Juergen Gross6ba286a2016-07-06 07:00:30 +020061 state = per_cpu_ptr(&xen_runstate, cpu);
Stefano Stabellini4ccefbe2015-11-05 15:15:07 +000062
Stefano Stabellini4ccefbe2015-11-05 15:15:07 +000063 do {
64 state_time = get64(&state->state_entry_time);
Juergen Gross6ba286a2016-07-06 07:00:30 +020065 rmb(); /* Hypervisor might update data. */
Stefano Stabellini2dd887e2015-11-20 15:02:44 +000066 *res = READ_ONCE(*state);
Juergen Gross6ba286a2016-07-06 07:00:30 +020067 rmb(); /* Hypervisor might update data. */
68 } while (get64(&state->state_entry_time) != state_time ||
69 (state_time & XEN_RUNSTATE_UPDATE));
70}
71
Dongli Zhang5e25f5d2017-11-01 09:46:33 +080072static void xen_get_runstate_snapshot_cpu(struct vcpu_runstate_info *res,
73 unsigned int cpu)
74{
75 int i;
76
77 xen_get_runstate_snapshot_cpu_delta(res, cpu);
78
79 for (i = 0; i < 4; i++)
80 res->time[i] += per_cpu(old_runstate_time, cpu)[i];
81}
82
83void xen_manage_runstate_time(int action)
84{
85 static struct vcpu_runstate_info *runstate_delta;
86 struct vcpu_runstate_info state;
87 int cpu, i;
88
89 switch (action) {
90 case -1: /* backup runstate time before suspend */
91 if (unlikely(runstate_delta))
92 pr_warn_once("%s: memory leak as runstate_delta is not NULL\n",
93 __func__);
94
95 runstate_delta = kmalloc_array(num_possible_cpus(),
96 sizeof(*runstate_delta),
97 GFP_ATOMIC);
98 if (unlikely(!runstate_delta)) {
99 pr_warn("%s: failed to allocate runstate_delta\n",
100 __func__);
101 return;
102 }
103
104 for_each_possible_cpu(cpu) {
105 xen_get_runstate_snapshot_cpu_delta(&state, cpu);
106 memcpy(runstate_delta[cpu].time, state.time,
107 sizeof(runstate_delta[cpu].time));
108 }
109
110 break;
111
112 case 0: /* backup runstate time after resume */
113 if (unlikely(!runstate_delta)) {
114 pr_warn("%s: cannot accumulate runstate time as runstate_delta is NULL\n",
115 __func__);
116 return;
117 }
118
119 for_each_possible_cpu(cpu) {
120 for (i = 0; i < 4; i++)
121 per_cpu(old_runstate_time, cpu)[i] +=
122 runstate_delta[cpu].time[i];
123 }
124
125 break;
126
127 default: /* do not accumulate runstate time for checkpointing */
128 break;
129 }
130
131 if (action != -1 && runstate_delta) {
132 kfree(runstate_delta);
133 runstate_delta = NULL;
134 }
135}
136
Juergen Gross6ba286a2016-07-06 07:00:30 +0200137/*
138 * Runstate accounting
139 */
140void xen_get_runstate_snapshot(struct vcpu_runstate_info *res)
141{
142 xen_get_runstate_snapshot_cpu(res, smp_processor_id());
Stefano Stabellini4ccefbe2015-11-05 15:15:07 +0000143}
144
145/* return true when a vcpu could run but has no real cpu to run on */
146bool xen_vcpu_stolen(int vcpu)
147{
148 return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable;
149}
150
Juergen Grossd34c30c2016-07-26 14:15:11 +0200151u64 xen_steal_clock(int cpu)
Juergen Grossecb23dc2016-05-20 09:26:48 +0200152{
153 struct vcpu_runstate_info state;
154
Juergen Gross6ba286a2016-07-06 07:00:30 +0200155 xen_get_runstate_snapshot_cpu(&state, cpu);
Juergen Grossecb23dc2016-05-20 09:26:48 +0200156 return state.time[RUNSTATE_runnable] + state.time[RUNSTATE_offline];
157}
158
Stefano Stabellini4ccefbe2015-11-05 15:15:07 +0000159void xen_setup_runstate_info(int cpu)
160{
161 struct vcpu_register_runstate_memory_area area;
162
163 area.addr.v = &per_cpu(xen_runstate, cpu);
164
165 if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area,
Vitaly Kuznetsovad5475f2016-06-30 17:56:38 +0200166 xen_vcpu_nr(cpu), &area))
Stefano Stabellini4ccefbe2015-11-05 15:15:07 +0000167 BUG();
168}
169
Juergen Grossecb23dc2016-05-20 09:26:48 +0200170void __init xen_time_setup_guest(void)
171{
Juergen Gross6ba286a2016-07-06 07:00:30 +0200172 bool xen_runstate_remote;
173
174 xen_runstate_remote = !HYPERVISOR_vm_assist(VMASST_CMD_enable,
175 VMASST_TYPE_runstate_update_flag);
176
Juergen Grossecb23dc2016-05-20 09:26:48 +0200177 pv_time_ops.steal_clock = xen_steal_clock;
178
179 static_key_slow_inc(&paravirt_steal_enabled);
Juergen Gross6ba286a2016-07-06 07:00:30 +0200180 if (xen_runstate_remote)
181 static_key_slow_inc(&paravirt_steal_rq_enabled);
Juergen Grossecb23dc2016-05-20 09:26:48 +0200182}