Merge branch 'x86-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-3.10.git] / arch / x86 / net / bpf_jit_comp.c
1 /* bpf_jit_comp.c : BPF JIT compiler
2  *
3  * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com)
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License
7  * as published by the Free Software Foundation; version 2
8  * of the License.
9  */
10 #include <linux/moduleloader.h>
11 #include <asm/cacheflush.h>
12 #include <linux/netdevice.h>
13 #include <linux/filter.h>
14 #include <linux/if_vlan.h>
15
16 /*
17  * Conventions :
18  *  EAX : BPF A accumulator
19  *  EBX : BPF X accumulator
20  *  RDI : pointer to skb   (first argument given to JIT function)
21  *  RBP : frame pointer (even if CONFIG_FRAME_POINTER=n)
22  *  ECX,EDX,ESI : scratch registers
23  *  r9d : skb->len - skb->data_len (headlen)
24  *  r8  : skb->data
25  * -8(RBP) : saved RBX value
26  * -16(RBP)..-80(RBP) : BPF_MEMWORDS values
27  */
28 int bpf_jit_enable __read_mostly;
29
30 /*
31  * assembly code in arch/x86/net/bpf_jit.S
32  */
33 extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[];
34 extern u8 sk_load_word_positive_offset[], sk_load_half_positive_offset[];
35 extern u8 sk_load_byte_positive_offset[], sk_load_byte_msh_positive_offset[];
36 extern u8 sk_load_word_negative_offset[], sk_load_half_negative_offset[];
37 extern u8 sk_load_byte_negative_offset[], sk_load_byte_msh_negative_offset[];
38
39 static inline u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)
40 {
41         if (len == 1)
42                 *ptr = bytes;
43         else if (len == 2)
44                 *(u16 *)ptr = bytes;
45         else {
46                 *(u32 *)ptr = bytes;
47                 barrier();
48         }
49         return ptr + len;
50 }
51
52 #define EMIT(bytes, len)        do { prog = emit_code(prog, bytes, len); } while (0)
53
54 #define EMIT1(b1)               EMIT(b1, 1)
55 #define EMIT2(b1, b2)           EMIT((b1) + ((b2) << 8), 2)
56 #define EMIT3(b1, b2, b3)       EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)
57 #define EMIT4(b1, b2, b3, b4)   EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4)
58 #define EMIT1_off32(b1, off)    do { EMIT1(b1); EMIT(off, 4);} while (0)
59
60 #define CLEAR_A() EMIT2(0x31, 0xc0) /* xor %eax,%eax */
61 #define CLEAR_X() EMIT2(0x31, 0xdb) /* xor %ebx,%ebx */
62
63 static inline bool is_imm8(int value)
64 {
65         return value <= 127 && value >= -128;
66 }
67
68 static inline bool is_near(int offset)
69 {
70         return offset <= 127 && offset >= -128;
71 }
72
73 #define EMIT_JMP(offset)                                                \
74 do {                                                                    \
75         if (offset) {                                                   \
76                 if (is_near(offset))                                    \
77                         EMIT2(0xeb, offset); /* jmp .+off8 */           \
78                 else                                                    \
79                         EMIT1_off32(0xe9, offset); /* jmp .+off32 */    \
80         }                                                               \
81 } while (0)
82
83 /* list of x86 cond jumps opcodes (. + s8)
84  * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
85  */
86 #define X86_JB  0x72
87 #define X86_JAE 0x73
88 #define X86_JE  0x74
89 #define X86_JNE 0x75
90 #define X86_JBE 0x76
91 #define X86_JA  0x77
92
93 #define EMIT_COND_JMP(op, offset)                               \
94 do {                                                            \
95         if (is_near(offset))                                    \
96                 EMIT2(op, offset); /* jxx .+off8 */             \
97         else {                                                  \
98                 EMIT2(0x0f, op + 0x10);                         \
99                 EMIT(offset, 4); /* jxx .+off32 */              \
100         }                                                       \
101 } while (0)
102
103 #define COND_SEL(CODE, TOP, FOP)        \
104         case CODE:                      \
105                 t_op = TOP;             \
106                 f_op = FOP;             \
107                 goto cond_branch
108
109
110 #define SEEN_DATAREF 1 /* might call external helpers */
111 #define SEEN_XREG    2 /* ebx is used */
112 #define SEEN_MEM     4 /* use mem[] for temporary storage */
113
114 static inline void bpf_flush_icache(void *start, void *end)
115 {
116         mm_segment_t old_fs = get_fs();
117
118         set_fs(KERNEL_DS);
119         smp_wmb();
120         flush_icache_range((unsigned long)start, (unsigned long)end);
121         set_fs(old_fs);
122 }
123
124 #define CHOOSE_LOAD_FUNC(K, func) \
125         ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
126
127 /* Helper to find the offset of pkt_type in sk_buff
128  * We want to make sure its still a 3bit field starting at a byte boundary.
129  */
130 #define PKT_TYPE_MAX 7
131 static int pkt_type_offset(void)
132 {
133         struct sk_buff skb_probe = {
134                 .pkt_type = ~0,
135         };
136         char *ct = (char *)&skb_probe;
137         unsigned int off;
138
139         for (off = 0; off < sizeof(struct sk_buff); off++) {
140                 if (ct[off] == PKT_TYPE_MAX)
141                         return off;
142         }
143         pr_err_once("Please fix pkt_type_offset(), as pkt_type couldn't be found\n");
144         return -1;
145 }
146
147 void bpf_jit_compile(struct sk_filter *fp)
148 {
149         u8 temp[64];
150         u8 *prog;
151         unsigned int proglen, oldproglen = 0;
152         int ilen, i;
153         int t_offset, f_offset;
154         u8 t_op, f_op, seen = 0, pass;
155         u8 *image = NULL;
156         u8 *func;
157         int pc_ret0 = -1; /* bpf index of first RET #0 instruction (if any) */
158         unsigned int cleanup_addr; /* epilogue code offset */
159         unsigned int *addrs;
160         const struct sock_filter *filter = fp->insns;
161         int flen = fp->len;
162
163         if (!bpf_jit_enable)
164                 return;
165
166         addrs = kmalloc(flen * sizeof(*addrs), GFP_KERNEL);
167         if (addrs == NULL)
168                 return;
169
170         /* Before first pass, make a rough estimation of addrs[]
171          * each bpf instruction is translated to less than 64 bytes
172          */
173         for (proglen = 0, i = 0; i < flen; i++) {
174                 proglen += 64;
175                 addrs[i] = proglen;
176         }
177         cleanup_addr = proglen; /* epilogue address */
178
179         for (pass = 0; pass < 10; pass++) {
180                 u8 seen_or_pass0 = (pass == 0) ? (SEEN_XREG | SEEN_DATAREF | SEEN_MEM) : seen;
181                 /* no prologue/epilogue for trivial filters (RET something) */
182                 proglen = 0;
183                 prog = temp;
184
185                 if (seen_or_pass0) {
186                         EMIT4(0x55, 0x48, 0x89, 0xe5); /* push %rbp; mov %rsp,%rbp */
187                         EMIT4(0x48, 0x83, 0xec, 96);    /* subq  $96,%rsp       */
188                         /* note : must save %rbx in case bpf_error is hit */
189                         if (seen_or_pass0 & (SEEN_XREG | SEEN_DATAREF))
190                                 EMIT4(0x48, 0x89, 0x5d, 0xf8); /* mov %rbx, -8(%rbp) */
191                         if (seen_or_pass0 & SEEN_XREG)
192                                 CLEAR_X(); /* make sure we dont leek kernel memory */
193
194                         /*
195                          * If this filter needs to access skb data,
196                          * loads r9 and r8 with :
197                          *  r9 = skb->len - skb->data_len
198                          *  r8 = skb->data
199                          */
200                         if (seen_or_pass0 & SEEN_DATAREF) {
201                                 if (offsetof(struct sk_buff, len) <= 127)
202                                         /* mov    off8(%rdi),%r9d */
203                                         EMIT4(0x44, 0x8b, 0x4f, offsetof(struct sk_buff, len));
204                                 else {
205                                         /* mov    off32(%rdi),%r9d */
206                                         EMIT3(0x44, 0x8b, 0x8f);
207                                         EMIT(offsetof(struct sk_buff, len), 4);
208                                 }
209                                 if (is_imm8(offsetof(struct sk_buff, data_len)))
210                                         /* sub    off8(%rdi),%r9d */
211                                         EMIT4(0x44, 0x2b, 0x4f, offsetof(struct sk_buff, data_len));
212                                 else {
213                                         EMIT3(0x44, 0x2b, 0x8f);
214                                         EMIT(offsetof(struct sk_buff, data_len), 4);
215                                 }
216
217                                 if (is_imm8(offsetof(struct sk_buff, data)))
218                                         /* mov off8(%rdi),%r8 */
219                                         EMIT4(0x4c, 0x8b, 0x47, offsetof(struct sk_buff, data));
220                                 else {
221                                         /* mov off32(%rdi),%r8 */
222                                         EMIT3(0x4c, 0x8b, 0x87);
223                                         EMIT(offsetof(struct sk_buff, data), 4);
224                                 }
225                         }
226                 }
227
228                 switch (filter[0].code) {
229                 case BPF_S_RET_K:
230                 case BPF_S_LD_W_LEN:
231                 case BPF_S_ANC_PROTOCOL:
232                 case BPF_S_ANC_IFINDEX:
233                 case BPF_S_ANC_MARK:
234                 case BPF_S_ANC_RXHASH:
235                 case BPF_S_ANC_CPU:
236                 case BPF_S_ANC_VLAN_TAG:
237                 case BPF_S_ANC_VLAN_TAG_PRESENT:
238                 case BPF_S_ANC_QUEUE:
239                 case BPF_S_ANC_PKTTYPE:
240                 case BPF_S_LD_W_ABS:
241                 case BPF_S_LD_H_ABS:
242                 case BPF_S_LD_B_ABS:
243                         /* first instruction sets A register (or is RET 'constant') */
244                         break;
245                 default:
246                         /* make sure we dont leak kernel information to user */
247                         CLEAR_A(); /* A = 0 */
248                 }
249
250                 for (i = 0; i < flen; i++) {
251                         unsigned int K = filter[i].k;
252
253                         switch (filter[i].code) {
254                         case BPF_S_ALU_ADD_X: /* A += X; */
255                                 seen |= SEEN_XREG;
256                                 EMIT2(0x01, 0xd8);              /* add %ebx,%eax */
257                                 break;
258                         case BPF_S_ALU_ADD_K: /* A += K; */
259                                 if (!K)
260                                         break;
261                                 if (is_imm8(K))
262                                         EMIT3(0x83, 0xc0, K);   /* add imm8,%eax */
263                                 else
264                                         EMIT1_off32(0x05, K);   /* add imm32,%eax */
265                                 break;
266                         case BPF_S_ALU_SUB_X: /* A -= X; */
267                                 seen |= SEEN_XREG;
268                                 EMIT2(0x29, 0xd8);              /* sub    %ebx,%eax */
269                                 break;
270                         case BPF_S_ALU_SUB_K: /* A -= K */
271                                 if (!K)
272                                         break;
273                                 if (is_imm8(K))
274                                         EMIT3(0x83, 0xe8, K); /* sub imm8,%eax */
275                                 else
276                                         EMIT1_off32(0x2d, K); /* sub imm32,%eax */
277                                 break;
278                         case BPF_S_ALU_MUL_X: /* A *= X; */
279                                 seen |= SEEN_XREG;
280                                 EMIT3(0x0f, 0xaf, 0xc3);        /* imul %ebx,%eax */
281                                 break;
282                         case BPF_S_ALU_MUL_K: /* A *= K */
283                                 if (is_imm8(K))
284                                         EMIT3(0x6b, 0xc0, K); /* imul imm8,%eax,%eax */
285                                 else {
286                                         EMIT2(0x69, 0xc0);              /* imul imm32,%eax */
287                                         EMIT(K, 4);
288                                 }
289                                 break;
290                         case BPF_S_ALU_DIV_X: /* A /= X; */
291                                 seen |= SEEN_XREG;
292                                 EMIT2(0x85, 0xdb);      /* test %ebx,%ebx */
293                                 if (pc_ret0 > 0) {
294                                         /* addrs[pc_ret0 - 1] is start address of target
295                                          * (addrs[i] - 4) is the address following this jmp
296                                          * ("xor %edx,%edx; div %ebx" being 4 bytes long)
297                                          */
298                                         EMIT_COND_JMP(X86_JE, addrs[pc_ret0 - 1] -
299                                                                 (addrs[i] - 4));
300                                 } else {
301                                         EMIT_COND_JMP(X86_JNE, 2 + 5);
302                                         CLEAR_A();
303                                         EMIT1_off32(0xe9, cleanup_addr - (addrs[i] - 4)); /* jmp .+off32 */
304                                 }
305                                 EMIT4(0x31, 0xd2, 0xf7, 0xf3); /* xor %edx,%edx; div %ebx */
306                                 break;
307                         case BPF_S_ALU_MOD_X: /* A %= X; */
308                                 seen |= SEEN_XREG;
309                                 EMIT2(0x85, 0xdb);      /* test %ebx,%ebx */
310                                 if (pc_ret0 > 0) {
311                                         /* addrs[pc_ret0 - 1] is start address of target
312                                          * (addrs[i] - 6) is the address following this jmp
313                                          * ("xor %edx,%edx; div %ebx;mov %edx,%eax" being 6 bytes long)
314                                          */
315                                         EMIT_COND_JMP(X86_JE, addrs[pc_ret0 - 1] -
316                                                                 (addrs[i] - 6));
317                                 } else {
318                                         EMIT_COND_JMP(X86_JNE, 2 + 5);
319                                         CLEAR_A();
320                                         EMIT1_off32(0xe9, cleanup_addr - (addrs[i] - 6)); /* jmp .+off32 */
321                                 }
322                                 EMIT2(0x31, 0xd2);      /* xor %edx,%edx */
323                                 EMIT2(0xf7, 0xf3);      /* div %ebx */
324                                 EMIT2(0x89, 0xd0);      /* mov %edx,%eax */
325                                 break;
326                         case BPF_S_ALU_MOD_K: /* A %= K; */
327                                 EMIT2(0x31, 0xd2);      /* xor %edx,%edx */
328                                 EMIT1(0xb9);EMIT(K, 4); /* mov imm32,%ecx */
329                                 EMIT2(0xf7, 0xf1);      /* div %ecx */
330                                 EMIT2(0x89, 0xd0);      /* mov %edx,%eax */
331                                 break;
332                         case BPF_S_ALU_DIV_K: /* A = reciprocal_divide(A, K); */
333                                 EMIT3(0x48, 0x69, 0xc0); /* imul imm32,%rax,%rax */
334                                 EMIT(K, 4);
335                                 EMIT4(0x48, 0xc1, 0xe8, 0x20); /* shr $0x20,%rax */
336                                 break;
337                         case BPF_S_ALU_AND_X:
338                                 seen |= SEEN_XREG;
339                                 EMIT2(0x21, 0xd8);              /* and %ebx,%eax */
340                                 break;
341                         case BPF_S_ALU_AND_K:
342                                 if (K >= 0xFFFFFF00) {
343                                         EMIT2(0x24, K & 0xFF); /* and imm8,%al */
344                                 } else if (K >= 0xFFFF0000) {
345                                         EMIT2(0x66, 0x25);      /* and imm16,%ax */
346                                         EMIT(K, 2);
347                                 } else {
348                                         EMIT1_off32(0x25, K);   /* and imm32,%eax */
349                                 }
350                                 break;
351                         case BPF_S_ALU_OR_X:
352                                 seen |= SEEN_XREG;
353                                 EMIT2(0x09, 0xd8);              /* or %ebx,%eax */
354                                 break;
355                         case BPF_S_ALU_OR_K:
356                                 if (is_imm8(K))
357                                         EMIT3(0x83, 0xc8, K); /* or imm8,%eax */
358                                 else
359                                         EMIT1_off32(0x0d, K);   /* or imm32,%eax */
360                                 break;
361                         case BPF_S_ANC_ALU_XOR_X: /* A ^= X; */
362                         case BPF_S_ALU_XOR_X:
363                                 seen |= SEEN_XREG;
364                                 EMIT2(0x31, 0xd8);              /* xor %ebx,%eax */
365                                 break;
366                         case BPF_S_ALU_XOR_K: /* A ^= K; */
367                                 if (K == 0)
368                                         break;
369                                 if (is_imm8(K))
370                                         EMIT3(0x83, 0xf0, K);   /* xor imm8,%eax */
371                                 else
372                                         EMIT1_off32(0x35, K);   /* xor imm32,%eax */
373                                 break;
374                         case BPF_S_ALU_LSH_X: /* A <<= X; */
375                                 seen |= SEEN_XREG;
376                                 EMIT4(0x89, 0xd9, 0xd3, 0xe0);  /* mov %ebx,%ecx; shl %cl,%eax */
377                                 break;
378                         case BPF_S_ALU_LSH_K:
379                                 if (K == 0)
380                                         break;
381                                 else if (K == 1)
382                                         EMIT2(0xd1, 0xe0); /* shl %eax */
383                                 else
384                                         EMIT3(0xc1, 0xe0, K);
385                                 break;
386                         case BPF_S_ALU_RSH_X: /* A >>= X; */
387                                 seen |= SEEN_XREG;
388                                 EMIT4(0x89, 0xd9, 0xd3, 0xe8);  /* mov %ebx,%ecx; shr %cl,%eax */
389                                 break;
390                         case BPF_S_ALU_RSH_K: /* A >>= K; */
391                                 if (K == 0)
392                                         break;
393                                 else if (K == 1)
394                                         EMIT2(0xd1, 0xe8); /* shr %eax */
395                                 else
396                                         EMIT3(0xc1, 0xe8, K);
397                                 break;
398                         case BPF_S_ALU_NEG:
399                                 EMIT2(0xf7, 0xd8);              /* neg %eax */
400                                 break;
401                         case BPF_S_RET_K:
402                                 if (!K) {
403                                         if (pc_ret0 == -1)
404                                                 pc_ret0 = i;
405                                         CLEAR_A();
406                                 } else {
407                                         EMIT1_off32(0xb8, K);   /* mov $imm32,%eax */
408                                 }
409                                 /* fallinto */
410                         case BPF_S_RET_A:
411                                 if (seen_or_pass0) {
412                                         if (i != flen - 1) {
413                                                 EMIT_JMP(cleanup_addr - addrs[i]);
414                                                 break;
415                                         }
416                                         if (seen_or_pass0 & SEEN_XREG)
417                                                 EMIT4(0x48, 0x8b, 0x5d, 0xf8);  /* mov  -8(%rbp),%rbx */
418                                         EMIT1(0xc9);            /* leaveq */
419                                 }
420                                 EMIT1(0xc3);            /* ret */
421                                 break;
422                         case BPF_S_MISC_TAX: /* X = A */
423                                 seen |= SEEN_XREG;
424                                 EMIT2(0x89, 0xc3);      /* mov    %eax,%ebx */
425                                 break;
426                         case BPF_S_MISC_TXA: /* A = X */
427                                 seen |= SEEN_XREG;
428                                 EMIT2(0x89, 0xd8);      /* mov    %ebx,%eax */
429                                 break;
430                         case BPF_S_LD_IMM: /* A = K */
431                                 if (!K)
432                                         CLEAR_A();
433                                 else
434                                         EMIT1_off32(0xb8, K); /* mov $imm32,%eax */
435                                 break;
436                         case BPF_S_LDX_IMM: /* X = K */
437                                 seen |= SEEN_XREG;
438                                 if (!K)
439                                         CLEAR_X();
440                                 else
441                                         EMIT1_off32(0xbb, K); /* mov $imm32,%ebx */
442                                 break;
443                         case BPF_S_LD_MEM: /* A = mem[K] : mov off8(%rbp),%eax */
444                                 seen |= SEEN_MEM;
445                                 EMIT3(0x8b, 0x45, 0xf0 - K*4);
446                                 break;
447                         case BPF_S_LDX_MEM: /* X = mem[K] : mov off8(%rbp),%ebx */
448                                 seen |= SEEN_XREG | SEEN_MEM;
449                                 EMIT3(0x8b, 0x5d, 0xf0 - K*4);
450                                 break;
451                         case BPF_S_ST: /* mem[K] = A : mov %eax,off8(%rbp) */
452                                 seen |= SEEN_MEM;
453                                 EMIT3(0x89, 0x45, 0xf0 - K*4);
454                                 break;
455                         case BPF_S_STX: /* mem[K] = X : mov %ebx,off8(%rbp) */
456                                 seen |= SEEN_XREG | SEEN_MEM;
457                                 EMIT3(0x89, 0x5d, 0xf0 - K*4);
458                                 break;
459                         case BPF_S_LD_W_LEN: /* A = skb->len; */
460                                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4);
461                                 if (is_imm8(offsetof(struct sk_buff, len)))
462                                         /* mov    off8(%rdi),%eax */
463                                         EMIT3(0x8b, 0x47, offsetof(struct sk_buff, len));
464                                 else {
465                                         EMIT2(0x8b, 0x87);
466                                         EMIT(offsetof(struct sk_buff, len), 4);
467                                 }
468                                 break;
469                         case BPF_S_LDX_W_LEN: /* X = skb->len; */
470                                 seen |= SEEN_XREG;
471                                 if (is_imm8(offsetof(struct sk_buff, len)))
472                                         /* mov off8(%rdi),%ebx */
473                                         EMIT3(0x8b, 0x5f, offsetof(struct sk_buff, len));
474                                 else {
475                                         EMIT2(0x8b, 0x9f);
476                                         EMIT(offsetof(struct sk_buff, len), 4);
477                                 }
478                                 break;
479                         case BPF_S_ANC_PROTOCOL: /* A = ntohs(skb->protocol); */
480                                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2);
481                                 if (is_imm8(offsetof(struct sk_buff, protocol))) {
482                                         /* movzwl off8(%rdi),%eax */
483                                         EMIT4(0x0f, 0xb7, 0x47, offsetof(struct sk_buff, protocol));
484                                 } else {
485                                         EMIT3(0x0f, 0xb7, 0x87); /* movzwl off32(%rdi),%eax */
486                                         EMIT(offsetof(struct sk_buff, protocol), 4);
487                                 }
488                                 EMIT2(0x86, 0xc4); /* ntohs() : xchg   %al,%ah */
489                                 break;
490                         case BPF_S_ANC_IFINDEX:
491                                 if (is_imm8(offsetof(struct sk_buff, dev))) {
492                                         /* movq off8(%rdi),%rax */
493                                         EMIT4(0x48, 0x8b, 0x47, offsetof(struct sk_buff, dev));
494                                 } else {
495                                         EMIT3(0x48, 0x8b, 0x87); /* movq off32(%rdi),%rax */
496                                         EMIT(offsetof(struct sk_buff, dev), 4);
497                                 }
498                                 EMIT3(0x48, 0x85, 0xc0);        /* test %rax,%rax */
499                                 EMIT_COND_JMP(X86_JE, cleanup_addr - (addrs[i] - 6));
500                                 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4);
501                                 EMIT2(0x8b, 0x80);      /* mov off32(%rax),%eax */
502                                 EMIT(offsetof(struct net_device, ifindex), 4);
503                                 break;
504                         case BPF_S_ANC_MARK:
505                                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
506                                 if (is_imm8(offsetof(struct sk_buff, mark))) {
507                                         /* mov off8(%rdi),%eax */
508                                         EMIT3(0x8b, 0x47, offsetof(struct sk_buff, mark));
509                                 } else {
510                                         EMIT2(0x8b, 0x87);
511                                         EMIT(offsetof(struct sk_buff, mark), 4);
512                                 }
513                                 break;
514                         case BPF_S_ANC_RXHASH:
515                                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, rxhash) != 4);
516                                 if (is_imm8(offsetof(struct sk_buff, rxhash))) {
517                                         /* mov off8(%rdi),%eax */
518                                         EMIT3(0x8b, 0x47, offsetof(struct sk_buff, rxhash));
519                                 } else {
520                                         EMIT2(0x8b, 0x87);
521                                         EMIT(offsetof(struct sk_buff, rxhash), 4);
522                                 }
523                                 break;
524                         case BPF_S_ANC_QUEUE:
525                                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2);
526                                 if (is_imm8(offsetof(struct sk_buff, queue_mapping))) {
527                                         /* movzwl off8(%rdi),%eax */
528                                         EMIT4(0x0f, 0xb7, 0x47, offsetof(struct sk_buff, queue_mapping));
529                                 } else {
530                                         EMIT3(0x0f, 0xb7, 0x87); /* movzwl off32(%rdi),%eax */
531                                         EMIT(offsetof(struct sk_buff, queue_mapping), 4);
532                                 }
533                                 break;
534                         case BPF_S_ANC_CPU:
535 #ifdef CONFIG_SMP
536                                 EMIT4(0x65, 0x8b, 0x04, 0x25); /* mov %gs:off32,%eax */
537                                 EMIT((u32)(unsigned long)&cpu_number, 4); /* A = smp_processor_id(); */
538 #else
539                                 CLEAR_A();
540 #endif
541                                 break;
542                         case BPF_S_ANC_VLAN_TAG:
543                         case BPF_S_ANC_VLAN_TAG_PRESENT:
544                                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2);
545                                 if (is_imm8(offsetof(struct sk_buff, vlan_tci))) {
546                                         /* movzwl off8(%rdi),%eax */
547                                         EMIT4(0x0f, 0xb7, 0x47, offsetof(struct sk_buff, vlan_tci));
548                                 } else {
549                                         EMIT3(0x0f, 0xb7, 0x87); /* movzwl off32(%rdi),%eax */
550                                         EMIT(offsetof(struct sk_buff, vlan_tci), 4);
551                                 }
552                                 BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000);
553                                 if (filter[i].code == BPF_S_ANC_VLAN_TAG) {
554                                         EMIT3(0x80, 0xe4, 0xef); /* and    $0xef,%ah */
555                                 } else {
556                                         EMIT3(0xc1, 0xe8, 0x0c); /* shr    $0xc,%eax */
557                                         EMIT3(0x83, 0xe0, 0x01); /* and    $0x1,%eax */
558                                 }
559                                 break;
560                         case BPF_S_ANC_PKTTYPE:
561                         {
562                                 int off = pkt_type_offset();
563
564                                 if (off < 0)
565                                         goto out;
566                                 if (is_imm8(off)) {
567                                         /* movzbl off8(%rdi),%eax */
568                                         EMIT4(0x0f, 0xb6, 0x47, off);
569                                 } else {
570                                         /* movbl off32(%rdi),%eax */
571                                         EMIT3(0x0f, 0xb6, 0x87);
572                                         EMIT(off, 4);
573                                 }
574                                 EMIT3(0x83, 0xe0, PKT_TYPE_MAX); /* and    $0x7,%eax */
575                                 break;
576                         }
577                         case BPF_S_LD_W_ABS:
578                                 func = CHOOSE_LOAD_FUNC(K, sk_load_word);
579 common_load:                    seen |= SEEN_DATAREF;
580                                 t_offset = func - (image + addrs[i]);
581                                 EMIT1_off32(0xbe, K); /* mov imm32,%esi */
582                                 EMIT1_off32(0xe8, t_offset); /* call */
583                                 break;
584                         case BPF_S_LD_H_ABS:
585                                 func = CHOOSE_LOAD_FUNC(K, sk_load_half);
586                                 goto common_load;
587                         case BPF_S_LD_B_ABS:
588                                 func = CHOOSE_LOAD_FUNC(K, sk_load_byte);
589                                 goto common_load;
590                         case BPF_S_LDX_B_MSH:
591                                 func = CHOOSE_LOAD_FUNC(K, sk_load_byte_msh);
592                                 seen |= SEEN_DATAREF | SEEN_XREG;
593                                 t_offset = func - (image + addrs[i]);
594                                 EMIT1_off32(0xbe, K);   /* mov imm32,%esi */
595                                 EMIT1_off32(0xe8, t_offset); /* call sk_load_byte_msh */
596                                 break;
597                         case BPF_S_LD_W_IND:
598                                 func = sk_load_word;
599 common_load_ind:                seen |= SEEN_DATAREF | SEEN_XREG;
600                                 t_offset = func - (image + addrs[i]);
601                                 if (K) {
602                                         if (is_imm8(K)) {
603                                                 EMIT3(0x8d, 0x73, K); /* lea imm8(%rbx), %esi */
604                                         } else {
605                                                 EMIT2(0x8d, 0xb3); /* lea imm32(%rbx),%esi */
606                                                 EMIT(K, 4);
607                                         }
608                                 } else {
609                                         EMIT2(0x89,0xde); /* mov %ebx,%esi */
610                                 }
611                                 EMIT1_off32(0xe8, t_offset);    /* call sk_load_xxx_ind */
612                                 break;
613                         case BPF_S_LD_H_IND:
614                                 func = sk_load_half;
615                                 goto common_load_ind;
616                         case BPF_S_LD_B_IND:
617                                 func = sk_load_byte;
618                                 goto common_load_ind;
619                         case BPF_S_JMP_JA:
620                                 t_offset = addrs[i + K] - addrs[i];
621                                 EMIT_JMP(t_offset);
622                                 break;
623                         COND_SEL(BPF_S_JMP_JGT_K, X86_JA, X86_JBE);
624                         COND_SEL(BPF_S_JMP_JGE_K, X86_JAE, X86_JB);
625                         COND_SEL(BPF_S_JMP_JEQ_K, X86_JE, X86_JNE);
626                         COND_SEL(BPF_S_JMP_JSET_K,X86_JNE, X86_JE);
627                         COND_SEL(BPF_S_JMP_JGT_X, X86_JA, X86_JBE);
628                         COND_SEL(BPF_S_JMP_JGE_X, X86_JAE, X86_JB);
629                         COND_SEL(BPF_S_JMP_JEQ_X, X86_JE, X86_JNE);
630                         COND_SEL(BPF_S_JMP_JSET_X,X86_JNE, X86_JE);
631
632 cond_branch:                    f_offset = addrs[i + filter[i].jf] - addrs[i];
633                                 t_offset = addrs[i + filter[i].jt] - addrs[i];
634
635                                 /* same targets, can avoid doing the test :) */
636                                 if (filter[i].jt == filter[i].jf) {
637                                         EMIT_JMP(t_offset);
638                                         break;
639                                 }
640
641                                 switch (filter[i].code) {
642                                 case BPF_S_JMP_JGT_X:
643                                 case BPF_S_JMP_JGE_X:
644                                 case BPF_S_JMP_JEQ_X:
645                                         seen |= SEEN_XREG;
646                                         EMIT2(0x39, 0xd8); /* cmp %ebx,%eax */
647                                         break;
648                                 case BPF_S_JMP_JSET_X:
649                                         seen |= SEEN_XREG;
650                                         EMIT2(0x85, 0xd8); /* test %ebx,%eax */
651                                         break;
652                                 case BPF_S_JMP_JEQ_K:
653                                         if (K == 0) {
654                                                 EMIT2(0x85, 0xc0); /* test   %eax,%eax */
655                                                 break;
656                                         }
657                                 case BPF_S_JMP_JGT_K:
658                                 case BPF_S_JMP_JGE_K:
659                                         if (K <= 127)
660                                                 EMIT3(0x83, 0xf8, K); /* cmp imm8,%eax */
661                                         else
662                                                 EMIT1_off32(0x3d, K); /* cmp imm32,%eax */
663                                         break;
664                                 case BPF_S_JMP_JSET_K:
665                                         if (K <= 0xFF)
666                                                 EMIT2(0xa8, K); /* test imm8,%al */
667                                         else if (!(K & 0xFFFF00FF))
668                                                 EMIT3(0xf6, 0xc4, K >> 8); /* test imm8,%ah */
669                                         else if (K <= 0xFFFF) {
670                                                 EMIT2(0x66, 0xa9); /* test imm16,%ax */
671                                                 EMIT(K, 2);
672                                         } else {
673                                                 EMIT1_off32(0xa9, K); /* test imm32,%eax */
674                                         }
675                                         break;
676                                 }
677                                 if (filter[i].jt != 0) {
678                                         if (filter[i].jf && f_offset)
679                                                 t_offset += is_near(f_offset) ? 2 : 5;
680                                         EMIT_COND_JMP(t_op, t_offset);
681                                         if (filter[i].jf)
682                                                 EMIT_JMP(f_offset);
683                                         break;
684                                 }
685                                 EMIT_COND_JMP(f_op, f_offset);
686                                 break;
687                         default:
688                                 /* hmm, too complex filter, give up with jit compiler */
689                                 goto out;
690                         }
691                         ilen = prog - temp;
692                         if (image) {
693                                 if (unlikely(proglen + ilen > oldproglen)) {
694                                         pr_err("bpb_jit_compile fatal error\n");
695                                         kfree(addrs);
696                                         module_free(NULL, image);
697                                         return;
698                                 }
699                                 memcpy(image + proglen, temp, ilen);
700                         }
701                         proglen += ilen;
702                         addrs[i] = proglen;
703                         prog = temp;
704                 }
705                 /* last bpf instruction is always a RET :
706                  * use it to give the cleanup instruction(s) addr
707                  */
708                 cleanup_addr = proglen - 1; /* ret */
709                 if (seen_or_pass0)
710                         cleanup_addr -= 1; /* leaveq */
711                 if (seen_or_pass0 & SEEN_XREG)
712                         cleanup_addr -= 4; /* mov  -8(%rbp),%rbx */
713
714                 if (image) {
715                         if (proglen != oldproglen)
716                                 pr_err("bpb_jit_compile proglen=%u != oldproglen=%u\n", proglen, oldproglen);
717                         break;
718                 }
719                 if (proglen == oldproglen) {
720                         image = module_alloc(max_t(unsigned int,
721                                                    proglen,
722                                                    sizeof(struct work_struct)));
723                         if (!image)
724                                 goto out;
725                 }
726                 oldproglen = proglen;
727         }
728         if (bpf_jit_enable > 1)
729                 pr_err("flen=%d proglen=%u pass=%d image=%p\n",
730                        flen, proglen, pass, image);
731
732         if (image) {
733                 if (bpf_jit_enable > 1)
734                         print_hex_dump(KERN_ERR, "JIT code: ", DUMP_PREFIX_ADDRESS,
735                                        16, 1, image, proglen, false);
736
737                 bpf_flush_icache(image, image + proglen);
738
739                 fp->bpf_func = (void *)image;
740         }
741 out:
742         kfree(addrs);
743         return;
744 }
745
746 static void jit_free_defer(struct work_struct *arg)
747 {
748         module_free(NULL, arg);
749 }
750
751 /* run from softirq, we must use a work_struct to call
752  * module_free() from process context
753  */
754 void bpf_jit_free(struct sk_filter *fp)
755 {
756         if (fp->bpf_func != sk_run_filter) {
757                 struct work_struct *work = (struct work_struct *)fp->bpf_func;
758
759                 INIT_WORK(work, jit_free_defer);
760                 schedule_work(work);
761         }
762 }