This source file includes following definitions.
- mca_patcher_base_patch_construct
- mca_patcher_base_patch_destruct
- addis
- ori
- oris
- mtspr
- bcctr
- rldicr
- PatchLoadImm
- flush_and_invalidate_cache
- ModifyMemoryProtection
- apply_patch
- mca_base_patcher_patch_unapply_binary
- mca_base_patcher_patch_apply_binary
- mca_patcher_base_patch_hook
1
2
3
4
5
6
7
8
9
10
11
12
13
14 #include "opal_config.h"
15
16 #include "opal/mca/patcher/patcher.h"
17 #include "opal/mca/patcher/base/base.h"
18 #include "opal/util/sys_limits.h"
19 #include "opal/prefetch.h"
20 #include <sys/mman.h>
21
22 static void mca_patcher_base_patch_construct (mca_patcher_base_patch_t *patch)
23 {
24 patch->patch_symbol = NULL;
25 patch->patch_data_size = 0;
26 }
27
28 static void mca_patcher_base_patch_destruct (mca_patcher_base_patch_t *patch)
29 {
30 free (patch->patch_symbol);
31 }
32
33 OBJ_CLASS_INSTANCE(mca_patcher_base_patch_t, opal_list_item_t,
34 mca_patcher_base_patch_construct,
35 mca_patcher_base_patch_destruct);
36
37 #if defined(__PPC__)
38
39
40
41 static unsigned int addis(unsigned int RT, unsigned int RS, unsigned int UI) {
42 return (15<<26) + (RT<<21) + (RS<<16) + (UI&0xffff);
43 }
44 static unsigned int ori(unsigned int RT, unsigned int RS, unsigned int UI) {
45 return (24<<26) + (RS<<21) + (RT<<16) + (UI&0xffff);
46 }
47 static unsigned int oris(unsigned int RT, unsigned int RS, unsigned int UI) {
48 return (25<<26) + (RS<<21) + (RT<<16) + (UI&0xffff);
49 }
50 static unsigned int mtspr(unsigned int SPR, unsigned int RS) {
51 return (31<<26) + (RS<<21) + ((SPR&0x1f)<<16) + ((SPR>>5)<<11) + (467<<1);
52 }
53 static unsigned int bcctr(unsigned int BO, unsigned int BI, unsigned int BH) {
54 return (19<<26) + (BO<<21) + (BI<<16) + (BH<<11) + (528<<1);
55 }
56 static unsigned int rldicr(unsigned int RT, unsigned int RS, unsigned int SH, unsigned int MB)
57 {
58 return (30<<26) + (RS<<21) + (RT<<16) + ((SH&0x1f)<<11) + ((SH>>5)<<1)
59 + ((MB&0x1f)<<6) + ((MB>>5)<<5) + (1<<2);
60 }
61
62 static int PatchLoadImm (uintptr_t addr, unsigned int reg, size_t value)
63 {
64 #if defined(__PPC64__)
65 *(unsigned int *) (addr + 0) = addis ( reg, 0, (value >> 48));
66 *(unsigned int *) (addr + 4) = ori ( reg, reg, (value >> 32));
67 *(unsigned int *) (addr + 8) = rldicr( reg, reg, 32, 31);
68 *(unsigned int *) (addr +12) = oris ( reg, reg, (value >> 16));
69 *(unsigned int *) (addr +16) = ori ( reg, reg, (value >> 0));
70 return 20;
71 #else
72 *(unsigned int *) (addr + 0) = addis ( reg, 0, (value >> 16));
73 *(unsigned int *) (addr + 4) = ori ( reg, reg, (value >> 0));
74 return 8;
75 #endif
76 }
77
78 #endif
79
80 #if !HAVE___CLEAR_CACHE
81 static void flush_and_invalidate_cache (unsigned long a)
82 {
83 #if OPAL_ASSEMBLY_ARCH == OPAL_IA32
84 static int have_clflush = -1;
85
86 if (OPAL_UNLIKELY(-1 == have_clflush)) {
87 int32_t cpuid1, cpuid2, tmp;
88 const int32_t level = 1;
89
90
91
92 __asm__ volatile ("xchgl %%ebx, %2\n"
93 "cpuid\n"
94 "xchgl %%ebx, %2\n":
95 "=a" (cpuid1), "=d" (cpuid2), "=r" (tmp) :
96 "a" (level) :
97 "ecx");
98
99 have_clflush = !!(cpuid2 & (1 << 19));
100 }
101
102 if (have_clflush) {
103
104 __asm__ volatile("mfence;clflush %0;mfence" : :"m" (*(char*)a));
105 }
106 #elif OPAL_ASSEMBLY_ARCH == OPAL_X86_64
107 __asm__ volatile("mfence;clflush %0;mfence" : :"m" (*(char*)a));
108 #elif OPAL_ASSEMBLY_ARCH == OPAL_IA64
109 __asm__ volatile ("fc %0;; sync.i;; srlz.i;;" : : "r"(a) : "memory");
110 #elif OPAL_ASSEMBLY_ARCH == OPAL_ARM64
111 __asm__ volatile ("dc cvau, %0\n\t"
112 "dsb ish\n\t"
113 "ic ivau, %0\n\t"
114 "dsb ish\n\t"
115 "isb":: "r" (a));
116 #endif
117 }
118 #endif
119
120
121 static void ModifyMemoryProtection (uintptr_t addr, size_t length, int prot)
122 {
123 long page_size = opal_getpagesize ();
124 uintptr_t base = (addr & ~(page_size-1));
125 uintptr_t bound = ((addr + length + page_size-1) & ~(page_size-1));
126
127 length = bound - base;
128
129 #if defined(__PPC__)
130
131 do {
132 if (mprotect((void *)base, page_size, prot))
133 perror("MemHook: mprotect failed");
134 base += page_size;
135 } while (base < bound);
136 #else
137 if (mprotect((void *) base, length, prot)) {
138 perror("MemHook: mprotect failed");
139 }
140 #endif
141 }
142
143 static inline void apply_patch (unsigned char *patch_data, uintptr_t address, size_t data_size)
144 {
145 ModifyMemoryProtection (address, data_size, PROT_EXEC|PROT_READ|PROT_WRITE);
146 memcpy ((void *) address, patch_data, data_size);
147 #if HAVE___CLEAR_CACHE
148
149 void __clear_cache(void* beg, void* end);
150
151 __clear_cache ((void *) address, (void *) (address + data_size));
152 #else
153 size_t offset_jump = 16;
154
155 #if OPAL_ASSEMBLY_ARCH == OPAL_ARM64
156 offset_jump = 32;
157 #endif
158
159
160 address &= ~(offset_jump - 1);
161
162 for (size_t i = 0 ; i < data_size ; i += offset_jump) {
163 flush_and_invalidate_cache (address + i);
164 }
165
166 #endif
167
168 ModifyMemoryProtection (address, data_size, PROT_EXEC|PROT_READ);
169 }
170
171 static void mca_base_patcher_patch_unapply_binary (mca_patcher_base_patch_t *patch)
172 {
173 apply_patch (patch->patch_orig_data, patch->patch_orig, patch->patch_data_size);
174 }
175
176 void mca_base_patcher_patch_apply_binary (mca_patcher_base_patch_t *patch)
177 {
178 memcpy (patch->patch_orig_data, (void *) patch->patch_orig, patch->patch_data_size);
179 apply_patch (patch->patch_data, patch->patch_orig, patch->patch_data_size);
180 patch->patch_restore = mca_base_patcher_patch_unapply_binary;
181 }
182
183
184 int mca_patcher_base_patch_hook (mca_patcher_base_module_t *module, uintptr_t hook_addr)
185 {
186 #if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64)
187 mca_patcher_base_patch_t *hook_patch;
188 const unsigned int nop = 0x60000000;
189
190 hook_patch = OBJ_NEW(mca_patcher_base_patch_t);
191 if (OPAL_UNLIKELY(NULL == hook_patch)) {
192 return OPAL_ERR_OUT_OF_RESOURCE;
193 }
194
195
196 for (unsigned int *nop_addr = (unsigned int *)hook_addr ; ; nop_addr++) {
197 if (nop_addr[0] == nop && nop_addr[1] == nop && nop_addr[2] == nop
198 && nop_addr[3] == nop && nop_addr[4] == nop) {
199 hook_patch->patch_orig = (uintptr_t) nop_addr;
200 break;
201 }
202 }
203
204
205 unsigned long toc;
206
207 asm volatile ("std 2, %0" : "=m" (toc));
208
209 hook_patch->patch_data_size = PatchLoadImm((uintptr_t)hook_patch->patch_data, 2, toc);
210
211
212 opal_list_append (&module->patch_list, &hook_patch->super);
213
214 mca_base_patcher_patch_apply_binary (hook_patch);
215 #endif
216
217 return OPAL_SUCCESS;
218 }