/home/mdboom/Work/builds/cpython/Python/specialize.c
Line | Count | Source (jump to first uncovered line) |
1 | #include "Python.h" |
2 | #include "pycore_code.h" |
3 | #include "pycore_dict.h" |
4 | #include "pycore_function.h" // _PyFunction_GetVersionForCurrentState() |
5 | #include "pycore_global_strings.h" // _Py_ID() |
6 | #include "pycore_long.h" |
7 | #include "pycore_moduleobject.h" |
8 | #include "pycore_object.h" |
9 | #include "pycore_opcode.h" // _PyOpcode_Caches |
10 | #include "structmember.h" // struct PyMemberDef, T_OFFSET_EX |
11 | #include "pycore_descrobject.h" |
12 | |
13 | #include <stdlib.h> // rand() |
14 | |
15 | /* For guidance on adding or extending families of instructions see |
16 | * ./adaptive.md |
17 | */ |
18 | |
19 | /* Map from opcode to adaptive opcode. |
20 | Values of zero are ignored. */ |
21 | uint8_t _PyOpcode_Adaptive[256] = { |
22 | [LOAD_ATTR] = LOAD_ATTR_ADAPTIVE, |
23 | [LOAD_GLOBAL] = LOAD_GLOBAL_ADAPTIVE, |
24 | [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE, |
25 | [STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE, |
26 | [CALL] = CALL_ADAPTIVE, |
27 | [STORE_ATTR] = STORE_ATTR_ADAPTIVE, |
28 | [BINARY_OP] = BINARY_OP_ADAPTIVE, |
29 | [COMPARE_OP] = COMPARE_OP_ADAPTIVE, |
30 | [UNPACK_SEQUENCE] = UNPACK_SEQUENCE_ADAPTIVE, |
31 | [FOR_ITER] = FOR_ITER_ADAPTIVE, |
32 | }; |
33 | |
34 | Py_ssize_t _Py_QuickenedCount = 0; |
35 | #ifdef Py_STATS |
36 | PyStats _py_stats_struct = { 0 }; |
37 | PyStats *_py_stats = &_py_stats_struct; |
38 | |
39 | #define ADD_STAT_TO_DICT(res, field) \ |
40 | do { \ |
41 | PyObject *val = PyLong_FromUnsignedLongLong(stats->field); \ |
42 | if (val == NULL) { \ |
43 | Py_DECREF(res); \ |
44 | return NULL; \ |
45 | } \ |
46 | if (PyDict_SetItemString(res, #field, val) == -1) { \ |
47 | Py_DECREF(res); \ |
48 | Py_DECREF(val); \ |
49 | return NULL; \ |
50 | } \ |
51 | Py_DECREF(val); \ |
52 | } while(0); |
53 | |
54 | static PyObject* |
55 | stats_to_dict(SpecializationStats *stats) |
56 | { |
57 | PyObject *res = PyDict_New(); |
58 | if (res == NULL) { |
59 | return NULL; |
60 | } |
61 | ADD_STAT_TO_DICT(res, success); |
62 | ADD_STAT_TO_DICT(res, failure); |
63 | ADD_STAT_TO_DICT(res, hit); |
64 | ADD_STAT_TO_DICT(res, deferred); |
65 | ADD_STAT_TO_DICT(res, miss); |
66 | ADD_STAT_TO_DICT(res, deopt); |
67 | PyObject *failure_kinds = PyTuple_New(SPECIALIZATION_FAILURE_KINDS); |
68 | if (failure_kinds == NULL) { |
69 | Py_DECREF(res); |
70 | return NULL; |
71 | } |
72 | for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) { |
73 | PyObject *stat = PyLong_FromUnsignedLongLong(stats->failure_kinds[i]); |
74 | if (stat == NULL) { |
75 | Py_DECREF(res); |
76 | Py_DECREF(failure_kinds); |
77 | return NULL; |
78 | } |
79 | PyTuple_SET_ITEM(failure_kinds, i, stat); |
80 | } |
81 | if (PyDict_SetItemString(res, "failure_kinds", failure_kinds)) { |
82 | Py_DECREF(res); |
83 | Py_DECREF(failure_kinds); |
84 | return NULL; |
85 | } |
86 | Py_DECREF(failure_kinds); |
87 | return res; |
88 | } |
89 | #undef ADD_STAT_TO_DICT |
90 | |
91 | static int |
92 | add_stat_dict( |
93 | PyObject *res, |
94 | int opcode, |
95 | const char *name) { |
96 | |
97 | SpecializationStats *stats = &_py_stats_struct.opcode_stats[opcode].specialization; |
98 | PyObject *d = stats_to_dict(stats); |
99 | if (d == NULL) { |
100 | return -1; |
101 | } |
102 | int err = PyDict_SetItemString(res, name, d); |
103 | Py_DECREF(d); |
104 | return err; |
105 | } |
106 | |
107 | #ifdef Py_STATS |
108 | PyObject* |
109 | _Py_GetSpecializationStats(void) { |
110 | PyObject *stats = PyDict_New(); |
111 | if (stats == NULL) { |
112 | return NULL; |
113 | } |
114 | int err = 0; |
115 | err += add_stat_dict(stats, LOAD_ATTR, "load_attr"); |
116 | err += add_stat_dict(stats, LOAD_GLOBAL, "load_global"); |
117 | err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr"); |
118 | err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr"); |
119 | err += add_stat_dict(stats, STORE_ATTR, "store_attr"); |
120 | err += add_stat_dict(stats, CALL, "call"); |
121 | err += add_stat_dict(stats, BINARY_OP, "binary_op"); |
122 | err += add_stat_dict(stats, COMPARE_OP, "compare_op"); |
123 | err += add_stat_dict(stats, UNPACK_SEQUENCE, "unpack_sequence"); |
124 | if (err < 0) { |
125 | Py_DECREF(stats); |
126 | return NULL; |
127 | } |
128 | return stats; |
129 | } |
130 | #endif |
131 | |
132 | |
133 | #define PRINT_STAT(i, field) \ |
134 | if (stats[i].field) { \ |
135 | fprintf(out, " opcode[%d]." #field " : %" PRIu64 "\n", i, stats[i].field); \ |
136 | } |
137 | |
138 | static void |
139 | print_spec_stats(FILE *out, OpcodeStats *stats) |
140 | { |
141 | /* Mark some opcodes as specializable for stats, |
142 | * even though we don't specialize them yet. */ |
143 | fprintf(out, "opcode[%d].specializable : 1\n", BINARY_SLICE); |
144 | fprintf(out, "opcode[%d].specializable : 1\n", STORE_SLICE); |
145 | for (int i = 0; i < 256; i++) { |
146 | if (_PyOpcode_Adaptive[i]) { |
147 | fprintf(out, "opcode[%d].specializable : 1\n", i); |
148 | } |
149 | PRINT_STAT(i, specialization.success); |
150 | PRINT_STAT(i, specialization.failure); |
151 | PRINT_STAT(i, specialization.hit); |
152 | PRINT_STAT(i, specialization.deferred); |
153 | PRINT_STAT(i, specialization.miss); |
154 | PRINT_STAT(i, specialization.deopt); |
155 | PRINT_STAT(i, execution_count); |
156 | for (int j = 0; j < SPECIALIZATION_FAILURE_KINDS; j++) { |
157 | uint64_t val = stats[i].specialization.failure_kinds[j]; |
158 | if (val) { |
159 | fprintf(out, " opcode[%d].specialization.failure_kinds[%d] : %" |
160 | PRIu64 "\n", i, j, val); |
161 | } |
162 | } |
163 | for(int j = 0; j < 256; j++) { |
164 | if (stats[i].pair_count[j]) { |
165 | fprintf(out, "opcode[%d].pair_count[%d] : %" PRIu64 "\n", |
166 | i, j, stats[i].pair_count[j]); |
167 | } |
168 | } |
169 | } |
170 | } |
171 | #undef PRINT_STAT |
172 | |
173 | |
174 | static void |
175 | print_call_stats(FILE *out, CallStats *stats) |
176 | { |
177 | fprintf(out, "Calls to PyEval_EvalDefault: %" PRIu64 "\n", stats->pyeval_calls); |
178 | fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls); |
179 | fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed); |
180 | fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created); |
181 | for (int i = 0; i < EVAL_CALL_KINDS; i++) { |
182 | fprintf(out, "Calls via PyEval_EvalFrame[%d] : %" PRIu64 "\n", i, stats->eval_calls[i]); |
183 | } |
184 | } |
185 | |
186 | static void |
187 | print_object_stats(FILE *out, ObjectStats *stats) |
188 | { |
189 | fprintf(out, "Object allocations from freelist: %" PRIu64 "\n", stats->from_freelist); |
190 | fprintf(out, "Object frees to freelist: %" PRIu64 "\n", stats->to_freelist); |
191 | fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations); |
192 | fprintf(out, "Object allocations to 512 bytes: %" PRIu64 "\n", stats->allocations512); |
193 | fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k); |
194 | fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big); |
195 | fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees); |
196 | fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values); |
197 | fprintf(out, "Object interpreter increfs: %" PRIu64 "\n", stats->interpreter_increfs); |
198 | fprintf(out, "Object interpreter decrefs: %" PRIu64 "\n", stats->interpreter_decrefs); |
199 | fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs); |
200 | fprintf(out, "Object decrefs: %" PRIu64 "\n", stats->decrefs); |
201 | fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request); |
202 | fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); |
203 | fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); |
204 | fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); |
205 | } |
206 | |
207 | static void |
208 | print_stats(FILE *out, PyStats *stats) { |
209 | print_spec_stats(out, stats->opcode_stats); |
210 | print_call_stats(out, &stats->call_stats); |
211 | print_object_stats(out, &stats->object_stats); |
212 | } |
213 | |
214 | void |
215 | _Py_StatsClear(void) |
216 | { |
217 | _py_stats_struct = (PyStats) { 0 }; |
218 | } |
219 | |
220 | void |
221 | _Py_PrintSpecializationStats(int to_file) |
222 | { |
223 | if (_py_stats == NULL) { |
224 | return; |
225 | } |
226 | FILE *out = stderr; |
227 | if (to_file) { |
228 | /* Write to a file instead of stderr. */ |
229 | # ifdef MS_WINDOWS |
230 | const char *dirname = "c:\\temp\\py_stats\\"; |
231 | # else |
232 | const char *dirname = "/tmp/py_stats/"; |
233 | # endif |
234 | /* Use random 160 bit number as file name, |
235 | * to avoid both accidental collisions and |
236 | * symlink attacks. */ |
237 | unsigned char rand[20]; |
238 | char hex_name[41]; |
239 | _PyOS_URandomNonblock(rand, 20); |
240 | for (int i = 0; i < 20; i++) { |
241 | hex_name[2*i] = "0123456789abcdef"[rand[i]&15]; |
242 | hex_name[2*i+1] = "0123456789abcdef"[(rand[i]>>4)&15]; |
243 | } |
244 | hex_name[40] = '\0'; |
245 | char buf[64]; |
246 | assert(strlen(dirname) + 40 + strlen(".txt") < 64); |
247 | sprintf(buf, "%s%s.txt", dirname, hex_name); |
248 | FILE *fout = fopen(buf, "w"); |
249 | if (fout) { |
250 | out = fout; |
251 | } |
252 | } |
253 | else { |
254 | fprintf(out, "Specialization stats:\n"); |
255 | } |
256 | print_stats(out, _py_stats); |
257 | if (out != stderr) { |
258 | fclose(out); |
259 | } |
260 | } |
261 | |
262 | #ifdef Py_STATS |
263 | |
264 | #define SPECIALIZATION_FAIL(opcode, kind) \ |
265 | do { \ |
266 | if (_py_stats) { \ |
267 | _py_stats->opcode_stats[opcode].specialization.failure_kinds[kind]++; \ |
268 | } \ |
269 | } while (0) |
270 | |
271 | #endif |
272 | #endif |
273 | |
274 | #ifndef SPECIALIZATION_FAIL |
275 | #define SPECIALIZATION_FAIL(opcode, kind) ((void)0) |
276 | #endif |
277 | |
278 | // Insert adaptive instructions and superinstructions. This cannot fail. |
279 | void |
280 | _PyCode_Quicken(PyCodeObject *code) |
281 | { |
282 | _Py_QuickenedCount++; |
283 | int previous_opcode = -1; |
284 | _Py_CODEUNIT *instructions = _PyCode_CODE(code); |
285 | for (int i = 0; i < Py_SIZE(code); i++2.03M ) { Branch (285:21): [True: 2.03M, False: 33.7k]
|
286 | int opcode = _Py_OPCODE(instructions[i]); |
287 | uint8_t adaptive_opcode = _PyOpcode_Adaptive[opcode]; |
288 | if (adaptive_opcode) { Branch (288:13): [True: 622k, False: 1.41M]
|
289 | _Py_SET_OPCODE(instructions[i], adaptive_opcode); |
290 | // Make sure the adaptive counter is zero: |
291 | assert(instructions[i + 1] == 0); |
292 | previous_opcode = -1; |
293 | i += _PyOpcode_Caches[opcode]; |
294 | } |
295 | else { |
296 | assert(!_PyOpcode_Caches[opcode]); |
297 | switch (opcode) { Branch (297:21): [True: 613k, False: 798k]
|
298 | case JUMP_BACKWARD: Branch (298:17): [True: 28.1k, False: 1.38M]
|
299 | _Py_SET_OPCODE(instructions[i], JUMP_BACKWARD_QUICK); |
300 | break; |
301 | case RESUME: Branch (301:17): [True: 36.4k, False: 1.37M]
|
302 | _Py_SET_OPCODE(instructions[i], RESUME_QUICK); |
303 | break; |
304 | case LOAD_FAST: Branch (304:17): [True: 397k, False: 1.01M]
|
305 | switch(previous_opcode) { Branch (305:28): [True: 260k, False: 137k]
|
306 | case LOAD_FAST: Branch (306:25): [True: 64.7k, False: 333k]
|
307 | _Py_SET_OPCODE(instructions[i - 1], |
308 | LOAD_FAST__LOAD_FAST); |
309 | break; |
310 | case STORE_FAST: Branch (310:25): [True: 48.5k, False: 349k]
|
311 | _Py_SET_OPCODE(instructions[i - 1], |
312 | STORE_FAST__LOAD_FAST); |
313 | break; |
314 | case LOAD_CONST: Branch (314:25): [True: 24.2k, False: 373k]
|
315 | _Py_SET_OPCODE(instructions[i - 1], |
316 | LOAD_CONST__LOAD_FAST); |
317 | break; |
318 | } |
319 | break; |
320 | case 107k STORE_FAST107k : Branch (320:17): [True: 107k, False: 1.30M]
|
321 | if (previous_opcode == STORE_FAST) { Branch (321:25): [True: 10.3k, False: 97.0k]
|
322 | _Py_SET_OPCODE(instructions[i - 1], |
323 | STORE_FAST__STORE_FAST); |
324 | } |
325 | break; |
326 | case LOAD_CONST: Branch (326:17): [True: 228k, False: 1.18M]
|
327 | if (previous_opcode == LOAD_FAST) { Branch (327:25): [True: 37.5k, False: 190k]
|
328 | _Py_SET_OPCODE(instructions[i - 1], |
329 | LOAD_FAST__LOAD_CONST); |
330 | } |
331 | break; |
332 | } |
333 | previous_opcode = opcode; |
334 | } |
335 | } |
336 | } |
337 | |
338 | static inline int |
339 | miss_counter_start(void) { |
340 | /* Starting value for the counter. |
341 | * This value needs to be not too low, otherwise |
342 | * it would cause excessive de-optimization. |
343 | * Neither should it be too high, or that would delay |
344 | * de-optimization excessively when it is needed. |
345 | * A value around 50 seems to work, and we choose a |
346 | * prime number to avoid artifacts. |
347 | */ |
348 | return 53; |
349 | } |
350 | |
351 | #define SIMPLE_FUNCTION 0 |
352 | |
353 | /* Common */ |
354 | |
355 | #define SPEC_FAIL_OTHER 0 |
356 | #define SPEC_FAIL_NO_DICT 1 |
357 | #define SPEC_FAIL_OVERRIDDEN 2 |
358 | #define SPEC_FAIL_OUT_OF_VERSIONS 3 |
359 | #define SPEC_FAIL_OUT_OF_RANGE 4 |
360 | #define SPEC_FAIL_EXPECTED_ERROR 5 |
361 | #define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS 6 |
362 | |
363 | #define SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT 18 |
364 | |
365 | /* Attributes */ |
366 | |
367 | #define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR 8 |
368 | #define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR 9 |
369 | #define SPEC_FAIL_ATTR_NOT_DESCRIPTOR 10 |
370 | #define SPEC_FAIL_ATTR_METHOD 11 |
371 | #define SPEC_FAIL_ATTR_MUTABLE_CLASS 12 |
372 | #define SPEC_FAIL_ATTR_PROPERTY 13 |
373 | #define SPEC_FAIL_ATTR_NON_OBJECT_SLOT 14 |
374 | #define SPEC_FAIL_ATTR_READ_ONLY 15 |
375 | #define SPEC_FAIL_ATTR_AUDITED_SLOT 16 |
376 | #define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 17 |
377 | #define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 18 |
378 | #define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 19 |
379 | |
380 | #define SPEC_FAIL_ATTR_SHADOWED 21 |
381 | #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22 |
382 | #define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23 |
383 | #define SPEC_FAIL_ATTR_OBJECT_SLOT 24 |
384 | #define SPEC_FAIL_ATTR_HAS_MANAGED_DICT 25 |
385 | #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26 |
386 | #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27 |
387 | |
388 | /* Binary subscr and store subscr */ |
389 | |
390 | #define SPEC_FAIL_SUBSCR_ARRAY_INT 8 |
391 | #define SPEC_FAIL_SUBSCR_ARRAY_SLICE 9 |
392 | #define SPEC_FAIL_SUBSCR_LIST_SLICE 10 |
393 | #define SPEC_FAIL_SUBSCR_TUPLE_SLICE 11 |
394 | #define SPEC_FAIL_SUBSCR_STRING_INT 12 |
395 | #define SPEC_FAIL_SUBSCR_STRING_SLICE 13 |
396 | #define SPEC_FAIL_SUBSCR_BUFFER_INT 15 |
397 | #define SPEC_FAIL_SUBSCR_BUFFER_SLICE 16 |
398 | #define SPEC_FAIL_SUBSCR_SEQUENCE_INT 17 |
399 | |
400 | /* Store subscr */ |
401 | #define SPEC_FAIL_SUBSCR_BYTEARRAY_INT 18 |
402 | #define SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE 19 |
403 | #define SPEC_FAIL_SUBSCR_PY_SIMPLE 20 |
404 | #define SPEC_FAIL_SUBSCR_PY_OTHER 21 |
405 | #define SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE 22 |
406 | #define SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE 23 |
407 | |
408 | /* Binary op */ |
409 | |
410 | #define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES 8 |
411 | #define SPEC_FAIL_BINARY_OP_ADD_OTHER 9 |
412 | #define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES 10 |
413 | #define SPEC_FAIL_BINARY_OP_AND_INT 11 |
414 | #define SPEC_FAIL_BINARY_OP_AND_OTHER 12 |
415 | #define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE 13 |
416 | #define SPEC_FAIL_BINARY_OP_LSHIFT 14 |
417 | #define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY 15 |
418 | #define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES 16 |
419 | #define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER 17 |
420 | #define SPEC_FAIL_BINARY_OP_OR 18 |
421 | #define SPEC_FAIL_BINARY_OP_POWER 19 |
422 | #define SPEC_FAIL_BINARY_OP_REMAINDER 20 |
423 | #define SPEC_FAIL_BINARY_OP_RSHIFT 21 |
424 | #define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES 22 |
425 | #define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER 23 |
426 | #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 24 |
427 | #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT 25 |
428 | #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER 26 |
429 | #define SPEC_FAIL_BINARY_OP_XOR 27 |
430 | |
431 | /* Calls */ |
432 | #define SPEC_FAIL_CALL_COMPLEX_PARAMETERS 9 |
433 | #define SPEC_FAIL_CALL_CO_NOT_OPTIMIZED 10 |
434 | /* SPEC_FAIL_METHOD defined as 11 above */ |
435 | |
436 | #define SPEC_FAIL_CALL_INSTANCE_METHOD 11 |
437 | #define SPEC_FAIL_CALL_CMETHOD 12 |
438 | #define SPEC_FAIL_CALL_PYCFUNCTION 13 |
439 | #define SPEC_FAIL_CALL_PYCFUNCTION_WITH_KEYWORDS 14 |
440 | #define SPEC_FAIL_CALL_PYCFUNCTION_FAST_WITH_KEYWORDS 15 |
441 | #define SPEC_FAIL_CALL_PYCFUNCTION_NOARGS 16 |
442 | #define SPEC_FAIL_CALL_BAD_CALL_FLAGS 17 |
443 | #define SPEC_FAIL_CALL_CLASS 18 |
444 | #define SPEC_FAIL_CALL_PYTHON_CLASS 19 |
445 | #define SPEC_FAIL_CALL_METHOD_DESCRIPTOR 20 |
446 | #define SPEC_FAIL_CALL_BOUND_METHOD 21 |
447 | #define SPEC_FAIL_CALL_STR 22 |
448 | #define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 23 |
449 | #define SPEC_FAIL_CALL_CLASS_MUTABLE 24 |
450 | #define SPEC_FAIL_CALL_KWNAMES 25 |
451 | #define SPEC_FAIL_CALL_METHOD_WRAPPER 26 |
452 | #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 27 |
453 | #define SPEC_FAIL_CALL_PYFUNCTION 28 |
454 | #define SPEC_FAIL_CALL_PEP_523 29 |
455 | |
456 | /* COMPARE_OP */ |
457 | #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 |
458 | #define SPEC_FAIL_COMPARE_OP_STRING 13 |
459 | #define SPEC_FAIL_COMPARE_OP_NOT_FOLLOWED_BY_COND_JUMP 14 |
460 | #define SPEC_FAIL_COMPARE_OP_BIG_INT 15 |
461 | #define SPEC_FAIL_COMPARE_OP_BYTES 16 |
462 | #define SPEC_FAIL_COMPARE_OP_TUPLE 17 |
463 | #define SPEC_FAIL_COMPARE_OP_LIST 18 |
464 | #define SPEC_FAIL_COMPARE_OP_SET 19 |
465 | #define SPEC_FAIL_COMPARE_OP_BOOL 20 |
466 | #define SPEC_FAIL_COMPARE_OP_BASEOBJECT 21 |
467 | #define SPEC_FAIL_COMPARE_OP_FLOAT_LONG 22 |
468 | #define SPEC_FAIL_COMPARE_OP_LONG_FLOAT 23 |
469 | #define SPEC_FAIL_COMPARE_OP_EXTENDED_ARG 24 |
470 | |
471 | /* FOR_ITER */ |
472 | #define SPEC_FAIL_FOR_ITER_GENERATOR 10 |
473 | #define SPEC_FAIL_FOR_ITER_COROUTINE 11 |
474 | #define SPEC_FAIL_FOR_ITER_ASYNC_GENERATOR 12 |
475 | #define SPEC_FAIL_FOR_ITER_LIST 13 |
476 | #define SPEC_FAIL_FOR_ITER_TUPLE 14 |
477 | #define SPEC_FAIL_FOR_ITER_SET 15 |
478 | #define SPEC_FAIL_FOR_ITER_STRING 16 |
479 | #define SPEC_FAIL_FOR_ITER_BYTES 17 |
480 | #define SPEC_FAIL_FOR_ITER_RANGE 18 |
481 | #define SPEC_FAIL_FOR_ITER_ITERTOOLS 19 |
482 | #define SPEC_FAIL_FOR_ITER_DICT_KEYS 20 |
483 | #define SPEC_FAIL_FOR_ITER_DICT_ITEMS 21 |
484 | #define SPEC_FAIL_FOR_ITER_DICT_VALUES 22 |
485 | #define SPEC_FAIL_FOR_ITER_ENUMERATE 23 |
486 | #define SPEC_FAIL_FOR_ITER_MAP 24 |
487 | #define SPEC_FAIL_FOR_ITER_ZIP 25 |
488 | #define SPEC_FAIL_FOR_ITER_SEQ_ITER 26 |
489 | #define SPEC_FAIL_FOR_ITER_REVERSED_LIST 27 |
490 | #define SPEC_FAIL_FOR_ITER_CALLABLE 28 |
491 | #define SPEC_FAIL_FOR_ITER_ASCII_STRING 29 |
492 | |
493 | // UNPACK_SEQUENCE |
494 | |
495 | #define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 8 |
496 | #define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 9 |
497 | |
498 | |
499 | static int |
500 | specialize_module_load_attr(PyObject *owner, _Py_CODEUNIT *instr, |
501 | PyObject *name, int opcode, int opcode_module) |
502 | { |
503 | _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); |
504 | PyModuleObject *m = (PyModuleObject *)owner; |
505 | PyObject *value = NULL; |
506 | assert((owner->ob_type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); |
507 | PyDictObject *dict = (PyDictObject *)m->md_dict; |
508 | if (dict == NULL) { Branch (508:9): [True: 0, False: 31.7k]
|
509 | SPECIALIZATION_FAIL(opcode, SPEC_FAIL_NO_DICT); |
510 | return -1; |
511 | } |
512 | if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { Branch (512:9): [True: 0, False: 31.7k]
|
513 | SPECIALIZATION_FAIL(opcode, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT); |
514 | return -1; |
515 | } |
516 | Py_ssize_t index = _PyDict_GetItemHint(dict, &_Py_ID(__getattr__), -1, |
517 | &value); |
518 | assert(index != DKIX_ERROR); |
519 | if (index != DKIX_EMPTY) { Branch (519:9): [True: 1.03k, False: 30.6k]
|
520 | SPECIALIZATION_FAIL(opcode, SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND); |
521 | return -1; |
522 | } |
523 | index = _PyDict_GetItemHint(dict, name, -1, &value); |
524 | assert (index != DKIX_ERROR); |
525 | if (index != (uint16_t)index) { Branch (525:9): [True: 1.57k, False: 29.1k]
|
526 | SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_RANGE); |
527 | return -1; |
528 | } |
529 | uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(dict->ma_keys); |
530 | if (keys_version == 0) { Branch (530:9): [True: 0, False: 29.1k]
|
531 | SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS); |
532 | return -1; |
533 | } |
534 | write_u32(cache->version, keys_version); |
535 | cache->index = (uint16_t)index; |
536 | _Py_SET_OPCODE(*instr, opcode_module); |
537 | return 0; |
538 | } |
539 | |
540 | |
541 | |
542 | /* Attribute specialization */ |
543 | |
544 | typedef enum { |
545 | OVERRIDING, /* Is an overriding descriptor, and will remain so. */ |
546 | METHOD, /* Attribute has Py_TPFLAGS_METHOD_DESCRIPTOR set */ |
547 | PROPERTY, /* Is a property */ |
548 | OBJECT_SLOT, /* Is an object slot descriptor */ |
549 | OTHER_SLOT, /* Is a slot descriptor of another type */ |
550 | NON_OVERRIDING, /* Is another non-overriding descriptor, and is an instance of an immutable class*/ |
551 | BUILTIN_CLASSMETHOD, /* Builtin methods with METH_CLASS */ |
552 | PYTHON_CLASSMETHOD, /* Python classmethod(func) object */ |
553 | NON_DESCRIPTOR, /* Is not a descriptor, and is an instance of an immutable class */ |
554 | MUTABLE, /* Instance of a mutable class; might, or might not, be a descriptor */ |
555 | ABSENT, /* Attribute is not present on the class */ |
556 | DUNDER_CLASS, /* __class__ attribute */ |
557 | GETSET_OVERRIDDEN /* __getattribute__ or __setattr__ has been overridden */ |
558 | } DescriptorClassification; |
559 | |
560 | |
561 | static DescriptorClassification |
562 | analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store) |
563 | { |
564 | if (store) { Branch (564:9): [True: 50.0k, False: 457k]
|
565 | if (type->tp_setattro != PyObject_GenericSetAttr) { Branch (565:13): [True: 3.17k, False: 46.8k]
|
566 | *descr = NULL; |
567 | return GETSET_OVERRIDDEN; |
568 | } |
569 | } |
570 | else { |
571 | if (type->tp_getattro != PyObject_GenericGetAttr) { Branch (571:13): [True: 19.2k, False: 438k]
|
572 | *descr = NULL; |
573 | return GETSET_OVERRIDDEN; |
574 | } |
575 | } |
576 | PyObject *descriptor = _PyType_Lookup(type, name); |
577 | *descr = descriptor; |
578 | if (descriptor == NULL) { Branch (578:9): [True: 214k, False: 270k]
|
579 | return ABSENT; |
580 | } |
581 | PyTypeObject *desc_cls = Py_TYPE(descriptor); |
582 | if (!(desc_cls->tp_flags & Py_TPFLAGS_IMMUTABLETYPE)) { Branch (582:9): [True: 4.19k, False: 266k]
|
583 | return MUTABLE; |
584 | } |
585 | if (desc_cls->tp_descr_set) { Branch (585:9): [True: 60.0k, False: 206k]
|
586 | if (desc_cls == &PyMemberDescr_Type) { Branch (586:13): [True: 14.4k, False: 45.6k]
|
587 | PyMemberDescrObject *member = (PyMemberDescrObject *)descriptor; |
588 | struct PyMemberDef *dmem = member->d_member; |
589 | if (dmem->type == T_OBJECT_EX) { Branch (589:17): [True: 7.01k, False: 7.46k]
|
590 | return OBJECT_SLOT; |
591 | } |
592 | return OTHER_SLOT; |
593 | } |
594 | if (desc_cls == &PyProperty_Type) { Branch (594:13): [True: 5.27k, False: 40.3k]
|
595 | return PROPERTY; |
596 | } |
597 | if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) { Branch (597:13): [True: 27.5k, False: 12.7k]
|
598 | if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) { Branch (598:17): [True: 27.5k, False: 0]
|
599 | return DUNDER_CLASS; |
600 | } |
601 | } |
602 | if (store) { Branch (602:13): [True: 1.11k, False: 11.6k]
|
603 | return OVERRIDING; |
604 | } |
605 | } |
606 | if (desc_cls->tp_descr_get) { Branch (606:9): [True: 168k, False: 48.9k]
|
607 | if (desc_cls->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR) { Branch (607:13): [True: 149k, False: 19.2k]
|
608 | return METHOD; |
609 | } |
610 | if (Py_IS_TYPE(descriptor, &PyClassMethodDescr_Type)) { |
611 | return BUILTIN_CLASSMETHOD; |
612 | } |
613 | if (Py_IS_TYPE(descriptor, &PyClassMethod_Type)) { |
614 | return PYTHON_CLASSMETHOD; |
615 | } |
616 | return NON_OVERRIDING; |
617 | } |
618 | return NON_DESCRIPTOR; |
619 | } |
620 | |
621 | static int |
622 | specialize_dict_access( |
623 | PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, |
624 | DescriptorClassification kind, PyObject *name, |
625 | int base_op, int values_op, int hint_op) |
626 | { |
627 | assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT || |
628 | kind == BUILTIN_CLASSMETHOD || kind == PYTHON_CLASSMETHOD); |
629 | // No descriptor, or non overriding. |
630 | if ((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { Branch (630:9): [True: 25.5k, False: 237k]
|
631 | SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); |
632 | return 0; |
633 | } |
634 | _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); |
635 | PyObject **dictptr = _PyObject_ManagedDictPointer(owner); |
636 | PyDictObject *dict = (PyDictObject *)*dictptr; |
637 | if (dict == NULL) { Branch (637:9): [True: 119k, False: 117k]
|
638 | // Virtual dictionary |
639 | PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; |
640 | assert(PyUnicode_CheckExact(name)); |
641 | Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); |
642 | assert (index != DKIX_ERROR); |
643 | if (index != (uint16_t)index) { Branch (643:13): [True: 15.7k, False: 104k]
|
644 | SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); |
645 | return 0; |
646 | } |
647 | write_u32(cache->version, type->tp_version_tag); |
648 | cache->index = (uint16_t)index; |
649 | _Py_SET_OPCODE(*instr, values_op); |
650 | } |
651 | else { |
652 | if (!PyDict_CheckExact(dict)) { Branch (652:13): [True: 0, False: 117k]
|
653 | SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); |
654 | return 0; |
655 | } |
656 | // We found an instance with a __dict__. |
657 | PyObject *value = NULL; |
658 | Py_ssize_t hint = |
659 | _PyDict_GetItemHint(dict, name, -1, &value); |
660 | if (hint != (uint16_t)hint) { Branch (660:13): [True: 4.57k, False: 113k]
|
661 | SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); |
662 | return 0; |
663 | } |
664 | cache->index = (uint16_t)hint; |
665 | write_u32(cache->version, type->tp_version_tag); |
666 | _Py_SET_OPCODE(*instr, hint_op); |
667 | } |
668 | return 1; |
669 | } |
670 | |
671 | static int specialize_attr_loadmethod(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, |
672 | PyObject* descr, DescriptorClassification kind); |
673 | static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name); |
674 | static int function_kind(PyCodeObject *code); |
675 | |
676 | int |
677 | _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) |
678 | { |
679 | assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR); |
680 | _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); |
681 | if (PyModule_CheckExact(owner)) { |
682 | int err = specialize_module_load_attr(owner, instr, name, LOAD_ATTR, |
683 | LOAD_ATTR_MODULE); |
684 | if (err) { Branch (684:13): [True: 2.60k, False: 29.1k]
|
685 | goto fail; |
686 | } |
687 | goto success; |
688 | } |
689 | if (PyType_Check(owner)) { |
690 | int err = specialize_class_load_attr(owner, instr, name); |
691 | if (err) { Branch (691:13): [True: 16.3k, False: 8.39k]
|
692 | goto fail; |
693 | } |
694 | goto success; |
695 | } |
696 | PyTypeObject *type = Py_TYPE(owner); |
697 | if (type->tp_dict == NULL) { Branch (697:9): [True: 0, False: 432k]
|
698 | if (PyType_Ready(type) < 0) { Branch (698:13): [True: 0, False: 0]
|
699 | return -1; |
700 | } |
701 | } |
702 | PyObject *descr = NULL; |
703 | DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0); |
704 | assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); |
705 | switch(kind) { Branch (705:12): [True: 0, False: 432k]
|
706 | case OVERRIDING: Branch (706:9): [True: 0, False: 432k]
|
707 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); |
708 | goto fail; |
709 | case METHOD: Branch (709:9): [True: 147k, False: 285k]
|
710 | { |
711 | int oparg = _Py_OPARG(*instr); |
712 | if (oparg & 1) { Branch (712:17): [True: 132k, False: 14.2k]
|
713 | if (specialize_attr_loadmethod(owner, instr, name, descr, kind)) { Branch (713:21): [True: 108k, False: 23.7k]
|
714 | goto success; |
715 | } |
716 | } |
717 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); |
718 | goto fail; |
719 | } |
720 | case PROPERTY: Branch (720:9): [True: 4.39k, False: 428k]
|
721 | { |
722 | _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1); |
723 | assert(Py_TYPE(descr) == &PyProperty_Type); |
724 | PyObject *fget = ((_PyPropertyObject *)descr)->prop_get; |
725 | if (fget == NULL) { Branch (725:17): [True: 0, False: 4.39k]
|
726 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); |
727 | goto fail; |
728 | } |
729 | if (Py_TYPE(fget) != &PyFunction_Type) { Branch (729:17): [True: 32, False: 4.36k]
|
730 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY); |
731 | goto fail; |
732 | } |
733 | PyFunctionObject *func = (PyFunctionObject *)fget; |
734 | PyCodeObject *fcode = (PyCodeObject *)func->func_code; |
735 | int kind = function_kind(fcode); |
736 | if (kind != SIMPLE_FUNCTION) { Branch (736:17): [True: 0, False: 4.36k]
|
737 | SPECIALIZATION_FAIL(LOAD_ATTR, kind); |
738 | goto fail; |
739 | } |
740 | if (fcode->co_argcount != 1) { Branch (740:17): [True: 0, False: 4.36k]
|
741 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); |
742 | goto fail; |
743 | } |
744 | int version = _PyFunction_GetVersionForCurrentState(func); |
745 | if (version == 0) { Branch (745:17): [True: 0, False: 4.36k]
|
746 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); |
747 | goto fail; |
748 | } |
749 | write_u32(lm_cache->keys_version, version); |
750 | assert(type->tp_version_tag != 0); |
751 | write_u32(lm_cache->type_version, type->tp_version_tag); |
752 | /* borrowed */ |
753 | write_obj(lm_cache->descr, fget); |
754 | _Py_SET_OPCODE(*instr, LOAD_ATTR_PROPERTY); |
755 | goto success; |
756 | } |
757 | case OBJECT_SLOT: Branch (757:9): [True: 4.64k, False: 428k]
|
758 | { |
759 | PyMemberDescrObject *member = (PyMemberDescrObject *)descr; |
760 | struct PyMemberDef *dmem = member->d_member; |
761 | Py_ssize_t offset = dmem->offset; |
762 | if (dmem->flags & PY_AUDIT_READ) { Branch (762:17): [True: 0, False: 4.64k]
|
763 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_AUDITED_SLOT); |
764 | goto fail; |
765 | } |
766 | if (offset != (uint16_t)offset) { Branch (766:17): [True: 0, False: 4.64k]
|
767 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); |
768 | goto fail; |
769 | } |
770 | assert(dmem->type == T_OBJECT_EX); |
771 | assert(offset > 0); |
772 | cache->index = (uint16_t)offset; |
773 | write_u32(cache->version, type->tp_version_tag); |
774 | _Py_SET_OPCODE(*instr, LOAD_ATTR_SLOT); |
775 | goto success; |
776 | } |
777 | case DUNDER_CLASS: Branch (777:9): [True: 27.5k, False: 405k]
|
778 | { |
779 | Py_ssize_t offset = offsetof(PyObject, ob_type); |
780 | assert(offset == (uint16_t)offset); |
781 | cache->index = (uint16_t)offset; |
782 | write_u32(cache->version, type->tp_version_tag); |
783 | _Py_SET_OPCODE(*instr, LOAD_ATTR_SLOT); |
784 | goto success; |
785 | } |
786 | case OTHER_SLOT: Branch (786:9): [True: 7.09k, False: 425k]
|
787 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT); |
788 | goto fail; |
789 | case MUTABLE: Branch (789:9): [True: 2.08k, False: 430k]
|
790 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS); |
791 | goto fail; |
792 | case GETSET_OVERRIDDEN: Branch (792:9): [True: 18.0k, False: 414k]
|
793 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OVERRIDDEN); |
794 | goto fail; |
795 | case BUILTIN_CLASSMETHOD: Branch (795:9): [True: 0, False: 432k]
|
796 | case PYTHON_CLASSMETHOD: Branch (796:9): [True: 475, False: 432k]
|
797 | case NON_OVERRIDING: Branch (797:9): [True: 10.9k, False: 421k]
|
798 | case NON_DESCRIPTOR: Branch (798:9): [True: 38.2k, False: 394k]
|
799 | case ABSENT: Branch (799:9): [True: 172k, False: 260k]
|
800 | break; |
801 | } |
802 | int err = specialize_dict_access( |
803 | owner, instr, type, kind, name, |
804 | LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT |
805 | ); |
806 | if (err < 0) { Branch (806:9): [True: 0, False: 222k]
|
807 | return -1; |
808 | } |
809 | if (err) { Branch (809:9): [True: 184k, False: 37.2k]
|
810 | goto success; |
811 | } |
812 | fail: |
813 | STAT_INC(LOAD_ATTR, failure); |
814 | assert(!PyErr_Occurred()); |
815 | cache->counter = adaptive_counter_backoff(cache->counter); |
816 | return 0; |
817 | success: |
818 | STAT_INC(LOAD_ATTR, success); |
819 | assert(!PyErr_Occurred()); |
820 | cache->counter = miss_counter_start(); |
821 | return 0; |
822 | } |
823 | |
824 | int |
825 | _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) |
826 | { |
827 | assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR); |
828 | _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); |
829 | PyTypeObject *type = Py_TYPE(owner); |
830 | if (PyModule_CheckExact(owner)) { |
831 | SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN); |
832 | goto fail; |
833 | } |
834 | PyObject *descr; |
835 | DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1); |
836 | switch(kind) { Branch (836:12): [True: 0, False: 50.0k]
|
837 | case OVERRIDING: Branch (837:9): [True: 1.11k, False: 48.9k]
|
838 | SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); |
839 | goto fail; |
840 | case METHOD: Branch (840:9): [True: 715, False: 49.3k]
|
841 | SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_METHOD); |
842 | goto fail; |
843 | case PROPERTY: Branch (843:9): [True: 878, False: 49.1k]
|
844 | SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_PROPERTY); |
845 | goto fail; |
846 | case OBJECT_SLOT: Branch (846:9): [True: 2.37k, False: 47.6k]
|
847 | { |
848 | PyMemberDescrObject *member = (PyMemberDescrObject *)descr; |
849 | struct PyMemberDef *dmem = member->d_member; |
850 | Py_ssize_t offset = dmem->offset; |
851 | if (dmem->flags & READONLY) { Branch (851:17): [True: 12, False: 2.35k]
|
852 | SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_READ_ONLY); |
853 | goto fail; |
854 | } |
855 | if (offset != (uint16_t)offset) { Branch (855:17): [True: 0, False: 2.35k]
|
856 | SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OUT_OF_RANGE); |
857 | goto fail; |
858 | } |
859 | assert(dmem->type == T_OBJECT_EX); |
860 | assert(offset > 0); |
861 | cache->index = (uint16_t)offset; |
862 | write_u32(cache->version, type->tp_version_tag); |
863 | _Py_SET_OPCODE(*instr, STORE_ATTR_SLOT); |
864 | goto success; |
865 | } |
866 | case DUNDER_CLASS: Branch (866:9): [True: 34, False: 49.9k]
|
867 | case OTHER_SLOT: Branch (867:9): [True: 350, False: 49.6k]
|
868 | SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT); |
869 | goto fail; |
870 | case MUTABLE: Branch (870:9): [True: 217, False: 49.7k]
|
871 | SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS); |
872 | goto fail; |
873 | case GETSET_OVERRIDDEN: Branch (873:9): [True: 3.17k, False: 46.8k]
|
874 | SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN); |
875 | goto fail; |
876 | case BUILTIN_CLASSMETHOD: Branch (876:9): [True: 0, False: 50.0k]
|
877 | case PYTHON_CLASSMETHOD: Branch (877:9): [True: 0, False: 50.0k]
|
878 | case NON_OVERRIDING: Branch (878:9): [True: 0, False: 50.0k]
|
879 | case NON_DESCRIPTOR: Branch (879:9): [True: 4.07k, False: 45.9k]
|
880 | case ABSENT: Branch (880:9): [True: 37.0k, False: 12.9k]
|
881 | break; |
882 | } |
883 | |
884 | int err = specialize_dict_access( |
885 | owner, instr, type, kind, name, |
886 | STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT |
887 | ); |
888 | if (err < 0) { Branch (888:9): [True: 0, False: 41.1k]
|
889 | return -1; |
890 | } |
891 | if (err) { Branch (891:9): [True: 32.5k, False: 8.65k]
|
892 | goto success; |
893 | } |
894 | fail: |
895 | STAT_INC(STORE_ATTR, failure); |
896 | assert(!PyErr_Occurred()); |
897 | cache->counter = adaptive_counter_backoff(cache->counter); |
898 | return 0; |
899 | success: |
900 | STAT_INC(STORE_ATTR, success); |
901 | assert(!PyErr_Occurred()); |
902 | cache->counter = miss_counter_start(); |
903 | return 0; |
904 | } |
905 | |
906 | |
907 | #ifdef Py_STATS |
908 | static int |
909 | load_attr_fail_kind(DescriptorClassification kind) |
910 | { |
911 | switch (kind) { |
912 | case OVERRIDING: |
913 | return SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR; |
914 | case METHOD: |
915 | return SPEC_FAIL_ATTR_METHOD; |
916 | case PROPERTY: |
917 | return SPEC_FAIL_ATTR_PROPERTY; |
918 | case OBJECT_SLOT: |
919 | return SPEC_FAIL_ATTR_OBJECT_SLOT; |
920 | case OTHER_SLOT: |
921 | return SPEC_FAIL_ATTR_NON_OBJECT_SLOT; |
922 | case DUNDER_CLASS: |
923 | return SPEC_FAIL_OTHER; |
924 | case MUTABLE: |
925 | return SPEC_FAIL_ATTR_MUTABLE_CLASS; |
926 | case GETSET_OVERRIDDEN: |
927 | return SPEC_FAIL_OVERRIDDEN; |
928 | case BUILTIN_CLASSMETHOD: |
929 | return SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD; |
930 | case PYTHON_CLASSMETHOD: |
931 | return SPEC_FAIL_ATTR_CLASS_METHOD_OBJ; |
932 | case NON_OVERRIDING: |
933 | return SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR; |
934 | case NON_DESCRIPTOR: |
935 | return SPEC_FAIL_ATTR_NOT_DESCRIPTOR; |
936 | case ABSENT: |
937 | return SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE; |
938 | } |
939 | Py_UNREACHABLE(); |
940 | } |
941 | #endif |
942 | |
943 | static int |
944 | specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, |
945 | PyObject *name) |
946 | { |
947 | _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1); |
948 | PyObject *descr = NULL; |
949 | DescriptorClassification kind = 0; |
950 | kind = analyze_descriptor((PyTypeObject *)owner, name, &descr, 0); |
951 | switch (kind) { |
952 | case METHOD: Branch (952:9): [True: 1.83k, False: 22.8k]
|
953 | case NON_DESCRIPTOR: Branch (953:9): [True: 6.56k, False: 18.1k]
|
954 | write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag); |
955 | write_obj(cache->descr, descr); |
956 | _Py_SET_OPCODE(*instr, LOAD_ATTR_CLASS); |
957 | return 0; |
958 | #ifdef Py_STATS |
959 | case ABSENT: |
960 | if (_PyType_Lookup(Py_TYPE(owner), name) != NULL) { |
961 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE); |
962 | } |
963 | else { |
964 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); |
965 | } |
966 | return -1; |
967 | #endif |
968 | default: Branch (968:9): [True: 16.3k, False: 8.39k]
|
969 | SPECIALIZATION_FAIL(LOAD_ATTR, load_attr_fail_kind(kind)); |
970 | return -1; |
971 | } |
972 | } |
973 | |
974 | typedef enum { |
975 | MANAGED_VALUES = 1, |
976 | MANAGED_DICT = 2, |
977 | OFFSET_DICT = 3, |
978 | NO_DICT = 4, |
979 | LAZY_DICT = 5, |
980 | } ObjectDictKind; |
981 | |
982 | // Please collect stats carefully before and after modifying. A subtle change |
983 | // can cause a significant drop in cache hits. A possible test is |
984 | // python.exe -m test_typing test_re test_dis test_zlib. |
985 | static int |
986 | specialize_attr_loadmethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, |
987 | PyObject *descr, DescriptorClassification kind) |
988 | { |
989 | _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1); |
990 | PyTypeObject *owner_cls = Py_TYPE(owner); |
991 | |
992 | assert(kind == METHOD && descr != NULL); |
993 | ObjectDictKind dictkind; |
994 | PyDictKeysObject *keys; |
995 | if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { Branch (995:9): [True: 89.4k, False: 43.3k]
|
996 | PyObject *dict = *_PyObject_ManagedDictPointer(owner); |
997 | keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; |
998 | if (dict == NULL) { Branch (998:13): [True: 66.5k, False: 22.8k]
|
999 | dictkind = MANAGED_VALUES; |
1000 | } |
1001 | else { |
1002 | dictkind = MANAGED_DICT; |
1003 | } |
1004 | } |
1005 | else { |
1006 | Py_ssize_t dictoffset = owner_cls->tp_dictoffset; |
1007 | if (dictoffset < 0 || dictoffset > INT16_MAX43.0k ) { Branch (1007:13): [True: 367, False: 43.0k]
Branch (1007:31): [True: 0, False: 43.0k]
|
1008 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); |
1009 | goto fail; |
1010 | } |
1011 | if (dictoffset == 0) { Branch (1011:13): [True: 37.9k, False: 5.10k]
|
1012 | dictkind = NO_DICT; |
1013 | keys = NULL; |
1014 | } |
1015 | else { |
1016 | PyObject *dict = *(PyObject **) ((char *)owner + dictoffset); |
1017 | if (dict == NULL) { Branch (1017:17): [True: 2.58k, False: 2.51k]
|
1018 | // This object will have a dict if user access __dict__ |
1019 | dictkind = LAZY_DICT; |
1020 | keys = NULL; |
1021 | } |
1022 | else { |
1023 | keys = ((PyDictObject *)dict)->ma_keys; |
1024 | dictkind = OFFSET_DICT; |
1025 | } |
1026 | } |
1027 | } |
1028 | if (dictkind == MANAGED_VALUES || dictkind == OFFSET_DICT65.8k ) { Branch (1028:9): [True: 66.5k, False: 65.8k]
Branch (1028:39): [True: 2.51k, False: 63.3k]
|
1029 | Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); |
1030 | if (index != DKIX_EMPTY) { Branch (1030:13): [True: 592, False: 68.5k]
|
1031 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED); |
1032 | goto fail; |
1033 | } |
1034 | uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(keys); |
1035 | if (keys_version == 0) { Branch (1035:13): [True: 0, False: 68.5k]
|
1036 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); |
1037 | goto fail; |
1038 | } |
1039 | write_u32(cache->keys_version, keys_version); |
1040 | } |
1041 | switch(dictkind) { Branch (1041:12): [True: 0, False: 131k]
|
1042 | case NO_DICT: Branch (1042:9): [True: 37.9k, False: 93.9k]
|
1043 | _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_NO_DICT); |
1044 | break; |
1045 | case MANAGED_VALUES: Branch (1045:9): [True: 65.9k, False: 65.8k]
|
1046 | _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_WITH_VALUES); |
1047 | break; |
1048 | case MANAGED_DICT: Branch (1048:9): [True: 22.8k, False: 108k]
|
1049 | SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); |
1050 | goto fail; |
1051 | case OFFSET_DICT: Branch (1051:9): [True: 2.50k, False: 129k]
|
1052 | assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); |
1053 | _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_WITH_DICT); |
1054 | break; |
1055 | case LAZY_DICT: Branch (1055:9): [True: 2.58k, False: 129k]
|
1056 | assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); |
1057 | _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_LAZY_DICT); |
1058 | break; |
1059 | } |
1060 | /* `descr` is borrowed. This is safe for methods (even inherited ones from |
1061 | * super classes!) as long as tp_version_tag is validated for two main reasons: |
1062 | * |
1063 | * 1. The class will always hold a reference to the method so it will |
1064 | * usually not be GC-ed. Should it be deleted in Python, e.g. |
1065 | * `del obj.meth`, tp_version_tag will be invalidated, because of reason 2. |
1066 | * |
1067 | * 2. The pre-existing type method cache (MCACHE) uses the same principles |
1068 | * of caching a borrowed descriptor. The MCACHE infrastructure does all the |
1069 | * heavy lifting for us. E.g. it invalidates tp_version_tag on any MRO |
1070 | * modification, on any type object change along said MRO, etc. (see |
1071 | * PyType_Modified usages in typeobject.c). The MCACHE has been |
1072 | * working since Python 2.6 and it's battle-tested. |
1073 | */ |
1074 | write_u32(cache->type_version, owner_cls->tp_version_tag); |
1075 | write_obj(cache->descr, descr); |
1076 | // Fall through. |
1077 | return 1; |
1078 | fail: |
1079 | return 0; |
1080 | } |
1081 | |
1082 | int |
1083 | _Py_Specialize_LoadGlobal( |
1084 | PyObject *globals, PyObject *builtins, |
1085 | _Py_CODEUNIT *instr, PyObject *name) |
1086 | { |
1087 | assert(_PyOpcode_Caches[LOAD_GLOBAL] == INLINE_CACHE_ENTRIES_LOAD_GLOBAL); |
1088 | /* Use inline cache */ |
1089 | _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)(instr + 1); |
1090 | assert(PyUnicode_CheckExact(name)); |
1091 | if (!PyDict_CheckExact(globals)) { Branch (1091:9): [True: 4.02k, False: 111k]
|
1092 | goto fail; |
1093 | } |
1094 | PyDictKeysObject * globals_keys = ((PyDictObject *)globals)->ma_keys; |
1095 | if (!DK_IS_UNICODE(globals_keys)) { Branch (1095:9): [True: 0, False: 111k]
|
1096 | SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT); |
1097 | goto fail; |
1098 | } |
1099 | Py_ssize_t index = _PyDictKeys_StringLookup(globals_keys, name); |
1100 | if (index == DKIX_ERROR) { Branch (1100:9): [True: 0, False: 111k]
|
1101 | SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT); |
1102 | goto fail; |
1103 | } |
1104 | if (index != DKIX_EMPTY) { Branch (1104:9): [True: 64.5k, False: 47.2k]
|
1105 | if (index != (uint16_t)index) { Branch (1105:13): [True: 0, False: 64.5k]
|
1106 | goto fail; |
1107 | } |
1108 | uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(globals_keys); |
1109 | if (keys_version == 0) { Branch (1109:13): [True: 0, False: 64.5k]
|
1110 | goto fail; |
1111 | } |
1112 | cache->index = (uint16_t)index; |
1113 | write_u32(cache->module_keys_version, keys_version); |
1114 | _Py_SET_OPCODE(*instr, LOAD_GLOBAL_MODULE); |
1115 | goto success; |
1116 | } |
1117 | if (!PyDict_CheckExact(builtins)) { Branch (1117:9): [True: 0, False: 47.2k]
|
1118 | goto fail; |
1119 | } |
1120 | PyDictKeysObject * builtin_keys = ((PyDictObject *)builtins)->ma_keys; |
1121 | if (!DK_IS_UNICODE(builtin_keys)) { Branch (1121:9): [True: 0, False: 47.2k]
|
1122 | SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT); |
1123 | goto fail; |
1124 | } |
1125 | index = _PyDictKeys_StringLookup(builtin_keys, name); |
1126 | if (index == DKIX_ERROR) { Branch (1126:9): [True: 0, False: 47.2k]
|
1127 | SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT); |
1128 | goto fail; |
1129 | } |
1130 | if (index != (uint16_t)index) { Branch (1130:9): [True: 3, False: 47.2k]
|
1131 | goto fail; |
1132 | } |
1133 | uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState(globals_keys); |
1134 | if (globals_version == 0) { Branch (1134:9): [True: 0, False: 47.2k]
|
1135 | SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); |
1136 | goto fail; |
1137 | } |
1138 | uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState(builtin_keys); |
1139 | if (builtins_version == 0) { Branch (1139:9): [True: 0, False: 47.2k]
|
1140 | SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); |
1141 | goto fail; |
1142 | } |
1143 | if (builtins_version > UINT16_MAX) { Branch (1143:9): [True: 0, False: 47.2k]
|
1144 | SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); |
1145 | goto fail; |
1146 | } |
1147 | cache->index = (uint16_t)index; |
1148 | write_u32(cache->module_keys_version, globals_version); |
1149 | cache->builtin_keys_version = (uint16_t)builtins_version; |
1150 | _Py_SET_OPCODE(*instr, LOAD_GLOBAL_BUILTIN); |
1151 | goto success; |
1152 | fail: |
1153 | STAT_INC(LOAD_GLOBAL, failure); |
1154 | assert(!PyErr_Occurred()); |
1155 | cache->counter = adaptive_counter_backoff(cache->counter); |
1156 | return 0; |
1157 | success: |
1158 | STAT_INC(LOAD_GLOBAL, success); |
1159 | assert(!PyErr_Occurred()); |
1160 | cache->counter = miss_counter_start(); |
1161 | return 0; |
1162 | } |
1163 | |
1164 | #ifdef Py_STATS |
1165 | static int |
1166 | binary_subscr_fail_kind(PyTypeObject *container_type, PyObject *sub) |
1167 | { |
1168 | if (container_type == &PyUnicode_Type) { |
1169 | if (PyLong_CheckExact(sub)) { |
1170 | return SPEC_FAIL_SUBSCR_STRING_INT; |
1171 | } |
1172 | if (PySlice_Check(sub)) { |
1173 | return SPEC_FAIL_SUBSCR_STRING_SLICE; |
1174 | } |
1175 | return SPEC_FAIL_OTHER; |
1176 | } |
1177 | else if (strcmp(container_type->tp_name, "array.array") == 0) { |
1178 | if (PyLong_CheckExact(sub)) { |
1179 | return SPEC_FAIL_SUBSCR_ARRAY_INT; |
1180 | } |
1181 | if (PySlice_Check(sub)) { |
1182 | return SPEC_FAIL_SUBSCR_ARRAY_SLICE; |
1183 | } |
1184 | return SPEC_FAIL_OTHER; |
1185 | } |
1186 | else if (container_type->tp_as_buffer) { |
1187 | if (PyLong_CheckExact(sub)) { |
1188 | return SPEC_FAIL_SUBSCR_BUFFER_INT; |
1189 | } |
1190 | if (PySlice_Check(sub)) { |
1191 | return SPEC_FAIL_SUBSCR_BUFFER_SLICE; |
1192 | } |
1193 | return SPEC_FAIL_OTHER; |
1194 | } |
1195 | else if (container_type->tp_as_sequence) { |
1196 | if (PyLong_CheckExact(sub) && container_type->tp_as_sequence->sq_item) { |
1197 | return SPEC_FAIL_SUBSCR_SEQUENCE_INT; |
1198 | } |
1199 | } |
1200 | return SPEC_FAIL_OTHER; |
1201 | } |
1202 | #endif |
1203 | |
1204 | static int |
1205 | function_kind(PyCodeObject *code) { |
1206 | int flags = code->co_flags; |
1207 | if ((flags & (CO_VARKEYWORDS | CO_VARARGS)) || code->co_kwonlyargcount147k ) { Branch (1207:9): [True: 29.8k, False: 147k]
Branch (1207:52): [True: 6.89k, False: 140k]
|
1208 | return SPEC_FAIL_CALL_COMPLEX_PARAMETERS; |
1209 | } |
1210 | if ((flags & CO_OPTIMIZED) == 0) { Branch (1210:9): [True: 9, False: 140k]
|
1211 | return SPEC_FAIL_CALL_CO_NOT_OPTIMIZED; |
1212 | } |
1213 | return SIMPLE_FUNCTION; |
1214 | } |
1215 | |
1216 | int |
1217 | _Py_Specialize_BinarySubscr( |
1218 | PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) |
1219 | { |
1220 | assert(_PyOpcode_Caches[BINARY_SUBSCR] == |
1221 | INLINE_CACHE_ENTRIES_BINARY_SUBSCR); |
1222 | _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)(instr + 1); |
1223 | PyTypeObject *container_type = Py_TYPE(container); |
1224 | if (container_type == &PyList_Type) { Branch (1224:9): [True: 62.5k, False: 38.1k]
|
1225 | if (PyLong_CheckExact(sub)) { |
1226 | _Py_SET_OPCODE(*instr, BINARY_SUBSCR_LIST_INT); |
1227 | goto success; |
1228 | } |
1229 | SPECIALIZATION_FAIL(BINARY_SUBSCR, |
1230 | PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_LIST_SLICE : SPEC_FAIL_OTHER); |
1231 | goto fail; |
1232 | } |
1233 | if (container_type == &PyTuple_Type) { Branch (1233:9): [True: 17.2k, False: 20.8k]
|
1234 | if (PyLong_CheckExact(sub)) { |
1235 | _Py_SET_OPCODE(*instr, BINARY_SUBSCR_TUPLE_INT); |
1236 | goto success; |
1237 | } |
1238 | SPECIALIZATION_FAIL(BINARY_SUBSCR, |
1239 | PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_TUPLE_SLICE : SPEC_FAIL_OTHER); |
1240 | goto fail; |
1241 | } |
1242 | if (container_type == &PyDict_Type) { Branch (1242:9): [True: 2.81k, False: 18.0k]
|
1243 | _Py_SET_OPCODE(*instr, BINARY_SUBSCR_DICT); |
1244 | goto success; |
1245 | } |
1246 | PyTypeObject *cls = Py_TYPE(container); |
1247 | PyObject *descriptor = _PyType_Lookup(cls, &_Py_ID(__getitem__)); |
1248 | if (descriptor && Py_TYPE18.0k (descriptor) == &PyFunction_Type18.0k ) { Branch (1248:9): [True: 18.0k, False: 65]
Branch (1248:23): [True: 1.10k, False: 16.9k]
|
1249 | if (!(container_type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { Branch (1249:13): [True: 0, False: 1.10k]
|
1250 | SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE); |
1251 | goto fail; |
1252 | } |
1253 | PyFunctionObject *func = (PyFunctionObject *)descriptor; |
1254 | PyCodeObject *fcode = (PyCodeObject *)func->func_code; |
1255 | int kind = function_kind(fcode); |
1256 | if (kind != SIMPLE_FUNCTION) { Branch (1256:13): [True: 137, False: 966]
|
1257 | SPECIALIZATION_FAIL(BINARY_SUBSCR, kind); |
1258 | goto fail; |
1259 | } |
1260 | if (fcode->co_argcount != 2) { Branch (1260:13): [True: 0, False: 966]
|
1261 | SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); |
1262 | goto fail; |
1263 | } |
1264 | assert(cls->tp_version_tag != 0); |
1265 | write_u32(cache->type_version, cls->tp_version_tag); |
1266 | int version = _PyFunction_GetVersionForCurrentState(func); |
1267 | if (version == 0 || version != (uint16_t)version) { Branch (1267:13): [True: 0, False: 966]
Branch (1267:29): [True: 0, False: 966]
|
1268 | SPECIALIZATION_FAIL(BINARY_SUBSCR, version == 0 ? |
1269 | SPEC_FAIL_OUT_OF_VERSIONS : SPEC_FAIL_OUT_OF_RANGE); |
1270 | goto fail; |
1271 | } |
1272 | cache->func_version = version; |
1273 | ((PyHeapTypeObject *)container_type)->_spec_cache.getitem = descriptor; |
1274 | _Py_SET_OPCODE(*instr, BINARY_SUBSCR_GETITEM); |
1275 | goto success; |
1276 | } |
1277 | SPECIALIZATION_FAIL(BINARY_SUBSCR, |
1278 | binary_subscr_fail_kind(container_type, sub)); |
1279 | fail: |
1280 | STAT_INC(BINARY_SUBSCR, failure); |
1281 | assert(!PyErr_Occurred()); |
1282 | cache->counter = adaptive_counter_backoff(cache->counter); |
1283 | return 0; |
1284 | success: |
1285 | STAT_INC(BINARY_SUBSCR, success); |
1286 | assert(!PyErr_Occurred()); |
1287 | cache->counter = miss_counter_start(); |
1288 | return 0; |
1289 | } |
1290 | |
1291 | int |
1292 | _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) |
1293 | { |
1294 | _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)(instr + 1); |
1295 | PyTypeObject *container_type = Py_TYPE(container); |
1296 | if (container_type == &PyList_Type) { Branch (1296:9): [True: 1.25k, False: 8.25k]
|
1297 | if (PyLong_CheckExact(sub)) { |
1298 | if ((Py_SIZE(sub) == 0 || Py_SIZE1.06k (sub) == 11.06k ) Branch (1298:18): [True: 105, False: 1.06k]
Branch (1298:39): [True: 486, False: 579]
|
1299 | && ((PyLongObject *)sub)->ob_digit[0] < (size_t)591 PyList_GET_SIZE591 (container)) Branch (1299:20): [True: 588, False: 3]
|
1300 | { |
1301 | _Py_SET_OPCODE(*instr, STORE_SUBSCR_LIST_INT); |
1302 | goto success; |
1303 | } |
1304 | else { |
1305 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); |
1306 | goto fail; |
1307 | } |
1308 | } |
1309 | else if (PySlice_Check(sub)) { |
1310 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_LIST_SLICE); |
1311 | goto fail; |
1312 | } |
1313 | else { |
1314 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); |
1315 | goto fail; |
1316 | } |
1317 | } |
1318 | if (container_type == &PyDict_Type) { Branch (1318:9): [True: 5.07k, False: 3.18k]
|
1319 | _Py_SET_OPCODE(*instr, STORE_SUBSCR_DICT); |
1320 | goto success; |
1321 | } |
1322 | #ifdef Py_STATS |
1323 | PyMappingMethods *as_mapping = container_type->tp_as_mapping; |
1324 | if (as_mapping && (as_mapping->mp_ass_subscript |
1325 | == PyDict_Type.tp_as_mapping->mp_ass_subscript)) { |
1326 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE); |
1327 | goto fail; |
1328 | } |
1329 | if (PyObject_CheckBuffer(container)) { |
1330 | if (PyLong_CheckExact(sub) && (((size_t)Py_SIZE(sub)) > 1)) { |
1331 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); |
1332 | } |
1333 | else if (strcmp(container_type->tp_name, "array.array") == 0) { |
1334 | if (PyLong_CheckExact(sub)) { |
1335 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_ARRAY_INT); |
1336 | } |
1337 | else if (PySlice_Check(sub)) { |
1338 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_ARRAY_SLICE); |
1339 | } |
1340 | else { |
1341 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); |
1342 | } |
1343 | } |
1344 | else if (PyByteArray_CheckExact(container)) { |
1345 | if (PyLong_CheckExact(sub)) { |
1346 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BYTEARRAY_INT); |
1347 | } |
1348 | else if (PySlice_Check(sub)) { |
1349 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE); |
1350 | } |
1351 | else { |
1352 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); |
1353 | } |
1354 | } |
1355 | else { |
1356 | if (PyLong_CheckExact(sub)) { |
1357 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BUFFER_INT); |
1358 | } |
1359 | else if (PySlice_Check(sub)) { |
1360 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BUFFER_SLICE); |
1361 | } |
1362 | else { |
1363 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); |
1364 | } |
1365 | } |
1366 | goto fail; |
1367 | } |
1368 | PyObject *descriptor = _PyType_Lookup(container_type, &_Py_ID(__setitem__)); |
1369 | if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) { |
1370 | PyFunctionObject *func = (PyFunctionObject *)descriptor; |
1371 | PyCodeObject *code = (PyCodeObject *)func->func_code; |
1372 | int kind = function_kind(code); |
1373 | if (kind == SIMPLE_FUNCTION) { |
1374 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_PY_SIMPLE); |
1375 | } |
1376 | else { |
1377 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_PY_OTHER); |
1378 | } |
1379 | goto fail; |
1380 | } |
1381 | #endif |
1382 | SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); |
1383 | fail: |
1384 | STAT_INC(STORE_SUBSCR, failure); |
1385 | assert(!PyErr_Occurred()); |
1386 | cache->counter = adaptive_counter_backoff(cache->counter); |
1387 | return 0; |
1388 | success: |
1389 | STAT_INC(STORE_SUBSCR, success); |
1390 | assert(!PyErr_Occurred()); |
1391 | cache->counter = miss_counter_start(); |
1392 | return 0; |
1393 | } |
1394 | |
1395 | static int |
1396 | specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, |
1397 | PyObject *kwnames) |
1398 | { |
1399 | assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE); |
1400 | PyTypeObject *tp = _PyType_CAST(callable); |
1401 | if (tp->tp_new == PyBaseObject_Type.tp_new) { Branch (1401:9): [True: 17.7k, False: 41.2k]
|
1402 | SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PYTHON_CLASS); |
1403 | return -1; |
1404 | } |
1405 | if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { Branch (1405:9): [True: 31.7k, False: 9.49k]
|
1406 | int oparg = _Py_OPARG(*instr); |
1407 | if (nargs == 1 && kwnames == NULL19.9k && oparg == 119.6k ) { Branch (1407:13): [True: 19.9k, False: 11.7k]
Branch (1407:27): [True: 19.6k, False: 372]
Branch (1407:46): [True: 19.1k, False: 461]
|
1408 | if (tp == &PyUnicode_Type) { Branch (1408:17): [True: 897, False: 18.2k]
|
1409 | _Py_SET_OPCODE(*instr, CALL_NO_KW_STR_1); |
1410 | return 0; |
1411 | } |
1412 | else if (tp == &PyType_Type) { Branch (1412:22): [True: 1.11k, False: 17.1k]
|
1413 | _Py_SET_OPCODE(*instr, CALL_NO_KW_TYPE_1); |
1414 | return 0; |
1415 | } |
1416 | else if (tp == &PyTuple_Type) { Branch (1416:22): [True: 472, False: 16.6k]
|
1417 | _Py_SET_OPCODE(*instr, CALL_NO_KW_TUPLE_1); |
1418 | return 0; |
1419 | } |
1420 | } |
1421 | if (tp->tp_vectorcall != NULL) { Branch (1421:13): [True: 6.42k, False: 22.8k]
|
1422 | _Py_SET_OPCODE(*instr, CALL_BUILTIN_CLASS); |
1423 | return 0; |
1424 | } |
1425 | SPECIALIZATION_FAIL(CALL, tp == &PyUnicode_Type ? |
1426 | SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL); |
1427 | return -1; |
1428 | } |
1429 | SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE); |
1430 | return -1; |
1431 | } |
1432 | |
1433 | #ifdef Py_STATS |
1434 | static int |
1435 | builtin_call_fail_kind(int ml_flags) |
1436 | { |
1437 | switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | |
1438 | METH_KEYWORDS | METH_METHOD)) { |
1439 | case METH_VARARGS: |
1440 | return SPEC_FAIL_CALL_PYCFUNCTION; |
1441 | case METH_VARARGS | METH_KEYWORDS: |
1442 | return SPEC_FAIL_CALL_PYCFUNCTION_WITH_KEYWORDS; |
1443 | case METH_FASTCALL | METH_KEYWORDS: |
1444 | return SPEC_FAIL_CALL_PYCFUNCTION_FAST_WITH_KEYWORDS; |
1445 | case METH_NOARGS: |
1446 | return SPEC_FAIL_CALL_PYCFUNCTION_NOARGS; |
1447 | /* This case should never happen with PyCFunctionObject -- only |
1448 | PyMethodObject. See zlib.compressobj()'s methods for an example. |
1449 | */ |
1450 | case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: |
1451 | default: |
1452 | return SPEC_FAIL_CALL_BAD_CALL_FLAGS; |
1453 | } |
1454 | } |
1455 | #endif |
1456 | |
1457 | static int |
1458 | specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, |
1459 | int nargs, PyObject *kwnames) |
1460 | { |
1461 | assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE); |
1462 | if (kwnames) { Branch (1462:9): [True: 1.42k, False: 48.2k]
|
1463 | SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); |
1464 | return -1; |
1465 | } |
1466 | |
1467 | switch (descr->d_method->ml_flags & Branch (1467:13): [True: 20.8k, False: 27.4k]
|
1468 | (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | |
1469 | METH_KEYWORDS | METH_METHOD)) { |
1470 | case METH_NOARGS: { Branch (1470:9): [True: 10.1k, False: 38.1k]
|
1471 | if (nargs != 1) { Branch (1471:17): [True: 0, False: 10.1k]
|
1472 | SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); |
1473 | return -1; |
1474 | } |
1475 | _Py_SET_OPCODE(*instr, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS); |
1476 | return 0; |
1477 | } |
1478 | case METH_O: { Branch (1478:9): [True: 9.56k, False: 38.7k]
|
1479 | if (nargs != 2) { Branch (1479:17): [True: 0, False: 9.56k]
|
1480 | SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); |
1481 | return -1; |
1482 | } |
1483 | PyInterpreterState *interp = _PyInterpreterState_GET(); |
1484 | PyObject *list_append = interp->callable_cache.list_append; |
1485 | _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_CALL + 1]; |
1486 | bool pop = (_Py_OPCODE(next) == POP_TOP); |
1487 | int oparg = _Py_OPARG(*instr); |
1488 | if ((PyObject *)descr == list_append && oparg == 13.16k && pop3.16k ) { Branch (1488:17): [True: 3.16k, False: 6.39k]
Branch (1488:53): [True: 3.16k, False: 3]
Branch (1488:67): [True: 3.15k, False: 6]
|
1489 | _Py_SET_OPCODE(*instr, CALL_NO_KW_LIST_APPEND); |
1490 | return 0; |
1491 | } |
1492 | _Py_SET_OPCODE(*instr, CALL_NO_KW_METHOD_DESCRIPTOR_O); |
1493 | return 0; |
1494 | } |
1495 | case METH_FASTCALL: { Branch (1495:9): [True: 4.91k, False: 43.3k]
|
1496 | _Py_SET_OPCODE(*instr, CALL_NO_KW_METHOD_DESCRIPTOR_FAST); |
1497 | return 0; |
1498 | } |
1499 | case METH_FASTCALL|METH_KEYWORDS: { Branch (1499:9): [True: 2.86k, False: 45.4k]
|
1500 | _Py_SET_OPCODE(*instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); |
1501 | return 0; |
1502 | } |
1503 | } |
1504 | SPECIALIZATION_FAIL(CALL, builtin_call_fail_kind(descr->d_method->ml_flags)); |
1505 | return -1; |
1506 | } |
1507 | |
1508 | static int |
1509 | specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, |
1510 | PyObject *kwnames, bool bound_method) |
1511 | { |
1512 | _PyCallCache *cache = (_PyCallCache *)(instr + 1); |
1513 | assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE); |
1514 | PyCodeObject *code = (PyCodeObject *)func->func_code; |
1515 | int kind = function_kind(code); |
1516 | /* Don't specialize if PEP 523 is active */ |
1517 | if (_PyInterpreterState_GET()->eval_frame) { Branch (1517:9): [True: 2, False: 171k]
|
1518 | SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); |
1519 | return -1; |
1520 | } |
1521 | if (kwnames) { Branch (1521:9): [True: 14.5k, False: 157k]
|
1522 | SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); |
1523 | return -1; |
1524 | } |
1525 | if (kind != SIMPLE_FUNCTION) { Branch (1525:9): [True: 28.9k, False: 128k]
|
1526 | SPECIALIZATION_FAIL(CALL, kind); |
1527 | return -1; |
1528 | } |
1529 | int argcount = code->co_argcount; |
1530 | int defcount = func->func_defaults == NULL ? 0104k : (int)23.7k PyTuple_GET_SIZE23.7k (func->func_defaults); Branch (1530:20): [True: 104k, False: 23.7k]
|
1531 | assert(defcount <= argcount); |
1532 | int min_args = argcount-defcount; |
1533 | if (nargs > argcount || nargs < min_args128k ) { Branch (1533:9): [True: 5, False: 128k]
Branch (1533:29): [True: 10, False: 128k]
|
1534 | SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); |
1535 | return -1; |
1536 | } |
1537 | assert(nargs <= argcount && nargs >= min_args); |
1538 | assert(min_args >= 0 && defcount >= 0); |
1539 | assert(defcount == 0 || func->func_defaults != NULL); |
1540 | if (min_args > 0xffff) { Branch (1540:9): [True: 0, False: 128k]
|
1541 | SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_RANGE); |
1542 | return -1; |
1543 | } |
1544 | int version = _PyFunction_GetVersionForCurrentState(func); |
1545 | if (version == 0) { Branch (1545:9): [True: 0, False: 128k]
|
1546 | SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); |
1547 | return -1; |
1548 | } |
1549 | write_u32(cache->func_version, version); |
1550 | cache->min_args = min_args; |
1551 | if (argcount == nargs) { Branch (1551:9): [True: 110k, False: 17.4k]
|
1552 | _Py_SET_OPCODE(*instr, bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS); |
1553 | } |
1554 | else if (bound_method) { Branch (1554:14): [True: 4.24k, False: 13.2k]
|
1555 | SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); |
1556 | return -1; |
1557 | } |
1558 | else { |
1559 | _Py_SET_OPCODE(*instr, CALL_PY_WITH_DEFAULTS); |
1560 | } |
1561 | return 0; |
1562 | } |
1563 | |
1564 | static int |
1565 | specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, |
1566 | PyObject *kwnames) |
1567 | { |
1568 | assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE); |
1569 | if (PyCFunction_GET_FUNCTION(callable) == NULL) { Branch (1569:9): [True: 0, False: 63.2k]
|
1570 | return 1; |
1571 | } |
1572 | switch (PyCFunction_GET_FLAGS(callable) & |
1573 | (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | |
1574 | METH_KEYWORDS | METH_METHOD)) { |
1575 | case METH_O: { Branch (1575:9): [True: 12.1k, False: 51.1k]
|
1576 | if (kwnames) { Branch (1576:17): [True: 0, False: 12.1k]
|
1577 | SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); |
1578 | return -1; |
1579 | } |
1580 | if (nargs != 1) { Branch (1580:17): [True: 0, False: 12.1k]
|
1581 | SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); |
1582 | return 1; |
1583 | } |
1584 | /* len(o) */ |
1585 | PyInterpreterState *interp = _PyInterpreterState_GET(); |
1586 | if (callable == interp->callable_cache.len) { Branch (1586:17): [True: 5.21k, False: 6.94k]
|
1587 | _Py_SET_OPCODE(*instr, CALL_NO_KW_LEN); |
1588 | return 0; |
1589 | } |
1590 | _Py_SET_OPCODE(*instr, CALL_NO_KW_BUILTIN_O); |
1591 | return 0; |
1592 | } |
1593 | case METH_FASTCALL: { Branch (1593:9): [True: 12.0k, False: 51.1k]
|
1594 | if (kwnames) { Branch (1594:17): [True: 1, False: 12.0k]
|
1595 | SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); |
1596 | return -1; |
1597 | } |
1598 | if (nargs == 2) { Branch (1598:17): [True: 8.11k, False: 3.97k]
|
1599 | /* isinstance(o1, o2) */ |
1600 | PyInterpreterState *interp = _PyInterpreterState_GET(); |
1601 | if (callable == interp->callable_cache.isinstance) { Branch (1601:21): [True: 3.49k, False: 4.62k]
|
1602 | _Py_SET_OPCODE(*instr, CALL_NO_KW_ISINSTANCE); |
1603 | return 0; |
1604 | } |
1605 | } |
1606 | _Py_SET_OPCODE(*instr, CALL_NO_KW_BUILTIN_FAST); |
1607 | return 0; |
1608 | } |
1609 | case METH_FASTCALL | METH_KEYWORDS: { Branch (1609:9): [True: 6.58k, False: 56.6k]
|
1610 | _Py_SET_OPCODE(*instr, CALL_BUILTIN_FAST_WITH_KEYWORDS); |
1611 | return 0; |
1612 | } |
1613 | default: Branch (1613:9): [True: 32.4k, False: 30.8k]
|
1614 | SPECIALIZATION_FAIL(CALL, |
1615 | builtin_call_fail_kind(PyCFunction_GET_FLAGS(callable))); |
1616 | return 1; |
1617 | } |
1618 | } |
1619 | |
1620 | #ifdef Py_STATS |
1621 | static int |
1622 | call_fail_kind(PyObject *callable) |
1623 | { |
1624 | if (PyCFunction_CheckExact(callable)) { |
1625 | return SPEC_FAIL_CALL_PYCFUNCTION; |
1626 | } |
1627 | else if (PyFunction_Check(callable)) { |
1628 | return SPEC_FAIL_CALL_PYFUNCTION; |
1629 | } |
1630 | else if (PyInstanceMethod_Check(callable)) { |
1631 | return SPEC_FAIL_CALL_INSTANCE_METHOD; |
1632 | } |
1633 | else if (PyMethod_Check(callable)) { |
1634 | return SPEC_FAIL_CALL_BOUND_METHOD; |
1635 | } |
1636 | // builtin method |
1637 | else if (PyCMethod_Check(callable)) { |
1638 | return SPEC_FAIL_CALL_CMETHOD; |
1639 | } |
1640 | else if (PyType_Check(callable)) { |
1641 | if (((PyTypeObject *)callable)->tp_new == PyBaseObject_Type.tp_new) { |
1642 | return SPEC_FAIL_CALL_PYTHON_CLASS; |
1643 | } |
1644 | else { |
1645 | return SPEC_FAIL_CALL_CLASS; |
1646 | } |
1647 | } |
1648 | else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) { |
1649 | return SPEC_FAIL_CALL_METHOD_DESCRIPTOR; |
1650 | } |
1651 | else if (Py_TYPE(callable) == &PyWrapperDescr_Type) { |
1652 | return SPEC_FAIL_CALL_OPERATOR_WRAPPER; |
1653 | } |
1654 | else if (Py_TYPE(callable) == &_PyMethodWrapper_Type) { |
1655 | return SPEC_FAIL_CALL_METHOD_WRAPPER; |
1656 | } |
1657 | return SPEC_FAIL_OTHER; |
1658 | } |
1659 | #endif |
1660 | |
1661 | |
1662 | /* TODO: |
1663 | - Specialize calling classes. |
1664 | */ |
1665 | int |
1666 | _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, |
1667 | PyObject *kwnames) |
1668 | { |
1669 | assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL); |
1670 | _PyCallCache *cache = (_PyCallCache *)(instr + 1); |
1671 | int fail; |
1672 | if (PyCFunction_CheckExact(callable)) { |
1673 | fail = specialize_c_call(callable, instr, nargs, kwnames); |
1674 | } |
1675 | else if (PyFunction_Check(callable)) { |
1676 | fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs, |
1677 | kwnames, false); |
1678 | } |
1679 | else if (PyType_Check(callable)) { |
1680 | fail = specialize_class_call(callable, instr, nargs, kwnames); |
1681 | } |
1682 | else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) { |
1683 | fail = specialize_method_descriptor((PyMethodDescrObject *)callable, |
1684 | instr, nargs, kwnames); |
1685 | } |
1686 | else if (Py_TYPE(callable) == &PyMethod_Type) { Branch (1686:14): [True: 24.4k, False: 6.66k]
|
1687 | PyObject *func = ((PyMethodObject *)callable)->im_func; |
1688 | if (PyFunction_Check(func)) { |
1689 | fail = specialize_py_call((PyFunctionObject *)func, |
1690 | instr, nargs+1, kwnames, true); |
1691 | } else { |
1692 | SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); |
1693 | fail = -1; |
1694 | } |
1695 | } |
1696 | else { |
1697 | SPECIALIZATION_FAIL(CALL, call_fail_kind(callable)); |
1698 | fail = -1; |
1699 | } |
1700 | if (fail) { Branch (1700:9): [True: 159k, False: 191k]
|
1701 | STAT_INC(CALL, failure); |
1702 | assert(!PyErr_Occurred()); |
1703 | cache->counter = adaptive_counter_backoff(cache->counter); |
1704 | } |
1705 | else { |
1706 | STAT_INC(CALL, success); |
1707 | assert(!PyErr_Occurred()); |
1708 | cache->counter = miss_counter_start(); |
1709 | } |
1710 | return 0; |
1711 | } |
1712 | |
1713 | #ifdef Py_STATS |
1714 | static int |
1715 | binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) |
1716 | { |
1717 | switch (oparg) { |
1718 | case NB_ADD: |
1719 | case NB_INPLACE_ADD: |
1720 | if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { |
1721 | return SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES; |
1722 | } |
1723 | return SPEC_FAIL_BINARY_OP_ADD_OTHER; |
1724 | case NB_AND: |
1725 | case NB_INPLACE_AND: |
1726 | if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { |
1727 | return SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES; |
1728 | } |
1729 | if (PyLong_CheckExact(lhs)) { |
1730 | return SPEC_FAIL_BINARY_OP_AND_INT; |
1731 | } |
1732 | return SPEC_FAIL_BINARY_OP_AND_OTHER; |
1733 | case NB_FLOOR_DIVIDE: |
1734 | case NB_INPLACE_FLOOR_DIVIDE: |
1735 | return SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE; |
1736 | case NB_LSHIFT: |
1737 | case NB_INPLACE_LSHIFT: |
1738 | return SPEC_FAIL_BINARY_OP_LSHIFT; |
1739 | case NB_MATRIX_MULTIPLY: |
1740 | case NB_INPLACE_MATRIX_MULTIPLY: |
1741 | return SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY; |
1742 | case NB_MULTIPLY: |
1743 | case NB_INPLACE_MULTIPLY: |
1744 | if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { |
1745 | return SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES; |
1746 | } |
1747 | return SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER; |
1748 | case NB_OR: |
1749 | case NB_INPLACE_OR: |
1750 | return SPEC_FAIL_BINARY_OP_OR; |
1751 | case NB_POWER: |
1752 | case NB_INPLACE_POWER: |
1753 | return SPEC_FAIL_BINARY_OP_POWER; |
1754 | case NB_REMAINDER: |
1755 | case NB_INPLACE_REMAINDER: |
1756 | return SPEC_FAIL_BINARY_OP_REMAINDER; |
1757 | case NB_RSHIFT: |
1758 | case NB_INPLACE_RSHIFT: |
1759 | return SPEC_FAIL_BINARY_OP_RSHIFT; |
1760 | case NB_SUBTRACT: |
1761 | case NB_INPLACE_SUBTRACT: |
1762 | if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { |
1763 | return SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES; |
1764 | } |
1765 | return SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER; |
1766 | case NB_TRUE_DIVIDE: |
1767 | case NB_INPLACE_TRUE_DIVIDE: |
1768 | if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { |
1769 | return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES; |
1770 | } |
1771 | if (PyFloat_CheckExact(lhs)) { |
1772 | return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT; |
1773 | } |
1774 | return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER; |
1775 | case NB_XOR: |
1776 | case NB_INPLACE_XOR: |
1777 | return SPEC_FAIL_BINARY_OP_XOR; |
1778 | } |
1779 | Py_UNREACHABLE(); |
1780 | } |
1781 | #endif |
1782 | |
1783 | void |
1784 | _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, |
1785 | int oparg, PyObject **locals) |
1786 | { |
1787 | assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP); |
1788 | _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1); |
1789 | switch (oparg) { |
1790 | case NB_ADD: Branch (1790:9): [True: 13.4k, False: 18.0k]
|
1791 | case NB_INPLACE_ADD: Branch (1791:9): [True: 2.97k, False: 28.4k]
|
1792 | if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { Branch (1792:17): [True: 1.33k, False: 15.0k]
|
1793 | break; |
1794 | } |
1795 | if (PyUnicode_CheckExact(lhs)) { |
1796 | _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_BINARY_OP + 1]; |
1797 | bool to_store = (_Py_OPCODE(next) == STORE_FAST || Branch (1797:34): [True: 685, False: 2.61k]
|
1798 | _Py_OPCODE2.61k (next) == 2.61k STORE_FAST__LOAD_FAST2.61k ); Branch (1798:34): [True: 466, False: 2.14k]
|
1799 | if (to_store && locals[1.15k _Py_OPARG1.15k (next)] == lhs) { Branch (1799:21): [True: 1.15k, False: 2.14k]
Branch (1799:33): [True: 623, False: 528]
|
1800 | _Py_SET_OPCODE(*instr, BINARY_OP_INPLACE_ADD_UNICODE); |
1801 | goto success; |
1802 | } |
1803 | _Py_SET_OPCODE(*instr, BINARY_OP_ADD_UNICODE); |
1804 | goto success; |
1805 | } |
1806 | if (PyLong_CheckExact(lhs)) { |
1807 | _Py_SET_OPCODE(*instr, BINARY_OP_ADD_INT); |
1808 | goto success; |
1809 | } |
1810 | if (PyFloat_CheckExact(lhs)) { |
1811 | _Py_SET_OPCODE(*instr, BINARY_OP_ADD_FLOAT); |
1812 | goto success; |
1813 | } |
1814 | break; |
1815 | case 5.18k NB_MULTIPLY5.18k : Branch (1815:9): [True: 5.18k, False: 26.2k]
|
1816 | case NB_INPLACE_MULTIPLY: Branch (1816:9): [True: 162, False: 31.2k]
|
1817 | if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { Branch (1817:17): [True: 4.16k, False: 1.17k]
|
1818 | break; |
1819 | } |
1820 | if (PyLong_CheckExact(lhs)) { |
1821 | _Py_SET_OPCODE(*instr, BINARY_OP_MULTIPLY_INT); |
1822 | goto success; |
1823 | } |
1824 | if (PyFloat_CheckExact(lhs)) { |
1825 | _Py_SET_OPCODE(*instr, BINARY_OP_MULTIPLY_FLOAT); |
1826 | goto success; |
1827 | } |
1828 | break; |
1829 | case NB_SUBTRACT: Branch (1829:9): [True: 3.50k, False: 27.9k]
|
1830 | case NB_INPLACE_SUBTRACT: Branch (1830:9): [True: 376, False: 31.0k]
|
1831 | if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { Branch (1831:17): [True: 681, False: 3.20k]
|
1832 | break; |
1833 | } |
1834 | if (PyLong_CheckExact(lhs)) { |
1835 | _Py_SET_OPCODE(*instr, BINARY_OP_SUBTRACT_INT); |
1836 | goto success; |
1837 | } |
1838 | if (PyFloat_CheckExact(lhs)) { |
1839 | _Py_SET_OPCODE(*instr, BINARY_OP_SUBTRACT_FLOAT); |
1840 | goto success; |
1841 | } |
1842 | break; |
1843 | #ifndef Py_STATS |
1844 | default: Branch (1844:9): [True: 5.80k, False: 25.6k]
|
1845 | // These operators don't have any available specializations. Rather |
1846 | // than repeatedly attempting to specialize them, just convert them |
1847 | // back to BINARY_OP (unless we're collecting stats, where it's more |
1848 | // important to get accurate hit counts for the unadaptive version |
1849 | // and each of the different failure types): |
1850 | _Py_SET_OPCODE(*instr, BINARY_OP); |
1851 | return; |
1852 | #endif |
1853 | } |
1854 | SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs)); |
1855 | STAT_INC(BINARY_OP, failure); |
1856 | cache->counter = adaptive_counter_backoff(cache->counter); |
1857 | return; |
1858 | success: |
1859 | STAT_INC(BINARY_OP, success); |
1860 | cache->counter = miss_counter_start(); |
1861 | } |
1862 | |
1863 | |
1864 | #ifdef Py_STATS |
1865 | static int |
1866 | compare_op_fail_kind(PyObject *lhs, PyObject *rhs) |
1867 | { |
1868 | if (Py_TYPE(lhs) != Py_TYPE(rhs)) { |
1869 | if (PyFloat_CheckExact(lhs) && PyLong_CheckExact(rhs)) { |
1870 | return SPEC_FAIL_COMPARE_OP_FLOAT_LONG; |
1871 | } |
1872 | if (PyLong_CheckExact(lhs) && PyFloat_CheckExact(rhs)) { |
1873 | return SPEC_FAIL_COMPARE_OP_LONG_FLOAT; |
1874 | } |
1875 | return SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES; |
1876 | } |
1877 | if (PyBytes_CheckExact(lhs)) { |
1878 | return SPEC_FAIL_COMPARE_OP_BYTES; |
1879 | } |
1880 | if (PyTuple_CheckExact(lhs)) { |
1881 | return SPEC_FAIL_COMPARE_OP_TUPLE; |
1882 | } |
1883 | if (PyList_CheckExact(lhs)) { |
1884 | return SPEC_FAIL_COMPARE_OP_LIST; |
1885 | } |
1886 | if (PySet_CheckExact(lhs) || PyFrozenSet_CheckExact(lhs)) { |
1887 | return SPEC_FAIL_COMPARE_OP_SET; |
1888 | } |
1889 | if (PyBool_Check(lhs)) { |
1890 | return SPEC_FAIL_COMPARE_OP_BOOL; |
1891 | } |
1892 | if (Py_TYPE(lhs)->tp_richcompare == PyBaseObject_Type.tp_richcompare) { |
1893 | return SPEC_FAIL_COMPARE_OP_BASEOBJECT; |
1894 | } |
1895 | return SPEC_FAIL_OTHER; |
1896 | } |
1897 | #endif |
1898 | |
1899 | |
1900 | static int compare_masks[] = { |
1901 | // 1-bit: jump if less than |
1902 | // 2-bit: jump if equal |
1903 | // 4-bit: jump if greater |
1904 | [Py_LT] = 1 | 0 | 0, |
1905 | [Py_LE] = 1 | 2 | 0, |
1906 | [Py_EQ] = 0 | 2 | 0, |
1907 | [Py_NE] = 1 | 0 | 4, |
1908 | [Py_GT] = 0 | 0 | 4, |
1909 | [Py_GE] = 0 | 2 | 4, |
1910 | }; |
1911 | |
1912 | void |
1913 | _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, |
1914 | int oparg) |
1915 | { |
1916 | assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); |
1917 | _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1); |
1918 | int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]); |
1919 | if (next_opcode != POP_JUMP_FORWARD_IF_FALSE && Branch (1919:9): [True: 13.9k, False: 30.8k]
|
1920 | next_opcode != 13.9k POP_JUMP_BACKWARD_IF_FALSE13.9k && Branch (1920:9): [True: 12.4k, False: 1.50k]
|
1921 | next_opcode != 12.4k POP_JUMP_FORWARD_IF_TRUE12.4k && Branch (1921:9): [True: 4.47k, False: 7.94k]
|
1922 | next_opcode != 4.47k POP_JUMP_BACKWARD_IF_TRUE4.47k ) { Branch (1922:9): [True: 2.52k, False: 1.94k]
|
1923 | // Can't ever combine, so don't don't bother being adaptive (unless |
1924 | // we're collecting stats, where it's more important to get accurate hit |
1925 | // counts for the unadaptive version and each of the different failure |
1926 | // types): |
1927 | #ifndef Py_STATS |
1928 | _Py_SET_OPCODE(*instr, COMPARE_OP); |
1929 | return; |
1930 | #else |
1931 | if (next_opcode == EXTENDED_ARG) { |
1932 | SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_EXTENDED_ARG); |
1933 | goto failure; |
1934 | } |
1935 | SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_NOT_FOLLOWED_BY_COND_JUMP); |
1936 | goto failure; |
1937 | #endif |
1938 | } |
1939 | assert(oparg <= Py_GE); |
1940 | int when_to_jump_mask = compare_masks[oparg]; |
1941 | if (next_opcode == POP_JUMP_FORWARD_IF_FALSE || Branch (1941:9): [True: 30.8k, False: 11.3k]
|
1942 | next_opcode == 11.3k POP_JUMP_BACKWARD_IF_FALSE11.3k ) { Branch (1942:9): [True: 1.50k, False: 9.88k]
|
1943 | when_to_jump_mask = (1 | 2 | 4) & ~when_to_jump_mask; |
1944 | } |
1945 | if (next_opcode == POP_JUMP_BACKWARD_IF_TRUE || Branch (1945:9): [True: 1.94k, False: 40.3k]
|
1946 | next_opcode == 40.3k POP_JUMP_BACKWARD_IF_FALSE40.3k ) { Branch (1946:9): [True: 1.50k, False: 38.8k]
|
1947 | when_to_jump_mask <<= 3; |
1948 | } |
1949 | if (Py_TYPE(lhs) != Py_TYPE(rhs)) { Branch (1949:9): [True: 2.90k, False: 39.3k]
|
1950 | SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); |
1951 | goto failure; |
1952 | } |
1953 | if (PyFloat_CheckExact(lhs)) { |
1954 | _Py_SET_OPCODE(*instr, COMPARE_OP_FLOAT_JUMP); |
1955 | cache->mask = when_to_jump_mask; |
1956 | goto success; |
1957 | } |
1958 | if (PyLong_CheckExact(lhs)) { |
1959 | if (Py_ABS(Py_SIZE(lhs)) <= 1 && Py_ABS16.9k (Py_SIZE(rhs)) <= 116.9k ) { Branch (1959:13): [True: 16.9k, False: 9.15k]
Branch (1959:42): [True: 16.0k, False: 889]
|
1960 | _Py_SET_OPCODE(*instr, COMPARE_OP_INT_JUMP); |
1961 | cache->mask = when_to_jump_mask; |
1962 | goto success; |
1963 | } |
1964 | else { |
1965 | SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_BIG_INT); |
1966 | goto failure; |
1967 | } |
1968 | } |
1969 | if (PyUnicode_CheckExact(lhs)) { |
1970 | if (oparg != Py_EQ && oparg != 1.18k Py_NE1.18k ) { Branch (1970:13): [True: 1.18k, False: 4.02k]
Branch (1970:31): [True: 162, False: 1.02k]
|
1971 | SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_STRING); |
1972 | goto failure; |
1973 | } |
1974 | else { |
1975 | _Py_SET_OPCODE(*instr, COMPARE_OP_STR_JUMP); |
1976 | cache->mask = when_to_jump_mask; |
1977 | goto success; |
1978 | } |
1979 | } |
1980 | SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); |
1981 | failure: |
1982 | STAT_INC(COMPARE_OP, failure); |
1983 | cache->counter = adaptive_counter_backoff(cache->counter); |
1984 | return; |
1985 | success: |
1986 | STAT_INC(COMPARE_OP, success); |
1987 | cache->counter = miss_counter_start(); |
1988 | } |
1989 | |
1990 | #ifdef Py_STATS |
1991 | static int |
1992 | unpack_sequence_fail_kind(PyObject *seq) |
1993 | { |
1994 | if (PySequence_Check(seq)) { |
1995 | return SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE; |
1996 | } |
1997 | if (PyIter_Check(seq)) { |
1998 | return SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR; |
1999 | } |
2000 | return SPEC_FAIL_OTHER; |
2001 | } |
2002 | #endif |
2003 | |
2004 | void |
2005 | _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) |
2006 | { |
2007 | assert(_PyOpcode_Caches[UNPACK_SEQUENCE] == |
2008 | INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); |
2009 | _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)(instr + 1); |
2010 | if (PyTuple_CheckExact(seq)) { |
2011 | if (PyTuple_GET_SIZE(seq) != oparg) { Branch (2011:13): [True: 1, False: 5.58k]
|
2012 | SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR); |
2013 | goto failure; |
2014 | } |
2015 | if (PyTuple_GET_SIZE(seq) == 2) { Branch (2015:13): [True: 4.46k, False: 1.12k]
|
2016 | _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_TWO_TUPLE); |
2017 | goto success; |
2018 | } |
2019 | _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_TUPLE); |
2020 | goto success; |
2021 | } |
2022 | if (PyList_CheckExact(seq)) { |
2023 | if (PyList_GET_SIZE(seq) != oparg) { Branch (2023:13): [True: 13, False: 142]
|
2024 | SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR); |
2025 | goto failure; |
2026 | } |
2027 | _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_LIST); |
2028 | goto success; |
2029 | } |
2030 | SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq)); |
2031 | failure: |
2032 | STAT_INC(UNPACK_SEQUENCE, failure); |
2033 | cache->counter = adaptive_counter_backoff(cache->counter); |
2034 | return; |
2035 | success: |
2036 | STAT_INC(UNPACK_SEQUENCE, success); |
2037 | cache->counter = miss_counter_start(); |
2038 | } |
2039 | |
2040 | #ifdef Py_STATS |
2041 | |
2042 | int |
2043 | _PySpecialization_ClassifyIterator(PyObject *iter) |
2044 | { |
2045 | if (PyGen_CheckExact(iter)) { |
2046 | return SPEC_FAIL_FOR_ITER_GENERATOR; |
2047 | } |
2048 | if (PyCoro_CheckExact(iter)) { |
2049 | return SPEC_FAIL_FOR_ITER_COROUTINE; |
2050 | } |
2051 | if (PyAsyncGen_CheckExact(iter)) { |
2052 | return SPEC_FAIL_FOR_ITER_ASYNC_GENERATOR; |
2053 | } |
2054 | PyTypeObject *t = Py_TYPE(iter); |
2055 | if (t == &PyListIter_Type) { |
2056 | return SPEC_FAIL_FOR_ITER_LIST; |
2057 | } |
2058 | if (t == &PyTupleIter_Type) { |
2059 | return SPEC_FAIL_FOR_ITER_TUPLE; |
2060 | } |
2061 | if (t == &PyDictIterKey_Type) { |
2062 | return SPEC_FAIL_FOR_ITER_DICT_KEYS; |
2063 | } |
2064 | if (t == &PyDictIterValue_Type) { |
2065 | return SPEC_FAIL_FOR_ITER_DICT_VALUES; |
2066 | } |
2067 | if (t == &PyDictIterItem_Type) { |
2068 | return SPEC_FAIL_FOR_ITER_DICT_ITEMS; |
2069 | } |
2070 | if (t == &PySetIter_Type) { |
2071 | return SPEC_FAIL_FOR_ITER_SET; |
2072 | } |
2073 | if (t == &PyUnicodeIter_Type) { |
2074 | return SPEC_FAIL_FOR_ITER_STRING; |
2075 | } |
2076 | if (t == &PyBytesIter_Type) { |
2077 | return SPEC_FAIL_FOR_ITER_BYTES; |
2078 | } |
2079 | if (t == &PyRangeIter_Type) { |
2080 | return SPEC_FAIL_FOR_ITER_RANGE; |
2081 | } |
2082 | if (t == &PyEnum_Type) { |
2083 | return SPEC_FAIL_FOR_ITER_ENUMERATE; |
2084 | } |
2085 | if (t == &PyMap_Type) { |
2086 | return SPEC_FAIL_FOR_ITER_MAP; |
2087 | } |
2088 | if (t == &PyZip_Type) { |
2089 | return SPEC_FAIL_FOR_ITER_ZIP; |
2090 | } |
2091 | if (t == &PySeqIter_Type) { |
2092 | return SPEC_FAIL_FOR_ITER_SEQ_ITER; |
2093 | } |
2094 | if (t == &PyListRevIter_Type) { |
2095 | return SPEC_FAIL_FOR_ITER_REVERSED_LIST; |
2096 | } |
2097 | if (t == &_PyUnicodeASCIIIter_Type) { |
2098 | return SPEC_FAIL_FOR_ITER_ASCII_STRING; |
2099 | } |
2100 | const char *name = t->tp_name; |
2101 | if (strncmp(name, "itertools", 9) == 0) { |
2102 | return SPEC_FAIL_FOR_ITER_ITERTOOLS; |
2103 | } |
2104 | if (strncmp(name, "callable_iterator", 17) == 0) { |
2105 | return SPEC_FAIL_FOR_ITER_CALLABLE; |
2106 | } |
2107 | return SPEC_FAIL_OTHER; |
2108 | } |
2109 | |
2110 | #endif |
2111 | |
2112 | void |
2113 | _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr) |
2114 | { |
2115 | assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER); |
2116 | _PyForIterCache *cache = (_PyForIterCache *)(instr + 1); |
2117 | PyTypeObject *tp = Py_TYPE(iter); |
2118 | _Py_CODEUNIT next = instr[1+INLINE_CACHE_ENTRIES_FOR_ITER]; |
2119 | int next_op = _PyOpcode_Deopt[_Py_OPCODE(next)]; |
2120 | if (tp == &PyListIter_Type) { Branch (2120:9): [True: 6.16k, False: 46.7k]
|
2121 | _Py_SET_OPCODE(*instr, FOR_ITER_LIST); |
2122 | goto success; |
2123 | } |
2124 | else if (tp == &PyRangeIter_Type && next_op == 2.47k STORE_FAST2.47k ) { Branch (2124:14): [True: 2.47k, False: 44.3k]
Branch (2124:41): [True: 2.15k, False: 321]
|
2125 | _Py_SET_OPCODE(*instr, FOR_ITER_RANGE); |
2126 | goto success; |
2127 | } |
2128 | else { |
2129 | SPECIALIZATION_FAIL(FOR_ITER, |
2130 | _PySpecialization_ClassifyIterator(iter)); |
2131 | goto failure; |
2132 | } |
2133 | failure: |
2134 | STAT_INC(FOR_ITER, failure); |
2135 | cache->counter = adaptive_counter_backoff(cache->counter); |
2136 | return; |
2137 | success: |
2138 | STAT_INC(FOR_ITER, success); |
2139 | cache->counter = miss_counter_start(); |
2140 | } |