Line data Source code
1 : #include "Python.h"
2 : #include "pycore_code.h"
3 : #include "pycore_dict.h"
4 : #include "pycore_function.h" // _PyFunction_GetVersionForCurrentState()
5 : #include "pycore_global_strings.h" // _Py_ID()
6 : #include "pycore_long.h"
7 : #include "pycore_moduleobject.h"
8 : #include "pycore_object.h"
9 : #include "pycore_opcode.h" // _PyOpcode_Caches
10 : #include "structmember.h" // struct PyMemberDef, T_OFFSET_EX
11 : #include "pycore_descrobject.h"
12 :
13 : #include <stdlib.h> // rand()
14 :
15 : /* For guidance on adding or extending families of instructions see
16 : * ./adaptive.md
17 : */
18 :
19 : /* Map from opcode to adaptive opcode.
20 : Values of zero are ignored. */
21 : uint8_t _PyOpcode_Adaptive[256] = {
22 : [LOAD_ATTR] = LOAD_ATTR_ADAPTIVE,
23 : [LOAD_GLOBAL] = LOAD_GLOBAL_ADAPTIVE,
24 : [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE,
25 : [STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE,
26 : [CALL] = CALL_ADAPTIVE,
27 : [STORE_ATTR] = STORE_ATTR_ADAPTIVE,
28 : [BINARY_OP] = BINARY_OP_ADAPTIVE,
29 : [COMPARE_OP] = COMPARE_OP_ADAPTIVE,
30 : [UNPACK_SEQUENCE] = UNPACK_SEQUENCE_ADAPTIVE,
31 : [FOR_ITER] = FOR_ITER_ADAPTIVE,
32 : };
33 :
34 : Py_ssize_t _Py_QuickenedCount = 0;
35 : #ifdef Py_STATS
36 : PyStats _py_stats_struct = { 0 };
37 : PyStats *_py_stats = &_py_stats_struct;
38 :
39 : #define ADD_STAT_TO_DICT(res, field) \
40 : do { \
41 : PyObject *val = PyLong_FromUnsignedLongLong(stats->field); \
42 : if (val == NULL) { \
43 : Py_DECREF(res); \
44 : return NULL; \
45 : } \
46 : if (PyDict_SetItemString(res, #field, val) == -1) { \
47 : Py_DECREF(res); \
48 : Py_DECREF(val); \
49 : return NULL; \
50 : } \
51 : Py_DECREF(val); \
52 : } while(0);
53 :
54 : static PyObject*
55 : stats_to_dict(SpecializationStats *stats)
56 : {
57 : PyObject *res = PyDict_New();
58 : if (res == NULL) {
59 : return NULL;
60 : }
61 : ADD_STAT_TO_DICT(res, success);
62 : ADD_STAT_TO_DICT(res, failure);
63 : ADD_STAT_TO_DICT(res, hit);
64 : ADD_STAT_TO_DICT(res, deferred);
65 : ADD_STAT_TO_DICT(res, miss);
66 : ADD_STAT_TO_DICT(res, deopt);
67 : PyObject *failure_kinds = PyTuple_New(SPECIALIZATION_FAILURE_KINDS);
68 : if (failure_kinds == NULL) {
69 : Py_DECREF(res);
70 : return NULL;
71 : }
72 : for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) {
73 : PyObject *stat = PyLong_FromUnsignedLongLong(stats->failure_kinds[i]);
74 : if (stat == NULL) {
75 : Py_DECREF(res);
76 : Py_DECREF(failure_kinds);
77 : return NULL;
78 : }
79 : PyTuple_SET_ITEM(failure_kinds, i, stat);
80 : }
81 : if (PyDict_SetItemString(res, "failure_kinds", failure_kinds)) {
82 : Py_DECREF(res);
83 : Py_DECREF(failure_kinds);
84 : return NULL;
85 : }
86 : Py_DECREF(failure_kinds);
87 : return res;
88 : }
89 : #undef ADD_STAT_TO_DICT
90 :
91 : static int
92 : add_stat_dict(
93 : PyObject *res,
94 : int opcode,
95 : const char *name) {
96 :
97 : SpecializationStats *stats = &_py_stats_struct.opcode_stats[opcode].specialization;
98 : PyObject *d = stats_to_dict(stats);
99 : if (d == NULL) {
100 : return -1;
101 : }
102 : int err = PyDict_SetItemString(res, name, d);
103 : Py_DECREF(d);
104 : return err;
105 : }
106 :
107 : #ifdef Py_STATS
108 : PyObject*
109 : _Py_GetSpecializationStats(void) {
110 : PyObject *stats = PyDict_New();
111 : if (stats == NULL) {
112 : return NULL;
113 : }
114 : int err = 0;
115 : err += add_stat_dict(stats, LOAD_ATTR, "load_attr");
116 : err += add_stat_dict(stats, LOAD_GLOBAL, "load_global");
117 : err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr");
118 : err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr");
119 : err += add_stat_dict(stats, STORE_ATTR, "store_attr");
120 : err += add_stat_dict(stats, CALL, "call");
121 : err += add_stat_dict(stats, BINARY_OP, "binary_op");
122 : err += add_stat_dict(stats, COMPARE_OP, "compare_op");
123 : err += add_stat_dict(stats, UNPACK_SEQUENCE, "unpack_sequence");
124 : if (err < 0) {
125 : Py_DECREF(stats);
126 : return NULL;
127 : }
128 : return stats;
129 : }
130 : #endif
131 :
132 :
133 : #define PRINT_STAT(i, field) \
134 : if (stats[i].field) { \
135 : fprintf(out, " opcode[%d]." #field " : %" PRIu64 "\n", i, stats[i].field); \
136 : }
137 :
138 : static void
139 : print_spec_stats(FILE *out, OpcodeStats *stats)
140 : {
141 : /* Mark some opcodes as specializable for stats,
142 : * even though we don't specialize them yet. */
143 : fprintf(out, "opcode[%d].specializable : 1\n", BINARY_SLICE);
144 : fprintf(out, "opcode[%d].specializable : 1\n", STORE_SLICE);
145 : for (int i = 0; i < 256; i++) {
146 : if (_PyOpcode_Adaptive[i]) {
147 : fprintf(out, "opcode[%d].specializable : 1\n", i);
148 : }
149 : PRINT_STAT(i, specialization.success);
150 : PRINT_STAT(i, specialization.failure);
151 : PRINT_STAT(i, specialization.hit);
152 : PRINT_STAT(i, specialization.deferred);
153 : PRINT_STAT(i, specialization.miss);
154 : PRINT_STAT(i, specialization.deopt);
155 : PRINT_STAT(i, execution_count);
156 : for (int j = 0; j < SPECIALIZATION_FAILURE_KINDS; j++) {
157 : uint64_t val = stats[i].specialization.failure_kinds[j];
158 : if (val) {
159 : fprintf(out, " opcode[%d].specialization.failure_kinds[%d] : %"
160 : PRIu64 "\n", i, j, val);
161 : }
162 : }
163 : for(int j = 0; j < 256; j++) {
164 : if (stats[i].pair_count[j]) {
165 : fprintf(out, "opcode[%d].pair_count[%d] : %" PRIu64 "\n",
166 : i, j, stats[i].pair_count[j]);
167 : }
168 : }
169 : }
170 : }
171 : #undef PRINT_STAT
172 :
173 :
174 : static void
175 : print_call_stats(FILE *out, CallStats *stats)
176 : {
177 : fprintf(out, "Calls to PyEval_EvalDefault: %" PRIu64 "\n", stats->pyeval_calls);
178 : fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls);
179 : fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed);
180 : fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created);
181 : for (int i = 0; i < EVAL_CALL_KINDS; i++) {
182 : fprintf(out, "Calls via PyEval_EvalFrame[%d] : %" PRIu64 "\n", i, stats->eval_calls[i]);
183 : }
184 : }
185 :
186 : static void
187 : print_object_stats(FILE *out, ObjectStats *stats)
188 : {
189 : fprintf(out, "Object allocations from freelist: %" PRIu64 "\n", stats->from_freelist);
190 : fprintf(out, "Object frees to freelist: %" PRIu64 "\n", stats->to_freelist);
191 : fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations);
192 : fprintf(out, "Object allocations to 512 bytes: %" PRIu64 "\n", stats->allocations512);
193 : fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k);
194 : fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big);
195 : fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);
196 : fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values);
197 : fprintf(out, "Object interpreter increfs: %" PRIu64 "\n", stats->interpreter_increfs);
198 : fprintf(out, "Object interpreter decrefs: %" PRIu64 "\n", stats->interpreter_decrefs);
199 : fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs);
200 : fprintf(out, "Object decrefs: %" PRIu64 "\n", stats->decrefs);
201 : fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request);
202 : fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);
203 : fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);
204 : fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass);
205 : }
206 :
207 : static void
208 : print_stats(FILE *out, PyStats *stats) {
209 : print_spec_stats(out, stats->opcode_stats);
210 : print_call_stats(out, &stats->call_stats);
211 : print_object_stats(out, &stats->object_stats);
212 : }
213 :
214 : void
215 : _Py_StatsClear(void)
216 : {
217 : _py_stats_struct = (PyStats) { 0 };
218 : }
219 :
220 : void
221 : _Py_PrintSpecializationStats(int to_file)
222 : {
223 : if (_py_stats == NULL) {
224 : return;
225 : }
226 : FILE *out = stderr;
227 : if (to_file) {
228 : /* Write to a file instead of stderr. */
229 : # ifdef MS_WINDOWS
230 : const char *dirname = "c:\\temp\\py_stats\\";
231 : # else
232 : const char *dirname = "/tmp/py_stats/";
233 : # endif
234 : /* Use random 160 bit number as file name,
235 : * to avoid both accidental collisions and
236 : * symlink attacks. */
237 : unsigned char rand[20];
238 : char hex_name[41];
239 : _PyOS_URandomNonblock(rand, 20);
240 : for (int i = 0; i < 20; i++) {
241 : hex_name[2*i] = "0123456789abcdef"[rand[i]&15];
242 : hex_name[2*i+1] = "0123456789abcdef"[(rand[i]>>4)&15];
243 : }
244 : hex_name[40] = '\0';
245 : char buf[64];
246 : assert(strlen(dirname) + 40 + strlen(".txt") < 64);
247 : sprintf(buf, "%s%s.txt", dirname, hex_name);
248 : FILE *fout = fopen(buf, "w");
249 : if (fout) {
250 : out = fout;
251 : }
252 : }
253 : else {
254 : fprintf(out, "Specialization stats:\n");
255 : }
256 : print_stats(out, _py_stats);
257 : if (out != stderr) {
258 : fclose(out);
259 : }
260 : }
261 :
262 : #ifdef Py_STATS
263 :
264 : #define SPECIALIZATION_FAIL(opcode, kind) \
265 : do { \
266 : if (_py_stats) { \
267 : _py_stats->opcode_stats[opcode].specialization.failure_kinds[kind]++; \
268 : } \
269 : } while (0)
270 :
271 : #endif
272 : #endif
273 :
274 : #ifndef SPECIALIZATION_FAIL
275 : #define SPECIALIZATION_FAIL(opcode, kind) ((void)0)
276 : #endif
277 :
278 : // Insert adaptive instructions and superinstructions. This cannot fail.
279 : void
280 617445 : _PyCode_Quicken(PyCodeObject *code)
281 : {
282 617445 : _Py_QuickenedCount++;
283 617445 : int previous_opcode = -1;
284 617445 : _Py_CODEUNIT *instructions = _PyCode_CODE(code);
285 48783900 : for (int i = 0; i < Py_SIZE(code); i++) {
286 48166400 : int opcode = _Py_OPCODE(instructions[i]);
287 48166400 : uint8_t adaptive_opcode = _PyOpcode_Adaptive[opcode];
288 48166400 : if (adaptive_opcode) {
289 11792800 : _Py_SET_OPCODE(instructions[i], adaptive_opcode);
290 : // Make sure the adaptive counter is zero:
291 11792800 : assert(instructions[i + 1] == 0);
292 11792800 : previous_opcode = -1;
293 11792800 : i += _PyOpcode_Caches[opcode];
294 : }
295 : else {
296 36373600 : assert(!_PyOpcode_Caches[opcode]);
297 36373600 : switch (opcode) {
298 554438 : case JUMP_BACKWARD:
299 554438 : _Py_SET_OPCODE(instructions[i], JUMP_BACKWARD_QUICK);
300 554438 : break;
301 648526 : case RESUME:
302 648526 : _Py_SET_OPCODE(instructions[i], RESUME_QUICK);
303 648526 : break;
304 8014580 : case LOAD_FAST:
305 : switch(previous_opcode) {
306 1461320 : case LOAD_FAST:
307 1461320 : _Py_SET_OPCODE(instructions[i - 1],
308 : LOAD_FAST__LOAD_FAST);
309 1461320 : break;
310 927382 : case STORE_FAST:
311 927382 : _Py_SET_OPCODE(instructions[i - 1],
312 : STORE_FAST__LOAD_FAST);
313 927382 : break;
314 451468 : case LOAD_CONST:
315 451468 : _Py_SET_OPCODE(instructions[i - 1],
316 : LOAD_CONST__LOAD_FAST);
317 451468 : break;
318 : }
319 8014580 : break;
320 2098320 : case STORE_FAST:
321 2098320 : if (previous_opcode == STORE_FAST) {
322 183821 : _Py_SET_OPCODE(instructions[i - 1],
323 : STORE_FAST__STORE_FAST);
324 : }
325 2098320 : break;
326 7440320 : case LOAD_CONST:
327 7440320 : if (previous_opcode == LOAD_FAST) {
328 783369 : _Py_SET_OPCODE(instructions[i - 1],
329 : LOAD_FAST__LOAD_CONST);
330 : }
331 7440320 : break;
332 : }
333 36373600 : previous_opcode = opcode;
334 : }
335 : }
336 617445 : }
337 :
338 : static inline int
339 6918700 : miss_counter_start(void) {
340 : /* Starting value for the counter.
341 : * This value needs to be not too low, otherwise
342 : * it would cause excessive de-optimization.
343 : * Neither should it be too high, or that would delay
344 : * de-optimization excessively when it is needed.
345 : * A value around 50 seems to work, and we choose a
346 : * prime number to avoid artifacts.
347 : */
348 6918700 : return 53;
349 : }
350 :
351 : #define SIMPLE_FUNCTION 0
352 :
353 : /* Common */
354 :
355 : #define SPEC_FAIL_OTHER 0
356 : #define SPEC_FAIL_NO_DICT 1
357 : #define SPEC_FAIL_OVERRIDDEN 2
358 : #define SPEC_FAIL_OUT_OF_VERSIONS 3
359 : #define SPEC_FAIL_OUT_OF_RANGE 4
360 : #define SPEC_FAIL_EXPECTED_ERROR 5
361 : #define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS 6
362 :
363 : #define SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT 18
364 :
365 : /* Attributes */
366 :
367 : #define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR 8
368 : #define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR 9
369 : #define SPEC_FAIL_ATTR_NOT_DESCRIPTOR 10
370 : #define SPEC_FAIL_ATTR_METHOD 11
371 : #define SPEC_FAIL_ATTR_MUTABLE_CLASS 12
372 : #define SPEC_FAIL_ATTR_PROPERTY 13
373 : #define SPEC_FAIL_ATTR_NON_OBJECT_SLOT 14
374 : #define SPEC_FAIL_ATTR_READ_ONLY 15
375 : #define SPEC_FAIL_ATTR_AUDITED_SLOT 16
376 : #define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 17
377 : #define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 18
378 : #define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 19
379 :
380 : #define SPEC_FAIL_ATTR_SHADOWED 21
381 : #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22
382 : #define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23
383 : #define SPEC_FAIL_ATTR_OBJECT_SLOT 24
384 : #define SPEC_FAIL_ATTR_HAS_MANAGED_DICT 25
385 : #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26
386 : #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27
387 :
388 : /* Binary subscr and store subscr */
389 :
390 : #define SPEC_FAIL_SUBSCR_ARRAY_INT 8
391 : #define SPEC_FAIL_SUBSCR_ARRAY_SLICE 9
392 : #define SPEC_FAIL_SUBSCR_LIST_SLICE 10
393 : #define SPEC_FAIL_SUBSCR_TUPLE_SLICE 11
394 : #define SPEC_FAIL_SUBSCR_STRING_INT 12
395 : #define SPEC_FAIL_SUBSCR_STRING_SLICE 13
396 : #define SPEC_FAIL_SUBSCR_BUFFER_INT 15
397 : #define SPEC_FAIL_SUBSCR_BUFFER_SLICE 16
398 : #define SPEC_FAIL_SUBSCR_SEQUENCE_INT 17
399 :
400 : /* Store subscr */
401 : #define SPEC_FAIL_SUBSCR_BYTEARRAY_INT 18
402 : #define SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE 19
403 : #define SPEC_FAIL_SUBSCR_PY_SIMPLE 20
404 : #define SPEC_FAIL_SUBSCR_PY_OTHER 21
405 : #define SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE 22
406 : #define SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE 23
407 :
408 : /* Binary op */
409 :
410 : #define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES 8
411 : #define SPEC_FAIL_BINARY_OP_ADD_OTHER 9
412 : #define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES 10
413 : #define SPEC_FAIL_BINARY_OP_AND_INT 11
414 : #define SPEC_FAIL_BINARY_OP_AND_OTHER 12
415 : #define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE 13
416 : #define SPEC_FAIL_BINARY_OP_LSHIFT 14
417 : #define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY 15
418 : #define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES 16
419 : #define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER 17
420 : #define SPEC_FAIL_BINARY_OP_OR 18
421 : #define SPEC_FAIL_BINARY_OP_POWER 19
422 : #define SPEC_FAIL_BINARY_OP_REMAINDER 20
423 : #define SPEC_FAIL_BINARY_OP_RSHIFT 21
424 : #define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES 22
425 : #define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER 23
426 : #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 24
427 : #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT 25
428 : #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER 26
429 : #define SPEC_FAIL_BINARY_OP_XOR 27
430 :
431 : /* Calls */
432 : #define SPEC_FAIL_CALL_COMPLEX_PARAMETERS 9
433 : #define SPEC_FAIL_CALL_CO_NOT_OPTIMIZED 10
434 : /* SPEC_FAIL_METHOD defined as 11 above */
435 :
436 : #define SPEC_FAIL_CALL_INSTANCE_METHOD 11
437 : #define SPEC_FAIL_CALL_CMETHOD 12
438 : #define SPEC_FAIL_CALL_PYCFUNCTION 13
439 : #define SPEC_FAIL_CALL_PYCFUNCTION_WITH_KEYWORDS 14
440 : #define SPEC_FAIL_CALL_PYCFUNCTION_FAST_WITH_KEYWORDS 15
441 : #define SPEC_FAIL_CALL_PYCFUNCTION_NOARGS 16
442 : #define SPEC_FAIL_CALL_BAD_CALL_FLAGS 17
443 : #define SPEC_FAIL_CALL_CLASS 18
444 : #define SPEC_FAIL_CALL_PYTHON_CLASS 19
445 : #define SPEC_FAIL_CALL_METHOD_DESCRIPTOR 20
446 : #define SPEC_FAIL_CALL_BOUND_METHOD 21
447 : #define SPEC_FAIL_CALL_STR 22
448 : #define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 23
449 : #define SPEC_FAIL_CALL_CLASS_MUTABLE 24
450 : #define SPEC_FAIL_CALL_KWNAMES 25
451 : #define SPEC_FAIL_CALL_METHOD_WRAPPER 26
452 : #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 27
453 : #define SPEC_FAIL_CALL_PYFUNCTION 28
454 : #define SPEC_FAIL_CALL_PEP_523 29
455 :
456 : /* COMPARE_OP */
457 : #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12
458 : #define SPEC_FAIL_COMPARE_OP_STRING 13
459 : #define SPEC_FAIL_COMPARE_OP_NOT_FOLLOWED_BY_COND_JUMP 14
460 : #define SPEC_FAIL_COMPARE_OP_BIG_INT 15
461 : #define SPEC_FAIL_COMPARE_OP_BYTES 16
462 : #define SPEC_FAIL_COMPARE_OP_TUPLE 17
463 : #define SPEC_FAIL_COMPARE_OP_LIST 18
464 : #define SPEC_FAIL_COMPARE_OP_SET 19
465 : #define SPEC_FAIL_COMPARE_OP_BOOL 20
466 : #define SPEC_FAIL_COMPARE_OP_BASEOBJECT 21
467 : #define SPEC_FAIL_COMPARE_OP_FLOAT_LONG 22
468 : #define SPEC_FAIL_COMPARE_OP_LONG_FLOAT 23
469 : #define SPEC_FAIL_COMPARE_OP_EXTENDED_ARG 24
470 :
471 : /* FOR_ITER */
472 : #define SPEC_FAIL_FOR_ITER_GENERATOR 10
473 : #define SPEC_FAIL_FOR_ITER_COROUTINE 11
474 : #define SPEC_FAIL_FOR_ITER_ASYNC_GENERATOR 12
475 : #define SPEC_FAIL_FOR_ITER_LIST 13
476 : #define SPEC_FAIL_FOR_ITER_TUPLE 14
477 : #define SPEC_FAIL_FOR_ITER_SET 15
478 : #define SPEC_FAIL_FOR_ITER_STRING 16
479 : #define SPEC_FAIL_FOR_ITER_BYTES 17
480 : #define SPEC_FAIL_FOR_ITER_RANGE 18
481 : #define SPEC_FAIL_FOR_ITER_ITERTOOLS 19
482 : #define SPEC_FAIL_FOR_ITER_DICT_KEYS 20
483 : #define SPEC_FAIL_FOR_ITER_DICT_ITEMS 21
484 : #define SPEC_FAIL_FOR_ITER_DICT_VALUES 22
485 : #define SPEC_FAIL_FOR_ITER_ENUMERATE 23
486 : #define SPEC_FAIL_FOR_ITER_MAP 24
487 : #define SPEC_FAIL_FOR_ITER_ZIP 25
488 : #define SPEC_FAIL_FOR_ITER_SEQ_ITER 26
489 : #define SPEC_FAIL_FOR_ITER_REVERSED_LIST 27
490 : #define SPEC_FAIL_FOR_ITER_CALLABLE 28
491 : #define SPEC_FAIL_FOR_ITER_ASCII_STRING 29
492 :
493 : // UNPACK_SEQUENCE
494 :
495 : #define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 8
496 : #define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 9
497 :
498 :
499 : static int
500 382953 : specialize_module_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
501 : PyObject *name, int opcode, int opcode_module)
502 : {
503 382953 : _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
504 382953 : PyModuleObject *m = (PyModuleObject *)owner;
505 382953 : PyObject *value = NULL;
506 382953 : assert((owner->ob_type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
507 382953 : PyDictObject *dict = (PyDictObject *)m->md_dict;
508 382953 : if (dict == NULL) {
509 : SPECIALIZATION_FAIL(opcode, SPEC_FAIL_NO_DICT);
510 0 : return -1;
511 : }
512 382953 : if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
513 : SPECIALIZATION_FAIL(opcode, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT);
514 0 : return -1;
515 : }
516 382953 : Py_ssize_t index = _PyDict_GetItemHint(dict, &_Py_ID(__getattr__), -1,
517 : &value);
518 382953 : assert(index != DKIX_ERROR);
519 382953 : if (index != DKIX_EMPTY) {
520 : SPECIALIZATION_FAIL(opcode, SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND);
521 5376 : return -1;
522 : }
523 377577 : index = _PyDict_GetItemHint(dict, name, -1, &value);
524 377577 : assert (index != DKIX_ERROR);
525 377577 : if (index != (uint16_t)index) {
526 : SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_RANGE);
527 36658 : return -1;
528 : }
529 340919 : uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(dict->ma_keys);
530 340919 : if (keys_version == 0) {
531 : SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS);
532 0 : return -1;
533 : }
534 340919 : write_u32(cache->version, keys_version);
535 340919 : cache->index = (uint16_t)index;
536 340919 : _Py_SET_OPCODE(*instr, opcode_module);
537 340919 : return 0;
538 : }
539 :
540 :
541 :
542 : /* Attribute specialization */
543 :
544 : typedef enum {
545 : OVERRIDING, /* Is an overriding descriptor, and will remain so. */
546 : METHOD, /* Attribute has Py_TPFLAGS_METHOD_DESCRIPTOR set */
547 : PROPERTY, /* Is a property */
548 : OBJECT_SLOT, /* Is an object slot descriptor */
549 : OTHER_SLOT, /* Is a slot descriptor of another type */
550 : NON_OVERRIDING, /* Is another non-overriding descriptor, and is an instance of an immutable class*/
551 : BUILTIN_CLASSMETHOD, /* Builtin methods with METH_CLASS */
552 : PYTHON_CLASSMETHOD, /* Python classmethod(func) object */
553 : NON_DESCRIPTOR, /* Is not a descriptor, and is an instance of an immutable class */
554 : MUTABLE, /* Instance of a mutable class; might, or might not, be a descriptor */
555 : ABSENT, /* Attribute is not present on the class */
556 : DUNDER_CLASS, /* __class__ attribute */
557 : GETSET_OVERRIDDEN /* __getattribute__ or __setattr__ has been overridden */
558 : } DescriptorClassification;
559 :
560 :
561 : static DescriptorClassification
562 3148650 : analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store)
563 : {
564 3148650 : if (store) {
565 609018 : if (type->tp_setattro != PyObject_GenericSetAttr) {
566 101742 : *descr = NULL;
567 101742 : return GETSET_OVERRIDDEN;
568 : }
569 : }
570 : else {
571 2539640 : if (type->tp_getattro != PyObject_GenericGetAttr) {
572 189917 : *descr = NULL;
573 189917 : return GETSET_OVERRIDDEN;
574 : }
575 : }
576 2856990 : PyObject *descriptor = _PyType_Lookup(type, name);
577 2856990 : *descr = descriptor;
578 2856990 : if (descriptor == NULL) {
579 1078360 : return ABSENT;
580 : }
581 1778630 : PyTypeObject *desc_cls = Py_TYPE(descriptor);
582 1778630 : if (!(desc_cls->tp_flags & Py_TPFLAGS_IMMUTABLETYPE)) {
583 46269 : return MUTABLE;
584 : }
585 1732360 : if (desc_cls->tp_descr_set) {
586 444303 : if (desc_cls == &PyMemberDescr_Type) {
587 203577 : PyMemberDescrObject *member = (PyMemberDescrObject *)descriptor;
588 203577 : struct PyMemberDef *dmem = member->d_member;
589 203577 : if (dmem->type == T_OBJECT_EX) {
590 22942 : return OBJECT_SLOT;
591 : }
592 180635 : return OTHER_SLOT;
593 : }
594 240726 : if (desc_cls == &PyProperty_Type) {
595 34482 : return PROPERTY;
596 : }
597 206244 : if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) {
598 52289 : if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) {
599 52289 : return DUNDER_CLASS;
600 : }
601 : }
602 153955 : if (store) {
603 23725 : return OVERRIDING;
604 : }
605 : }
606 1418290 : if (desc_cls->tp_descr_get) {
607 1093960 : if (desc_cls->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR) {
608 818009 : return METHOD;
609 : }
610 275952 : if (Py_IS_TYPE(descriptor, &PyClassMethodDescr_Type)) {
611 29343 : return BUILTIN_CLASSMETHOD;
612 : }
613 246609 : if (Py_IS_TYPE(descriptor, &PyClassMethod_Type)) {
614 73090 : return PYTHON_CLASSMETHOD;
615 : }
616 173519 : return NON_OVERRIDING;
617 : }
618 324327 : return NON_DESCRIPTOR;
619 : }
620 :
621 : static int
622 1196730 : specialize_dict_access(
623 : PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
624 : DescriptorClassification kind, PyObject *name,
625 : int base_op, int values_op, int hint_op)
626 : {
627 1196730 : assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT ||
628 : kind == BUILTIN_CLASSMETHOD || kind == PYTHON_CLASSMETHOD);
629 : // No descriptor, or non overriding.
630 1196730 : if ((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
631 : SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
632 275981 : return 0;
633 : }
634 920751 : _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
635 920751 : PyObject **dictptr = _PyObject_ManagedDictPointer(owner);
636 920751 : PyDictObject *dict = (PyDictObject *)*dictptr;
637 920751 : if (dict == NULL) {
638 : // Virtual dictionary
639 773717 : PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
640 773717 : assert(PyUnicode_CheckExact(name));
641 773717 : Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
642 773717 : assert (index != DKIX_ERROR);
643 773717 : if (index != (uint16_t)index) {
644 : SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
645 52146 : return 0;
646 : }
647 721571 : write_u32(cache->version, type->tp_version_tag);
648 721571 : cache->index = (uint16_t)index;
649 721571 : _Py_SET_OPCODE(*instr, values_op);
650 : }
651 : else {
652 147034 : if (!PyDict_CheckExact(dict)) {
653 : SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
654 12249 : return 0;
655 : }
656 : // We found an instance with a __dict__.
657 147034 : PyObject *value = NULL;
658 : Py_ssize_t hint =
659 147034 : _PyDict_GetItemHint(dict, name, -1, &value);
660 147034 : if (hint != (uint16_t)hint) {
661 : SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
662 12249 : return 0;
663 : }
664 134785 : cache->index = (uint16_t)hint;
665 134785 : write_u32(cache->version, type->tp_version_tag);
666 134785 : _Py_SET_OPCODE(*instr, hint_op);
667 : }
668 856356 : return 1;
669 : }
670 :
671 : static int specialize_attr_loadmethod(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name,
672 : PyObject* descr, DescriptorClassification kind);
673 : static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name);
674 : static int function_kind(PyCodeObject *code);
675 :
676 : int
677 2922590 : _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
678 : {
679 2922590 : assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR);
680 2922590 : _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
681 2922590 : if (PyModule_CheckExact(owner)) {
682 382953 : int err = specialize_module_load_attr(owner, instr, name, LOAD_ATTR,
683 : LOAD_ATTR_MODULE);
684 382953 : if (err) {
685 42034 : goto fail;
686 : }
687 340919 : goto success;
688 : }
689 2539640 : if (PyType_Check(owner)) {
690 563131 : int err = specialize_class_load_attr(owner, instr, name);
691 563131 : if (err) {
692 359801 : goto fail;
693 : }
694 203330 : goto success;
695 : }
696 1976500 : PyTypeObject *type = Py_TYPE(owner);
697 1976500 : if (type->tp_dict == NULL) {
698 0 : if (PyType_Ready(type) < 0) {
699 0 : return -1;
700 : }
701 : }
702 1976500 : PyObject *descr = NULL;
703 1976500 : DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0);
704 1976500 : assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
705 1976500 : switch(kind) {
706 0 : case OVERRIDING:
707 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
708 0 : goto fail;
709 776576 : case METHOD:
710 : {
711 776576 : int oparg = _Py_OPARG(*instr);
712 776576 : if (oparg & 1) {
713 628015 : if (specialize_attr_loadmethod(owner, instr, name, descr, kind)) {
714 566177 : goto success;
715 : }
716 : }
717 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
718 210399 : goto fail;
719 : }
720 27281 : case PROPERTY:
721 : {
722 27281 : _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1);
723 27281 : assert(Py_TYPE(descr) == &PyProperty_Type);
724 27281 : PyObject *fget = ((_PyPropertyObject *)descr)->prop_get;
725 27281 : if (fget == NULL) {
726 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
727 0 : goto fail;
728 : }
729 27281 : if (Py_TYPE(fget) != &PyFunction_Type) {
730 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY);
731 20 : goto fail;
732 : }
733 27261 : PyFunctionObject *func = (PyFunctionObject *)fget;
734 27261 : PyCodeObject *fcode = (PyCodeObject *)func->func_code;
735 27261 : int kind = function_kind(fcode);
736 27261 : if (kind != SIMPLE_FUNCTION) {
737 : SPECIALIZATION_FAIL(LOAD_ATTR, kind);
738 0 : goto fail;
739 : }
740 27261 : if (fcode->co_argcount != 1) {
741 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
742 0 : goto fail;
743 : }
744 27261 : int version = _PyFunction_GetVersionForCurrentState(func);
745 27261 : if (version == 0) {
746 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
747 0 : goto fail;
748 : }
749 27261 : write_u32(lm_cache->keys_version, version);
750 27261 : assert(type->tp_version_tag != 0);
751 27261 : write_u32(lm_cache->type_version, type->tp_version_tag);
752 : /* borrowed */
753 27261 : write_obj(lm_cache->descr, fget);
754 27261 : _Py_SET_OPCODE(*instr, LOAD_ATTR_PROPERTY);
755 27261 : goto success;
756 : }
757 11466 : case OBJECT_SLOT:
758 : {
759 11466 : PyMemberDescrObject *member = (PyMemberDescrObject *)descr;
760 11466 : struct PyMemberDef *dmem = member->d_member;
761 11466 : Py_ssize_t offset = dmem->offset;
762 11466 : if (dmem->flags & PY_AUDIT_READ) {
763 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_AUDITED_SLOT);
764 0 : goto fail;
765 : }
766 11466 : if (offset != (uint16_t)offset) {
767 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
768 0 : goto fail;
769 : }
770 11466 : assert(dmem->type == T_OBJECT_EX);
771 11466 : assert(offset > 0);
772 11466 : cache->index = (uint16_t)offset;
773 11466 : write_u32(cache->version, type->tp_version_tag);
774 11466 : _Py_SET_OPCODE(*instr, LOAD_ATTR_SLOT);
775 11466 : goto success;
776 : }
777 52180 : case DUNDER_CLASS:
778 : {
779 52180 : Py_ssize_t offset = offsetof(PyObject, ob_type);
780 52180 : assert(offset == (uint16_t)offset);
781 52180 : cache->index = (uint16_t)offset;
782 52180 : write_u32(cache->version, type->tp_version_tag);
783 52180 : _Py_SET_OPCODE(*instr, LOAD_ATTR_SLOT);
784 52180 : goto success;
785 : }
786 148612 : case OTHER_SLOT:
787 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT);
788 148612 : goto fail;
789 26657 : case MUTABLE:
790 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);
791 26657 : goto fail;
792 159255 : case GETSET_OVERRIDDEN:
793 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OVERRIDDEN);
794 159255 : goto fail;
795 774477 : case BUILTIN_CLASSMETHOD:
796 : case PYTHON_CLASSMETHOD:
797 : case NON_OVERRIDING:
798 : case NON_DESCRIPTOR:
799 : case ABSENT:
800 774477 : break;
801 : }
802 774477 : int err = specialize_dict_access(
803 : owner, instr, type, kind, name,
804 : LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT
805 : );
806 774477 : if (err < 0) {
807 0 : return -1;
808 : }
809 774477 : if (err) {
810 541699 : goto success;
811 : }
812 232778 : fail:
813 : STAT_INC(LOAD_ATTR, failure);
814 1179560 : assert(!PyErr_Occurred());
815 1179560 : cache->counter = adaptive_counter_backoff(cache->counter);
816 1179560 : return 0;
817 1743030 : success:
818 : STAT_INC(LOAD_ATTR, success);
819 1743030 : assert(!PyErr_Occurred());
820 1743030 : cache->counter = miss_counter_start();
821 1743030 : return 0;
822 : }
823 :
824 : int
825 704814 : _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
826 : {
827 704814 : assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR);
828 704814 : _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
829 704814 : PyTypeObject *type = Py_TYPE(owner);
830 704814 : if (PyModule_CheckExact(owner)) {
831 : SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
832 95796 : goto fail;
833 : }
834 : PyObject *descr;
835 609018 : DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1);
836 609018 : switch(kind) {
837 23725 : case OVERRIDING:
838 : SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
839 23725 : goto fail;
840 894 : case METHOD:
841 : SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_METHOD);
842 894 : goto fail;
843 7201 : case PROPERTY:
844 : SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_PROPERTY);
845 7201 : goto fail;
846 11476 : case OBJECT_SLOT:
847 : {
848 11476 : PyMemberDescrObject *member = (PyMemberDescrObject *)descr;
849 11476 : struct PyMemberDef *dmem = member->d_member;
850 11476 : Py_ssize_t offset = dmem->offset;
851 11476 : if (dmem->flags & READONLY) {
852 : SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_READ_ONLY);
853 12 : goto fail;
854 : }
855 11464 : if (offset != (uint16_t)offset) {
856 : SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OUT_OF_RANGE);
857 0 : goto fail;
858 : }
859 11464 : assert(dmem->type == T_OBJECT_EX);
860 11464 : assert(offset > 0);
861 11464 : cache->index = (uint16_t)offset;
862 11464 : write_u32(cache->version, type->tp_version_tag);
863 11464 : _Py_SET_OPCODE(*instr, STORE_ATTR_SLOT);
864 11464 : goto success;
865 : }
866 32068 : case DUNDER_CLASS:
867 : case OTHER_SLOT:
868 : SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT);
869 32068 : goto fail;
870 9657 : case MUTABLE:
871 : SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);
872 9657 : goto fail;
873 101742 : case GETSET_OVERRIDDEN:
874 : SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
875 101742 : goto fail;
876 422255 : case BUILTIN_CLASSMETHOD:
877 : case PYTHON_CLASSMETHOD:
878 : case NON_OVERRIDING:
879 : case NON_DESCRIPTOR:
880 : case ABSENT:
881 422255 : break;
882 : }
883 :
884 422255 : int err = specialize_dict_access(
885 : owner, instr, type, kind, name,
886 : STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT
887 : );
888 422255 : if (err < 0) {
889 0 : return -1;
890 : }
891 422255 : if (err) {
892 314657 : goto success;
893 : }
894 107598 : fail:
895 : STAT_INC(STORE_ATTR, failure);
896 378693 : assert(!PyErr_Occurred());
897 378693 : cache->counter = adaptive_counter_backoff(cache->counter);
898 378693 : return 0;
899 326121 : success:
900 : STAT_INC(STORE_ATTR, success);
901 326121 : assert(!PyErr_Occurred());
902 326121 : cache->counter = miss_counter_start();
903 326121 : return 0;
904 : }
905 :
906 :
907 : #ifdef Py_STATS
908 : static int
909 : load_attr_fail_kind(DescriptorClassification kind)
910 : {
911 : switch (kind) {
912 : case OVERRIDING:
913 : return SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR;
914 : case METHOD:
915 : return SPEC_FAIL_ATTR_METHOD;
916 : case PROPERTY:
917 : return SPEC_FAIL_ATTR_PROPERTY;
918 : case OBJECT_SLOT:
919 : return SPEC_FAIL_ATTR_OBJECT_SLOT;
920 : case OTHER_SLOT:
921 : return SPEC_FAIL_ATTR_NON_OBJECT_SLOT;
922 : case DUNDER_CLASS:
923 : return SPEC_FAIL_OTHER;
924 : case MUTABLE:
925 : return SPEC_FAIL_ATTR_MUTABLE_CLASS;
926 : case GETSET_OVERRIDDEN:
927 : return SPEC_FAIL_OVERRIDDEN;
928 : case BUILTIN_CLASSMETHOD:
929 : return SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD;
930 : case PYTHON_CLASSMETHOD:
931 : return SPEC_FAIL_ATTR_CLASS_METHOD_OBJ;
932 : case NON_OVERRIDING:
933 : return SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR;
934 : case NON_DESCRIPTOR:
935 : return SPEC_FAIL_ATTR_NOT_DESCRIPTOR;
936 : case ABSENT:
937 : return SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE;
938 : }
939 : Py_UNREACHABLE();
940 : }
941 : #endif
942 :
943 : static int
944 563131 : specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
945 : PyObject *name)
946 : {
947 563131 : _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
948 563131 : PyObject *descr = NULL;
949 563131 : DescriptorClassification kind = 0;
950 563131 : kind = analyze_descriptor((PyTypeObject *)owner, name, &descr, 0);
951 563131 : switch (kind) {
952 203330 : case METHOD:
953 : case NON_DESCRIPTOR:
954 203330 : write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag);
955 203330 : write_obj(cache->descr, descr);
956 203330 : _Py_SET_OPCODE(*instr, LOAD_ATTR_CLASS);
957 203330 : return 0;
958 : #ifdef Py_STATS
959 : case ABSENT:
960 : if (_PyType_Lookup(Py_TYPE(owner), name) != NULL) {
961 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE);
962 : }
963 : else {
964 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
965 : }
966 : return -1;
967 : #endif
968 359801 : default:
969 : SPECIALIZATION_FAIL(LOAD_ATTR, load_attr_fail_kind(kind));
970 359801 : return -1;
971 : }
972 : }
973 :
974 : typedef enum {
975 : MANAGED_VALUES = 1,
976 : MANAGED_DICT = 2,
977 : OFFSET_DICT = 3,
978 : NO_DICT = 4,
979 : LAZY_DICT = 5,
980 : } ObjectDictKind;
981 :
982 : // Please collect stats carefully before and after modifying. A subtle change
983 : // can cause a significant drop in cache hits. A possible test is
984 : // python.exe -m test_typing test_re test_dis test_zlib.
985 : static int
986 628015 : specialize_attr_loadmethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
987 : PyObject *descr, DescriptorClassification kind)
988 : {
989 628015 : _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
990 628015 : PyTypeObject *owner_cls = Py_TYPE(owner);
991 :
992 628015 : assert(kind == METHOD && descr != NULL);
993 : ObjectDictKind dictkind;
994 : PyDictKeysObject *keys;
995 628015 : if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
996 276554 : PyObject *dict = *_PyObject_ManagedDictPointer(owner);
997 276554 : keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys;
998 276554 : if (dict == NULL) {
999 229413 : dictkind = MANAGED_VALUES;
1000 : }
1001 : else {
1002 47141 : dictkind = MANAGED_DICT;
1003 : }
1004 : }
1005 : else {
1006 351461 : Py_ssize_t dictoffset = owner_cls->tp_dictoffset;
1007 351461 : if (dictoffset < 0 || dictoffset > INT16_MAX) {
1008 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
1009 13731 : goto fail;
1010 : }
1011 337730 : if (dictoffset == 0) {
1012 325687 : dictkind = NO_DICT;
1013 325687 : keys = NULL;
1014 : }
1015 : else {
1016 12043 : PyObject *dict = *(PyObject **) ((char *)owner + dictoffset);
1017 12043 : if (dict == NULL) {
1018 : // This object will have a dict if user access __dict__
1019 8000 : dictkind = LAZY_DICT;
1020 8000 : keys = NULL;
1021 : }
1022 : else {
1023 4043 : keys = ((PyDictObject *)dict)->ma_keys;
1024 4043 : dictkind = OFFSET_DICT;
1025 : }
1026 : }
1027 : }
1028 614284 : if (dictkind == MANAGED_VALUES || dictkind == OFFSET_DICT) {
1029 233456 : Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
1030 233456 : if (index != DKIX_EMPTY) {
1031 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED);
1032 966 : goto fail;
1033 : }
1034 232490 : uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(keys);
1035 232490 : if (keys_version == 0) {
1036 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
1037 0 : goto fail;
1038 : }
1039 232490 : write_u32(cache->keys_version, keys_version);
1040 : }
1041 613318 : switch(dictkind) {
1042 325687 : case NO_DICT:
1043 325687 : _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_NO_DICT);
1044 325687 : break;
1045 228458 : case MANAGED_VALUES:
1046 228458 : _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_WITH_VALUES);
1047 228458 : break;
1048 47141 : case MANAGED_DICT:
1049 : SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT);
1050 47141 : goto fail;
1051 4032 : case OFFSET_DICT:
1052 4032 : assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX);
1053 4032 : _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_WITH_DICT);
1054 4032 : break;
1055 8000 : case LAZY_DICT:
1056 8000 : assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX);
1057 8000 : _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_LAZY_DICT);
1058 8000 : break;
1059 : }
1060 : /* `descr` is borrowed. This is safe for methods (even inherited ones from
1061 : * super classes!) as long as tp_version_tag is validated for two main reasons:
1062 : *
1063 : * 1. The class will always hold a reference to the method so it will
1064 : * usually not be GC-ed. Should it be deleted in Python, e.g.
1065 : * `del obj.meth`, tp_version_tag will be invalidated, because of reason 2.
1066 : *
1067 : * 2. The pre-existing type method cache (MCACHE) uses the same principles
1068 : * of caching a borrowed descriptor. The MCACHE infrastructure does all the
1069 : * heavy lifting for us. E.g. it invalidates tp_version_tag on any MRO
1070 : * modification, on any type object change along said MRO, etc. (see
1071 : * PyType_Modified usages in typeobject.c). The MCACHE has been
1072 : * working since Python 2.6 and it's battle-tested.
1073 : */
1074 566177 : write_u32(cache->type_version, owner_cls->tp_version_tag);
1075 566177 : write_obj(cache->descr, descr);
1076 : // Fall through.
1077 566177 : return 1;
1078 61838 : fail:
1079 61838 : return 0;
1080 : }
1081 :
1082 : int
1083 1942840 : _Py_Specialize_LoadGlobal(
1084 : PyObject *globals, PyObject *builtins,
1085 : _Py_CODEUNIT *instr, PyObject *name)
1086 : {
1087 1942840 : assert(_PyOpcode_Caches[LOAD_GLOBAL] == INLINE_CACHE_ENTRIES_LOAD_GLOBAL);
1088 : /* Use inline cache */
1089 1942840 : _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)(instr + 1);
1090 1942840 : assert(PyUnicode_CheckExact(name));
1091 1942840 : if (!PyDict_CheckExact(globals)) {
1092 4021 : goto fail;
1093 : }
1094 1938820 : PyDictKeysObject * globals_keys = ((PyDictObject *)globals)->ma_keys;
1095 1938820 : if (!DK_IS_UNICODE(globals_keys)) {
1096 : SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT);
1097 0 : goto fail;
1098 : }
1099 1938820 : Py_ssize_t index = _PyDictKeys_StringLookup(globals_keys, name);
1100 1938820 : if (index == DKIX_ERROR) {
1101 : SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT);
1102 0 : goto fail;
1103 : }
1104 1938820 : if (index != DKIX_EMPTY) {
1105 1175520 : if (index != (uint16_t)index) {
1106 0 : goto fail;
1107 : }
1108 1175520 : uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(globals_keys);
1109 1175520 : if (keys_version == 0) {
1110 0 : goto fail;
1111 : }
1112 1175520 : cache->index = (uint16_t)index;
1113 1175520 : write_u32(cache->module_keys_version, keys_version);
1114 1175520 : _Py_SET_OPCODE(*instr, LOAD_GLOBAL_MODULE);
1115 1175520 : goto success;
1116 : }
1117 763292 : if (!PyDict_CheckExact(builtins)) {
1118 0 : goto fail;
1119 : }
1120 763292 : PyDictKeysObject * builtin_keys = ((PyDictObject *)builtins)->ma_keys;
1121 763292 : if (!DK_IS_UNICODE(builtin_keys)) {
1122 : SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT);
1123 0 : goto fail;
1124 : }
1125 763292 : index = _PyDictKeys_StringLookup(builtin_keys, name);
1126 763292 : if (index == DKIX_ERROR) {
1127 : SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT);
1128 0 : goto fail;
1129 : }
1130 763292 : if (index != (uint16_t)index) {
1131 3 : goto fail;
1132 : }
1133 763289 : uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState(globals_keys);
1134 763289 : if (globals_version == 0) {
1135 : SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS);
1136 0 : goto fail;
1137 : }
1138 763289 : uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState(builtin_keys);
1139 763289 : if (builtins_version == 0) {
1140 : SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS);
1141 0 : goto fail;
1142 : }
1143 763289 : if (builtins_version > UINT16_MAX) {
1144 : SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
1145 0 : goto fail;
1146 : }
1147 763289 : cache->index = (uint16_t)index;
1148 763289 : write_u32(cache->module_keys_version, globals_version);
1149 763289 : cache->builtin_keys_version = (uint16_t)builtins_version;
1150 763289 : _Py_SET_OPCODE(*instr, LOAD_GLOBAL_BUILTIN);
1151 763289 : goto success;
1152 4024 : fail:
1153 : STAT_INC(LOAD_GLOBAL, failure);
1154 4024 : assert(!PyErr_Occurred());
1155 4024 : cache->counter = adaptive_counter_backoff(cache->counter);
1156 4024 : return 0;
1157 1938810 : success:
1158 : STAT_INC(LOAD_GLOBAL, success);
1159 1938810 : assert(!PyErr_Occurred());
1160 1938810 : cache->counter = miss_counter_start();
1161 1938810 : return 0;
1162 : }
1163 :
1164 : #ifdef Py_STATS
1165 : static int
1166 : binary_subscr_fail_kind(PyTypeObject *container_type, PyObject *sub)
1167 : {
1168 : if (container_type == &PyUnicode_Type) {
1169 : if (PyLong_CheckExact(sub)) {
1170 : return SPEC_FAIL_SUBSCR_STRING_INT;
1171 : }
1172 : if (PySlice_Check(sub)) {
1173 : return SPEC_FAIL_SUBSCR_STRING_SLICE;
1174 : }
1175 : return SPEC_FAIL_OTHER;
1176 : }
1177 : else if (strcmp(container_type->tp_name, "array.array") == 0) {
1178 : if (PyLong_CheckExact(sub)) {
1179 : return SPEC_FAIL_SUBSCR_ARRAY_INT;
1180 : }
1181 : if (PySlice_Check(sub)) {
1182 : return SPEC_FAIL_SUBSCR_ARRAY_SLICE;
1183 : }
1184 : return SPEC_FAIL_OTHER;
1185 : }
1186 : else if (container_type->tp_as_buffer) {
1187 : if (PyLong_CheckExact(sub)) {
1188 : return SPEC_FAIL_SUBSCR_BUFFER_INT;
1189 : }
1190 : if (PySlice_Check(sub)) {
1191 : return SPEC_FAIL_SUBSCR_BUFFER_SLICE;
1192 : }
1193 : return SPEC_FAIL_OTHER;
1194 : }
1195 : else if (container_type->tp_as_sequence) {
1196 : if (PyLong_CheckExact(sub) && container_type->tp_as_sequence->sq_item) {
1197 : return SPEC_FAIL_SUBSCR_SEQUENCE_INT;
1198 : }
1199 : }
1200 : return SPEC_FAIL_OTHER;
1201 : }
1202 : #endif
1203 :
1204 : static int
1205 1235460 : function_kind(PyCodeObject *code) {
1206 1235460 : int flags = code->co_flags;
1207 1235460 : if ((flags & (CO_VARKEYWORDS | CO_VARARGS)) || code->co_kwonlyargcount) {
1208 428007 : return SPEC_FAIL_CALL_COMPLEX_PARAMETERS;
1209 : }
1210 807450 : if ((flags & CO_OPTIMIZED) == 0) {
1211 9 : return SPEC_FAIL_CALL_CO_NOT_OPTIMIZED;
1212 : }
1213 807441 : return SIMPLE_FUNCTION;
1214 : }
1215 :
1216 : int
1217 599947 : _Py_Specialize_BinarySubscr(
1218 : PyObject *container, PyObject *sub, _Py_CODEUNIT *instr)
1219 : {
1220 599947 : assert(_PyOpcode_Caches[BINARY_SUBSCR] ==
1221 : INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
1222 599947 : _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)(instr + 1);
1223 599947 : PyTypeObject *container_type = Py_TYPE(container);
1224 599947 : if (container_type == &PyList_Type) {
1225 155545 : if (PyLong_CheckExact(sub)) {
1226 145462 : _Py_SET_OPCODE(*instr, BINARY_SUBSCR_LIST_INT);
1227 145462 : goto success;
1228 : }
1229 : SPECIALIZATION_FAIL(BINARY_SUBSCR,
1230 : PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_LIST_SLICE : SPEC_FAIL_OTHER);
1231 10083 : goto fail;
1232 : }
1233 444402 : if (container_type == &PyTuple_Type) {
1234 131718 : if (PyLong_CheckExact(sub)) {
1235 131361 : _Py_SET_OPCODE(*instr, BINARY_SUBSCR_TUPLE_INT);
1236 131361 : goto success;
1237 : }
1238 : SPECIALIZATION_FAIL(BINARY_SUBSCR,
1239 : PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_TUPLE_SLICE : SPEC_FAIL_OTHER);
1240 357 : goto fail;
1241 : }
1242 312684 : if (container_type == &PyDict_Type) {
1243 62035 : _Py_SET_OPCODE(*instr, BINARY_SUBSCR_DICT);
1244 62035 : goto success;
1245 : }
1246 250649 : PyTypeObject *cls = Py_TYPE(container);
1247 250649 : PyObject *descriptor = _PyType_Lookup(cls, &_Py_ID(__getitem__));
1248 250649 : if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) {
1249 20405 : if (!(container_type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
1250 : SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE);
1251 0 : goto fail;
1252 : }
1253 20405 : PyFunctionObject *func = (PyFunctionObject *)descriptor;
1254 20405 : PyCodeObject *fcode = (PyCodeObject *)func->func_code;
1255 20405 : int kind = function_kind(fcode);
1256 20405 : if (kind != SIMPLE_FUNCTION) {
1257 : SPECIALIZATION_FAIL(BINARY_SUBSCR, kind);
1258 696 : goto fail;
1259 : }
1260 19709 : if (fcode->co_argcount != 2) {
1261 : SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1262 0 : goto fail;
1263 : }
1264 19709 : assert(cls->tp_version_tag != 0);
1265 19709 : write_u32(cache->type_version, cls->tp_version_tag);
1266 19709 : int version = _PyFunction_GetVersionForCurrentState(func);
1267 19709 : if (version == 0 || version != (uint16_t)version) {
1268 : SPECIALIZATION_FAIL(BINARY_SUBSCR, version == 0 ?
1269 : SPEC_FAIL_OUT_OF_VERSIONS : SPEC_FAIL_OUT_OF_RANGE);
1270 0 : goto fail;
1271 : }
1272 19709 : cache->func_version = version;
1273 19709 : ((PyHeapTypeObject *)container_type)->_spec_cache.getitem = descriptor;
1274 19709 : _Py_SET_OPCODE(*instr, BINARY_SUBSCR_GETITEM);
1275 19709 : goto success;
1276 : }
1277 : SPECIALIZATION_FAIL(BINARY_SUBSCR,
1278 : binary_subscr_fail_kind(container_type, sub));
1279 230244 : fail:
1280 : STAT_INC(BINARY_SUBSCR, failure);
1281 241380 : assert(!PyErr_Occurred());
1282 241380 : cache->counter = adaptive_counter_backoff(cache->counter);
1283 241380 : return 0;
1284 358567 : success:
1285 : STAT_INC(BINARY_SUBSCR, success);
1286 358567 : assert(!PyErr_Occurred());
1287 358567 : cache->counter = miss_counter_start();
1288 358567 : return 0;
1289 : }
1290 :
1291 : int
1292 277798 : _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr)
1293 : {
1294 277798 : _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)(instr + 1);
1295 277798 : PyTypeObject *container_type = Py_TYPE(container);
1296 277798 : if (container_type == &PyList_Type) {
1297 30858 : if (PyLong_CheckExact(sub)) {
1298 30071 : if ((Py_SIZE(sub) == 0 || Py_SIZE(sub) == 1)
1299 20955 : && ((PyLongObject *)sub)->ob_digit[0] < (size_t)PyList_GET_SIZE(container))
1300 : {
1301 20952 : _Py_SET_OPCODE(*instr, STORE_SUBSCR_LIST_INT);
1302 20952 : goto success;
1303 : }
1304 : else {
1305 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
1306 9119 : goto fail;
1307 : }
1308 : }
1309 787 : else if (PySlice_Check(sub)) {
1310 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_LIST_SLICE);
1311 787 : goto fail;
1312 : }
1313 : else {
1314 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1315 0 : goto fail;
1316 : }
1317 : }
1318 246940 : if (container_type == &PyDict_Type) {
1319 169235 : _Py_SET_OPCODE(*instr, STORE_SUBSCR_DICT);
1320 169235 : goto success;
1321 : }
1322 : #ifdef Py_STATS
1323 : PyMappingMethods *as_mapping = container_type->tp_as_mapping;
1324 : if (as_mapping && (as_mapping->mp_ass_subscript
1325 : == PyDict_Type.tp_as_mapping->mp_ass_subscript)) {
1326 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE);
1327 : goto fail;
1328 : }
1329 : if (PyObject_CheckBuffer(container)) {
1330 : if (PyLong_CheckExact(sub) && (((size_t)Py_SIZE(sub)) > 1)) {
1331 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
1332 : }
1333 : else if (strcmp(container_type->tp_name, "array.array") == 0) {
1334 : if (PyLong_CheckExact(sub)) {
1335 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_ARRAY_INT);
1336 : }
1337 : else if (PySlice_Check(sub)) {
1338 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_ARRAY_SLICE);
1339 : }
1340 : else {
1341 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1342 : }
1343 : }
1344 : else if (PyByteArray_CheckExact(container)) {
1345 : if (PyLong_CheckExact(sub)) {
1346 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BYTEARRAY_INT);
1347 : }
1348 : else if (PySlice_Check(sub)) {
1349 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE);
1350 : }
1351 : else {
1352 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1353 : }
1354 : }
1355 : else {
1356 : if (PyLong_CheckExact(sub)) {
1357 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BUFFER_INT);
1358 : }
1359 : else if (PySlice_Check(sub)) {
1360 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BUFFER_SLICE);
1361 : }
1362 : else {
1363 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1364 : }
1365 : }
1366 : goto fail;
1367 : }
1368 : PyObject *descriptor = _PyType_Lookup(container_type, &_Py_ID(__setitem__));
1369 : if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) {
1370 : PyFunctionObject *func = (PyFunctionObject *)descriptor;
1371 : PyCodeObject *code = (PyCodeObject *)func->func_code;
1372 : int kind = function_kind(code);
1373 : if (kind == SIMPLE_FUNCTION) {
1374 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_PY_SIMPLE);
1375 : }
1376 : else {
1377 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_PY_OTHER);
1378 : }
1379 : goto fail;
1380 : }
1381 : #endif
1382 : SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1383 77705 : fail:
1384 : STAT_INC(STORE_SUBSCR, failure);
1385 87611 : assert(!PyErr_Occurred());
1386 87611 : cache->counter = adaptive_counter_backoff(cache->counter);
1387 87611 : return 0;
1388 190187 : success:
1389 : STAT_INC(STORE_SUBSCR, success);
1390 190187 : assert(!PyErr_Occurred());
1391 190187 : cache->counter = miss_counter_start();
1392 190187 : return 0;
1393 : }
1394 :
1395 : static int
1396 605989 : specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
1397 : PyObject *kwnames)
1398 : {
1399 605989 : assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE);
1400 605989 : PyTypeObject *tp = _PyType_CAST(callable);
1401 605989 : if (tp->tp_new == PyBaseObject_Type.tp_new) {
1402 : SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PYTHON_CLASS);
1403 219238 : return -1;
1404 : }
1405 386751 : if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) {
1406 336203 : int oparg = _Py_OPARG(*instr);
1407 336203 : if (nargs == 1 && kwnames == NULL && oparg == 1) {
1408 185172 : if (tp == &PyUnicode_Type) {
1409 9251 : _Py_SET_OPCODE(*instr, CALL_NO_KW_STR_1);
1410 9251 : return 0;
1411 : }
1412 175921 : else if (tp == &PyType_Type) {
1413 19606 : _Py_SET_OPCODE(*instr, CALL_NO_KW_TYPE_1);
1414 19606 : return 0;
1415 : }
1416 156315 : else if (tp == &PyTuple_Type) {
1417 15723 : _Py_SET_OPCODE(*instr, CALL_NO_KW_TUPLE_1);
1418 15723 : return 0;
1419 : }
1420 : }
1421 291623 : if (tp->tp_vectorcall != NULL) {
1422 103836 : _Py_SET_OPCODE(*instr, CALL_BUILTIN_CLASS);
1423 103836 : return 0;
1424 : }
1425 : SPECIALIZATION_FAIL(CALL, tp == &PyUnicode_Type ?
1426 : SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL);
1427 187787 : return -1;
1428 : }
1429 : SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE);
1430 50548 : return -1;
1431 : }
1432 :
1433 : #ifdef Py_STATS
1434 : static int
1435 : builtin_call_fail_kind(int ml_flags)
1436 : {
1437 : switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
1438 : METH_KEYWORDS | METH_METHOD)) {
1439 : case METH_VARARGS:
1440 : return SPEC_FAIL_CALL_PYCFUNCTION;
1441 : case METH_VARARGS | METH_KEYWORDS:
1442 : return SPEC_FAIL_CALL_PYCFUNCTION_WITH_KEYWORDS;
1443 : case METH_FASTCALL | METH_KEYWORDS:
1444 : return SPEC_FAIL_CALL_PYCFUNCTION_FAST_WITH_KEYWORDS;
1445 : case METH_NOARGS:
1446 : return SPEC_FAIL_CALL_PYCFUNCTION_NOARGS;
1447 : /* This case should never happen with PyCFunctionObject -- only
1448 : PyMethodObject. See zlib.compressobj()'s methods for an example.
1449 : */
1450 : case METH_METHOD | METH_FASTCALL | METH_KEYWORDS:
1451 : default:
1452 : return SPEC_FAIL_CALL_BAD_CALL_FLAGS;
1453 : }
1454 : }
1455 : #endif
1456 :
1457 : static int
1458 629803 : specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr,
1459 : int nargs, PyObject *kwnames)
1460 : {
1461 629803 : assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE);
1462 629803 : if (kwnames) {
1463 : SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES);
1464 3945 : return -1;
1465 : }
1466 :
1467 625858 : switch (descr->d_method->ml_flags &
1468 : (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
1469 : METH_KEYWORDS | METH_METHOD)) {
1470 52211 : case METH_NOARGS: {
1471 52211 : if (nargs != 1) {
1472 : SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1473 0 : return -1;
1474 : }
1475 52211 : _Py_SET_OPCODE(*instr, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS);
1476 52211 : return 0;
1477 : }
1478 129729 : case METH_O: {
1479 129729 : if (nargs != 2) {
1480 : SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1481 0 : return -1;
1482 : }
1483 129729 : PyInterpreterState *interp = _PyInterpreterState_GET();
1484 129729 : PyObject *list_append = interp->callable_cache.list_append;
1485 129729 : _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_CALL + 1];
1486 129729 : bool pop = (_Py_OPCODE(next) == POP_TOP);
1487 129729 : int oparg = _Py_OPARG(*instr);
1488 129729 : if ((PyObject *)descr == list_append && oparg == 1 && pop) {
1489 58623 : _Py_SET_OPCODE(*instr, CALL_NO_KW_LIST_APPEND);
1490 58623 : return 0;
1491 : }
1492 71106 : _Py_SET_OPCODE(*instr, CALL_NO_KW_METHOD_DESCRIPTOR_O);
1493 71106 : return 0;
1494 : }
1495 78429 : case METH_FASTCALL: {
1496 78429 : _Py_SET_OPCODE(*instr, CALL_NO_KW_METHOD_DESCRIPTOR_FAST);
1497 78429 : return 0;
1498 : }
1499 13273 : case METH_FASTCALL|METH_KEYWORDS: {
1500 13273 : _Py_SET_OPCODE(*instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS);
1501 13273 : return 0;
1502 : }
1503 : }
1504 : SPECIALIZATION_FAIL(CALL, builtin_call_fail_kind(descr->d_method->ml_flags));
1505 352216 : return -1;
1506 : }
1507 :
1508 : static int
1509 1187790 : specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs,
1510 : PyObject *kwnames, bool bound_method)
1511 : {
1512 1187790 : _PyCallCache *cache = (_PyCallCache *)(instr + 1);
1513 1187790 : assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE);
1514 1187790 : PyCodeObject *code = (PyCodeObject *)func->func_code;
1515 1187790 : int kind = function_kind(code);
1516 : /* Don't specialize if PEP 523 is active */
1517 1187790 : if (_PyInterpreterState_GET()->eval_frame) {
1518 : SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523);
1519 2 : return -1;
1520 : }
1521 1187790 : if (kwnames) {
1522 : SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES);
1523 110171 : return -1;
1524 : }
1525 1077620 : if (kind != SIMPLE_FUNCTION) {
1526 : SPECIALIZATION_FAIL(CALL, kind);
1527 353118 : return -1;
1528 : }
1529 724500 : int argcount = code->co_argcount;
1530 724500 : int defcount = func->func_defaults == NULL ? 0 : (int)PyTuple_GET_SIZE(func->func_defaults);
1531 724500 : assert(defcount <= argcount);
1532 724500 : int min_args = argcount-defcount;
1533 724500 : if (nargs > argcount || nargs < min_args) {
1534 : SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1535 17 : return -1;
1536 : }
1537 724483 : assert(nargs <= argcount && nargs >= min_args);
1538 724483 : assert(min_args >= 0 && defcount >= 0);
1539 724483 : assert(defcount == 0 || func->func_defaults != NULL);
1540 724483 : if (min_args > 0xffff) {
1541 : SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_RANGE);
1542 0 : return -1;
1543 : }
1544 724483 : int version = _PyFunction_GetVersionForCurrentState(func);
1545 724483 : if (version == 0) {
1546 : SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS);
1547 0 : return -1;
1548 : }
1549 724483 : write_u32(cache->func_version, version);
1550 724483 : cache->min_args = min_args;
1551 724483 : if (argcount == nargs) {
1552 678295 : _Py_SET_OPCODE(*instr, bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS);
1553 : }
1554 46188 : else if (bound_method) {
1555 : SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD);
1556 6730 : return -1;
1557 : }
1558 : else {
1559 39458 : _Py_SET_OPCODE(*instr, CALL_PY_WITH_DEFAULTS);
1560 : }
1561 717753 : return 0;
1562 : }
1563 :
1564 : static int
1565 965660 : specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
1566 : PyObject *kwnames)
1567 : {
1568 965660 : assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE);
1569 965660 : if (PyCFunction_GET_FUNCTION(callable) == NULL) {
1570 0 : return 1;
1571 : }
1572 965660 : switch (PyCFunction_GET_FLAGS(callable) &
1573 : (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
1574 : METH_KEYWORDS | METH_METHOD)) {
1575 263050 : case METH_O: {
1576 263050 : if (kwnames) {
1577 : SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES);
1578 0 : return -1;
1579 : }
1580 263050 : if (nargs != 1) {
1581 : SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1582 0 : return 1;
1583 : }
1584 : /* len(o) */
1585 263050 : PyInterpreterState *interp = _PyInterpreterState_GET();
1586 263050 : if (callable == interp->callable_cache.len) {
1587 105997 : _Py_SET_OPCODE(*instr, CALL_NO_KW_LEN);
1588 105997 : return 0;
1589 : }
1590 157053 : _Py_SET_OPCODE(*instr, CALL_NO_KW_BUILTIN_O);
1591 157053 : return 0;
1592 : }
1593 267696 : case METH_FASTCALL: {
1594 267696 : if (kwnames) {
1595 : SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES);
1596 1 : return -1;
1597 : }
1598 267695 : if (nargs == 2) {
1599 : /* isinstance(o1, o2) */
1600 186584 : PyInterpreterState *interp = _PyInterpreterState_GET();
1601 186584 : if (callable == interp->callable_cache.isinstance) {
1602 83040 : _Py_SET_OPCODE(*instr, CALL_NO_KW_ISINSTANCE);
1603 83040 : return 0;
1604 : }
1605 : }
1606 184655 : _Py_SET_OPCODE(*instr, CALL_NO_KW_BUILTIN_FAST);
1607 184655 : return 0;
1608 : }
1609 77207 : case METH_FASTCALL | METH_KEYWORDS: {
1610 77207 : _Py_SET_OPCODE(*instr, CALL_BUILTIN_FAST_WITH_KEYWORDS);
1611 77207 : return 0;
1612 : }
1613 357707 : default:
1614 : SPECIALIZATION_FAIL(CALL,
1615 : builtin_call_fail_kind(PyCFunction_GET_FLAGS(callable)));
1616 357707 : return 1;
1617 : }
1618 : }
1619 :
1620 : #ifdef Py_STATS
1621 : static int
1622 : call_fail_kind(PyObject *callable)
1623 : {
1624 : if (PyCFunction_CheckExact(callable)) {
1625 : return SPEC_FAIL_CALL_PYCFUNCTION;
1626 : }
1627 : else if (PyFunction_Check(callable)) {
1628 : return SPEC_FAIL_CALL_PYFUNCTION;
1629 : }
1630 : else if (PyInstanceMethod_Check(callable)) {
1631 : return SPEC_FAIL_CALL_INSTANCE_METHOD;
1632 : }
1633 : else if (PyMethod_Check(callable)) {
1634 : return SPEC_FAIL_CALL_BOUND_METHOD;
1635 : }
1636 : // builtin method
1637 : else if (PyCMethod_Check(callable)) {
1638 : return SPEC_FAIL_CALL_CMETHOD;
1639 : }
1640 : else if (PyType_Check(callable)) {
1641 : if (((PyTypeObject *)callable)->tp_new == PyBaseObject_Type.tp_new) {
1642 : return SPEC_FAIL_CALL_PYTHON_CLASS;
1643 : }
1644 : else {
1645 : return SPEC_FAIL_CALL_CLASS;
1646 : }
1647 : }
1648 : else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) {
1649 : return SPEC_FAIL_CALL_METHOD_DESCRIPTOR;
1650 : }
1651 : else if (Py_TYPE(callable) == &PyWrapperDescr_Type) {
1652 : return SPEC_FAIL_CALL_OPERATOR_WRAPPER;
1653 : }
1654 : else if (Py_TYPE(callable) == &_PyMethodWrapper_Type) {
1655 : return SPEC_FAIL_CALL_METHOD_WRAPPER;
1656 : }
1657 : return SPEC_FAIL_OTHER;
1658 : }
1659 : #endif
1660 :
1661 :
1662 : /* TODO:
1663 : - Specialize calling classes.
1664 : */
1665 : int
1666 3467280 : _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
1667 : PyObject *kwnames)
1668 : {
1669 3467280 : assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL);
1670 3467280 : _PyCallCache *cache = (_PyCallCache *)(instr + 1);
1671 : int fail;
1672 3467280 : if (PyCFunction_CheckExact(callable)) {
1673 965660 : fail = specialize_c_call(callable, instr, nargs, kwnames);
1674 : }
1675 2501620 : else if (PyFunction_Check(callable)) {
1676 1007360 : fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs,
1677 : kwnames, false);
1678 : }
1679 1494250 : else if (PyType_Check(callable)) {
1680 605989 : fail = specialize_class_call(callable, instr, nargs, kwnames);
1681 : }
1682 888265 : else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) {
1683 629803 : fail = specialize_method_descriptor((PyMethodDescrObject *)callable,
1684 : instr, nargs, kwnames);
1685 : }
1686 258462 : else if (Py_TYPE(callable) == &PyMethod_Type) {
1687 180484 : PyObject *func = ((PyMethodObject *)callable)->im_func;
1688 180484 : if (PyFunction_Check(func)) {
1689 180428 : fail = specialize_py_call((PyFunctionObject *)func,
1690 : instr, nargs+1, kwnames, true);
1691 : } else {
1692 : SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD);
1693 56 : fail = -1;
1694 : }
1695 : }
1696 : else {
1697 : SPECIALIZATION_FAIL(CALL, call_fail_kind(callable));
1698 77978 : fail = -1;
1699 : }
1700 3467280 : if (fail) {
1701 : STAT_INC(CALL, failure);
1702 1719510 : assert(!PyErr_Occurred());
1703 1719510 : cache->counter = adaptive_counter_backoff(cache->counter);
1704 : }
1705 : else {
1706 : STAT_INC(CALL, success);
1707 1747760 : assert(!PyErr_Occurred());
1708 1747760 : cache->counter = miss_counter_start();
1709 : }
1710 3467280 : return 0;
1711 : }
1712 :
1713 : #ifdef Py_STATS
1714 : static int
1715 : binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs)
1716 : {
1717 : switch (oparg) {
1718 : case NB_ADD:
1719 : case NB_INPLACE_ADD:
1720 : if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1721 : return SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES;
1722 : }
1723 : return SPEC_FAIL_BINARY_OP_ADD_OTHER;
1724 : case NB_AND:
1725 : case NB_INPLACE_AND:
1726 : if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1727 : return SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES;
1728 : }
1729 : if (PyLong_CheckExact(lhs)) {
1730 : return SPEC_FAIL_BINARY_OP_AND_INT;
1731 : }
1732 : return SPEC_FAIL_BINARY_OP_AND_OTHER;
1733 : case NB_FLOOR_DIVIDE:
1734 : case NB_INPLACE_FLOOR_DIVIDE:
1735 : return SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE;
1736 : case NB_LSHIFT:
1737 : case NB_INPLACE_LSHIFT:
1738 : return SPEC_FAIL_BINARY_OP_LSHIFT;
1739 : case NB_MATRIX_MULTIPLY:
1740 : case NB_INPLACE_MATRIX_MULTIPLY:
1741 : return SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY;
1742 : case NB_MULTIPLY:
1743 : case NB_INPLACE_MULTIPLY:
1744 : if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1745 : return SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES;
1746 : }
1747 : return SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER;
1748 : case NB_OR:
1749 : case NB_INPLACE_OR:
1750 : return SPEC_FAIL_BINARY_OP_OR;
1751 : case NB_POWER:
1752 : case NB_INPLACE_POWER:
1753 : return SPEC_FAIL_BINARY_OP_POWER;
1754 : case NB_REMAINDER:
1755 : case NB_INPLACE_REMAINDER:
1756 : return SPEC_FAIL_BINARY_OP_REMAINDER;
1757 : case NB_RSHIFT:
1758 : case NB_INPLACE_RSHIFT:
1759 : return SPEC_FAIL_BINARY_OP_RSHIFT;
1760 : case NB_SUBTRACT:
1761 : case NB_INPLACE_SUBTRACT:
1762 : if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1763 : return SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES;
1764 : }
1765 : return SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER;
1766 : case NB_TRUE_DIVIDE:
1767 : case NB_INPLACE_TRUE_DIVIDE:
1768 : if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1769 : return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES;
1770 : }
1771 : if (PyFloat_CheckExact(lhs)) {
1772 : return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT;
1773 : }
1774 : return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER;
1775 : case NB_XOR:
1776 : case NB_INPLACE_XOR:
1777 : return SPEC_FAIL_BINARY_OP_XOR;
1778 : }
1779 : Py_UNREACHABLE();
1780 : }
1781 : #endif
1782 :
1783 : void
1784 389375 : _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
1785 : int oparg, PyObject **locals)
1786 : {
1787 389375 : assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP);
1788 389375 : _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1);
1789 389375 : switch (oparg) {
1790 168718 : case NB_ADD:
1791 : case NB_INPLACE_ADD:
1792 168718 : if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1793 8590 : break;
1794 : }
1795 160128 : if (PyUnicode_CheckExact(lhs)) {
1796 58819 : _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_BINARY_OP + 1];
1797 104843 : bool to_store = (_Py_OPCODE(next) == STORE_FAST ||
1798 46024 : _Py_OPCODE(next) == STORE_FAST__LOAD_FAST);
1799 58819 : if (to_store && locals[_Py_OPARG(next)] == lhs) {
1800 13131 : _Py_SET_OPCODE(*instr, BINARY_OP_INPLACE_ADD_UNICODE);
1801 13131 : goto success;
1802 : }
1803 45688 : _Py_SET_OPCODE(*instr, BINARY_OP_ADD_UNICODE);
1804 45688 : goto success;
1805 : }
1806 101309 : if (PyLong_CheckExact(lhs)) {
1807 55554 : _Py_SET_OPCODE(*instr, BINARY_OP_ADD_INT);
1808 55554 : goto success;
1809 : }
1810 45755 : if (PyFloat_CheckExact(lhs)) {
1811 350 : _Py_SET_OPCODE(*instr, BINARY_OP_ADD_FLOAT);
1812 350 : goto success;
1813 : }
1814 45405 : break;
1815 34709 : case NB_MULTIPLY:
1816 : case NB_INPLACE_MULTIPLY:
1817 34709 : if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1818 24484 : break;
1819 : }
1820 10225 : if (PyLong_CheckExact(lhs)) {
1821 9293 : _Py_SET_OPCODE(*instr, BINARY_OP_MULTIPLY_INT);
1822 9293 : goto success;
1823 : }
1824 932 : if (PyFloat_CheckExact(lhs)) {
1825 899 : _Py_SET_OPCODE(*instr, BINARY_OP_MULTIPLY_FLOAT);
1826 899 : goto success;
1827 : }
1828 33 : break;
1829 65771 : case NB_SUBTRACT:
1830 : case NB_INPLACE_SUBTRACT:
1831 65771 : if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1832 18246 : break;
1833 : }
1834 47525 : if (PyLong_CheckExact(lhs)) {
1835 44819 : _Py_SET_OPCODE(*instr, BINARY_OP_SUBTRACT_INT);
1836 44819 : goto success;
1837 : }
1838 2706 : if (PyFloat_CheckExact(lhs)) {
1839 363 : _Py_SET_OPCODE(*instr, BINARY_OP_SUBTRACT_FLOAT);
1840 363 : goto success;
1841 : }
1842 2343 : break;
1843 : #ifndef Py_STATS
1844 120177 : default:
1845 : // These operators don't have any available specializations. Rather
1846 : // than repeatedly attempting to specialize them, just convert them
1847 : // back to BINARY_OP (unless we're collecting stats, where it's more
1848 : // important to get accurate hit counts for the unadaptive version
1849 : // and each of the different failure types):
1850 120177 : _Py_SET_OPCODE(*instr, BINARY_OP);
1851 120177 : return;
1852 : #endif
1853 : }
1854 : SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs));
1855 : STAT_INC(BINARY_OP, failure);
1856 99101 : cache->counter = adaptive_counter_backoff(cache->counter);
1857 99101 : return;
1858 170097 : success:
1859 : STAT_INC(BINARY_OP, success);
1860 170097 : cache->counter = miss_counter_start();
1861 : }
1862 :
1863 :
1864 : #ifdef Py_STATS
1865 : static int
1866 : compare_op_fail_kind(PyObject *lhs, PyObject *rhs)
1867 : {
1868 : if (Py_TYPE(lhs) != Py_TYPE(rhs)) {
1869 : if (PyFloat_CheckExact(lhs) && PyLong_CheckExact(rhs)) {
1870 : return SPEC_FAIL_COMPARE_OP_FLOAT_LONG;
1871 : }
1872 : if (PyLong_CheckExact(lhs) && PyFloat_CheckExact(rhs)) {
1873 : return SPEC_FAIL_COMPARE_OP_LONG_FLOAT;
1874 : }
1875 : return SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES;
1876 : }
1877 : if (PyBytes_CheckExact(lhs)) {
1878 : return SPEC_FAIL_COMPARE_OP_BYTES;
1879 : }
1880 : if (PyTuple_CheckExact(lhs)) {
1881 : return SPEC_FAIL_COMPARE_OP_TUPLE;
1882 : }
1883 : if (PyList_CheckExact(lhs)) {
1884 : return SPEC_FAIL_COMPARE_OP_LIST;
1885 : }
1886 : if (PySet_CheckExact(lhs) || PyFrozenSet_CheckExact(lhs)) {
1887 : return SPEC_FAIL_COMPARE_OP_SET;
1888 : }
1889 : if (PyBool_Check(lhs)) {
1890 : return SPEC_FAIL_COMPARE_OP_BOOL;
1891 : }
1892 : if (Py_TYPE(lhs)->tp_richcompare == PyBaseObject_Type.tp_richcompare) {
1893 : return SPEC_FAIL_COMPARE_OP_BASEOBJECT;
1894 : }
1895 : return SPEC_FAIL_OTHER;
1896 : }
1897 : #endif
1898 :
1899 :
1900 : static int compare_masks[] = {
1901 : // 1-bit: jump if less than
1902 : // 2-bit: jump if equal
1903 : // 4-bit: jump if greater
1904 : [Py_LT] = 1 | 0 | 0,
1905 : [Py_LE] = 1 | 2 | 0,
1906 : [Py_EQ] = 0 | 2 | 0,
1907 : [Py_NE] = 1 | 0 | 4,
1908 : [Py_GT] = 0 | 0 | 4,
1909 : [Py_GE] = 0 | 2 | 4,
1910 : };
1911 :
1912 : void
1913 424297 : _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
1914 : int oparg)
1915 : {
1916 424297 : assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP);
1917 424297 : _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1);
1918 424297 : int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]);
1919 424297 : if (next_opcode != POP_JUMP_FORWARD_IF_FALSE &&
1920 82467 : next_opcode != POP_JUMP_BACKWARD_IF_FALSE &&
1921 47408 : next_opcode != POP_JUMP_FORWARD_IF_TRUE &&
1922 : next_opcode != POP_JUMP_BACKWARD_IF_TRUE) {
1923 : // Can't ever combine, so don't don't bother being adaptive (unless
1924 : // we're collecting stats, where it's more important to get accurate hit
1925 : // counts for the unadaptive version and each of the different failure
1926 : // types):
1927 : #ifndef Py_STATS
1928 44667 : _Py_SET_OPCODE(*instr, COMPARE_OP);
1929 44667 : return;
1930 : #else
1931 : if (next_opcode == EXTENDED_ARG) {
1932 : SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_EXTENDED_ARG);
1933 : goto failure;
1934 : }
1935 : SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_NOT_FOLLOWED_BY_COND_JUMP);
1936 : goto failure;
1937 : #endif
1938 : }
1939 379630 : assert(oparg <= Py_GE);
1940 379630 : int when_to_jump_mask = compare_masks[oparg];
1941 379630 : if (next_opcode == POP_JUMP_FORWARD_IF_FALSE ||
1942 : next_opcode == POP_JUMP_BACKWARD_IF_FALSE) {
1943 341830 : when_to_jump_mask = (1 | 2 | 4) & ~when_to_jump_mask;
1944 : }
1945 379630 : if (next_opcode == POP_JUMP_BACKWARD_IF_TRUE ||
1946 : next_opcode == POP_JUMP_BACKWARD_IF_FALSE) {
1947 19300 : when_to_jump_mask <<= 3;
1948 : }
1949 379630 : if (Py_TYPE(lhs) != Py_TYPE(rhs)) {
1950 : SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs));
1951 30622 : goto failure;
1952 : }
1953 349008 : if (PyFloat_CheckExact(lhs)) {
1954 3817 : _Py_SET_OPCODE(*instr, COMPARE_OP_FLOAT_JUMP);
1955 3817 : cache->mask = when_to_jump_mask;
1956 3817 : goto success;
1957 : }
1958 345191 : if (PyLong_CheckExact(lhs)) {
1959 184937 : if (Py_ABS(Py_SIZE(lhs)) <= 1 && Py_ABS(Py_SIZE(rhs)) <= 1) {
1960 119374 : _Py_SET_OPCODE(*instr, COMPARE_OP_INT_JUMP);
1961 119374 : cache->mask = when_to_jump_mask;
1962 119374 : goto success;
1963 : }
1964 : else {
1965 : SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_BIG_INT);
1966 65563 : goto failure;
1967 : }
1968 : }
1969 160254 : if (PyUnicode_CheckExact(lhs)) {
1970 99910 : if (oparg != Py_EQ && oparg != Py_NE) {
1971 : SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_STRING);
1972 316 : goto failure;
1973 : }
1974 : else {
1975 99594 : _Py_SET_OPCODE(*instr, COMPARE_OP_STR_JUMP);
1976 99594 : cache->mask = when_to_jump_mask;
1977 99594 : goto success;
1978 : }
1979 : }
1980 : SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs));
1981 60344 : failure:
1982 : STAT_INC(COMPARE_OP, failure);
1983 156845 : cache->counter = adaptive_counter_backoff(cache->counter);
1984 156845 : return;
1985 222785 : success:
1986 : STAT_INC(COMPARE_OP, success);
1987 222785 : cache->counter = miss_counter_start();
1988 : }
1989 :
1990 : #ifdef Py_STATS
1991 : static int
1992 : unpack_sequence_fail_kind(PyObject *seq)
1993 : {
1994 : if (PySequence_Check(seq)) {
1995 : return SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE;
1996 : }
1997 : if (PyIter_Check(seq)) {
1998 : return SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR;
1999 : }
2000 : return SPEC_FAIL_OTHER;
2001 : }
2002 : #endif
2003 :
2004 : void
2005 109875 : _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg)
2006 : {
2007 109875 : assert(_PyOpcode_Caches[UNPACK_SEQUENCE] ==
2008 : INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
2009 109875 : _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)(instr + 1);
2010 109875 : if (PyTuple_CheckExact(seq)) {
2011 107588 : if (PyTuple_GET_SIZE(seq) != oparg) {
2012 : SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR);
2013 2 : goto failure;
2014 : }
2015 107586 : if (PyTuple_GET_SIZE(seq) == 2) {
2016 85625 : _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_TWO_TUPLE);
2017 85625 : goto success;
2018 : }
2019 21961 : _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_TUPLE);
2020 21961 : goto success;
2021 : }
2022 2287 : if (PyList_CheckExact(seq)) {
2023 466 : if (PyList_GET_SIZE(seq) != oparg) {
2024 : SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR);
2025 14 : goto failure;
2026 : }
2027 452 : _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_LIST);
2028 452 : goto success;
2029 : }
2030 : SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq));
2031 1821 : failure:
2032 : STAT_INC(UNPACK_SEQUENCE, failure);
2033 1837 : cache->counter = adaptive_counter_backoff(cache->counter);
2034 1837 : return;
2035 108038 : success:
2036 : STAT_INC(UNPACK_SEQUENCE, success);
2037 108038 : cache->counter = miss_counter_start();
2038 : }
2039 :
2040 : #ifdef Py_STATS
2041 :
2042 : int
2043 : _PySpecialization_ClassifyIterator(PyObject *iter)
2044 : {
2045 : if (PyGen_CheckExact(iter)) {
2046 : return SPEC_FAIL_FOR_ITER_GENERATOR;
2047 : }
2048 : if (PyCoro_CheckExact(iter)) {
2049 : return SPEC_FAIL_FOR_ITER_COROUTINE;
2050 : }
2051 : if (PyAsyncGen_CheckExact(iter)) {
2052 : return SPEC_FAIL_FOR_ITER_ASYNC_GENERATOR;
2053 : }
2054 : PyTypeObject *t = Py_TYPE(iter);
2055 : if (t == &PyListIter_Type) {
2056 : return SPEC_FAIL_FOR_ITER_LIST;
2057 : }
2058 : if (t == &PyTupleIter_Type) {
2059 : return SPEC_FAIL_FOR_ITER_TUPLE;
2060 : }
2061 : if (t == &PyDictIterKey_Type) {
2062 : return SPEC_FAIL_FOR_ITER_DICT_KEYS;
2063 : }
2064 : if (t == &PyDictIterValue_Type) {
2065 : return SPEC_FAIL_FOR_ITER_DICT_VALUES;
2066 : }
2067 : if (t == &PyDictIterItem_Type) {
2068 : return SPEC_FAIL_FOR_ITER_DICT_ITEMS;
2069 : }
2070 : if (t == &PySetIter_Type) {
2071 : return SPEC_FAIL_FOR_ITER_SET;
2072 : }
2073 : if (t == &PyUnicodeIter_Type) {
2074 : return SPEC_FAIL_FOR_ITER_STRING;
2075 : }
2076 : if (t == &PyBytesIter_Type) {
2077 : return SPEC_FAIL_FOR_ITER_BYTES;
2078 : }
2079 : if (t == &PyRangeIter_Type) {
2080 : return SPEC_FAIL_FOR_ITER_RANGE;
2081 : }
2082 : if (t == &PyEnum_Type) {
2083 : return SPEC_FAIL_FOR_ITER_ENUMERATE;
2084 : }
2085 : if (t == &PyMap_Type) {
2086 : return SPEC_FAIL_FOR_ITER_MAP;
2087 : }
2088 : if (t == &PyZip_Type) {
2089 : return SPEC_FAIL_FOR_ITER_ZIP;
2090 : }
2091 : if (t == &PySeqIter_Type) {
2092 : return SPEC_FAIL_FOR_ITER_SEQ_ITER;
2093 : }
2094 : if (t == &PyListRevIter_Type) {
2095 : return SPEC_FAIL_FOR_ITER_REVERSED_LIST;
2096 : }
2097 : if (t == &_PyUnicodeASCIIIter_Type) {
2098 : return SPEC_FAIL_FOR_ITER_ASCII_STRING;
2099 : }
2100 : const char *name = t->tp_name;
2101 : if (strncmp(name, "itertools", 9) == 0) {
2102 : return SPEC_FAIL_FOR_ITER_ITERTOOLS;
2103 : }
2104 : if (strncmp(name, "callable_iterator", 17) == 0) {
2105 : return SPEC_FAIL_FOR_ITER_CALLABLE;
2106 : }
2107 : return SPEC_FAIL_OTHER;
2108 : }
2109 :
2110 : #endif
2111 :
2112 : void
2113 813756 : _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr)
2114 : {
2115 813756 : assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER);
2116 813756 : _PyForIterCache *cache = (_PyForIterCache *)(instr + 1);
2117 813756 : PyTypeObject *tp = Py_TYPE(iter);
2118 813756 : _Py_CODEUNIT next = instr[1+INLINE_CACHE_ENTRIES_FOR_ITER];
2119 813756 : int next_op = _PyOpcode_Deopt[_Py_OPCODE(next)];
2120 813756 : if (tp == &PyListIter_Type) {
2121 97585 : _Py_SET_OPCODE(*instr, FOR_ITER_LIST);
2122 97585 : goto success;
2123 : }
2124 716171 : else if (tp == &PyRangeIter_Type && next_op == STORE_FAST) {
2125 15713 : _Py_SET_OPCODE(*instr, FOR_ITER_RANGE);
2126 15713 : goto success;
2127 : }
2128 : else {
2129 : SPECIALIZATION_FAIL(FOR_ITER,
2130 : _PySpecialization_ClassifyIterator(iter));
2131 700458 : goto failure;
2132 : }
2133 700458 : failure:
2134 : STAT_INC(FOR_ITER, failure);
2135 700458 : cache->counter = adaptive_counter_backoff(cache->counter);
2136 700458 : return;
2137 113298 : success:
2138 : STAT_INC(FOR_ITER, success);
2139 113298 : cache->counter = miss_counter_start();
2140 : }
|
