diff options
31 files changed, 1031 insertions, 183 deletions
diff --git a/dotviewer/drawgraph.py b/dotviewer/drawgraph.py index db19d85920..c688688a71 100644 --- a/dotviewer/drawgraph.py +++ b/dotviewer/drawgraph.py @@ -423,20 +423,43 @@ class GraphRenderer: else: for line in lines: raw_line = line.replace('\\l','').replace('\r','') or ' ' - img = TextSnippet(self, raw_line, (0, 0, 0), bgcolor) - w, h = img.get_size() - if w>wmax: wmax = w - if raw_line.strip(): - if line.endswith('\\l'): - def cmd(img=img, y=hmax): - img.draw(xleft, ytop+y) - elif line.endswith('\r'): - def cmd(img=img, y=hmax, w=w): - img.draw(xright-w, ytop+y) - else: - def cmd(img=img, y=hmax, w=w): - img.draw(xcenter-w//2, ytop+y) + if '\f' in raw_line: # grayed out parts of the line + imgs = [] + graytext = True + h = 16 + w_total = 0 + for linepart in raw_line.split('\f'): + graytext = not graytext + if not linepart.strip(): + continue + if graytext: + fgcolor = (128, 160, 160) + else: + fgcolor = (0, 0, 0) + img = TextSnippet(self, linepart, fgcolor, bgcolor) + imgs.append((w_total, img)) + w, h = img.get_size() + w_total += w + if w_total > wmax: wmax = w_total + def cmd(imgs=imgs, y=hmax): + for x, img in imgs: + img.draw(xleft+x, ytop+y) commands.append(cmd) + else: + img = TextSnippet(self, raw_line, (0, 0, 0), bgcolor) + w, h = img.get_size() + if w>wmax: wmax = w + if raw_line.strip(): + if line.endswith('\\l'): + def cmd(img=img, y=hmax): + img.draw(xleft, ytop+y) + elif line.endswith('\r'): + def cmd(img=img, y=hmax, w=w): + img.draw(xright-w, ytop+y) + else: + def cmd(img=img, y=hmax, w=w): + img.draw(xcenter-w//2, ytop+y) + commands.append(cmd) hmax += h #hmax += 8 diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py index 1b046ced27..6543874344 100644 --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -1068,6 +1068,9 @@ class RegAlloc(object): def consider_debug_merge_point(self, op): pass + def consider_jit_debug(self, op): + pass + def get_mark_gc_roots(self, gcrootmap): shape = gcrootmap.get_basic_shape(IS_X86_64) for v, val in self.fm.frame_bindings.items(): diff --git a/pypy/jit/codewriter/call.py b/pypy/jit/codewriter/call.py index e1db938e2f..658f97b550 100644 --- a/pypy/jit/codewriter/call.py +++ b/pypy/jit/codewriter/call.py @@ -277,3 +277,11 @@ class CallControl(object): return seen.pop() else: return None + + def could_be_green_field(self, GTYPE, fieldname): + GTYPE_fieldname = (GTYPE, fieldname) + for jd in self.jitdrivers_sd: + if jd.greenfield_info is not None: + if GTYPE_fieldname in jd.greenfield_info.green_fields: + return True + return False diff --git a/pypy/jit/codewriter/codewriter.py b/pypy/jit/codewriter/codewriter.py index e7f40ee006..9013ed9615 100644 --- a/pypy/jit/codewriter/codewriter.py +++ b/pypy/jit/codewriter/codewriter.py @@ -95,18 +95,18 @@ class CodeWriter(object): print '%s:' % (ssarepr.name,) print format_assembler(ssarepr) else: - dir = udir.ensure("jitcodes", dir=1) - if portal_jitdriver: - name = "%02d_portal_runner" % (portal_jitdriver.index,) - elif ssarepr.name and ssarepr.name != '?': - name = ssarepr.name - else: - name = 'unnamed' % id(ssarepr) - i = 1 - extra = '' - while name+extra in self._seen_files: - i += 1 - extra = '.%d' % i - self._seen_files.add(name+extra) - dir.join(name+extra).write(format_assembler(ssarepr)) log.dot() + dir = udir.ensure("jitcodes", dir=1) + if portal_jitdriver: + name = "%02d_portal_runner" % (portal_jitdriver.index,) + elif ssarepr.name and ssarepr.name != '?': + name = ssarepr.name + else: + name = 'unnamed' % id(ssarepr) + i = 1 + extra = '' + while name+extra in self._seen_files: + i += 1 + extra = '.%d' % i + self._seen_files.add(name+extra) + dir.join(name+extra).write(format_assembler(ssarepr)) diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py index 23bd17e01f..8130f8d2c4 100644 --- a/pypy/jit/codewriter/jtransform.py +++ b/pypy/jit/codewriter/jtransform.py @@ -320,6 +320,8 @@ class Transformer(object): prepare = self._handle_str2unicode_call elif oopspec_name.startswith('virtual_ref'): prepare = self._handle_virtual_ref_call + elif oopspec_name.startswith('jit.'): + prepare = self._handle_jit_call elif oopspec_name.startswith('libffi_'): prepare = self._handle_libffi_call else: @@ -523,7 +525,12 @@ class Transformer(object): # check for deepfrozen structures that force constant-folding immut = v_inst.concretetype.TO._immutable_field(c_fieldname.value) if immut: - pure = '_pure' + if (self.callcontrol is not None and + self.callcontrol.could_be_green_field(v_inst.concretetype.TO, + c_fieldname.value)): + pure = '_greenfield' + else: + pure = '_pure' if immut == "[*]": self.immutable_arrays[op.result] = True else: @@ -856,6 +863,15 @@ class Transformer(object): (self.graph,)) return [] + def _handle_jit_call(self, op, oopspec_name, args): + if oopspec_name == 'jit.debug': + return SpaceOperation('jit_debug', args, None) + elif oopspec_name == 'jit.assert_green': + kind = getkind(args[0].concretetype) + return SpaceOperation('%s_assert_green' % kind, args, None) + else: + raise AssertionError("missing support for %r" % oopspec_name) + # ---------- # Lists. diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py index 34d0afe34a..c3ec285510 100644 --- a/pypy/jit/codewriter/support.py +++ b/pypy/jit/codewriter/support.py @@ -61,7 +61,7 @@ def getgraph(func, values): return rtyper.annotator.translator.graphs[0] def split_before_jit_merge_point(graph, portalblock, portalopindex): - """Find the block with 'jit_merge_point' and split just before, + """Split the block just before the 'jit_merge_point', making sure the input args are in the canonical order. """ # split the block just before the jit_merge_point() diff --git a/pypy/jit/codewriter/test/test_codewriter.py b/pypy/jit/codewriter/test/test_codewriter.py index 978f3c75ee..6077be722f 100644 --- a/pypy/jit/codewriter/test/test_codewriter.py +++ b/pypy/jit/codewriter/test/test_codewriter.py @@ -45,6 +45,7 @@ class FakeJitDriverSD: self.portal_graph = portal_graph self.portal_runner_ptr = "???" self.virtualizable_info = None + self.greenfield_info = None def test_loop(): diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py index 9c11067007..967381cefb 100644 --- a/pypy/jit/codewriter/test/test_jtransform.py +++ b/pypy/jit/codewriter/test/test_jtransform.py @@ -687,6 +687,79 @@ def test_promote_2(): assert block.operations[1].result is None assert block.exits[0].args == [v1] +def test_jit_merge_point_1(): + class FakeJitDriverSD: + index = 42 + class jitdriver: + greens = ['green1', 'green2', 'voidgreen3'] + reds = ['red1', 'red2', 'voidred3'] + jd = FakeJitDriverSD() + v1 = varoftype(lltype.Signed) + v2 = varoftype(lltype.Signed) + vvoid1 = varoftype(lltype.Void) + v3 = varoftype(lltype.Signed) + v4 = varoftype(lltype.Signed) + vvoid2 = varoftype(lltype.Void) + v5 = varoftype(lltype.Void) + op = SpaceOperation('jit_marker', + [Constant('jit_merge_point', lltype.Void), + Constant(jd.jitdriver, lltype.Void), + v1, v2, vvoid1, v3, v4, vvoid2], v5) + tr = Transformer() + tr.portal_jd = jd + oplist = tr.rewrite_operation(op) + assert len(oplist) == 6 + assert oplist[0].opname == '-live-' + assert oplist[1].opname == 'int_guard_value' + assert oplist[1].args == [v1] + assert oplist[2].opname == '-live-' + assert oplist[3].opname == 'int_guard_value' + assert oplist[3].args == [v2] + assert oplist[4].opname == 'jit_merge_point' + assert oplist[4].args[0].value == 42 + assert list(oplist[4].args[1]) == [v1, v2] + assert list(oplist[4].args[4]) == [v3, v4] + assert oplist[5].opname == '-live-' + +def test_getfield_gc(): + S = lltype.GcStruct('S', ('x', lltype.Char)) + v1 = varoftype(lltype.Ptr(S)) + v2 = varoftype(lltype.Char) + op = SpaceOperation('getfield', [v1, Constant('x', lltype.Void)], v2) + op1 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'getfield_gc_i' + assert op1.args == [v1, ('fielddescr', S, 'x')] + assert op1.result == v2 + +def test_getfield_gc_pure(): + S = lltype.GcStruct('S', ('x', lltype.Char), + hints={'immutable': True}) + v1 = varoftype(lltype.Ptr(S)) + v2 = varoftype(lltype.Char) + op = SpaceOperation('getfield', [v1, Constant('x', lltype.Void)], v2) + op1 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'getfield_gc_i_pure' + assert op1.args == [v1, ('fielddescr', S, 'x')] + assert op1.result == v2 + +def test_getfield_gc_greenfield(): + class FakeCC: + def get_vinfo(self, v): + return None + def could_be_green_field(self, S1, name1): + assert S1 is S + assert name1 == 'x' + return True + S = lltype.GcStruct('S', ('x', lltype.Char), + hints={'immutable': True}) + v1 = varoftype(lltype.Ptr(S)) + v2 = varoftype(lltype.Char) + op = SpaceOperation('getfield', [v1, Constant('x', lltype.Void)], v2) + op1 = Transformer(FakeCPU(), FakeCC()).rewrite_operation(op) + assert op1.opname == 'getfield_gc_i_greenfield' + assert op1.args == [v1, ('fielddescr', S, 'x')] + assert op1.result == v2 + def test_int_abs(): v1 = varoftype(lltype.Signed) v2 = varoftype(lltype.Signed) diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py index b106d488ad..700280971b 100644 --- a/pypy/jit/metainterp/blackhole.py +++ b/pypy/jit/metainterp/blackhole.py @@ -760,6 +760,20 @@ class BlackholeInterpreter(object): def bhimpl_debug_fatalerror(msg): llop.debug_fatalerror(lltype.Void, msg) + @arguments("r", "i", "i", "i", "i") + def bhimpl_jit_debug(string, arg1=0, arg2=0, arg3=0, arg4=0): + pass + + @arguments("i") + def bhimpl_int_assert_green(x): + pass + @arguments("r") + def bhimpl_ref_assert_green(x): + pass + @arguments("f") + def bhimpl_float_assert_green(x): + pass + # ---------- # the main hints and recursive calls @@ -1073,6 +1087,10 @@ class BlackholeInterpreter(object): bhimpl_getfield_vable_r = bhimpl_getfield_gc_r bhimpl_getfield_vable_f = bhimpl_getfield_gc_f + bhimpl_getfield_gc_i_greenfield = bhimpl_getfield_gc_i + bhimpl_getfield_gc_r_greenfield = bhimpl_getfield_gc_r + bhimpl_getfield_gc_f_greenfield = bhimpl_getfield_gc_f + @arguments("cpu", "i", "d", returns="i") def bhimpl_getfield_raw_i(cpu, struct, fielddescr): return cpu.bh_getfield_raw_i(struct, fielddescr) diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py index dc2ac8d548..d007ce33e7 100644 --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -370,7 +370,8 @@ class ResumeGuardForcedDescr(ResumeGuardDescr): from pypy.jit.metainterp.resume import force_from_resumedata metainterp_sd = self.metainterp_sd vinfo = self.jitdriver_sd.virtualizable_info - all_virtuals = force_from_resumedata(metainterp_sd, self, vinfo) + ginfo = self.jitdriver_sd.greenfield_info + all_virtuals = force_from_resumedata(metainterp_sd, self, vinfo, ginfo) # The virtualizable data was stored on the real virtualizable above. # Handle all_virtuals: keep them for later blackholing from the # future failure of the GUARD_NOT_FORCED diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py index 941c9f514b..cb5b6cda8c 100644 --- a/pypy/jit/metainterp/executor.py +++ b/pypy/jit/metainterp/executor.py @@ -307,6 +307,7 @@ def _make_execute_list(): rop.CALL_ASSEMBLER, rop.COND_CALL_GC_WB, rop.DEBUG_MERGE_POINT, + rop.JIT_DEBUG, rop.SETARRAYITEM_RAW, ): # list of opcodes never executed by pyjitpl continue diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py index f647276328..b506f9f5b7 100644 --- a/pypy/jit/metainterp/graphpage.py +++ b/pypy/jit/metainterp/graphpage.py @@ -153,7 +153,7 @@ class ResOpGen(object): opindex = opstartindex while True: op = operations[opindex] - lines.append(repr(op)) + lines.append(op.repr(graytext=True)) if is_interesting_guard(op): tgt = op.getdescr()._debug_suboperations[0] tgt_g, tgt_i = self.all_operations[tgt] diff --git a/pypy/jit/metainterp/greenfield.py b/pypy/jit/metainterp/greenfield.py new file mode 100644 index 0000000000..e2410436a3 --- /dev/null +++ b/pypy/jit/metainterp/greenfield.py @@ -0,0 +1,27 @@ +from pypy.jit.metainterp.typesystem import deref + + +class GreenFieldInfo(object): + + def __init__(self, cpu, jd): + self.cpu = cpu + self.jitdriver_sd = jd + # XXX for now, only supports a single instance, + # but several fields of it can be green + seen = set() + for name in jd.jitdriver.greens: + if '.' in name: + objname, fieldname = name.split('.') + seen.add(objname) + assert len(seen) == 1, ( + "Current limitation: you can only give one instance with green " + "fields. Found %r" % seen.keys()) + self.red_index = jd.jitdriver.reds.index(objname) + # + # a list of (GTYPE, fieldname) + self.green_fields = jd.jitdriver.ll_greenfields.values() + self.green_field_descrs = [cpu.fielddescrof(GTYPE, fieldname) + for GTYPE, fieldname in self.green_fields] + + def _freeze_(self): + return True diff --git a/pypy/jit/metainterp/jitdriver.py b/pypy/jit/metainterp/jitdriver.py index edfe083ba3..d2b6694bec 100644 --- a/pypy/jit/metainterp/jitdriver.py +++ b/pypy/jit/metainterp/jitdriver.py @@ -13,8 +13,10 @@ class JitDriverStaticData: # self.num_red_args ... pypy.jit.metainterp.warmspot # self.result_type ... pypy.jit.metainterp.warmspot # self.virtualizable_info... pypy.jit.metainterp.warmspot + # self.greenfield_info ... pypy.jit.metainterp.warmspot # self.warmstate ... pypy.jit.metainterp.warmspot # self.handle_jitexc_from_bh pypy.jit.metainterp.warmspot + # self.no_loop_header ... pypy.jit.metainterp.warmspot # self.portal_finishtoken... pypy.jit.metainterp.pyjitpl # self.index ... pypy.jit.codewriter.call # self.mainjitcode ... pypy.jit.codewriter.call diff --git a/pypy/jit/metainterp/optimizeopt/string.py b/pypy/jit/metainterp/optimizeopt/string.py index 5b10a2962f..6962f37576 100644 --- a/pypy/jit/metainterp/optimizeopt/string.py +++ b/pypy/jit/metainterp/optimizeopt/string.py @@ -12,7 +12,7 @@ from pypy.jit.metainterp.optimizeutil import _findall from pypy.jit.codewriter.effectinfo import EffectInfo, callinfo_for_oopspec from pypy.jit.codewriter import heaptracker from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.objectmodel import specialize +from pypy.rlib.objectmodel import specialize, we_are_translated class StrOrUnicode(object): @@ -107,7 +107,10 @@ class VAbstractStringValue(virtualize.AbstractVirtualValue): self.box = box = self.source_op.result newoperations = self.optimizer.newoperations lengthbox = self.getstrlen(newoperations, self.mode) - newoperations.append(ResOperation(self.mode.NEWSTR, [lengthbox], box)) + op = ResOperation(self.mode.NEWSTR, [lengthbox], box) + if not we_are_translated(): + op.name = 'FORCE' + newoperations.append(op) self.string_copy_parts(newoperations, box, CONST_0, self.mode) diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py index 0db7171a27..da1ea634ce 100644 --- a/pypy/jit/metainterp/optimizeopt/virtualize.py +++ b/pypy/jit/metainterp/optimizeopt/virtualize.py @@ -74,6 +74,8 @@ class AbstractVirtualStructValue(AbstractVirtualValue): assert self.source_op is not None # ^^^ This case should not occur any more (see test_bug_3). # + if not we_are_translated(): + self.source_op.name = 'FORCE ' + self.source_op.name newoperations = self.optimizer.newoperations newoperations.append(self.source_op) self.box = box = self.source_op.result @@ -170,6 +172,8 @@ class VArrayValue(AbstractVirtualValue): def _really_force(self): assert self.source_op is not None + if not we_are_translated(): + self.source_op.name = 'FORCE ' + self.source_op.name newoperations = self.optimizer.newoperations newoperations.append(self.source_op) self.box = box = self.source_op.result diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py index dd5d86d41b..6f73c6b43b 100644 --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1,4 +1,4 @@ -import py, os +import py, os, sys from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable @@ -498,6 +498,22 @@ class MIFrame(object): opimpl_getfield_gc_r_pure = _opimpl_getfield_gc_pure_any opimpl_getfield_gc_f_pure = _opimpl_getfield_gc_pure_any + @arguments("orgpc", "box", "descr") + def _opimpl_getfield_gc_greenfield_any(self, pc, box, fielddescr): + ginfo = self.metainterp.jitdriver_sd.greenfield_info + if (ginfo is not None and fielddescr in ginfo.green_field_descrs + and not self._nonstandard_virtualizable(pc, box)): + # fetch the result, but consider it as a Const box and don't + # record any operation + resbox = executor.execute(self.metainterp.cpu, self.metainterp, + rop.GETFIELD_GC_PURE, fielddescr, box) + return resbox.constbox() + # fall-back + return self.execute_with_descr(rop.GETFIELD_GC_PURE, fielddescr, box) + opimpl_getfield_gc_i_greenfield = _opimpl_getfield_gc_greenfield_any + opimpl_getfield_gc_r_greenfield = _opimpl_getfield_gc_greenfield_any + opimpl_getfield_gc_f_greenfield = _opimpl_getfield_gc_greenfield_any + @arguments("box", "descr", "box") def _opimpl_setfield_gc_any(self, box, fielddescr, valuebox): self.execute_with_descr(rop.SETFIELD_GC, fielddescr, box, valuebox) @@ -529,7 +545,8 @@ class MIFrame(object): def _nonstandard_virtualizable(self, pc, box): # returns True if 'box' is actually not the "standard" virtualizable # that is stored in metainterp.virtualizable_boxes[-1] - if self.metainterp.jitdriver_sd.virtualizable_info is None: + if (self.metainterp.jitdriver_sd.virtualizable_info is None and + self.metainterp.jitdriver_sd.greenfield_info is None): return True # can occur in case of multiple JITs standard_box = self.metainterp.virtualizable_boxes[-1] if standard_box is box: @@ -799,12 +816,16 @@ class MIFrame(object): @arguments("orgpc", "int", "boxes3", "boxes3") def opimpl_jit_merge_point(self, orgpc, jdindex, greenboxes, redboxes): + any_operation = len(self.metainterp.history.operations) > 0 jitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex] self.verify_green_args(jitdriver_sd, greenboxes) # xxx we may disable the following line in some context later self.debug_merge_point(jitdriver_sd, greenboxes) if self.metainterp.seen_loop_header_for_jdindex < 0: - return + if not jitdriver_sd.no_loop_header or not any_operation: + return + # automatically add a loop_header if there is none + self.metainterp.seen_loop_header_for_jdindex = jdindex # assert self.metainterp.seen_loop_header_for_jdindex == jdindex, ( "found a loop_header for a JitDriver that does not match " @@ -893,6 +914,40 @@ class MIFrame(object): msg = box.getref(lltype.Ptr(rstr.STR)) lloperation.llop.debug_fatalerror(msg) + @arguments("box", "box", "box", "box", "box") + def opimpl_jit_debug(self, stringbox, arg1box, arg2box, arg3box, arg4box): + from pypy.rpython.lltypesystem import rstr + from pypy.rpython.annlowlevel import hlstr + msg = stringbox.getref(lltype.Ptr(rstr.STR)) + debug_print('jit_debug:', hlstr(msg), + arg1box.getint(), arg2box.getint(), + arg3box.getint(), arg4box.getint()) + args = [stringbox, arg1box, arg2box, arg3box, arg4box] + i = 4 + while i > 0 and args[i].getint() == -sys.maxint-1: + i -= 1 + assert i >= 0 + op = self.metainterp.history.record(rop.JIT_DEBUG, args[:i+1], None) + self.metainterp.attach_debug_info(op) + + @arguments("box") + def _opimpl_assert_green(self, box): + if not isinstance(box, Const): + msg = "assert_green failed at %s:%d" % ( + self.jitcode.name, + self.pc) + if we_are_translated(): + from pypy.rpython.annlowlevel import llstr + from pypy.rpython.lltypesystem import lloperation + lloperation.llop.debug_fatalerror(lltype.Void, llstr(msg)) + else: + from pypy.rlib.jit import AssertGreenFailed + raise AssertGreenFailed(msg) + + opimpl_int_assert_green = _opimpl_assert_green + opimpl_ref_assert_green = _opimpl_assert_green + opimpl_float_assert_green = _opimpl_assert_green + @arguments("box") def opimpl_virtual_ref(self, box): # Details on the content of metainterp.virtualref_boxes: @@ -998,7 +1053,8 @@ class MIFrame(object): guard_op = metainterp.history.record(opnum, moreargs, None, descr=resumedescr) virtualizable_boxes = None - if metainterp.jitdriver_sd.virtualizable_info is not None: + if (metainterp.jitdriver_sd.virtualizable_info is not None or + metainterp.jitdriver_sd.greenfield_info is not None): virtualizable_boxes = metainterp.virtualizable_boxes saved_pc = self.pc if resumepc >= 0: @@ -1646,6 +1702,7 @@ class MetaInterp(object): duplicates) live_arg_boxes += self.virtualizable_boxes live_arg_boxes.pop() + # assert len(self.virtualref_boxes) == 0, "missing virtual_ref_finish()?" # Called whenever we reach the 'loop_header' hint. # First, attempt to make a bridge: @@ -1832,6 +1889,7 @@ class MetaInterp(object): f.setup_call(original_boxes) assert self.in_recursion == 0 self.virtualref_boxes = [] + self.initialize_withgreenfields(original_boxes) self.initialize_virtualizable(original_boxes) def initialize_state_from_guard_failure(self, resumedescr): @@ -1856,6 +1914,14 @@ class MetaInterp(object): self.virtualizable_boxes.append(virtualizable_box) self.initialize_virtualizable_enter() + def initialize_withgreenfields(self, original_boxes): + ginfo = self.jitdriver_sd.greenfield_info + if ginfo is not None: + assert self.jitdriver_sd.virtualizable_info is None + index = (self.jitdriver_sd.num_green_args + + ginfo.red_index) + self.virtualizable_boxes = [original_boxes[index]] + def initialize_virtualizable_enter(self): vinfo = self.jitdriver_sd.virtualizable_info virtualizable_box = self.virtualizable_boxes[-1] @@ -1949,8 +2015,10 @@ class MetaInterp(object): def rebuild_state_after_failure(self, resumedescr): vinfo = self.jitdriver_sd.virtualizable_info + ginfo = self.jitdriver_sd.greenfield_info self.framestack = [] - boxlists = resume.rebuild_from_resumedata(self, resumedescr, vinfo) + boxlists = resume.rebuild_from_resumedata(self, resumedescr, vinfo, + ginfo) inputargs_and_holes, virtualizable_boxes, virtualref_boxes = boxlists # # virtual refs: make the vrefs point to the freshly allocated virtuals @@ -1975,6 +2043,12 @@ class MetaInterp(object): assert not virtualizable.vable_token # fill the virtualizable with the local boxes self.synchronize_virtualizable() + # + elif self.jitdriver_sd.greenfield_info: + self.virtualizable_boxes = virtualizable_boxes + else: + assert not virtualizable_boxes + # return inputargs_and_holes def check_synchronized_virtualizable(self): @@ -2048,7 +2122,8 @@ class MetaInterp(object): for i in range(len(boxes)): if boxes[i] is oldbox: boxes[i] = newbox - if self.jitdriver_sd.virtualizable_info is not None: + if (self.jitdriver_sd.virtualizable_info is not None or + self.jitdriver_sd.greenfield_info is not None): boxes = self.virtualizable_boxes for i in range(len(boxes)): if boxes[i] is oldbox: diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py index d5387d5356..36e55214a5 100644 --- a/pypy/jit/metainterp/resoperation.py +++ b/pypy/jit/metainterp/resoperation.py @@ -93,7 +93,7 @@ class AbstractResOp(object): def __repr__(self): return self.repr() - def repr(self): + def repr(self, graytext=False): # RPython-friendly version if self.result is not None: sres = '%s = ' % (self.result,) @@ -101,6 +101,8 @@ class AbstractResOp(object): sres = '' if self.name: prefix = "%s:%s " % (self.name, self.pc) + if graytext: + prefix = "\f%s\f" % prefix else: prefix = "" args = self.getarglist() @@ -457,6 +459,7 @@ _oplist = [ #'RUNTIMENEW/1', # ootype operation 'COND_CALL_GC_WB/2d', # [objptr, newvalue] (for the write barrier) 'DEBUG_MERGE_POINT/1', # debugging only + 'JIT_DEBUG/*', # debugging only 'VIRTUAL_REF_FINISH/2', # removed before it's passed to the backend 'COPYSTRCONTENT/5', # src, dst, srcstart, dststart, length 'COPYUNICODECONTENT/5', diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py index 51cc186682..b6df2c84dc 100644 --- a/pypy/jit/metainterp/resume.py +++ b/pypy/jit/metainterp/resume.py @@ -689,9 +689,11 @@ class AbstractResumeDataReader(object): # ---------- when resuming for pyjitpl.py, make boxes ---------- -def rebuild_from_resumedata(metainterp, storage, virtualizable_info): +def rebuild_from_resumedata(metainterp, storage, virtualizable_info, + greenfield_info): resumereader = ResumeDataBoxReader(storage, metainterp) - boxes = resumereader.consume_vref_and_vable_boxes(virtualizable_info) + boxes = resumereader.consume_vref_and_vable_boxes(virtualizable_info, + greenfield_info) virtualizable_boxes, virtualref_boxes = boxes frameinfo = storage.rd_frame_info_list while True: @@ -736,15 +738,18 @@ class ResumeDataBoxReader(AbstractResumeDataReader): assert (end & 1) == 0 return [self.decode_ref(nums[i]) for i in range(end)] - def consume_vref_and_vable_boxes(self, vinfo): + def consume_vref_and_vable_boxes(self, vinfo, ginfo): nums = self.cur_numb.nums self.cur_numb = self.cur_numb.prev - if vinfo is None: - virtualizable_boxes = None - end = len(nums) - else: + if vinfo is not None: virtualizable_boxes = self.consume_virtualizable_boxes(vinfo, nums) end = len(nums) - len(virtualizable_boxes) + elif ginfo is not None: + virtualizable_boxes = [self.decode_ref(nums[-1])] + end = len(nums) - 1 + else: + virtualizable_boxes = None + end = len(nums) virtualref_boxes = self.consume_virtualref_boxes(nums, end) return virtualizable_boxes, virtualref_boxes @@ -901,8 +906,9 @@ def blackhole_from_resumedata(blackholeinterpbuilder, jitdriver_sd, storage, resumereader = ResumeDataDirectReader(blackholeinterpbuilder.cpu, storage, all_virtuals) vinfo = jitdriver_sd.virtualizable_info + ginfo = jitdriver_sd.greenfield_info vrefinfo = blackholeinterpbuilder.metainterp_sd.virtualref_info - resumereader.consume_vref_and_vable(vrefinfo, vinfo) + resumereader.consume_vref_and_vable(vrefinfo, vinfo, ginfo) # # First get a chain of blackhole interpreters whose length is given # by the depth of rd_frame_info_list. The first one we get must be @@ -932,11 +938,11 @@ def blackhole_from_resumedata(blackholeinterpbuilder, jitdriver_sd, storage, resumereader.done() return firstbh -def force_from_resumedata(metainterp_sd, storage, vinfo=None): +def force_from_resumedata(metainterp_sd, storage, vinfo, ginfo): resumereader = ResumeDataDirectReader(metainterp_sd.cpu, storage) resumereader.handling_async_forcing() vrefinfo = metainterp_sd.virtualref_info - resumereader.consume_vref_and_vable(vrefinfo, vinfo) + resumereader.consume_vref_and_vable(vrefinfo, vinfo, ginfo) return resumereader.force_all_virtuals() class ResumeDataDirectReader(AbstractResumeDataReader): @@ -1011,11 +1017,12 @@ class ResumeDataDirectReader(AbstractResumeDataReader): return specialize_value(TYPE, x) load_value_of_type._annspecialcase_ = 'specialize:arg(1)' - def consume_vref_and_vable(self, vrefinfo, vinfo): + def consume_vref_and_vable(self, vrefinfo, vinfo, ginfo): nums = self.cur_numb.nums self.cur_numb = self.cur_numb.prev if self.resume_after_guard_not_forced != 2: end_vref = self.consume_vable_info(vinfo, nums) + if ginfo is not None: end_vref -= 1 self.consume_virtualref_info(vrefinfo, nums, end_vref) def allocate_with_vtable(self, known_class): diff --git a/pypy/jit/metainterp/test/test_basic.py b/pypy/jit/metainterp/test/test_basic.py index e03d0be598..4f1a507cee 100644 --- a/pypy/jit/metainterp/test/test_basic.py +++ b/pypy/jit/metainterp/test/test_basic.py @@ -2,6 +2,7 @@ import py import sys from pypy.rlib.jit import JitDriver, we_are_jitted, hint, dont_look_inside from pypy.rlib.jit import OPTIMIZER_FULL, OPTIMIZER_SIMPLE, loop_invariant +from pypy.rlib.jit import jit_debug, assert_green, AssertGreenFailed from pypy.jit.metainterp.warmspot import ll_meta_interp, get_stats from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp import pyjitpl, history @@ -44,6 +45,7 @@ def _get_jitcodes(testself, CPUClass, func, values, type_system): num_green_args = 0 portal_graph = graphs[0] virtualizable_info = None + greenfield_info = None result_type = result_kind portal_runner_ptr = "???" @@ -1644,6 +1646,33 @@ class BasicTests: res = self.interp_operations(f, [10, 3.5]) assert res == 3.5 + def test_jit_debug(self): + myjitdriver = JitDriver(greens = [], reds = ['x']) + class A: + pass + def f(x): + while x > 0: + myjitdriver.can_enter_jit(x=x) + myjitdriver.jit_merge_point(x=x) + jit_debug("hi there:", x) + jit_debug("foobar") + x -= 1 + return x + res = self.meta_interp(f, [8]) + assert res == 0 + self.check_loops(jit_debug=2) + + def test_assert_green(self): + def f(x, promote): + if promote: + x = hint(x, promote=True) + assert_green(x) + return x + res = self.interp_operations(f, [8, 1]) + assert res == 8 + py.test.raises(AssertGreenFailed, self.interp_operations, f, [8, 0]) + + class TestOOtype(BasicTests, OOJitMixin): def test_oohash(self): diff --git a/pypy/jit/metainterp/test/test_greenfield.py b/pypy/jit/metainterp/test/test_greenfield.py new file mode 100644 index 0000000000..b9fbbf50a8 --- /dev/null +++ b/pypy/jit/metainterp/test/test_greenfield.py @@ -0,0 +1,60 @@ +from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin +from pypy.rlib.jit import JitDriver + + +class GreenFieldsTests: + + def test_green_field_1(self): + myjitdriver = JitDriver(greens=['ctx.x'], reds=['ctx']) + class Ctx(object): + _immutable_fields_ = ['x'] + def __init__(self, x, y): + self.x = x + self.y = y + def f(x, y): + ctx = Ctx(x, y) + while 1: + myjitdriver.can_enter_jit(ctx=ctx) + myjitdriver.jit_merge_point(ctx=ctx) + ctx.y -= 1 + if ctx.y < 0: + return ctx.y + def g(y): + return f(5, y) + f(6, y) + # + res = self.meta_interp(g, [7]) + assert res == -2 + self.check_loop_count(2) + self.check_loops(guard_value=0) + + def test_green_field_2(self): + myjitdriver = JitDriver(greens=['ctx.x'], reds=['ctx']) + class Ctx(object): + _immutable_fields_ = ['x'] + def __init__(self, x, y): + self.x = x + self.y = y + def f(x, y): + ctx = Ctx(x, y) + while 1: + myjitdriver.can_enter_jit(ctx=ctx) + myjitdriver.jit_merge_point(ctx=ctx) + ctx.y -= 1 + if ctx.y < 0: + pass # to just make two paths + if ctx.y < -10: + return ctx.y + def g(y): + return f(5, y) + f(6, y) + # + res = self.meta_interp(g, [7]) + assert res == -22 + self.check_loop_count(4) + self.check_loops(guard_value=0) + + +class TestLLtypeGreenFieldsTests(GreenFieldsTests, LLJitMixin): + pass + +class TestOOtypeGreenFieldsTests(GreenFieldsTests, OOJitMixin): + pass diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py index f81998fcfb..097f9f99c9 100644 --- a/pypy/jit/metainterp/test/test_virtualref.py +++ b/pypy/jit/metainterp/test/test_virtualref.py @@ -93,7 +93,7 @@ class VRefTests: lst = [] vrefinfo.continue_tracing = lambda vref, virtual: \ lst.append((vref, virtual)) - resumereader.consume_vref_and_vable(vrefinfo, None) + resumereader.consume_vref_and_vable(vrefinfo, None, None) del vrefinfo.continue_tracing assert len(lst) == 1 lltype.cast_opaque_ptr(lltype.Ptr(JIT_VIRTUAL_REF), diff --git a/pypy/jit/metainterp/test/test_warmspot.py b/pypy/jit/metainterp/test/test_warmspot.py index b8c2bbafdd..3fedae8d39 100644 --- a/pypy/jit/metainterp/test/test_warmspot.py +++ b/pypy/jit/metainterp/test/test_warmspot.py @@ -296,6 +296,69 @@ class WarmspotTests(object): assert res == 1 self.check_loops(int_add=1) # I get 13 without the loop_header() + def test_omit_can_enter_jit(self): + # Simple test comparing the effects of always giving a can_enter_jit(), + # or not giving any. Mostly equivalent, except that if given, it is + # ignored the first time, and so it ends up taking one extra loop to + # start JITting. + mydriver = JitDriver(greens=[], reds=['m']) + # + for i2 in range(10): + def f2(m): + while m > 0: + mydriver.jit_merge_point(m=m) + m -= 1 + self.meta_interp(f2, [i2]) + try: + self.check_tree_loop_count(1) + break + except AssertionError: + print "f2: no loop generated for i2==%d" % i2 + else: + raise # re-raise the AssertionError: check_loop_count never 1 + # + for i1 in range(10): + def f1(m): + while m > 0: + mydriver.can_enter_jit(m=m) + mydriver.jit_merge_point(m=m) + m -= 1 + self.meta_interp(f1, [i1]) + try: + self.check_tree_loop_count(1) + break + except AssertionError: + print "f1: no loop generated for i1==%d" % i1 + else: + raise # re-raise the AssertionError: check_loop_count never 1 + # + assert i1 - 1 == i2 + + def test_no_loop_at_all(self): + mydriver = JitDriver(greens=[], reds=['m']) + def f2(m): + mydriver.jit_merge_point(m=m) + return m - 1 + def f1(m): + while m > 0: + m = f2(m) + self.meta_interp(f1, [8]) + # it should generate one "loop" only, which ends in a FINISH + # corresponding to the return from f2. + self.check_tree_loop_count(1) + self.check_loop_count(0) + + def test_simple_loop(self): + mydriver = JitDriver(greens=[], reds=['m']) + def f1(m): + while m > 0: + mydriver.jit_merge_point(m=m) + m = m - 1 + self.meta_interp(f1, [8]) + self.check_loop_count(1) + self.check_loops({'int_sub': 1, 'int_gt': 1, 'guard_true': 1, + 'jump': 1}) + class TestLLWarmspot(WarmspotTests, LLJitMixin): CPUClass = runner.LLtypeCPU diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py index b978ba3dc0..1e3ce0dabf 100644 --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -115,10 +115,10 @@ def _find_jit_marker(graphs, marker_name): return results def find_can_enter_jit(graphs): - results = _find_jit_marker(graphs, 'can_enter_jit') - if not results: - raise Exception("no can_enter_jit found!") - return results + return _find_jit_marker(graphs, 'can_enter_jit') + +def find_loop_headers(graphs): + return _find_jit_marker(graphs, 'loop_header') def find_jit_merge_points(graphs): results = _find_jit_marker(graphs, 'jit_merge_point') @@ -211,9 +211,9 @@ class WarmRunnerDesc(object): "there are multiple jit_merge_points with the same jitdriver" def split_graph_and_record_jitdriver(self, graph, block, pos): - jd = JitDriverStaticData() - jd._jit_merge_point_pos = (graph, block, pos) op = block.operations[pos] + jd = JitDriverStaticData() + jd._jit_merge_point_pos = (graph, op) args = op.args[2:] s_binding = self.translator.annotator.binding jd._portal_args_s = [s_binding(v) for v in args] @@ -286,10 +286,20 @@ class WarmRunnerDesc(object): def make_virtualizable_infos(self): vinfos = {} for jd in self.jitdrivers_sd: + # + jd.greenfield_info = None + for name in jd.jitdriver.greens: + if '.' in name: + from pypy.jit.metainterp.greenfield import GreenFieldInfo + jd.greenfield_info = GreenFieldInfo(self.cpu, jd) + break + # if not jd.jitdriver.virtualizables: jd.virtualizable_info = None jd.index_of_virtualizable = -1 continue + else: + assert jd.greenfield_info is None, "XXX not supported yet" # jitdriver = jd.jitdriver assert len(jitdriver.virtualizables) == 1 # for now @@ -457,8 +467,7 @@ class WarmRunnerDesc(object): self.make_args_specification(jd) def make_args_specification(self, jd): - graph, block, index = jd._jit_merge_point_pos - op = block.operations[index] + graph, op = jd._jit_merge_point_pos greens_v, reds_v = support.decode_hp_hint_args(op) ALLARGS = [v.concretetype for v in (greens_v + reds_v)] jd._green_args_spec = [v.concretetype for v in greens_v] @@ -474,26 +483,37 @@ class WarmRunnerDesc(object): [lltype.Signed, llmemory.GCREF], RESTYPE) def rewrite_can_enter_jits(self): - can_enter_jits = find_can_enter_jit(self.translator.graphs) sublists = {} for jd in self.jitdrivers_sd: - sublists[jd.jitdriver] = [] + sublists[jd.jitdriver] = jd, [] + jd.no_loop_header = True + # + loop_headers = find_loop_headers(self.translator.graphs) + for graph, block, index in loop_headers: + op = block.operations[index] + jitdriver = op.args[1].value + assert jitdriver in sublists, \ + "loop_header with no matching jit_merge_point" + jd, sublist = sublists[jitdriver] + jd.no_loop_header = False + # + can_enter_jits = find_can_enter_jit(self.translator.graphs) for graph, block, index in can_enter_jits: op = block.operations[index] jitdriver = op.args[1].value assert jitdriver in sublists, \ "can_enter_jit with no matching jit_merge_point" + jd, sublist = sublists[jitdriver] origportalgraph = jd._jit_merge_point_pos[0] if graph is not origportalgraph: - sublists[jitdriver].append((graph, block, index)) + sublist.append((graph, block, index)) + jd.no_loop_header = False else: pass # a 'can_enter_jit' before the 'jit-merge_point', but # originally in the same function: we ignore it here # see e.g. test_jitdriver.test_simple for jd in self.jitdrivers_sd: - sublist = sublists[jd.jitdriver] - assert len(sublist) > 0, \ - "found no can_enter_jit for %r" % (jd.jitdriver,) + _, sublist = sublists[jd.jitdriver] self.rewrite_can_enter_jit(jd, sublist) def rewrite_can_enter_jit(self, jd, can_enter_jits): @@ -501,6 +521,19 @@ class WarmRunnerDesc(object): FUNCPTR = jd._PTR_JIT_ENTER_FUNCTYPE jit_enter_fnptr = self.helper_func(FUNCPTR, jd._maybe_enter_jit_fn) + if len(can_enter_jits) == 0: + # see test_warmspot.test_no_loop_at_all + operations = jd.portal_graph.startblock.operations + op1 = operations[0] + assert (op1.opname == 'jit_marker' and + op1.args[0].value == 'jit_merge_point') + op0 = SpaceOperation( + 'jit_marker', + [Constant('can_enter_jit', lltype.Void)] + op1.args[1:], + None) + operations.insert(0, op0) + can_enter_jits = [(jd.portal_graph, jd.portal_graph.startblock, 0)] + for graph, block, index in can_enter_jits: if graph is jd._jit_merge_point_pos[0]: continue @@ -709,8 +742,14 @@ class WarmRunnerDesc(object): # ____________________________________________________________ # Now mutate origportalgraph to end with a call to portal_runner_ptr # - _, origblock, origindex = jd._jit_merge_point_pos - op = origblock.operations[origindex] + _, op = jd._jit_merge_point_pos + for origblock in origportalgraph.iterblocks(): + if op in origblock.operations: + break + else: + assert False, "lost the operation %r in the graph %r" % ( + op, origportalgraph) + origindex = origblock.operations.index(op) assert op.opname == 'jit_marker' assert op.args[0].value == 'jit_merge_point' greens_v, reds_v = support.decode_hp_hint_args(op) diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py index 476c09da8d..acaf7d2274 100644 --- a/pypy/rlib/jit.py +++ b/pypy/rlib/jit.py @@ -145,6 +145,24 @@ class Entry(ExtRegistryEntry): return hop.inputconst(lltype.Signed, _we_are_jitted) +def jit_debug(string, arg1=-sys.maxint-1, arg2=-sys.maxint-1, + arg3=-sys.maxint-1, arg4=-sys.maxint-1): + """When JITted, cause an extra operation DEBUG_MERGE_POINT to appear in + the graphs. Should not be left after debugging.""" + keepalive_until_here(string) # otherwise the whole function call is removed +jit_debug.oopspec = 'jit.debug(string, arg1, arg2, arg3, arg4)' + +def assert_green(value): + """Very strong assert: checks that 'value' is a green + (a JIT compile-time constant).""" + keepalive_until_here(value) +assert_green._annspecialcase_ = 'specialize:argtype(0)' +assert_green.oopspec = 'jit.assert_green(value)' + +class AssertGreenFailed(Exception): + pass + + ##def force_virtualizable(virtualizable): ## pass @@ -272,7 +290,8 @@ class JitDriver: self.virtualizables = virtualizables for v in self.virtualizables: assert v in self.reds - self._alllivevars = dict.fromkeys(self.greens + self.reds) + self._alllivevars = dict.fromkeys( + [name for name in self.greens + self.reds if '.' not in name]) self._make_extregistryentries() self.get_jitcell_at = get_jitcell_at self.set_jitcell_at = set_jitcell_at @@ -364,7 +383,8 @@ class ExtEnterLeaveMarker(ExtRegistryEntry): driver = self.instance.im_self keys = kwds_s.keys() keys.sort() - expected = ['s_' + name for name in driver.greens + driver.reds] + expected = ['s_' + name for name in driver.greens + driver.reds + if '.' not in name] expected.sort() if keys != expected: raise JitHintError("%s expects the following keyword " @@ -409,7 +429,13 @@ class ExtEnterLeaveMarker(ExtRegistryEntry): uniquekey = 'jitdriver.%s' % func.func_name args_s = args_s[:] for name in variables: - s_arg = kwds_s['s_' + name] + if '.' not in name: + s_arg = kwds_s['s_' + name] + else: + objname, fieldname = name.split('.') + s_instance = kwds_s['s_' + objname] + s_arg = s_instance.classdef.about_attribute(fieldname) + assert s_arg is not None args_s.append(s_arg) bk.emulate_pbc_call(uniquekey, s_func, args_s) @@ -422,9 +448,42 @@ class ExtEnterLeaveMarker(ExtRegistryEntry): greens_v = [] reds_v = [] for name in driver.greens: - i = kwds_i['i_' + name] - r_green = hop.args_r[i] - v_green = hop.inputarg(r_green, arg=i) + if '.' not in name: + i = kwds_i['i_' + name] + r_green = hop.args_r[i] + v_green = hop.inputarg(r_green, arg=i) + else: + if hop.rtyper.type_system.name == 'ootypesystem': + py.test.skip("lltype only") + objname, fieldname = name.split('.') # see test_green_field + assert objname in driver.reds + i = kwds_i['i_' + objname] + s_red = hop.args_s[i] + r_red = hop.args_r[i] + while True: + try: + mangled_name, r_field = r_red._get_field(fieldname) + break + except KeyError: + pass + assert r_red.rbase is not None, ( + "field %r not found in %r" % (name, + r_red.lowleveltype.TO)) + r_red = r_red.rbase + GTYPE = r_red.lowleveltype.TO + assert GTYPE._immutable_field(mangled_name), ( + "field %r must be declared as immutable" % name) + if not hasattr(driver, 'll_greenfields'): + driver.ll_greenfields = {} + driver.ll_greenfields[name] = GTYPE, mangled_name + # + v_red = hop.inputarg(r_red, arg=i) + c_llname = hop.inputconst(lltype.Void, mangled_name) + v_green = hop.genop('getfield', [v_red, c_llname], + resulttype = r_field) + s_green = s_red.classdef.about_attribute(fieldname) + assert s_green is not None + hop.rtyper.annotator.setbinding(v_green, s_green) greens_v.append(v_green) for name in driver.reds: i = kwds_i['i_' + name] diff --git a/pypy/rlib/rsre/rsre_char.py b/pypy/rlib/rsre/rsre_char.py index 87029d7331..6eaa610b15 100644 --- a/pypy/rlib/rsre/rsre_char.py +++ b/pypy/rlib/rsre/rsre_char.py @@ -4,6 +4,7 @@ Character categories and charsets. import sys from pypy.rlib.rlocale import tolower, isalnum from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib import jit # Note: the unicode parts of this module require you to call # rsre_char.set_unicode_db() first, to select one of the modules @@ -43,6 +44,7 @@ BIG_ENDIAN = sys.byteorder == "big" # XXX can we import those safely from sre_constants? SRE_INFO_PREFIX = 1 SRE_INFO_LITERAL = 2 +SRE_INFO_CHARSET = 4 SRE_FLAG_LOCALE = 4 # honour system locale SRE_FLAG_UNICODE = 32 # use unicode locale OPCODE_INFO = 17 @@ -64,33 +66,27 @@ def getlower(char_ord, flags): #### Category helpers -ascii_char_info = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 2, -2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, -0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 25, 25, 25, 25, 25, 25, 25, -25, 25, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, -24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, -0, 0, 16, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, -24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0 ] - +is_a_word = [(chr(i).isalnum() or chr(i) == '_') for i in range(256)] linebreak = ord("\n") underline = ord("_") def is_digit(code): - return code < 128 and (ascii_char_info[code] & 1 != 0) + return code <= 57 and code >= 48 def is_uni_digit(code): assert unicodedb is not None return unicodedb.isdigit(code) def is_space(code): - return code < 128 and (ascii_char_info[code] & 2 != 0) + return code == 32 or (code <= 13 and code >= 9) def is_uni_space(code): assert unicodedb is not None return unicodedb.isspace(code) def is_word(code): - return code < 128 and (ascii_char_info[code] & 16 != 0) + assert code >= 0 + return code < 256 and is_a_word[code] def is_uni_word(code): assert unicodedb is not None @@ -142,6 +138,7 @@ category_dispatch_unroll = unrolling_iterable(category_dispatch_table) SET_OK = -1 SET_NOT_OK = -2 +@jit.unroll_safe def check_charset(pattern, ppos, char_code): """Checks whether a character matches set of arbitrary length. The set starts at pattern[ppos].""" diff --git a/pypy/rlib/rsre/rsre_core.py b/pypy/rlib/rsre/rsre_core.py index f48da84ebd..5a7b9c39d0 100644 --- a/pypy/rlib/rsre/rsre_core.py +++ b/pypy/rlib/rsre/rsre_core.py @@ -4,6 +4,8 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rsre import rsre_char from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib import jit +from pypy.rlib.rsre.rsre_jit import install_jitdriver, install_jitdriver_spec OPCODE_FAILURE = 0 @@ -56,16 +58,19 @@ def specializectx(func): _seen_specname[specname] = True # Install a copy of the function under the name '_spec_funcname' in each # concrete subclass + specialized_methods = [] for prefix, concreteclass in [('str', StrMatchContext), ('uni', UnicodeMatchContext)]: newfunc = func_with_new_name(func, prefix + specname) assert not hasattr(concreteclass, specname) setattr(concreteclass, specname, newfunc) + specialized_methods.append(newfunc) # Return a dispatcher function, specialized on the exact type of 'ctx' def dispatch(ctx, *args): return getattr(ctx, specname)(*args) dispatch._annspecialcase_ = 'specialize:argtype(0)' - return dispatch + dispatch._specialized_methods_ = specialized_methods + return func_with_new_name(dispatch, specname) # ____________________________________________________________ @@ -75,6 +80,7 @@ class Error(Exception): class AbstractMatchContext(object): """Abstract base class""" + _immutable_fields_ = ['pattern[*]', 'flags', 'end'] match_start = 0 match_end = 0 match_marks = None @@ -238,8 +244,9 @@ class BranchMatchResult(MatchResult): self.start_ptr = ptr self.start_marks = marks + @jit.unroll_safe def find_first_result(self, ctx): - ppos = self.ppos + ppos = jit.hint(self.ppos, promote=True) while ctx.pat(ppos): result = sre_match(ctx, ppos + 1, self.start_ptr, self.start_marks) ppos += ctx.pat(ppos) @@ -250,6 +257,10 @@ class BranchMatchResult(MatchResult): find_next_result = find_first_result class RepeatOneMatchResult(MatchResult): + install_jitdriver('RepeatOne', + greens=['nextppos', 'ctx.pattern'], + reds=['ptr', 'self', 'ctx'], + debugprint=(1, 0)) # indices in 'greens' def __init__(self, nextppos, minptr, ptr, marks): self.nextppos = nextppos @@ -259,8 +270,11 @@ class RepeatOneMatchResult(MatchResult): def find_first_result(self, ctx): ptr = self.start_ptr + nextppos = self.nextppos while ptr >= self.minptr: - result = sre_match(ctx, self.nextppos, ptr, self.start_marks) + ctx.jitdriver_RepeatOne.jit_merge_point( + self=self, ptr=ptr, ctx=ctx, nextppos=nextppos) + result = sre_match(ctx, nextppos, ptr, self.start_marks) ptr -= 1 if result is not None: self.subresult = result @@ -270,6 +284,10 @@ class RepeatOneMatchResult(MatchResult): class MinRepeatOneMatchResult(MatchResult): + install_jitdriver('MinRepeatOne', + greens=['nextppos', 'ppos3', 'ctx.pattern'], + reds=['ptr', 'self', 'ctx'], + debugprint=(2, 0)) # indices in 'greens' def __init__(self, nextppos, ppos3, maxptr, ptr, marks): self.nextppos = nextppos @@ -280,29 +298,32 @@ class MinRepeatOneMatchResult(MatchResult): def find_first_result(self, ctx): ptr = self.start_ptr + nextppos = self.nextppos + ppos3 = self.ppos3 while ptr <= self.maxptr: - result = sre_match(ctx, self.nextppos, ptr, self.start_marks) + ctx.jitdriver_MinRepeatOne.jit_merge_point( + self=self, ptr=ptr, ctx=ctx, nextppos=nextppos, ppos3=ppos3) + result = sre_match(ctx, nextppos, ptr, self.start_marks) if result is not None: self.subresult = result self.start_ptr = ptr return self - if not self.next_char_ok(ctx, ptr): + if not self.next_char_ok(ctx, ptr, ppos3): break ptr += 1 def find_next_result(self, ctx): ptr = self.start_ptr - if not self.next_char_ok(ctx, ptr): + if not self.next_char_ok(ctx, ptr, self.ppos3): return self.start_ptr = ptr + 1 return self.find_first_result(ctx) - def next_char_ok(self, ctx, ptr): + def next_char_ok(self, ctx, ptr, ppos): if ptr == ctx.end: return False - ppos = self.ppos3 op = ctx.pat(ppos) - for op1, (checkerfn, _) in unroll_char_checker: + for op1, checkerfn in unroll_char_checker: if op1 == op: return checkerfn(ctx, ptr, ppos) raise Error("next_char_ok[%d]" % op) @@ -325,41 +346,34 @@ class Pending(object): self.next = next # chained list class MaxUntilMatchResult(AbstractUntilMatchResult): + install_jitdriver('MaxUntil', + greens=['ppos', 'tailppos', 'match_more', 'ctx.pattern'], + reds=['ptr', 'marks', 'self', 'ctx'], + debugprint=(3, 0, 2)) def find_first_result(self, ctx): - enum = sre_match(ctx, self.ppos + 3, self.cur_ptr, self.cur_marks) - return self.search_next(ctx, enum, resume=False) + return self.search_next(ctx, match_more=True) def find_next_result(self, ctx): - return self.search_next(ctx, None, resume=True) + return self.search_next(ctx, match_more=False) - def search_next(self, ctx, enum, resume): + def search_next(self, ctx, match_more): ppos = self.ppos - min = ctx.pat(ppos+1) - max = ctx.pat(ppos+2) + tailppos = self.tailppos ptr = self.cur_ptr marks = self.cur_marks while True: - while True: - if (enum is not None and - (ptr != ctx.match_end or self.num_pending < min)): - # ^^^^^^^^^^ zero-width match protection - # matched one more 'item'. record it and continue. - self.pending = Pending(ptr, marks, enum, self.pending) - self.num_pending += 1 - ptr = ctx.match_end - marks = ctx.match_marks - break - # 'item' no longer matches. - if not resume and self.num_pending >= min: - # try to match 'tail' if we have enough 'item' - result = sre_match(ctx, self.tailppos, ptr, marks) - if result is not None: - self.subresult = result - self.cur_ptr = ptr - self.cur_marks = marks - return self - resume = False + ctx.jitdriver_MaxUntil.jit_merge_point( + ppos=ppos, tailppos=tailppos, match_more=match_more, + ptr=ptr, marks=marks, self=self, ctx=ctx) + if match_more: + max = ctx.pat(ppos+2) + if max == 65535 or self.num_pending < max: + # try to match one more 'item' + enum = sre_match(ctx, ppos + 3, ptr, marks) + else: + enum = None # 'max' reached, no more matches + else: p = self.pending if p is None: return @@ -369,11 +383,27 @@ class MaxUntilMatchResult(AbstractUntilMatchResult): marks = p.marks enum = p.enum.move_to_next_result(ctx) # - if max == 65535 or self.num_pending < max: - # try to match one more 'item' - enum = sre_match(ctx, ppos + 3, ptr, marks) + min = ctx.pat(ppos+1) + if (enum is not None and + (ptr != ctx.match_end or self.num_pending < min)): + # ^^^^^^^^^^ zero-width match protection + # matched one more 'item'. record it and continue. + self.pending = Pending(ptr, marks, enum, self.pending) + self.num_pending += 1 + ptr = ctx.match_end + marks = ctx.match_marks + match_more = True else: - enum = None # 'max' reached, no more matches + # 'item' no longer matches. + if self.num_pending >= min: + # try to match 'tail' if we have enough 'item' + result = sre_match(ctx, tailppos, ptr, marks) + if result is not None: + self.subresult = result + self.cur_ptr = ptr + self.cur_marks = marks + return self + match_more = False class MinUntilMatchResult(AbstractUntilMatchResult): @@ -384,6 +414,7 @@ class MinUntilMatchResult(AbstractUntilMatchResult): return self.search_next(ctx, resume=True) def search_next(self, ctx, resume): + # XXX missing jit support here ppos = self.ppos min = ctx.pat(ppos+1) max = ctx.pat(ppos+2) @@ -429,6 +460,7 @@ class MinUntilMatchResult(AbstractUntilMatchResult): # ____________________________________________________________ @specializectx +@jit.unroll_safe def sre_match(ctx, ppos, ptr, marks): """Returns either None or a MatchResult object. Usually we only need the first result, but there is the case of REPEAT...UNTIL where we @@ -438,6 +470,12 @@ def sre_match(ctx, ppos, ptr, marks): op = ctx.pat(ppos) ppos += 1 + #jit.jit_debug("sre_match", op, ppos, ptr) + # + # When using the JIT, calls to sre_match() must always have a constant + # (green) argument for 'ppos'. If not, the following assert fails. + jit.assert_green(op) + if op == OPCODE_FAILURE: return @@ -712,13 +750,23 @@ def match_repeated_ignore(ctx, ptr, oldptr, length): @specializectx def find_repetition_end(ctx, ppos, ptr, maxcount): end = ctx.end - # adjust end - if maxcount != 65535: + if maxcount <= 1: + if maxcount == 1 and ptr < end: + # Relatively common case: maxcount == 1. If we are not at the + # end of the string, it's done by a single direct check. + op = ctx.pat(ppos) + for op1, checkerfn in unroll_char_checker: + if op1 == op: + if checkerfn(ctx, ptr, ppos): + return ptr + 1 + return ptr + elif maxcount != 65535: + # adjust end end1 = ptr + maxcount if end1 <= end: end = end1 op = ctx.pat(ppos) - for op1, (_, fre) in unroll_char_checker: + for op1, fre in unroll_fre_checker: if op1 == op: return fre(ctx, ptr, end, ppos) raise Error("rsre.find_repetition_end[%d]" % op) @@ -751,23 +799,60 @@ def _make_fre(checkerfn): if checkerfn == match_ANY_ALL: def fre(ctx, ptr, end, ppos): return end + elif checkerfn == match_IN: + install_jitdriver_spec('MatchIn', + greens=['ppos', 'ctx.pattern'], + reds=['ptr', 'end', 'ctx'], + debugprint=(1, 0)) + @specializectx + def fre(ctx, ptr, end, ppos): + while True: + ctx.jitdriver_MatchIn.jit_merge_point(ctx=ctx, ptr=ptr, + end=end, ppos=ppos) + if ptr < end and checkerfn(ctx, ptr, ppos): + ptr += 1 + else: + return ptr + elif checkerfn == match_IN_IGNORE: + install_jitdriver_spec('MatchInIgnore', + greens=['ppos', 'ctx.pattern'], + reds=['ptr', 'end', 'ctx'], + debugprint=(1, 0)) + @specializectx + def fre(ctx, ptr, end, ppos): + while True: + ctx.jitdriver_MatchInIgnore.jit_merge_point(ctx=ctx, ptr=ptr, + end=end, ppos=ppos) + if ptr < end and checkerfn(ctx, ptr, ppos): + ptr += 1 + else: + return ptr else: + # in the other cases, the fre() function is not JITted at all + # and is present as a residual call. + @specializectx def fre(ctx, ptr, end, ppos): while ptr < end and checkerfn(ctx, ptr, ppos): ptr += 1 return ptr - return checkerfn, fre - -unroll_char_checker = unrolling_iterable([ - (OPCODE_ANY, _make_fre(match_ANY)), - (OPCODE_ANY_ALL, _make_fre(match_ANY_ALL)), - (OPCODE_IN, _make_fre(match_IN)), - (OPCODE_IN_IGNORE, _make_fre(match_IN_IGNORE)), - (OPCODE_LITERAL, _make_fre(match_LITERAL)), - (OPCODE_LITERAL_IGNORE, _make_fre(match_LITERAL_IGNORE)), - (OPCODE_NOT_LITERAL, _make_fre(match_NOT_LITERAL)), - (OPCODE_NOT_LITERAL_IGNORE, _make_fre(match_NOT_LITERAL_IGNORE)), - ]) + fre = func_with_new_name(fre, 'fre_' + checkerfn.__name__) + return fre + +unroll_char_checker = [ + (OPCODE_ANY, match_ANY), + (OPCODE_ANY_ALL, match_ANY_ALL), + (OPCODE_IN, match_IN), + (OPCODE_IN_IGNORE, match_IN_IGNORE), + (OPCODE_LITERAL, match_LITERAL), + (OPCODE_LITERAL_IGNORE, match_LITERAL_IGNORE), + (OPCODE_NOT_LITERAL, match_NOT_LITERAL), + (OPCODE_NOT_LITERAL_IGNORE, match_NOT_LITERAL_IGNORE), + ] +unroll_fre_checker = [(_op, _make_fre(_fn)) + for (_op, _fn) in unroll_char_checker] + +unroll_char_checker = unrolling_iterable(unroll_char_checker) +unroll_fre_checker = unrolling_iterable(unroll_fre_checker) ##### At dispatch @@ -873,74 +958,139 @@ def search(pattern, string, start=0, end=sys.maxint, flags=0): else: return None +install_jitdriver('Match', + greens=['ctx.pattern'], reds=['ctx'], + debugprint=(0,)) + def match_context(ctx): ctx.original_pos = ctx.match_start if ctx.end < ctx.match_start: return False + ctx.jitdriver_Match.jit_merge_point(ctx=ctx) return sre_match(ctx, 0, ctx.match_start, None) is not None def search_context(ctx): ctx.original_pos = ctx.match_start if ctx.end < ctx.match_start: return False - if ctx.pat(0) == OPCODE_INFO: - if ctx.pat(2) & rsre_char.SRE_INFO_PREFIX and ctx.pat(5) > 1: - return fast_search(ctx) - return regular_search(ctx) - -def regular_search(ctx): + base = 0 + charset = False + if ctx.pat(base) == OPCODE_INFO: + flags = ctx.pat(2) + if flags & rsre_char.SRE_INFO_PREFIX: + if ctx.pat(5) > 1: + return fast_search(ctx) + else: + charset = (flags & rsre_char.SRE_INFO_CHARSET) + base += 1 + ctx.pat(1) + if ctx.pat(base) == OPCODE_LITERAL: + return literal_search(ctx, base) + if charset: + return charset_search(ctx, base) + return regular_search(ctx, base) + +install_jitdriver('RegularSearch', + greens=['base', 'ctx.pattern'], + reds=['start', 'ctx'], + debugprint=(1, 0)) + +def regular_search(ctx, base): start = ctx.match_start while start <= ctx.end: - if sre_match(ctx, 0, start, None) is not None: + ctx.jitdriver_RegularSearch.jit_merge_point(ctx=ctx, start=start, + base=base) + if sre_match(ctx, base, start, None) is not None: ctx.match_start = start return True start += 1 return False +install_jitdriver_spec("LiteralSearch", + greens=['base', 'character', 'ctx.pattern'], + reds=['start', 'ctx'], + debugprint=(2, 0, 1)) +@specializectx +def literal_search(ctx, base): + # pattern starts with a literal character. this is used + # for short prefixes, and if fast search is disabled + character = ctx.pat(base + 1) + base += 2 + start = ctx.match_start + while start < ctx.end: + ctx.jitdriver_LiteralSearch.jit_merge_point(ctx=ctx, start=start, + base=base, character=character) + if ctx.str(start) == character: + if sre_match(ctx, base, start + 1, None) is not None: + ctx.match_start = start + return True + start += 1 + return False + +install_jitdriver_spec("CharsetSearch", + greens=['base', 'ctx.pattern'], + reds=['start', 'ctx'], + debugprint=(1, 0)) +@specializectx +def charset_search(ctx, base): + # pattern starts with a character from a known set + start = ctx.match_start + while start < ctx.end: + ctx.jitdriver_CharsetSearch.jit_merge_point(ctx=ctx, start=start, + base=base) + if rsre_char.check_charset(ctx.pattern, 5, ctx.str(start)): + if sre_match(ctx, base, start, None) is not None: + ctx.match_start = start + return True + start += 1 + return False + +install_jitdriver_spec('FastSearch', + greens=['i', 'prefix_len', 'ctx.pattern'], + reds=['string_position', 'ctx'], + debugprint=(2, 0)) @specializectx def fast_search(ctx): # skips forward in a string as fast as possible using information from # an optimization info block # <INFO> <1=skip> <2=flags> <3=min> <4=...> # <5=length> <6=skip> <7=prefix data> <overlap data> - flags = ctx.pat(2) + string_position = ctx.match_start + if string_position >= ctx.end: + return False prefix_len = ctx.pat(5) assert prefix_len >= 0 - prefix_skip = ctx.pat(6) - assert prefix_skip >= 0 - overlap_offset = 7 + prefix_len - 1 - assert overlap_offset >= 0 - pattern_offset = ctx.pat(1) + 1 - ppos_start = pattern_offset + 2 * prefix_skip - assert ppos_start >= 0 i = 0 - string_position = ctx.match_start - end = ctx.end - while string_position < end: - while True: - char_ord = ctx.str(string_position) - if char_ord != ctx.pat(7 + i): - if i == 0: - break - else: - i = ctx.pat(overlap_offset + i) - else: - i += 1 - if i == prefix_len: - # found a potential match - start = string_position + 1 - prefix_len - assert start >= 0 - ptr = start + prefix_skip - if flags & rsre_char.SRE_INFO_LITERAL: - # matched all of pure literal pattern - ctx.match_start = start - ctx.match_end = ptr - ctx.match_marks = None - return True - if sre_match(ctx, ppos_start, ptr, None) is not None: - ctx.match_start = start - return True - i = ctx.pat(overlap_offset + i) - break + while True: + ctx.jitdriver_FastSearch.jit_merge_point(ctx=ctx, + string_position=string_position, i=i, prefix_len=prefix_len) + char_ord = ctx.str(string_position) + if char_ord != ctx.pat(7 + i): + if i > 0: + overlap_offset = prefix_len + (7 - 1) + i = ctx.pat(overlap_offset + i) + continue + else: + i += 1 + if i == prefix_len: + # found a potential match + start = string_position + 1 - prefix_len + assert start >= 0 + prefix_skip = ctx.pat(6) + ptr = start + prefix_skip + #flags = ctx.pat(2) + #if flags & rsre_char.SRE_INFO_LITERAL: + # # matched all of pure literal pattern + # ctx.match_start = start + # ctx.match_end = ptr + # ctx.match_marks = None + # return True + pattern_offset = ctx.pat(1) + 1 + ppos_start = pattern_offset + 2 * prefix_skip + if sre_match(ctx, ppos_start, ptr, None) is not None: + ctx.match_start = start + return True + overlap_offset = prefix_len + (7 - 1) + i = ctx.pat(overlap_offset + i) string_position += 1 - return False + if string_position >= ctx.end: + return False diff --git a/pypy/rlib/rsre/rsre_jit.py b/pypy/rlib/rsre/rsre_jit.py new file mode 100644 index 0000000000..97d91d2203 --- /dev/null +++ b/pypy/rlib/rsre/rsre_jit.py @@ -0,0 +1,40 @@ +from pypy.rlib.jit import JitDriver + + +class RSreJitDriver(JitDriver): + + def __init__(self, name, debugprint, **kwds): + JitDriver.__init__(self, **kwds) + # + def get_printable_location(*args): + # we print based on indices in 'args'. We first print + # 'ctx.pattern' from the arg number debugprint[0]. + pattern = args[debugprint[0]] + s = str(pattern) + if len(s) > 120: + s = s[:110] + '...' + if len(debugprint) > 1: + # then we print numbers from the args number + # debugprint[1] and possibly debugprint[2] + info = ' at %d' % (args[debugprint[1]],) + if len(debugprint) > 2: + info = '%s/%d' % (info, args[debugprint[2]]) + else: + info = '' + return '%s%s %s' % (name, info, s) + # + self.get_printable_location = get_printable_location + + +def install_jitdriver(name, **kwds): + from pypy.rlib.rsre.rsre_core import AbstractMatchContext + jitdriver = RSreJitDriver(name, **kwds) + setattr(AbstractMatchContext, 'jitdriver_' + name, jitdriver) + +def install_jitdriver_spec(name, **kwds): + from pypy.rlib.rsre.rsre_core import StrMatchContext + from pypy.rlib.rsre.rsre_core import UnicodeMatchContext + for prefix, concreteclass in [('Str', StrMatchContext), + ('Uni', UnicodeMatchContext)]: + jitdriver = RSreJitDriver(prefix + name, **kwds) + setattr(concreteclass, 'jitdriver_' + name, jitdriver) diff --git a/pypy/rlib/rsre/test/conftest.py b/pypy/rlib/rsre/test/conftest.py new file mode 100644 index 0000000000..05178c7d44 --- /dev/null +++ b/pypy/rlib/rsre/test/conftest.py @@ -0,0 +1,5 @@ +# import the option --viewloops from the JIT + +def pytest_addoption(parser): + from pypy.jit.conftest import pytest_addoption + pytest_addoption(parser) diff --git a/pypy/rlib/rsre/test/test_zjit.py b/pypy/rlib/rsre/test/test_zjit.py new file mode 100644 index 0000000000..b832bdceee --- /dev/null +++ b/pypy/rlib/rsre/test/test_zjit.py @@ -0,0 +1,120 @@ +from pypy.jit.metainterp.test import test_basic +from pypy.rlib.nonconst import NonConstant +from pypy.rlib.debug import make_sure_not_modified +from pypy.rlib.rsre.test.test_match import get_code +from pypy.rlib.rsre import rsre_core +from pypy.rpython.lltypesystem import lltype +from pypy.rpython.annlowlevel import llstr, hlstr + +def entrypoint1(r, string, repeat): + r = array2list(r) + string = hlstr(string) + make_sure_not_modified(r) + match = None + for i in range(repeat): + match = rsre_core.match(r, string) + if match is None: + return -1 + else: + return match.match_end + +def entrypoint2(r, string, repeat): + r = array2list(r) + string = hlstr(string) + make_sure_not_modified(r) + match = None + for i in range(repeat): + match = rsre_core.search(r, string) + if match is None: + return -1 + else: + return match.match_start + +def list2array(lst): + a = lltype.malloc(lltype.GcArray(lltype.Signed), len(lst)) + for i, x in enumerate(lst): + a[i] = x + return a + +def array2list(a): + return [a[i] for i in range(len(a))] + + +def test_jit_unroll_safe(): + # test that the decorators are applied in the right order + assert not hasattr(rsre_core.sre_match, '_jit_unroll_safe_') + for m in rsre_core.sre_match._specialized_methods_: + assert m._jit_unroll_safe_ + + +class TestJitRSre(test_basic.LLJitMixin): + + def meta_interp_match(self, pattern, string, repeat=1): + r = get_code(pattern) + return self.meta_interp(entrypoint1, [list2array(r), llstr(string), + repeat], + listcomp=True, backendopt=True) + + def meta_interp_search(self, pattern, string, repeat=1): + r = get_code(pattern) + return self.meta_interp(entrypoint2, [list2array(r), llstr(string), + repeat], + listcomp=True, backendopt=True) + + def test_simple_match_1(self): + res = self.meta_interp_match(r"ab*bbbbbbbc", "abbbbbbbbbcdef") + assert res == 11 + + def test_simple_match_2(self): + res = self.meta_interp_match(r".*abc", "xxabcyyyyyyyyyyyyy") + assert res == 5 + + def test_simple_match_repeated(self): + res = self.meta_interp_match(r"abcdef", "abcdef", repeat=10) + assert res == 6 + self.check_tree_loop_count(1) + + def test_match_minrepeat_1(self): + res = self.meta_interp_match(r".*?abc", "xxxxxxxxxxxxxxabc") + assert res == 17 + + #def test_match_maxuntil_1(self): + # res = self.meta_interp_match(r"(ab)*c", "ababababababababc") + # assert res == 17 + + def test_branch_1(self): + res = self.meta_interp_match(r".*?(ab|x)c", "xxxxxxxxxxxxxxabc") + assert res == 17 + + def test_match_minrepeat_2(self): + s = ("xxxxxxxxxxabbbbbbbbbb" + + "xxxxxxxxxxabbbbbbbbbb" + + "xxxxxxxxxxabbbbbbbbbb" + + "xxxxxxxxxxabbbbbbbbbbc") + res = self.meta_interp_match(r".*?ab+?c", s) + assert res == len(s) + + + def test_fast_search(self): + res = self.meta_interp_search(r"<foo\w+>", "e<f<f<foxd<f<fh<foobar>ua") + assert res == 15 + self.check_loops(guard_value=0) + + def test_regular_search(self): + res = self.meta_interp_search(r"<\w+>", "eiofweoxdiwhdoh<foobar>ua") + assert res == 15 + + def test_regular_search_upcase(self): + res = self.meta_interp_search(r"<\w+>", "EIOFWEOXDIWHDOH<FOOBAR>UA") + assert res == 15 + + def test_max_until_1(self): + res = self.meta_interp_match(r"(ab)*abababababc", + "ababababababababababc") + assert res == 21 + + def test_example_1(self): + res = self.meta_interp_search( + r"Active\s+20\d\d-\d\d-\d\d\s+[[]\d+[]]([^[]+)", + "Active"*20 + "Active 2010-04-07 [42] Foobar baz boz blah[43]") + assert res == 6*20 diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py index af77f477e8..5a458e69ab 100644 --- a/pypy/rlib/test/test_jit.py +++ b/pypy/rlib/test/test_jit.py @@ -1,4 +1,5 @@ import py +from pypy.conftest import option from pypy.rlib.jit import hint, we_are_jitted, JitDriver, purefunction_promote from pypy.rlib.jit import JitHintError, oopspec from pypy.translator.translator import TranslationContext, graphof @@ -110,6 +111,26 @@ class BaseTestJIT(BaseRtypingTest): return n py.test.raises(JitHintError, self.gengraph, fn, [int]) + def test_green_field(self): + def get_printable_location(xfoo): + return str(ord(xfoo)) # xfoo must be annotated as a character + myjitdriver = JitDriver(greens=['x.foo'], reds=['n', 'x'], + get_printable_location=get_printable_location) + class A(object): + _immutable_fields_ = ['foo'] + def fn(n): + x = A() + x.foo = chr(n) + while n > 0: + myjitdriver.can_enter_jit(x=x, n=n) + myjitdriver.jit_merge_point(x=x, n=n) + n -= 1 + return n + t = self.gengraph(fn, [int])[0] + if option.view: + t.view() + # assert did not raise + class TestJITLLtype(BaseTestJIT, LLRtypeMixin): pass |