X Tutup
Skip to content

Commit da2ecaf

Browse files
pitroupitrou
committed
Merged revisions 77241 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/trunk ........ r77241 | antoine.pitrou | 2010-01-02 22:12:58 +0100 (sam., 02 janv. 2010) | 4 lines Issue python#7462: Implement the stringlib fast search algorithm for the `rfind`, `rindex`, `rsplit` and `rpartition` methods. Patch by Florent Xicluna. ........
1 parent 2952148 commit da2ecaf

File tree

10 files changed

+132
-116
lines changed

10 files changed

+132
-116
lines changed

‎Lib/collections.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -767,6 +767,8 @@ def replace(self, old, new, maxsplit=-1):
767767
new = new.data
768768
return self.__class__(self.data.replace(old, new, maxsplit))
769769
def rfind(self, sub, start=0, end=_sys.maxsize):
770+
if isinstance(sub, UserString):
771+
sub = sub.data
770772
return self.data.rfind(sub, start, end)
771773
def rindex(self, sub, start=0, end=_sys.maxsize):
772774
return self.data.rindex(sub, start, end)

‎Lib/test/string_tests.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,30 @@ def test_rfind(self):
216216
self.checkraises(TypeError, 'hello', 'rfind')
217217
self.checkraises(TypeError, 'hello', 'rfind', 42)
218218

219+
# For a variety of combinations,
220+
# verify that str.rfind() matches __contains__
221+
# and that the found substring is really at that location
222+
charset = ['', 'a', 'b', 'c']
223+
digits = 5
224+
base = len(charset)
225+
teststrings = set()
226+
for i in range(base ** digits):
227+
entry = []
228+
for j in range(digits):
229+
i, m = divmod(i, base)
230+
entry.append(charset[m])
231+
teststrings.add(''.join(entry))
232+
teststrings = [self.fixtype(ts) for ts in teststrings]
233+
for i in teststrings:
234+
for j in teststrings:
235+
loc = i.rfind(j)
236+
r1 = (loc != -1)
237+
r2 = j in i
238+
if r1 != r2:
239+
self.assertEqual(r1, r2)
240+
if loc != -1:
241+
self.assertEqual(i[loc:loc+len(j)], j)
242+
219243
def test_index(self):
220244
self.checkequal(0, 'abcdefghiabc', 'index', '')
221245
self.checkequal(3, 'abcdefghiabc', 'index', 'def')

‎Misc/NEWS‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ What's New in Python 3.2 Alpha 1?
1212
Core and Builtins
1313
-----------------
1414

15+
- Issue #7462: Implement the stringlib fast search algorithm for the `rfind`,
16+
`rindex`, `rsplit` and `rpartition` methods. Patch by Florent Xicluna.
17+
1518
- Issue #7604: Deleting an unset slotted attribute did not raise an
1619
AttributeError.
1720

‎Objects/bytearrayobject.c‎

Lines changed: 9 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1035,7 +1035,6 @@ bytearray_dealloc(PyByteArrayObject *self)
10351035
/* Methods */
10361036

10371037
#define STRINGLIB_CHAR char
1038-
#define STRINGLIB_CMP memcmp
10391038
#define STRINGLIB_LEN PyByteArray_GET_SIZE
10401039
#define STRINGLIB_STR PyByteArray_AS_STRING
10411040
#define STRINGLIB_NEW PyByteArray_FromStringAndSize
@@ -2214,14 +2213,11 @@ If maxsplit is given, at most maxsplit splits are done.");
22142213
static PyObject *
22152214
bytearray_split(PyByteArrayObject *self, PyObject *args)
22162215
{
2217-
Py_ssize_t len = PyByteArray_GET_SIZE(self), n, i, j;
2216+
Py_ssize_t len = PyByteArray_GET_SIZE(self), n, i, j, pos;
22182217
Py_ssize_t maxsplit = -1, count = 0;
22192218
const char *s = PyByteArray_AS_STRING(self), *sub;
22202219
PyObject *list, *str, *subobj = Py_None;
22212220
Py_buffer vsub;
2222-
#ifdef USE_FAST
2223-
Py_ssize_t pos;
2224-
#endif
22252221

22262222
if (!PyArg_ParseTuple(args, "|On:split", &subobj, &maxsplit))
22272223
return NULL;
@@ -2253,7 +2249,6 @@ bytearray_split(PyByteArrayObject *self, PyObject *args)
22532249
return NULL;
22542250
}
22552251

2256-
#ifdef USE_FAST
22572252
i = j = 0;
22582253
while (maxsplit-- > 0) {
22592254
pos = fastsearch(s+i, len-i, sub, n, FAST_SEARCH);
@@ -2263,18 +2258,6 @@ bytearray_split(PyByteArrayObject *self, PyObject *args)
22632258
SPLIT_ADD(s, i, j);
22642259
i = j + n;
22652260
}
2266-
#else
2267-
i = j = 0;
2268-
while ((j+n <= len) && (maxsplit-- > 0)) {
2269-
for (; j+n <= len; j++) {
2270-
if (Py_STRING_MATCH(s, j, sub, n)) {
2271-
SPLIT_ADD(s, i, j);
2272-
i = j = j + n;
2273-
break;
2274-
}
2275-
}
2276-
}
2277-
#endif
22782261
SPLIT_ADD(s, i, len);
22792262
FIX_PREALLOC_SIZE(list);
22802263
PyBuffer_Release(&vsub);
@@ -2452,7 +2435,7 @@ If maxsplit is given, at most maxsplit splits are done.");
24522435
static PyObject *
24532436
bytearray_rsplit(PyByteArrayObject *self, PyObject *args)
24542437
{
2455-
Py_ssize_t len = PyByteArray_GET_SIZE(self), n, i, j;
2438+
Py_ssize_t len = PyByteArray_GET_SIZE(self), n, j, pos;
24562439
Py_ssize_t maxsplit = -1, count = 0;
24572440
const char *s = PyByteArray_AS_STRING(self), *sub;
24582441
PyObject *list, *str, *subobj = Py_None;
@@ -2489,17 +2472,13 @@ bytearray_rsplit(PyByteArrayObject *self, PyObject *args)
24892472
}
24902473

24912474
j = len;
2492-
i = j - n;
2493-
2494-
while ( (i >= 0) && (maxsplit-- > 0) ) {
2495-
for (; i>=0; i--) {
2496-
if (Py_STRING_MATCH(s, i, sub, n)) {
2497-
SPLIT_ADD(s, i + n, j);
2498-
j = i;
2499-
i -= n;
2500-
break;
2501-
}
2502-
}
2475+
2476+
while (maxsplit-- > 0) {
2477+
pos = fastsearch(s, j, sub, n, FAST_RSEARCH);
2478+
if (pos < 0)
2479+
break;
2480+
SPLIT_ADD(s, pos + n, j);
2481+
j = pos;
25032482
}
25042483
SPLIT_ADD(s, 0, j);
25052484
FIX_PREALLOC_SIZE(list);

‎Objects/stringlib/README.txt‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,6 @@ STRINGLIB_EMPTY
1515

1616
a PyObject representing the empty string
1717

18-
int STRINGLIB_CMP(STRINGLIB_CHAR*, STRINGLIB_CHAR*, Py_ssize_t)
19-
20-
compares two strings. returns 0 if they match, and non-zero if not.
21-
2218
Py_ssize_t STRINGLIB_LEN(PyObject*)
2319

2420
returns the length of the given string object (which must be of the

‎Objects/stringlib/fastsearch.h‎

Lines changed: 77 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
/* fast search/count implementation, based on a mix between boyer-
77
moore and horspool, with a few more bells and whistles on the top.
8-
for some more background, see: http://effbot.org/stringlib.htm */
8+
for some more background, see: http://effbot.org/zone/stringlib.htm */
99

1010
/* note: fastsearch may access s[n], which isn't a problem when using
1111
Python's ordinary string types, but may cause problems if you're
@@ -16,6 +16,7 @@
1616

1717
#define FAST_COUNT 0
1818
#define FAST_SEARCH 1
19+
#define FAST_RSEARCH 2
1920

2021
Py_LOCAL_INLINE(Py_ssize_t)
2122
fastsearch(const STRINGLIB_CHAR* s, Py_ssize_t n,
@@ -41,51 +42,92 @@ fastsearch(const STRINGLIB_CHAR* s, Py_ssize_t n,
4142
if (s[i] == p[0])
4243
count++;
4344
return count;
44-
} else {
45+
} else if (mode == FAST_SEARCH) {
4546
for (i = 0; i < n; i++)
4647
if (s[i] == p[0])
4748
return i;
49+
} else { /* FAST_RSEARCH */
50+
for (i = n - 1; i > -1; i--)
51+
if (s[i] == p[0])
52+
return i;
4853
}
4954
return -1;
5055
}
5156

5257
mlast = m - 1;
53-
54-
/* create compressed boyer-moore delta 1 table */
5558
skip = mlast - 1;
56-
/* process pattern[:-1] */
57-
for (mask = i = 0; i < mlast; i++) {
58-
mask |= (1 << (p[i] & 0x1F));
59-
if (p[i] == p[mlast])
60-
skip = mlast - i - 1;
61-
}
62-
/* process pattern[-1] outside the loop */
63-
mask |= (1 << (p[mlast] & 0x1F));
64-
65-
for (i = 0; i <= w; i++) {
66-
/* note: using mlast in the skip path slows things down on x86 */
67-
if (s[i+m-1] == p[m-1]) {
68-
/* candidate match */
69-
for (j = 0; j < mlast; j++)
70-
if (s[i+j] != p[j])
71-
break;
72-
if (j == mlast) {
73-
/* got a match! */
74-
if (mode != FAST_COUNT)
59+
60+
if (mode != FAST_RSEARCH) {
61+
62+
/* create compressed boyer-moore delta 1 table */
63+
64+
/* process pattern[:-1] */
65+
for (mask = i = 0; i < mlast; i++) {
66+
mask |= (1 << (p[i] & 0x1F));
67+
if (p[i] == p[mlast])
68+
skip = mlast - i - 1;
69+
}
70+
/* process pattern[-1] outside the loop */
71+
mask |= (1 << (p[mlast] & 0x1F));
72+
73+
for (i = 0; i <= w; i++) {
74+
/* note: using mlast in the skip path slows things down on x86 */
75+
if (s[i+m-1] == p[m-1]) {
76+
/* candidate match */
77+
for (j = 0; j < mlast; j++)
78+
if (s[i+j] != p[j])
79+
break;
80+
if (j == mlast) {
81+
/* got a match! */
82+
if (mode != FAST_COUNT)
83+
return i;
84+
count++;
85+
i = i + mlast;
86+
continue;
87+
}
88+
/* miss: check if next character is part of pattern */
89+
if (!(mask & (1 << (s[i+m] & 0x1F))))
90+
i = i + m;
91+
else
92+
i = i + skip;
93+
} else {
94+
/* skip: check if next character is part of pattern */
95+
if (!(mask & (1 << (s[i+m] & 0x1F))))
96+
i = i + m;
97+
}
98+
}
99+
} else { /* FAST_RSEARCH */
100+
101+
/* create compressed boyer-moore delta 1 table */
102+
103+
/* process pattern[0] outside the loop */
104+
mask = (1 << (p[0] & 0x1F));
105+
/* process pattern[:0:-1] */
106+
for (i = mlast; i > 0; i--) {
107+
mask |= (1 << (p[i] & 0x1F));
108+
if (p[i] == p[0])
109+
skip = i - 1;
110+
}
111+
112+
for (i = w; i >= 0; i--) {
113+
if (s[i] == p[0]) {
114+
/* candidate match */
115+
for (j = mlast; j > 0; j--)
116+
if (s[i+j] != p[j])
117+
break;
118+
if (j == 0)
119+
/* got a match! */
75120
return i;
76-
count++;
77-
i = i + mlast;
78-
continue;
121+
/* miss: check if previous character is part of pattern */
122+
if (!(mask & (1 << (s[i-1] & 0x1F))))
123+
i = i - m;
124+
else
125+
i = i - skip;
126+
} else {
127+
/* skip: check if previous character is part of pattern */
128+
if (!(mask & (1 << (s[i-1] & 0x1F))))
129+
i = i - m;
79130
}
80-
/* miss: check if next character is part of pattern */
81-
if (!(mask & (1 << (s[i+m] & 0x1F))))
82-
i = i + m;
83-
else
84-
i = i + skip;
85-
} else {
86-
/* skip: check if next character is part of pattern */
87-
if (!(mask & (1 << (s[i+m] & 0x1F))))
88-
i = i + m;
89131
}
90132
}
91133

‎Objects/stringlib/find.h‎

Lines changed: 14 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -32,20 +32,19 @@ stringlib_rfind(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
3232
const STRINGLIB_CHAR* sub, Py_ssize_t sub_len,
3333
Py_ssize_t offset)
3434
{
35-
/* XXX - create reversefastsearch helper! */
36-
if (sub_len == 0) {
37-
if (str_len < 0)
38-
return -1;
39-
return str_len + offset;
40-
} else {
41-
Py_ssize_t j, pos = -1;
42-
for (j = str_len - sub_len; j >= 0; --j)
43-
if (STRINGLIB_CMP(str+j, sub, sub_len) == 0) {
44-
pos = j + offset;
45-
break;
46-
}
47-
return pos;
48-
}
35+
Py_ssize_t pos;
36+
37+
if (str_len < 0)
38+
return -1;
39+
if (sub_len == 0)
40+
return str_len + offset;
41+
42+
pos = fastsearch(str, str_len, sub, sub_len, FAST_RSEARCH);
43+
44+
if (pos >= 0)
45+
pos += offset;
46+
47+
return pos;
4948
}
5049

5150
Py_LOCAL_INLINE(Py_ssize_t)
@@ -64,10 +63,7 @@ stringlib_find_slice(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
6463
if (end < 0)
6564
end = 0;
6665

67-
return stringlib_find(
68-
str + start, end - start,
69-
sub, sub_len, start
70-
);
66+
return stringlib_find(str + start, end - start, sub, sub_len, start);
7167
}
7268

7369
Py_LOCAL_INLINE(Py_ssize_t)

‎Objects/stringlib/partition.h‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ stringlib_rpartition(
5858
)
5959
{
6060
PyObject* out;
61-
Py_ssize_t pos, j;
61+
Py_ssize_t pos;
6262

6363
if (sep_len == 0) {
6464
PyErr_SetString(PyExc_ValueError, "empty separator");
@@ -69,20 +69,14 @@ stringlib_rpartition(
6969
if (!out)
7070
return NULL;
7171

72-
/* XXX - create reversefastsearch helper! */
73-
pos = -1;
74-
for (j = str_len - sep_len; j >= 0; --j)
75-
if (STRINGLIB_CMP(str+j, sep, sep_len) == 0) {
76-
pos = j;
77-
break;
78-
}
72+
pos = fastsearch(str, str_len, sep, sep_len, FAST_RSEARCH);
7973

8074
if (pos < 0) {
8175
Py_INCREF(STRINGLIB_EMPTY);
8276
PyTuple_SET_ITEM(out, 0, (PyObject*) STRINGLIB_EMPTY);
8377
Py_INCREF(STRINGLIB_EMPTY);
8478
PyTuple_SET_ITEM(out, 1, (PyObject*) STRINGLIB_EMPTY);
85-
Py_INCREF(str_obj);
79+
Py_INCREF(str_obj);
8680
PyTuple_SET_ITEM(out, 2, (PyObject*) str_obj);
8781
return out;
8882
}

‎Objects/stringlib/stringdefs.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
#define STRINGLIB_RESIZE _PyBytes_Resize
2323
#define STRINGLIB_CHECK PyBytes_Check
2424
#define STRINGLIB_CHECK_EXACT PyBytes_CheckExact
25-
#define STRINGLIB_CMP memcmp
2625
#define STRINGLIB_TOSTR PyObject_Str
2726
#define STRINGLIB_GROUPING _PyBytes_InsertThousandsGrouping
2827
#define STRINGLIB_GROUPING_LOCALE _PyBytes_InsertThousandsGroupingLocale

‎Objects/stringlib/unicodedefs.h‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -35,23 +35,4 @@
3535

3636
#define STRINGLIB_WANT_CONTAINS_OBJ 1
3737

38-
/* STRINGLIB_CMP was defined as:
39-
40-
Py_LOCAL_INLINE(int)
41-
STRINGLIB_CMP(const Py_UNICODE* str, const Py_UNICODE* other, Py_ssize_t len)
42-
{
43-
if (str[0] != other[0])
44-
return 1;
45-
return memcmp((void*) str, (void*) other, len * sizeof(Py_UNICODE));
46-
}
47-
48-
but unfortunately that gives a error if the function isn't used in a file that
49-
includes this file. So, reluctantly convert it to a macro instead. */
50-
51-
#define STRINGLIB_CMP(str, other, len) \
52-
(((str)[0] != (other)[0]) ? \
53-
1 : \
54-
memcmp((void*) (str), (void*) (other), (len) * sizeof(Py_UNICODE)))
55-
56-
5738
#endif /* !STRINGLIB_UNICODEDEFS_H */

0 commit comments

Comments
 (0)
X Tutup