Merge branch 'fix-unicode'

thomasballinger · thomasballinger · commit 1b6fba864058 · 2015-02-18T13:01:35.000-05:00
Updates curtsies version to 2.0, use unicode for all output
diff --git a/bpython/_py3compat.py b/bpython/_py3compat.py
@@ -49,3 +49,13 @@ def prepare_for_exec(arg, encoding=None):
 else:
     def prepare_for_exec(arg, encoding=None):
         return arg.encode(encoding)
+
+
+def try_decode(s, encoding):
+    """Try to decode s which is str names. Return None if not decodable"""
+    if not py3 and not isinstance(s, unicode):
+        try:
+            return s.decode(encoding)
+        except UnicodeDecodeError:
+            return None
+    return s
diff --git a/bpython/autocomplete.py b/bpython/autocomplete.py
@@ -34,7 +34,7 @@
 from bpython import inspection
 from bpython import importcompletion
 from bpython import line as lineparts
-from bpython._py3compat import py3
+from bpython._py3compat import py3, try_decode
 from bpython.lazyre import LazyReCompile
 
 
@@ -286,11 +286,17 @@ def matches(self, cursor_offset, line, **kwargs):
         n = len(text)
         for word in keyword.kwlist:
             if method_match(word, n, text):
+                if not py3:
+                    word = word.decode('ascii')  # py2 keywords are all ascii
                 matches.add(word)
         for nspace in [builtins.__dict__, locals_]:
             for word, val in nspace.items():
                 if (method_match(word, len(text), text) and
                         word != "__builtins__"):
+                    word = try_decode(word, 'ascii')
+                    # if identifier isn't ascii, don't complete (syntax error)
+                    if word is None:
+                        continue
                     matches.add(_callable_postfix(val, word))
         return matches
 
diff --git a/bpython/curtsiesfrontend/replpainter.py b/bpython/curtsiesfrontend/replpainter.py
@@ -108,12 +108,14 @@ def formatted_argspec(argspec, columns, config):
             color = bolds[color]
 
         if not py3:
-            s += color(inspect.strseq(arg, str))
+            s += color(inspect.strseq(arg, unicode))
         else:
             s += color(arg)
 
         if kw is not None:
             s += punctuation_color('=')
+            if not py3:
+                kw = kw.decode('ascii', 'replace')
             s += token_color(kw)
 
         if i != len(args) - 1:
diff --git a/bpython/importcompletion.py b/bpython/importcompletion.py
@@ -20,7 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-from bpython._py3compat import py3
+from bpython._py3compat import py3, try_decode
 from bpython.line import current_word, current_import, \
     current_from_import_from, current_from_import_import
 
@@ -42,17 +42,6 @@
 fully_loaded = False
 
 
-def try_decode_module(module, encoding):
-    """Try to decode module names."""
-    if not py3 and not isinstance(module, unicode):
-        try:
-            return module.decode(encoding)
-        except UnicodeDecodeError:
-            # Not importable anyway, ignore it
-            return None
-    return module
-
-
 def module_matches(cw, prefix=''):
     """Modules names to replace cw with"""
     full = '%s.%s' % (prefix, cw) if prefix else cw
@@ -83,7 +72,7 @@ def attr_matches(cw, prefix='', only_modules=False):
     if module_part:
         matches = ('%s.%s' % (module_part, m) for m in matches)
 
-    generator = (try_decode_module(match, 'ascii') for match in matches)
+    generator = (try_decode(match, 'ascii') for match in matches)
     return set(filter(lambda x: x is not None, generator))
 
 
@@ -177,14 +166,15 @@ def find_all_modules(path=None):
     """Return a list with all modules in `path`, which should be a list of
     directory names. If path is not given, sys.path will be used."""
     if path is None:
-        modules.update(sys.builtin_module_names)
+        modules.update(try_decode(m, 'ascii')
+                       for m in sys.builtin_module_names)
         path = sys.path
 
     for p in path:
         if not p:
             p = os.curdir
         for module in find_modules(p):
-            module = try_decode_module(module, sys.getfilesystemencoding())
+            module = try_decode(module, 'ascii')
             if module is None:
                 continue
             modules.add(module)
diff --git a/bpython/test/test_autocomplete.py b/bpython/test/test_autocomplete.py
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 from collections import namedtuple
 import inspect
 from bpython._py3compat import py3
@@ -273,6 +275,15 @@ def function():
                                           locals_={'function': function}),
                          set(('function(', )))
 
+    def test_completions_are_unicode(self):
+        for m in self.com.matches(1, 'a', locals_={'abc': 10}):
+            self.assertIsInstance(m, type(u''))
+
+    @unittest.skipIf(py3, "in Python 3 invalid identifiers are passed through")
+    def test_ignores_nonascii_encodable(self):
+        self.assertSetEqual(self.com.matches(1, 'abc', locals_={'abcß': 10}),
+                            set())
+
 
 class TestParameterNameCompletion(unittest.TestCase):
     def test_set_of_params_returns_when_matches_found(self):
diff --git a/bpython/test/test_importcompletion.py b/bpython/test/test_importcompletion.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from bpython import importcompletion
 
 try:
@@ -29,7 +31,8 @@ class TestRealComplete(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        [_ for _ in importcompletion.find_iterator]
+        for _ in importcompletion.find_iterator:
+            pass
         __import__('sys')
         __import__('os')
 
diff --git a/setup.py b/setup.py
@@ -210,7 +210,7 @@ def initialize_options(self):
 install_requires = [
     'pygments',
     'requests',
-    'curtsies >=0.1.18, <0.2.0',
+    'curtsies >=0.1.18',
     'greenlet',
     'six >=1.4'
 ]

Original file line number	Diff line number	Diff line change
`@@ -210,7 +210,7 @@ def initialize_options(self):`
`210`	`210`	`install_requires = [`
`211`	`211`	`'pygments',`
`212`	`212`	`'requests',`
`213`		`- 'curtsies >=0.1.18, <0.2.0',`
	`213`	`+ 'curtsies >=0.1.18',`
`214`	`214`	`'greenlet',`
`215`	`215`	`'six >=1.4'`
`216`	`216`	`]`