Merge branch 'u/embray/python3/string-fixes' in 8.2.b4

src/sage/cpython/string.pxd

@@ -15,7 +15,6 @@ from libc.string cimport strlen
 from cpython.bytes cimport PyBytes_AS_STRING, PyBytes_FromString
 from cpython.unicode cimport PyUnicode_Decode, PyUnicode_AsEncodedString
 
-
 cdef extern from "Python.h":
     # Missing from cpython.unicode in Cython 0.27.3
     char* PyUnicode_AsUTF8(object s)

src/sage/libs/ecl.pyx

@@ -1,4 +1,4 @@
-r"""
+"""
 Library interface to Embeddable Common Lisp (ECL)
 """
 #*****************************************************************************
@@ -23,6 +23,7 @@ cimport cysignals.signals
 
 from sage.libs.gmp.types cimport mpz_t
 from sage.misc.misc import ECL_TMP
+from sage.cpython.string cimport str_to_bytes, char_to_str
 from sage.rings.integer cimport Integer
 from sage.rings.rational cimport Rational
 from cpython.object cimport Py_EQ, Py_NE
@@ -279,46 +280,46 @@ def init_ecl():
     #initialise list of objects and bind to global variable
     # *SAGE-LIST-OF-OBJECTS* to make it rooted in the reachable tree for the GC
     list_of_objects=cl_cons(Cnil,cl_cons(Cnil,Cnil))
-    cl_set(string_to_object("*SAGE-LIST-OF-OBJECTS*"),list_of_objects)
+    cl_set(string_to_object(b"*SAGE-LIST-OF-OBJECTS*"), list_of_objects)
 
-    cl_eval(string_to_object("""
+    cl_eval(string_to_object(b"""
         (setf (logical-pathname-translations "TMP")
               '(("**;*.*" "%s/**/*.*")))
-        """ % ECL_TMP))
+        """ % str_to_bytes(str(ECL_TMP))))
 
     # We define our own error catching eval, apply and funcall/
     # Presently these routines are only converted to byte-code. If they
     # ever turn out to be a bottle neck, it should be easy to properly
     # compile them.
 
-    read_from_string_clobj=cl_eval(string_to_object("(symbol-function 'read-from-string)"))
+    read_from_string_clobj=cl_eval(string_to_object(b"(symbol-function 'read-from-string)"))
 
-    cl_eval(string_to_object("""
+    cl_eval(string_to_object(b"""
         (defun sage-safe-eval (form)
             (handler-case
                 (values (eval form))
                 (serious-condition (cnd)
                     (values nil (princ-to-string cnd)))))
         """))
-    safe_eval_clobj=cl_eval(string_to_object("(symbol-function 'sage-safe-eval)"))
+    safe_eval_clobj=cl_eval(string_to_object(b"(symbol-function 'sage-safe-eval)"))
 
-    cl_eval(string_to_object("""
+    cl_eval(string_to_object(b"""
         (defun sage-safe-apply (func args)
             (handler-case
                 (values (apply func args))
                 (serious-condition (cnd)
                     (values nil (princ-to-string cnd)))))
         """))
 
-    safe_apply_clobj=cl_eval(string_to_object("(symbol-function 'sage-safe-apply)"))
-    cl_eval(string_to_object("""
+    safe_apply_clobj=cl_eval(string_to_object(b"(symbol-function 'sage-safe-apply)"))
+    cl_eval(string_to_object(b"""
         (defun sage-safe-funcall (func arg)
             (handler-case
                 (values (funcall func arg))
                 (serious-condition (cnd)
                     (values nil (princ-to-string cnd)))))
         """))
-    safe_funcall_clobj=cl_eval(string_to_object("(symbol-function 'sage-safe-funcall)"))
+    safe_funcall_clobj=cl_eval(string_to_object(b"(symbol-function 'sage-safe-funcall)"))
 
     ecl_has_booted = 1
 
@@ -346,7 +347,8 @@ cdef cl_object ecl_safe_eval(cl_object form) except NULL:
 
     if ecl_nvalues > 1:
         s = si_coerce_to_base_string(ecl_values(1))
-        raise RuntimeError("ECL says: "+ecl_base_string_pointer_safe(s))
+        raise RuntimeError("ECL says: {}".format(
+            char_to_str(ecl_base_string_pointer_safe(s))))
     else:
         return ecl_values(0)
 
@@ -360,7 +362,8 @@ cdef cl_object ecl_safe_funcall(cl_object func, cl_object arg) except NULL:
 
     if ecl_nvalues > 1:
         s = si_coerce_to_base_string(ecl_values(1))
-        raise RuntimeError("ECL says: "+ecl_base_string_pointer_safe(s))
+        raise RuntimeError("ECL says: {}".format(
+            char_to_str(ecl_base_string_pointer_safe(s))))
     else:
         return ecl_values(0)
 
@@ -372,7 +375,8 @@ cdef cl_object ecl_safe_apply(cl_object func, cl_object args) except NULL:
 
     if ecl_nvalues > 1:
         s = si_coerce_to_base_string(ecl_values(1))
-        raise RuntimeError("ECL says: "+ecl_base_string_pointer_safe(s))
+        raise RuntimeError("ECL says: {}".format(
+            char_to_str(ecl_base_string_pointer_safe(s))))
     else:
         return ecl_values(0)
 
@@ -426,7 +430,7 @@ def print_objects():
     c = list_of_objects
     while True:
         s = si_coerce_to_base_string(cl_write_to_string(1,cl_car(c)))
-        print(ecl_base_string_pointer_safe(s))
+        print(char_to_str(ecl_base_string_pointer_safe(s)))
         c = cl_cadr(c)
         if c == Cnil:
             break
@@ -458,7 +462,7 @@ cdef cl_object python_to_ecl(pyobj) except NULL:
     elif isinstance(pyobj,float):
         return ecl_make_doublefloat(pyobj)
     elif isinstance(pyobj,unicode):
-        s=<bytes>(str(pyobj))
+        s=str_to_bytes(pyobj)
         return ecl_safe_read_string(s)
     elif isinstance(pyobj,bytes):
         s=<bytes>pyobj
@@ -537,7 +541,7 @@ cdef ecl_to_python(cl_object o):
         return L
     else:
         s = si_coerce_to_base_string(cl_write_to_string(1,o))
-        return ecl_base_string_pointer_safe(s)
+        return char_to_str(ecl_base_string_pointer_safe(s))
 
 #Maxima's BFLOAT multiprecision float type can be read with:
 #def bfloat_to_python(e):
@@ -754,7 +758,7 @@ cdef class EclObject:
         """
         cdef cl_object s
         s = si_coerce_to_base_string(cl_write_to_string(1,self.obj))
-        return ecl_base_string_pointer_safe(s)
+        return char_to_str(ecl_base_string_pointer_safe(s))
 
     def __hash__(self):
         r"""
@@ -1319,7 +1323,7 @@ cdef EclObject ecl_wrap(cl_object o):
     return obj
 
 #convenience routine to more easily evaluate strings
-cpdef EclObject ecl_eval(bytes s):
+cpdef EclObject ecl_eval(str s):
     """
     Read and evaluate string in Lisp and return the result
 
@@ -1333,7 +1337,7 @@ cpdef EclObject ecl_eval(bytes s):
 
     """
     cdef cl_object o
-    o=ecl_safe_read_string(s)
+    o=ecl_safe_read_string(str_to_bytes(s))
     o=ecl_safe_eval(o)
     return ecl_wrap(o)
 

src/sage/libs/mpmath/utils.pyx

@@ -18,6 +18,8 @@ from sage.libs.gmp.all cimport *
 from sage.rings.complex_field import ComplexField
 from sage.rings.real_mpfr cimport RealField
 
+from sage.cpython.string import str_to_bytes
+
 cpdef int bitcount(n):
     """
     Bitcount of a Sage Integer or Python int/long.

src/sage/libs/pynac/constant.pyx

@@ -19,6 +19,7 @@ from __future__ import absolute_import, division, print_function
 from .pynac cimport *
 from sage.symbolic.expression cimport new_Expression_from_GEx
 from sage.symbolic.ring import SR
+from sage.cpython.string import str_to_bytes
 
 
 cdef class PynacConstant:
@@ -65,7 +66,8 @@ cdef class PynacConstant:
         elif self._name == "NaN":
             self.pointer = <GConstant *>&g_NaN
         else:
-            self._object = new GConstant(name, ConstantEvalf, texname, domain)
+            self._object = new GConstant(str_to_bytes(name), ConstantEvalf,
+                                         str_to_bytes(texname), domain)
             self.pointer = self._object
 
     def __dealloc__(self):

src/sage/libs/pynac/pynac.pyx

@@ -27,6 +27,8 @@ from sage.libs.gsl.gamma cimport gsl_sf_lngamma_complex_e
 from sage.libs.mpmath import utils as mpmath_utils
 from sage.libs.pari.all import pari
 
+from sage.cpython.string import str_to_bytes
+
 from sage.arith.all import gcd, lcm, is_prime, factorial, bernoulli
 
 from sage.structure.element cimport Element, parent, coercion_model
@@ -370,6 +372,10 @@ cdef stdstring* string_from_pystr(py_str) except NULL:
     cdef bytes s
     if isinstance(py_str, bytes):
         s = <bytes>py_str
+    elif isinstance(py_str, str):
+        # Note: This should only by the case on Python 3 since on Python 2
+        # bytes is str
+        s = str_to_bytes(py_str)
     else:
         s = b"(INVALID)"  # Avoid segfaults for invalid input
     return new stdstring(s)

src/sage/libs/singular/singular.pyx

@@ -37,6 +37,7 @@ from sage.rings.finite_rings.finite_field_givaro import FiniteField_givaro
 from sage.rings.finite_rings.finite_field_ntl_gf2e import FiniteField_ntl_gf2e
 from sage.libs.pari.all import pari
 from sage.libs.gmp.all cimport *
+from sage.cpython.string import FS_ENCODING, str_to_bytes
 
 from sage.rings.polynomial.multi_polynomial_libsingular cimport MPolynomial_libsingular
 
@@ -774,7 +775,9 @@ cdef init_libsingular():
     if not os.path.exists(lib):
         raise ImportError("cannot locate Singular library ({})".format(lib))
 
-    handle = dlopen(lib, RTLD_GLOBAL|RTLD_LAZY)   
+    lib = str_to_bytes(lib, FS_ENCODING, "surrogateescape")
+
+    handle = dlopen(lib, RTLD_GLOBAL|RTLD_LAZY)
     if not handle:
         err = dlerror()
         raise ImportError("cannot load Singular library ({})".format(err))

src/sage/rings/integer.pyx

@@ -158,6 +158,7 @@ from sage.libs.gmp.mpz cimport *
 from sage.libs.gmp.mpq cimport *
 from sage.misc.superseded import deprecated_function_alias
 from sage.arith.long cimport pyobject_to_long, integer_check_long
+from sage.cpython.string cimport char_to_str, str_to_bytes
 
 from cpython.list cimport *
 from cpython.number cimport *
@@ -687,9 +688,12 @@ cdef class Integer(sage.structure.element.EuclideanDomainElement):
                     except AttributeError:
                         pass
 
-                elif isinstance(x, basestring):
+                elif isinstance(x, bytes):
                     mpz_set_str_python(self.value, x, base)
                     return
+                elif isinstance(x, unicode):
+                    mpz_set_str_python(self.value, str_to_bytes(x), base)
+                    return
 
                 elif (isinstance(x, list) or isinstance(x, tuple)) and base > 1:
                     b = the_integer_ring(base)
@@ -752,7 +756,7 @@ cdef class Integer(sage.structure.element.EuclideanDomainElement):
            Integers are supposed to be immutable, so you should not
            use this function.
         """
-        mpz_set_str(self.value, s, 32)
+        mpz_set_str(self.value, str_to_bytes(s), 32)
 
     def __index__(self):
         """
@@ -1084,7 +1088,7 @@ cdef class Integer(sage.structure.element.EuclideanDomainElement):
         sig_on()
         mpz_get_str(s, base, self.value)
         sig_off()
-        k = <bytes>s
+        k = char_to_str(s)
         sig_free(s)
         return k
 
@@ -6896,7 +6900,7 @@ cdef int mpz_set_str_python(mpz_ptr z, char* s, int base) except -1:
 
     assert base >= 2
     if mpz_set_str(z, x, base) != 0:
-        raise TypeError("unable to convert %r to an integer" % s)
+        raise TypeError("unable to convert %r to an integer" % char_to_str(s))
     if sign < 0:
         mpz_neg(z, z)
     if warnoctal and mpz_sgn(z) != 0:
@@ -6983,7 +6987,7 @@ def make_integer(s):
         sage: make_integer(29)
         Traceback (most recent call last):
         ...
-        TypeError: expected string or Unicode object, sage.rings.integer.Integer found
+        TypeError: expected str...Integer found
     """
     cdef Integer r = PY_NEW(Integer)
     r._reduce_set(s)

src/sage/rings/rational.pyx

@@ -58,6 +58,7 @@ import fractions
 
 from sage.misc.mathml import mathml
 from sage.arith.long cimport pyobject_to_long
+from sage.cpython.string cimport char_to_str
 
 import sage.misc.misc as misc
 from sage.structure.sage_object cimport SageObject
@@ -2073,7 +2074,7 @@ cdef class Rational(sage.structure.element.FieldElement):
         sig_on()
         mpq_get_str(s, base, self.value)
         sig_off()
-        k = str(s)
+        k = char_to_str(s)
         PyMem_Free(s)
         return k
 

src/sage/rings/real_arb.pyx

@@ -225,6 +225,7 @@ from sage.rings.integer_ring import ZZ
 from sage.rings.rational_field import QQ
 from sage.rings.real_mpfi import RealIntervalField, RealIntervalField_class
 from sage.structure.unique_representation import UniqueRepresentation
+from sage.cpython.string cimport char_to_str
 
 cdef void mpfi_to_arb(arb_t target, const mpfi_t source, const long precision):
     """
@@ -1412,11 +1413,10 @@ cdef class RealBall(RingElement):
            [2e+0 +/- 0.101]
         """
         cdef char* c_result
-        cdef bytes py_string
 
         c_result = arb_get_str(self.value, (prec(self) * 31) // 100, 0)
         try:
-            py_string = c_result
+            py_string = char_to_str(c_result)
         finally:
             flint_free(c_result)
 

src/sage/rings/real_mpfi.pyx

@@ -267,6 +267,7 @@ cimport sage.rings.real_mpfr as real_mpfr
 import math # for log
 import sys
 import operator
+from sage.cpython.string cimport char_to_str
 
 import sage.rings.complex_field
 import sage.rings.infinity
@@ -1908,7 +1909,7 @@ cdef class RealIntervalFieldElement(RingElement):
                 sig_on()
                 mpz_get_str(zz_str, base, self_zz)
                 sig_off()
-                v = str(zz_str)
+                v = char_to_str(zz_str)
                 PyMem_Free(zz_str)
                 return v
 
@@ -2141,7 +2142,7 @@ cdef class RealIntervalFieldElement(RingElement):
             if tmp_cstr == NULL:
                 raise MemoryError("Unable to allocate memory for the error of an interval")
             mpz_get_str(tmp_cstr, 10, cur_error)
-            error_string = str(tmp_cstr)
+            error_string = char_to_str(tmp_cstr)
             PyMem_Free(tmp_cstr)
 
         mpz_clear(lower_mpz)

src/sage/rings/real_mpfr.pyx

@@ -126,6 +126,7 @@ from sage.ext.stdsage cimport PY_NEW
 from sage.libs.gmp.mpz cimport *
 from sage.libs.mpfr cimport *
 from sage.misc.randstate cimport randstate, current_randstate
+from sage.cpython.string cimport char_to_str
 
 from sage.structure.element cimport RingElement, Element, ModuleElement
 from sage.structure.richcmp cimport rich_to_bool_sgn
@@ -1974,7 +1975,7 @@ cdef class RealNumber(sage.structure.element.RingElement):
         sig_off()
         if s is NULL:
             raise RuntimeError("unable to convert an mpfr number to a string")
-        t = str(s)
+        t = char_to_str(s)
         mpfr_free_str(s)
 
         if skip_zeroes: