Fix more 0.7 deprecations (#505)

* silence cfunction deprecations

* silence some more deprecations

* define and use constructors rather than convert where possible, but also define convert methods

* pymethod is used in only one place, so inline it so that we can avoid using local vars in at-cfunction

* PyError is no longer mutable

* detect changes to Conda location that may occur in Pkg3

* whoops, be consistent

* separate precompile step to get more Travis info, I hope

* missing Conda check

* don't use at-cfunction interpolation in PyGetSetDef (which can be removed entirely for now, as it is unused)

* rm another unused function

* Revert "define and use constructors rather than convert where possible, but also define convert methods"

This reverts commit 65cb003.

* add constructors for convert methods in 0.7

* rm redundant constructor methods

* defining constructor methods seems to give a huge load time slowdown even in 0.7

* more undef deprecations

* PYTHON=Conda to force conda

* make sure bytes conversions work for codeunits arrays

Author: stevengj

.travis.yml

@@ -16,13 +16,18 @@ env:
   matrix:
     - PYTHON=python  # python 2.7
     - PYTHON=python3 # python 3.5
-    - PYTHON=Conda-python # not actually a python version, here to test Conda.jl's python
+    - PYTHON=Conda # not actually a python version, here to test Conda.jl's python
 matrix:
   exclude:
     - os: osx
       env: PYTHON=python3 # I'm not sure how to install Python 3 on Travis OSX
 before_install:
-    - test "x$TRAVIS_OS_NAME" = xosx -o "x$PYTHON" = xConda-python || (sudo apt-get -qq update && sudo apt-get install -y $PYTHON)
+    - test "x$TRAVIS_OS_NAME" = xosx -o "x$PYTHON" = xConda || (sudo apt-get -qq update && sudo apt-get install -y $PYTHON)
+script:
+    - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
+    - julia -e 'Pkg.clone(pwd()); Pkg.build("PyCall")'
+    - julia -e 'import PyCall' # precompile
+    - julia -e 'Pkg.test("PyCall"; coverage=true)'
 after_success:
     - julia -e 'cd(Pkg.dir("PyCall")); Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder())'
 notifications:

REQUIRE

@@ -1,5 +1,5 @@
 julia 0.6
-Compat 0.62.0
+Compat 0.68.0
 Conda 0.2
 MacroTools 0.3
 VersionParsing

appveyor.yml

@@ -16,7 +16,7 @@ environment:
 
   - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe"
     PYTHONDIR: "use_conda"
-    PYTHON: "use_conda"
+    PYTHON: "Conda"
 
   - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe"
     PYTHONDIR: "C:\\Python27-x64"
@@ -36,7 +36,7 @@ environment:
 
   - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe"
     PYTHONDIR: "use_conda"
-    PYTHON: "use_conda"
+    PYTHON: "Conda"
 
 # Nightlies
   - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe"
@@ -57,7 +57,7 @@ environment:
 
   - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe"
     PYTHONDIR: "use_conda"
-    PYTHON: "use_conda"
+    PYTHON: "Conda"
 
   - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe"
     PYTHONDIR: "C:\\Python27-x64"
@@ -77,7 +77,7 @@ environment:
 
   - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe"
     PYTHONDIR: "use_conda"
-    PYTHON: "use_conda"
+    PYTHON: "Conda"
 
 branches:
   only:

deps/build.jl

@@ -161,9 +161,9 @@ try # make sure deps.jl file is removed on error
     python = try
         let py = get(ENV, "PYTHON", isfile("PYTHON") ? readchomp("PYTHON") :
                      (Compat.Sys.isunix() && !Compat.Sys.isapple()) || Sys.ARCH ∉ (:i686, :x86_64) ? "python" : ""),
-            vers = isempty(py) ? v"0.0" : vparse(pyconfigvar(py,"VERSION","0.0"))
+            vers = isempty(py) || py == "Conda" ? v"0.0" : vparse(pyconfigvar(py,"VERSION","0.0"))
             if vers < v"2.7"
-                if isempty(py)
+                if isempty(py) || py == "Conda"
                     throw(UseCondaPython())
                 else
                     error("Python version $vers < 2.7 is not supported")
@@ -242,7 +242,7 @@ try # make sure deps.jl file is removed on error
     """)
 
     # Make subsequent builds (e.g. Pkg.update) use the same Python by default:
-    writeifchanged("PYTHON", isfile(programname) ? programname : python)
+    writeifchanged("PYTHON", use_conda ? "Conda" : isfile(programname) ? programname : python)
 
     #########################################################################
 

src/PyCall.jl

@@ -155,22 +155,13 @@ PyObject(o::PyObject) = o
 
 #########################################################################
 
-include("pyinit.jl")
 include("exception.jl")
 include("gui.jl")
 
 pytypeof(o::PyObject) = ispynull(o) ? throw(ArgumentError("NULL PyObjects have no Python type")) : PyObject(@pycheckn ccall(@pysym(:PyObject_Type), PyPtr, (PyPtr,), o))
 
 #########################################################################
 
-include("gc.jl")
-
-# make a PyObject that embeds a reference to keep, to prevent Julia
-# from garbage-collecting keep until o is finalized.
-PyObject(o::PyPtr, keep::Any) = pyembed(PyObject(o), keep)
-
-#########################################################################
-
 const TypeTuple = Union{Type,NTuple{N, Type}} where {N}
 include("pybuffer.jl")
 include("conversions.jl")
@@ -180,6 +171,14 @@ include("pyclass.jl")
 include("callback.jl")
 include("io.jl")
 
+#########################################################################
+
+include("gc.jl")
+
+# make a PyObject that embeds a reference to keep, to prevent Julia
+# from garbage-collecting keep until o is finalized.
+PyObject(o::PyPtr, keep::Any) = pyembed(PyObject(o), keep)
+
 #########################################################################
 # Pretty-printing PyObject
 
@@ -333,8 +332,8 @@ function pywrap(o::PyObject, mname::Symbol=:__anon__)
               catch
                   [Symbol(x[1]) for x in filter(x -> x[1][1] != '_', members)]
               end
-    eval(m, Expr(:toplevel, consts..., :(pymember(s) = $(getindex)($(o), s)),
-                 Expr(:export, exports...)))
+    Core.eval(m, Expr(:toplevel, consts..., :(pymember(s) = $(getindex)($(o), s)),
+                                              Expr(:export, exports...)))
     m
 end
 
@@ -469,7 +468,7 @@ or alternatively you can use the Conda package directly (via
 `using Conda` followed by `Conda.add` etcetera).
 """
                 end
-                e.msg *= "\n\n" * msg * "\n"
+                e = PyError(string(e.msg, "\n\n", msg, "\n"), e)
             end
             throw(e)
         else
@@ -907,13 +906,17 @@ end
 #########################################################################
 # Expose Python docstrings to the Julia doc system
 
-Docs.getdoc(o::PyObject) = Text(String(o["__doc__"]))
+Docs.getdoc(o::PyObject) = Text(convert(String, o["__doc__"]))
 
 #########################################################################
 
 include("pyeval.jl")
 include("serialize.jl")
 
+#########################################################################
+
+include("pyinit.jl")
+
 #########################################################################
 # Precompilation: just an optimization to speed up initialization.
 # Here, we precompile functions that are passed to cfunction by __init__,

src/callback.jl

@@ -7,20 +7,6 @@
 
 ################################################################
 
-# Define a Python method/function object from f(PyPtr,PyPtr)::PyPtr.
-# Requires f to be a top-level function.
-function pymethod(f::Function, name::AbstractString, flags::Integer)
-    # Python expects the PyMethodDef structure to be a *constant*,
-    # so we define an anonymous global to hold it.
-    def = gensym("PyMethodDef")
-    @eval const $def = PyMethodDef[PyMethodDef($name, $f, $flags)]
-    PyObject(@pycheckn ccall((@pysym :PyCFunction_NewEx), PyPtr,
-                             (Ptr{PyMethodDef}, Ptr{Cvoid}, Ptr{Cvoid}),
-                             eval(def), C_NULL, C_NULL))
-end
-
-################################################################
-
 # To pass an arbitrary Julia object to Python, we wrap it
 # in a jlwrap Python class, where jlwrap.__call__
 # executes the pyjlwrap_call function in Julia, which

src/conversions.jl

@@ -123,9 +123,17 @@ convert(::Type{Symbol}, po::PyObject) = Symbol(convert(AbstractString, po))
 #########################################################################
 # ByteArray conversions
 
-PyObject(a::Vector{UInt8}) =
+function PyObject(a::DenseVector{UInt8})
+  if stride(a,1) != 1
+    try
+        return NpyArray(a, true)
+    catch
+        return array2py(a) # fallback to non-NumPy version
+    end
+  end
   PyObject(@pycheckn ccall((@pysym :PyByteArray_FromStringAndSize),
                            PyPtr, (Ptr{UInt8}, Int), a, length(a)))
+end
 
 ispybytearray(po::PyObject) =
   pyisinstance(po, @pyglobalobj :PyByteArray_Type)

src/exception.jl

@@ -1,8 +1,8 @@
 #########################################################################
 # Wrapper around Python exceptions
 
-mutable struct PyError <: Exception
-    msg::AbstractString # message string from Julia context, or "" if none
+struct PyError <: Exception
+    msg::String # message string from Julia context, or "" if none
 
     # info returned by PyErr_Fetch/PyErr_Normalize
     T::PyObject
@@ -20,8 +20,10 @@ mutable struct PyError <: Exception
               pexc, pexc + sizeof(PyPtr), pexc + 2*sizeof(PyPtr))
         ccall((@pysym :PyErr_NormalizeException), Cvoid, (UInt,UInt,UInt),
               pexc, pexc + sizeof(PyPtr), pexc + 2*sizeof(PyPtr))
-        new(msg, exc[1], exc[2], exc[3])
+        new(msg, PyObject(exc[1]), PyObject(exc[2]), PyObject(exc[3]))
     end
+
+    PyError(msg::AbstractString, e::PyError) = new(msg, e.T, e.val, e.traceback)
 end
 
 function show(io::IO, e::PyError)

src/gc.jl

@@ -21,20 +21,22 @@ end
 
 const weakref_callback_obj = PyNULL() # weakref_callback Python method
 
-function pygc_finalize()
-    pydecref(weakref_callback_obj)
-    empty!(pycall_gc)
-end
+# Python expects the PyMethodDef structure to be a constant, so
+# we put it in a global to prevent gc.
+const weakref_callback_meth = Ref{PyMethodDef}()
 
 # "embed" a reference to jo in po, using the weak-reference mechanism
 function pyembed(po::PyObject, jo::Any)
     # If there's a need to support immutable embedding,
     # the API needs to be changed to return the pointer.
     isimmutable(jo) && ArgumentError("pyembed: immutable argument not allowed")
     if ispynull(weakref_callback_obj)
-        weakref_callback_obj.o = pyincref(pymethod(weakref_callback,
-                                                   "weakref_callback",
-                                                   METH_O)).o
+        cf = @cfunction(weakref_callback, PyPtr, (PyPtr,PyPtr))
+        weakref_callback_meth[] = PyMethodDef("weakref_callback", cf, METH_O)
+        copy!(weakref_callback_obj,
+              PyObject(@pycheckn ccall((@pysym :PyCFunction_NewEx), PyPtr,
+                                       (Ref{PyMethodDef}, Ptr{Cvoid}, Ptr{Cvoid}),
+                                       weakref_callback_meth, C_NULL, C_NULL)))
     end
     wo = @pycheckn ccall((@pysym :PyWeakref_NewRef), PyPtr, (PyPtr,PyPtr),
                          po, weakref_callback_obj)

src/numpy.jl

@@ -282,7 +282,11 @@ function f_contiguous(T::Type, sz::Vector{Int}, st::Vector{Int})
 end
 
 f_contiguous(i::PyArray_Info) = f_contiguous(i.T, i.sz, i.st)
-c_contiguous(i::PyArray_Info) = f_contiguous(i.T, flipdim(i.sz,1), flipdim(i.st,1))
+@static if VERSION >= v"0.7.0-DEV.4534" # julia#26369
+    c_contiguous(i::PyArray_Info) = f_contiguous(i.T, reverse(i.sz,dims=1), reverse(i.st,dims=1))
+else
+    c_contiguous(i::PyArray_Info) = f_contiguous(i.T, flipdim(i.sz,1), flipdim(i.st,1))
+end
 
 #########################################################################
 # PyArray: no-copy wrapper around NumPy ndarray

src/pyeval.jl

@@ -196,7 +196,7 @@ macro py_str(code, options...)
                                   :(m[$v] = PyObject($(esc(ex)))) :
                                   nothing) for (v,ex) in locals]...)
     code_expr = Expr(:call, esc(:(Base.string)))
-    i0 = start(code)
+    i0 = firstindex(code)
     for i in sort!(collect(filter(k -> isa(k,Integer), keys(locals))))
         push!(code_expr.args, code[i0:prevind(code,i)], esc(locals[i]))
         i0 = i

src/pyinit.jl

@@ -21,9 +21,54 @@ const c_void_p_Type = PyNULL()
 const pynothing = Ref{PyPtr}(0)
 const pyxrange = Ref{PyPtr}(0)
 
+#########################################################################
+# initialize jlWrapType for pytype.jl
+
+function pyjlwrap_init()
+    # PyMemberDef stores explicit pointers, hence must be initialized at runtime
+    push!(pyjlwrap_members, PyMemberDef(pyjlwrap_membername,
+                                        T_PYSSIZET, sizeof_PyObject_HEAD, READONLY,
+                                        pyjlwrap_doc),
+                            PyMemberDef(C_NULL,0,0,0,C_NULL))
+
+    # all cfunctions must be compiled at runtime
+    pyjlwrap_dealloc_ptr = @cfunction(pyjlwrap_dealloc, Cvoid, (PyPtr,))
+    pyjlwrap_repr_ptr = @cfunction(pyjlwrap_repr, PyPtr, (PyPtr,))
+    pyjlwrap_hash_ptr = @cfunction(pyjlwrap_hash, UInt, (PyPtr,))
+    pyjlwrap_hash32_ptr = @cfunction(pyjlwrap_hash32, UInt32, (PyPtr,))
+    pyjlwrap_call_ptr = @cfunction(pyjlwrap_call, PyPtr, (PyPtr,PyPtr,PyPtr))
+    pyjlwrap_getattr_ptr = @cfunction(pyjlwrap_getattr, PyPtr, (PyPtr,PyPtr))
+    pyjlwrap_getiter_ptr = @cfunction(pyjlwrap_getiter, PyPtr, (PyPtr,))
+
+    # detect at runtime whether we are using Stackless Python
+    try
+        pyimport("stackless")
+        Py_TPFLAGS_HAVE_STACKLESS_EXTENSION[] = Py_TPFLAGS_HAVE_STACKLESS_EXTENSION_
+    end
+
+    PyTypeObject!(jlWrapType, "PyCall.jlwrap", sizeof(Py_jlWrap)) do t::PyTypeObject
+        t.tp_flags |= Py_TPFLAGS_BASETYPE
+        t.tp_members = pointer(pyjlwrap_members);
+        t.tp_dealloc = pyjlwrap_dealloc_ptr
+        t.tp_repr = pyjlwrap_repr_ptr
+        t.tp_call = pyjlwrap_call_ptr
+        t.tp_getattro = pyjlwrap_getattr_ptr
+        t.tp_iter = pyjlwrap_getiter_ptr
+        t.tp_hash = sizeof(Py_hash_t) < sizeof(Int) ?
+                    pyjlwrap_hash32_ptr : pyjlwrap_hash_ptr
+    end
+end
+
 #########################################################################
 
 function __init__()
+    # sanity check: in Pkg for Julia 0.7+, the location of Conda can change
+    # if e.g. you checkout Conda master, and we'll need to re-build PyCall
+    # for something like pyimport_conda to continue working.
+    if conda && dirname(python) != abspath(Conda.PYTHONDIR)
+        error("Using Conda.jl python, but location of $python seems to have moved to $(Conda.PYTHONDIR).  Re-run Pkg.build(\"PyCall\") and restart Julia.")
+    end
+
     # issue #189
     libpy_handle = libpython === nothing ? C_NULL :
         Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL)

src/pyiterator.jl

@@ -42,20 +42,6 @@ Base.collect(o::PyObject) = collect(Any, o)
 
 const jlWrapIteratorType = PyTypeObject()
 
-# Given a Julia object o, return a jlwrap_iterator Python iterator object
-# that wraps the Julia iteration protocol with the Python iteration protocol.
-# Internally, the jlwrap_iterator object stores the tuple (o, Ref(start(o))),
-# where the Ref is used to store the iterator state (which updates during iteration).
-function jlwrap_iterator(o::Any)
-    if jlWrapIteratorType.tp_name == C_NULL # lazily initialize
-        pyjlwrap_type!(jlWrapIteratorType, "PyCall.jlwrap_iterator") do t
-            t.tp_iter = cfunction(pyincref_, PyPtr, Tuple{PyPtr}) # new reference to same object
-            t.tp_iternext = cfunction(pyjlwrap_iternext, PyPtr, Tuple{PyPtr})
-        end
-    end
-    return pyjlwrap_new(jlWrapIteratorType, (o, Ref(start(o))))
-end
-
 # tp_iternext object of a jlwrap_iterator object, similar to PyIter_Next
 function pyjlwrap_iternext(self_::PyPtr)
     try
@@ -84,6 +70,20 @@ function pyjlwrap_getiter(self_::PyPtr)
     return PyPtr_NULL
 end
 
+# Given a Julia object o, return a jlwrap_iterator Python iterator object
+# that wraps the Julia iteration protocol with the Python iteration protocol.
+# Internally, the jlwrap_iterator object stores the tuple (o, Ref(start(o))),
+# where the Ref is used to store the iterator state (which updates during iteration).
+function jlwrap_iterator(o::Any)
+    if jlWrapIteratorType.tp_name == C_NULL # lazily initialize
+        pyjlwrap_type!(jlWrapIteratorType, "PyCall.jlwrap_iterator") do t
+            t.tp_iter = @cfunction(pyincref_, PyPtr, (PyPtr,)) # new reference to same object
+            t.tp_iternext = @cfunction(pyjlwrap_iternext, PyPtr, (PyPtr,))
+        end
+    end
+    return pyjlwrap_new(jlWrapIteratorType, (o, Ref(start(o))))
+end
+
 #########################################################################
 # Broadcasting: if the object is iterable, return collect(o), and otherwise
 #               return o.