From aa93103e13d9e7c4b038da687f9b23c4325580e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Helmut=20H=C3=A4nsel?= Date: Sat, 1 Jul 2023 01:20:23 +0200 Subject: [PATCH 1/9] support conversion of datetime64 --- src/concrete/datetime.jl | 4 ++++ src/convert.jl | 1 + src/pywrap/PyArray.jl | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/concrete/datetime.jl b/src/concrete/datetime.jl index 1e0c41a8..29de5410 100644 --- a/src/concrete/datetime.jl +++ b/src/concrete/datetime.jl @@ -93,3 +93,7 @@ function pyconvert_rule_datetime(::Type{DateTime}, x::Py) iszero(mod(microseconds, 1000)) || return pyconvert_unconverted() return pyconvert_return(_base_datetime + Millisecond(div(microseconds, 1000) + 1000 * (seconds + 60 * 60 * 24 * days))) end + +function pyconvert_rule_datetime64(::Type{DateTime}, x::Py) + pyconvert(DateTime, pyimport("pandas").to_datetime(x)) +end \ No newline at end of file diff --git a/src/convert.jl b/src/convert.jl index 498293a5..723c32f2 100644 --- a/src/convert.jl +++ b/src/convert.jl @@ -420,6 +420,7 @@ function init_pyconvert() pyconvert_add_rule("datetime:datetime", DateTime, pyconvert_rule_datetime, priority) pyconvert_add_rule("datetime:date", Date, pyconvert_rule_date, priority) pyconvert_add_rule("datetime:time", Time, pyconvert_rule_time, priority) + pyconvert_add_rule("numpy:datetime64", DateTime, pyconvert_rule_datetime64, priority) pyconvert_add_rule("builtins:BaseException", PyException, pyconvert_rule_exception, priority) priority = PYCONVERT_PRIORITY_NORMAL diff --git a/src/pywrap/PyArray.jl b/src/pywrap/PyArray.jl index af7933cf..3819b07e 100644 --- a/src/pywrap/PyArray.jl +++ b/src/pywrap/PyArray.jl @@ -435,7 +435,7 @@ function pyarray_get_R(src::PyArraySource_ArrayStruct) elseif kind == 109 # m = timedelta error("timedelta not supported") elseif kind == 77 # M = datetime - error("datetime not supported") + return DateTime elseif kind == 79 # O = object if size == sizeof(C.PyPtr) return UnsafePyObject From c270f87b22d9d0f1a2755b10ecac649faac12a3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Helmut=20H=C3=A4nsel?= Date: Sat, 1 Jul 2023 11:07:33 +0200 Subject: [PATCH 2/9] change priority for datetime64 --- src/convert.jl | 6 +++--- src/pywrap/PyArray.jl | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/convert.jl b/src/convert.jl index 723c32f2..8f38f168 100644 --- a/src/convert.jl +++ b/src/convert.jl @@ -390,13 +390,14 @@ function init_pyconvert() priority = PYCONVERT_PRIORITY_WRAP pyconvert_add_rule("juliacall:ValueBase", Any, pyconvert_rule_jlvalue, priority) - + priority = PYCONVERT_PRIORITY_ARRAY pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) - + pyconvert_add_rule("numpy:datetime64", DateTime, pyconvert_rule_datetime64, priority) + priority = PYCONVERT_PRIORITY_CANONICAL pyconvert_add_rule("builtins:NoneType", Nothing, pyconvert_rule_none, priority) pyconvert_add_rule("builtins:bool", Bool, pyconvert_rule_bool, priority) @@ -420,7 +421,6 @@ function init_pyconvert() pyconvert_add_rule("datetime:datetime", DateTime, pyconvert_rule_datetime, priority) pyconvert_add_rule("datetime:date", Date, pyconvert_rule_date, priority) pyconvert_add_rule("datetime:time", Time, pyconvert_rule_time, priority) - pyconvert_add_rule("numpy:datetime64", DateTime, pyconvert_rule_datetime64, priority) pyconvert_add_rule("builtins:BaseException", PyException, pyconvert_rule_exception, priority) priority = PYCONVERT_PRIORITY_NORMAL diff --git a/src/pywrap/PyArray.jl b/src/pywrap/PyArray.jl index 3819b07e..32ca236a 100644 --- a/src/pywrap/PyArray.jl +++ b/src/pywrap/PyArray.jl @@ -435,7 +435,7 @@ function pyarray_get_R(src::PyArraySource_ArrayStruct) elseif kind == 109 # m = timedelta error("timedelta not supported") elseif kind == 77 # M = datetime - return DateTime + error("datetime64 not supported") elseif kind == 79 # O = object if size == sizeof(C.PyPtr) return UnsafePyObject From 4bcd3f2922b09016708374901f653ce851dbdf5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Helmut=20H=C3=A4nsel?= Date: Sat, 1 Jul 2023 11:15:36 +0200 Subject: [PATCH 3/9] add test for DataFrame conversion --- Project.toml | 3 ++- test/pywrap.jl | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index b7770094..ecf1f953 100644 --- a/Project.toml +++ b/Project.toml @@ -26,8 +26,9 @@ julia = "1.6.1" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" [targets] -test = ["Aqua", "Test", "TestItemRunner"] +test = ["Aqua", "Test", "TestItemRunner", "DataFrames"] diff --git a/test/pywrap.jl b/test/pywrap.jl index 13b4d3a1..5bbba2e9 100644 --- a/test/pywrap.jl +++ b/test/pywrap.jl @@ -339,6 +339,15 @@ end end @testitem "PyPandasDataFrame" begin + using Dates + using DataFrames + using CondaPkg + CondaPkg.add("pandas") + jdf = DataFrame(x = [now() + Second(rand(1:1000)) for _ in 1:100]) + pdf = pytable(jdf) + @test PyTable(pdf) isa PyPandasDataFrame + jdf2 = DataFrame(PyTable(pdf)) + @test all((jdf .== jdf2).x) end @testitem "PySet" begin From ac141a46d89997a043174028cd5f80024b9696bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Helmut=20H=C3=A4nsel?= Date: Sun, 2 Jul 2023 23:23:10 +0200 Subject: [PATCH 4/9] add support for timedelta and timedelta64 --- src/concrete/datetime.jl | 13 +++++++++++++ src/convert.jl | 2 ++ 2 files changed, 15 insertions(+) diff --git a/src/concrete/datetime.jl b/src/concrete/datetime.jl index 29de5410..687c6781 100644 --- a/src/concrete/datetime.jl +++ b/src/concrete/datetime.jl @@ -96,4 +96,17 @@ end function pyconvert_rule_datetime64(::Type{DateTime}, x::Py) pyconvert(DateTime, pyimport("pandas").to_datetime(x)) +end + +function pyconvert_rule_timedelta(::Type{<:Dates.CompoundPeriod}, x::Py) + days = pyconvert(Int, x.days) + seconds = pyconvert(Int, x.seconds) + microseconds = pyconvert(Int, x.microseconds) + nanoseconds = pyconvert(Int, x.nanoseconds) + iszero(mod(microseconds, 1000)) || return pyconvert_unconverted() + return pyconvert_return(Day(days) + Second(seconds) + Microsecond(microseconds) + Nanosecond(nanoseconds)) +end + +function pyconvert_rule_timedelta64(::Type{Dates.CompoundPeriod}, x::Py) + pyconvert_rule_timedelta(Dates.CompoundPeriod, pyimport("pandas").to_timedelta(x)) end \ No newline at end of file diff --git a/src/convert.jl b/src/convert.jl index 8f38f168..8bdbf9b9 100644 --- a/src/convert.jl +++ b/src/convert.jl @@ -397,6 +397,7 @@ function init_pyconvert() pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) pyconvert_add_rule("numpy:datetime64", DateTime, pyconvert_rule_datetime64, priority) + pyconvert_add_rule("numpy:timedelta64", Dates.CompoundPeriod, pyconvert_rule_timedelta64, priority) priority = PYCONVERT_PRIORITY_CANONICAL pyconvert_add_rule("builtins:NoneType", Nothing, pyconvert_rule_none, priority) @@ -421,6 +422,7 @@ function init_pyconvert() pyconvert_add_rule("datetime:datetime", DateTime, pyconvert_rule_datetime, priority) pyconvert_add_rule("datetime:date", Date, pyconvert_rule_date, priority) pyconvert_add_rule("datetime:time", Time, pyconvert_rule_time, priority) + pyconvert_add_rule("datetime:timedelta", Dates.CompoundPeriod, pyconvert_rule_timedelta, priority) pyconvert_add_rule("builtins:BaseException", PyException, pyconvert_rule_exception, priority) priority = PYCONVERT_PRIORITY_NORMAL From 89b0cf735d21e255d772a4bfa81d67351dfd54d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Helmut=20H=C3=A4nsel?= Date: Mon, 3 Jul 2023 09:45:12 +0200 Subject: [PATCH 5/9] finetune timedelta conversion with respect to ns --- src/concrete/datetime.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/concrete/datetime.jl b/src/concrete/datetime.jl index 687c6781..730868cf 100644 --- a/src/concrete/datetime.jl +++ b/src/concrete/datetime.jl @@ -102,9 +102,9 @@ function pyconvert_rule_timedelta(::Type{<:Dates.CompoundPeriod}, x::Py) days = pyconvert(Int, x.days) seconds = pyconvert(Int, x.seconds) microseconds = pyconvert(Int, x.microseconds) - nanoseconds = pyconvert(Int, x.nanoseconds) - iszero(mod(microseconds, 1000)) || return pyconvert_unconverted() - return pyconvert_return(Day(days) + Second(seconds) + Microsecond(microseconds) + Nanosecond(nanoseconds)) + nanoseconds = pyhasattr(x, "nanoseconds") ? pyconvert(Int, x.nanoseconds) : 0 + timedelta = Day(days) + Second(seconds) + Microsecond(microseconds) + Nanosecond(nanoseconds) + return pyconvert_return(timedelta) end function pyconvert_rule_timedelta64(::Type{Dates.CompoundPeriod}, x::Py) From cf2bbaacb0405072c8cdb576f35aed5e273d173d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Helmut=20H=C3=A4nsel?= Date: Mon, 3 Jul 2023 11:58:49 +0200 Subject: [PATCH 6/9] add conversion from Periods/CompoundPeriods to timedelta64 and tests for timedelta64 --- src/concrete/datetime.jl | 28 ++++++++++++++++++++++++++++ test/pywrap.jl | 5 ++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/concrete/datetime.jl b/src/concrete/datetime.jl index 730868cf..c8700baa 100644 --- a/src/concrete/datetime.jl +++ b/src/concrete/datetime.jl @@ -33,6 +33,34 @@ end pydatetime(x::Date) = pydatetime(year(x), month(x), day(x)) export pydatetime +pytimedelta64(_year=0, _month=0, _day=0, _hour=0, _minute=0, _second=0, _microsecond=0, _nanosecond=0; year=_year, month=_month, day=_day, hour=_hour, minute=_minute, second=_second, microsecond=_microsecond, nanosecond=_nanosecond) = _pytimedelta64(year, month, day, hour, minute, second, microsecond, nanosecond) +function pytimedelta64(@nospecialize(x::T)) where T <: Period + unit = if T==Year + "Y" + elseif T==Month + "M" + elseif T==Day + "D" + elseif T==Hour + "h" + elseif T==Minute + "m" + elseif T==Second + "s" + elseif T==Millisecond + "ms" + elseif T==Microsecond + "us" + elseif T==Nanosecond + "ns" + else + "" + end + pyimport("numpy").timedelta64(x.value, unit) +end +pytimedelta64(x::Dates.CompoundPeriod) = isempty(x.periods) ? pytimedelta64(Second(0)) : sum(pytimedelta64.(x.periods)) +export pytimedelta64 + function pytime_isaware(x) tzinfo = pygetattr(x, "tzinfo") if pyisnone(tzinfo) diff --git a/test/pywrap.jl b/test/pywrap.jl index 5bbba2e9..85b573f2 100644 --- a/test/pywrap.jl +++ b/test/pywrap.jl @@ -343,11 +343,14 @@ end using DataFrames using CondaPkg CondaPkg.add("pandas") - jdf = DataFrame(x = [now() + Second(rand(1:1000)) for _ in 1:100]) + jdf = DataFrame(x = [now() + Second(rand(1:1000)) for _ in 1:100], y = [Second(n) for n in 1:100]) pdf = pytable(jdf) @test PyTable(pdf) isa PyPandasDataFrame + @test pyconvert_typename(pt.x[0]) == "pandas._libs.tslibs.timestamps:" + @test pyconvert_typename(pt.y[0]) == "pandas._libs.tslibs.timedeltas:" jdf2 = DataFrame(PyTable(pdf)) @test all((jdf .== jdf2).x) + @test all((jdf .== jdf2).y) end @testitem "PySet" begin From 3f87fb6ba518de5336886a6f79ea8bf213bcc650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Helmut=20H=C3=A4nsel?= Date: Tue, 4 Jul 2023 00:44:48 +0200 Subject: [PATCH 7/9] add more conversion rules for periods --- src/concrete/datetime.jl | 8 ++++++++ src/convert.jl | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/concrete/datetime.jl b/src/concrete/datetime.jl index c8700baa..1b6c44fd 100644 --- a/src/concrete/datetime.jl +++ b/src/concrete/datetime.jl @@ -135,6 +135,14 @@ function pyconvert_rule_timedelta(::Type{<:Dates.CompoundPeriod}, x::Py) return pyconvert_return(timedelta) end +function pyconvert_rule_timedelta(::Type{T}, x::Py) where T<:Period + pyconvert_return(convert(T, pyconvert_rule_timedelta(Dates.CompoundPeriod, x))) +end + function pyconvert_rule_timedelta64(::Type{Dates.CompoundPeriod}, x::Py) pyconvert_rule_timedelta(Dates.CompoundPeriod, pyimport("pandas").to_timedelta(x)) +end + +function pyconvert_rule_timedelta64(::Type{T}, x::Py) where T<:Period + pyconvert_return(convert(T, pyconvert_rule_timedelta64(Dates.CompoundPeriod, x))) end \ No newline at end of file diff --git a/src/convert.jl b/src/convert.jl index 8bdbf9b9..402b01fd 100644 --- a/src/convert.jl +++ b/src/convert.jl @@ -398,7 +398,14 @@ function init_pyconvert() pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) pyconvert_add_rule("numpy:datetime64", DateTime, pyconvert_rule_datetime64, priority) pyconvert_add_rule("numpy:timedelta64", Dates.CompoundPeriod, pyconvert_rule_timedelta64, priority) - + pyconvert_add_rule("numpy:timedelta64", Year, pyconvert_rule_timedelta64, priority) + pyconvert_add_rule("numpy:timedelta64", Month, pyconvert_rule_timedelta64, priority) + pyconvert_add_rule("numpy:timedelta64", Day, pyconvert_rule_timedelta64, priority) + pyconvert_add_rule("numpy:timedelta64", Second, pyconvert_rule_timedelta64, priority) + pyconvert_add_rule("numpy:timedelta64", Millisecond, pyconvert_rule_timedelta64, priority) + pyconvert_add_rule("numpy:timedelta64", Microsecond, pyconvert_rule_timedelta64, priority) + pyconvert_add_rule("numpy:timedelta64", Nanosecond, pyconvert_rule_timedelta64, priority) + priority = PYCONVERT_PRIORITY_CANONICAL pyconvert_add_rule("builtins:NoneType", Nothing, pyconvert_rule_none, priority) pyconvert_add_rule("builtins:bool", Bool, pyconvert_rule_bool, priority) From 1ed5229049f947b0bbcb079c0551f3e1570c9187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Helmut=20H=C3=A4nsel?= Date: Tue, 4 Jul 2023 00:45:53 +0200 Subject: [PATCH 8/9] add deltatime64 conversion from Julia to Py, fix respective test --- src/Py.jl | 1 + test/pywrap.jl | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Py.jl b/src/Py.jl index 13fbdb02..b815ca9a 100644 --- a/src/Py.jl +++ b/src/Py.jl @@ -148,6 +148,7 @@ Py(x::AbstractRange{<:Union{Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UI Py(x::Date) = pydate(x) Py(x::Time) = pytime(x) Py(x::DateTime) = pydatetime(x) +Py(x::Union{Period, Dates.CompoundPeriod}) = pytimedelta64(x) Py(x) = ispy(x) ? throw(MethodError(Py, (x,))) : pyjl(x) Base.string(x::Py) = pyisnull(x) ? "" : pystr(String, x) diff --git a/test/pywrap.jl b/test/pywrap.jl index 85b573f2..4c53b1dc 100644 --- a/test/pywrap.jl +++ b/test/pywrap.jl @@ -346,8 +346,8 @@ end jdf = DataFrame(x = [now() + Second(rand(1:1000)) for _ in 1:100], y = [Second(n) for n in 1:100]) pdf = pytable(jdf) @test PyTable(pdf) isa PyPandasDataFrame - @test pyconvert_typename(pt.x[0]) == "pandas._libs.tslibs.timestamps:" - @test pyconvert_typename(pt.y[0]) == "pandas._libs.tslibs.timedeltas:" + @test PythonCall.pyconvert_typename(pdf.x[0]) == "pandas._libs.tslibs.timestamps:" + @test PythonCall.pyconvert_typename(pdf.y[0]) == "pandas._libs.tslibs.timedeltas:" jdf2 = DataFrame(PyTable(pdf)) @test all((jdf .== jdf2).x) @test all((jdf .== jdf2).y) From 37b6213f945e4b2693ed99050f3b29dc491bbef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Helmut=20H=C3=A4nsel?= Date: Wed, 5 Jul 2023 16:52:35 +0200 Subject: [PATCH 9/9] fix pytimedelta with kwargs --- src/concrete/datetime.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/concrete/datetime.jl b/src/concrete/datetime.jl index 1b6c44fd..76fbe50a 100644 --- a/src/concrete/datetime.jl +++ b/src/concrete/datetime.jl @@ -33,7 +33,13 @@ end pydatetime(x::Date) = pydatetime(year(x), month(x), day(x)) export pydatetime -pytimedelta64(_year=0, _month=0, _day=0, _hour=0, _minute=0, _second=0, _microsecond=0, _nanosecond=0; year=_year, month=_month, day=_day, hour=_hour, minute=_minute, second=_second, microsecond=_microsecond, nanosecond=_nanosecond) = _pytimedelta64(year, month, day, hour, minute, second, microsecond, nanosecond) +function pytimedelta64(_year=0, _month=0, _day=0, _hour=0, _minute=0, _second=0, _millisecond=0, _microsecond=0, _nanosecond=0; year=_year, month=_month, day=_day, hour=_hour, minute=_minute, second=_second, microsecond=_microsecond, millisecond=_millisecond, nanosecond=_nanosecond) + pytimedelta64(sum(( + Year(year), Month(month), Day(day), Hour(hour), + Minute(minute), Second(second), Millisecond(millisecond), Microsecond(microsecond), Nanosecond(nanosecond)) + )) +end + function pytimedelta64(@nospecialize(x::T)) where T <: Period unit = if T==Year "Y"