Merge branch 'main' into dataframe-display-config

Author: kosiew

Cargo.lock

@@ -17,7 +17,7 @@
 
 [package]
 name = "datafusion-python"
-version = "45.2.0"
+version = "46.0.0"
 homepage = "https://datafusion.apache.org/python"
 repository = "https://github.com/apache/datafusion-python"
 authors = ["Apache DataFusion <[email protected]>"]

Cargo.toml

@@ -0,0 +1,73 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Python 46.0.0 Changelog
+
+This release consists of 21 commits from 11 contributors. See credits at the end of this changelog for more information.
+
+**Implemented enhancements:**
+
+- feat: reads using global ctx [#982](https://github.com/apache/datafusion-python/pull/982) (ion-elgreco)
+- feat: Implementation of udf and udaf decorator [#1040](https://github.com/apache/datafusion-python/pull/1040) (CrystalZhou0529)
+- feat: expose regex_count function [#1066](https://github.com/apache/datafusion-python/pull/1066) (nirnayroy)
+- feat: Update DataFusion dependency to 46 [#1079](https://github.com/apache/datafusion-python/pull/1079) (timsaucer)
+
+**Fixed bugs:**
+
+- fix: add to_timestamp_nanos [#1020](https://github.com/apache/datafusion-python/pull/1020) (chenkovsky)
+- fix: type checking [#993](https://github.com/apache/datafusion-python/pull/993) (chenkovsky)
+
+**Other:**
+
+- [infra] Fail Clippy on rust build warnings [#1029](https://github.com/apache/datafusion-python/pull/1029) (kevinjqliu)
+- Add user documentation for the FFI approach [#1031](https://github.com/apache/datafusion-python/pull/1031) (timsaucer)
+- build(deps): bump arrow from 54.1.0 to 54.2.0 [#1035](https://github.com/apache/datafusion-python/pull/1035) (dependabot[bot])
+- Chore: Release datafusion-python 45 [#1024](https://github.com/apache/datafusion-python/pull/1024) (timsaucer)
+- Enable Dataframe to be converted into views which can be used in register_table [#1016](https://github.com/apache/datafusion-python/pull/1016) (kosiew)
+- Add ruff check for missing futures import [#1052](https://github.com/apache/datafusion-python/pull/1052) (timsaucer)
+- Enable take comments to assign issues to users [#1058](https://github.com/apache/datafusion-python/pull/1058) (timsaucer)
+- Update python min version to 3.9 [#1043](https://github.com/apache/datafusion-python/pull/1043) (kevinjqliu)
+- feat/improve ruff test coverage [#1055](https://github.com/apache/datafusion-python/pull/1055) (timsaucer)
+- feat/making global context accessible for users [#1060](https://github.com/apache/datafusion-python/pull/1060) (jsai28)
+- Renaming Internal Structs [#1059](https://github.com/apache/datafusion-python/pull/1059) (Spaarsh)
+- test: add pytest asyncio tests [#1063](https://github.com/apache/datafusion-python/pull/1063) (jsai28)
+- Add decorator for udwf [#1061](https://github.com/apache/datafusion-python/pull/1061) (kosiew)
+- Add additional ruff suggestions [#1062](https://github.com/apache/datafusion-python/pull/1062) (Spaarsh)
+- Improve collection during repr and repr_html [#1036](https://github.com/apache/datafusion-python/pull/1036) (timsaucer)
+
+## Credits
+
+Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor.
+
+```
+     7	Tim Saucer
+     2	Kevin Liu
+     2	Spaarsh
+     2	jsai28
+     2	kosiew
+     1	Chen Chongchen
+     1	Chongchen Chen
+     1	Crystal Zhou
+     1	Ion Koutsouris
+     1	Nirnay Roy
+     1	dependabot[bot]
+```
+
+Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release.
+

dev/changelog/46.0.0.md

@@ -20,8 +20,8 @@
 Concepts
 ========
 
-In this section, we will cover a basic example to introduce a few key concepts. We will use the same
-source file as described in the :ref:`Introduction <guide>`, the Pokemon data set.
+In this section, we will cover a basic example to introduce a few key concepts. We will use the
+2021 Yellow Taxi Trip Records ([download](https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-01.parquet)), from the [TLC Trip Record Data](https://www.nyc.gov/site/tlc/about/tlc-trip-record-data.page).
 
 .. ipython:: python
 

docs/source/user-guide/basics.rst

@@ -26,8 +26,9 @@
 except ImportError:
     import importlib_metadata
 
-# Local module imports
-from . import functions, object_store, substrait
+from . import functions, object_store, substrait, unparser
+
+# The following imports are okay to remain as opaque to the user.
 from ._internal import Config
 from .catalog import Catalog, Database, Table
 from .common import DFSchema
@@ -85,6 +86,7 @@
     "udaf",
     "udf",
     "udwf",
+    "unparser",
 ]
 
 

python/datafusion/__init__.py

@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""This module provides support for unparsing datafusion plans to SQL.
+
+For additional information about unparsing, see https://docs.rs/datafusion-sql/latest/datafusion_sql/unparser/index.html
+"""
+
+from ._internal import unparser as unparser_internal
+from .plan import LogicalPlan
+
+
+class Dialect:
+    """DataFusion data catalog."""
+
+    def __init__(self, dialect: unparser_internal.Dialect) -> None:
+        """This constructor is not typically called by the end user."""
+        self.dialect = dialect
+
+    @staticmethod
+    def default() -> "Dialect":
+        """Create a new default dialect."""
+        return Dialect(unparser_internal.Dialect.default())
+
+    @staticmethod
+    def mysql() -> "Dialect":
+        """Create a new MySQL dialect."""
+        return Dialect(unparser_internal.Dialect.mysql())
+
+    @staticmethod
+    def postgres() -> "Dialect":
+        """Create a new PostgreSQL dialect."""
+        return Dialect(unparser_internal.Dialect.postgres())
+
+    @staticmethod
+    def sqlite() -> "Dialect":
+        """Create a new SQLite dialect."""
+        return Dialect(unparser_internal.Dialect.sqlite())
+
+    @staticmethod
+    def duckdb() -> "Dialect":
+        """Create a new DuckDB dialect."""
+        return Dialect(unparser_internal.Dialect.duckdb())
+
+
+class Unparser:
+    """DataFusion unparser."""
+
+    def __init__(self, dialect: Dialect) -> None:
+        """This constructor is not typically called by the end user."""
+        self.unparser = unparser_internal.Unparser(dialect.dialect)
+
+    def plan_to_sql(self, plan: LogicalPlan) -> str:
+        """Convert a logical plan to a SQL string."""
+        return self.unparser.plan_to_sql(plan._raw_plan)
+
+    def with_pretty(self, pretty: bool) -> "Unparser":
+        """Set the pretty flag."""
+        self.unparser = self.unparser.with_pretty(pretty)
+        return self
+
+
+__all__ = [
+    "Dialect",
+    "Unparser",
+]

python/datafusion/unparser.py

@@ -1380,8 +1380,7 @@ def test_display_config_affects_repr(data):
     # The representation should show truncated data (3 rows as specified)
     assert (
         # 5 = 1 header row + 3 separator line + 1 truncation message
-        repr_str.count("\n")
-        <= max_table_rows_in_repr + 5
+        repr_str.count("\n") <= max_table_rows_in_repr + 5
     )
     assert "Data truncated" in repr_str
 
@@ -1397,8 +1396,7 @@ def test_display_config_affects_repr(data):
     # Should show all data without truncation message
     assert (
         # 4 = 1 header row + 3 separator lines
-        repr_str2.count("\n")
-        == max_table_rows_in_repr + 4
+        repr_str2.count("\n") == max_table_rows_in_repr + 4
     )  # All rows should be shown
     assert "Data truncated" not in repr_str2
 

python/tests/test_dataframe.py

@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from datafusion.context import SessionContext
+from datafusion.unparser import Dialect, Unparser
+
+
+def test_unparser():
+    ctx = SessionContext()
+    df = ctx.sql("SELECT 1")
+    for dialect in [
+        Dialect.mysql(),
+        Dialect.postgres(),
+        Dialect.sqlite(),
+        Dialect.duckdb(),
+    ]:
+        unparser = Unparser(dialect)
+        sql = unparser.plan_to_sql(df.logical_plan())
+        assert sql == "SELECT 1"

python/tests/test_unparser.py

@@ -52,6 +52,7 @@ pub mod pyarrow_util;
 mod record_batch;
 pub mod sql;
 pub mod store;
+pub mod unparser;
 
 #[cfg(feature = "substrait")]
 pub mod substrait;
@@ -104,6 +105,10 @@ fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> {
     expr::init_module(&expr)?;
     m.add_submodule(&expr)?;
 
+    let unparser = PyModule::new(py, "unparser")?;
+    unparser::init_module(&unparser)?;
+    m.add_submodule(&unparser)?;
+
     // Register the functions as a submodule
     let funcs = PyModule::new(py, "functions")?;
     functions::init_module(&funcs)?;

src/lib.rs

@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use datafusion::sql::unparser::dialect::{
+    DefaultDialect, Dialect, DuckDBDialect, MySqlDialect, PostgreSqlDialect, SqliteDialect,
+};
+use pyo3::prelude::*;
+
+#[pyclass(name = "Dialect", module = "datafusion.unparser", subclass)]
+#[derive(Clone)]
+pub struct PyDialect {
+    pub dialect: Arc<dyn Dialect>,
+}
+
+#[pymethods]
+impl PyDialect {
+    #[staticmethod]
+    pub fn default() -> Self {
+        Self {
+            dialect: Arc::new(DefaultDialect {}),
+        }
+    }
+    #[staticmethod]
+    pub fn postgres() -> Self {
+        Self {
+            dialect: Arc::new(PostgreSqlDialect {}),
+        }
+    }
+    #[staticmethod]
+    pub fn mysql() -> Self {
+        Self {
+            dialect: Arc::new(MySqlDialect {}),
+        }
+    }
+    #[staticmethod]
+    pub fn sqlite() -> Self {
+        Self {
+            dialect: Arc::new(SqliteDialect {}),
+        }
+    }
+    #[staticmethod]
+    pub fn duckdb() -> Self {
+        Self {
+            dialect: Arc::new(DuckDBDialect::new()),
+        }
+    }
+}