From 517c837a44c84ed01628906ca1fc28f2a4025582 Mon Sep 17 00:00:00 2001 From: Gary Helmling Date: Mon, 17 Mar 2025 13:20:12 -0700 Subject: [PATCH 1/2] Add ConnectorDeleteTableHandle to ConnectorProtocol (#24721) Summary: Pull Request resolved: https://github.com/prestodb/presto/pull/24721 Differential Revision: D70209547 --- .../connector/hive/HiveConnectorProtocol.h | 3 +- .../iceberg/IcebergConnectorProtocol.h | 1 + .../iceberg/presto_protocol_iceberg.cpp | 83 +++++++++++++++++ .../iceberg/presto_protocol_iceberg.h | 12 +++ .../iceberg/presto_protocol_iceberg.yml | 2 + .../special/IcebergInsertTableHandle.hpp.inc | 1 + .../special/IcebergOutputTableHandle.hpp.inc | 1 + .../presto_protocol/core/ConnectorProtocol.h | 20 +++- .../core/presto_protocol_core.cpp | 91 ++++++++++++++++++- .../core/presto_protocol_core.h | 16 +++- .../core/presto_protocol_core.yml | 5 + .../ConnectorDeleteTableHandle.cpp.inc | 33 +++++++ .../presto_protocol/java-to-struct-json.py | 32 ++++--- .../presto_protocol/presto_protocol.yml | 5 + 14 files changed, 285 insertions(+), 20 deletions(-) create mode 100644 presto-native-execution/presto_cpp/presto_protocol/core/special/ConnectorDeleteTableHandle.cpp.inc diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/hive/HiveConnectorProtocol.h b/presto-native-execution/presto_cpp/presto_protocol/connector/hive/HiveConnectorProtocol.h index 46e8f0045b537..f6feb5dd46c79 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/hive/HiveConnectorProtocol.h +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/hive/HiveConnectorProtocol.h @@ -25,5 +25,6 @@ using HiveConnectorProtocol = ConnectorProtocolTemplate< HiveSplit, HivePartitioningHandle, HiveTransactionHandle, - HiveMetadataUpdateHandle>; + HiveMetadataUpdateHandle, + NotImplemented>; } // namespace facebook::presto::protocol::hive diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/IcebergConnectorProtocol.h b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/IcebergConnectorProtocol.h index 36ced202d5c58..8fbfa5c378110 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/IcebergConnectorProtocol.h +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/IcebergConnectorProtocol.h @@ -27,6 +27,7 @@ using IcebergConnectorProtocol = ConnectorProtocolTemplate< IcebergSplit, NotImplemented, hive::HiveTransactionHandle, + NotImplemented, NotImplemented>; } // namespace facebook::presto::protocol::iceberg diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp index 33b0ec8df4ace..b4f0a998dc2b8 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp @@ -658,6 +658,33 @@ void from_json(const json& j, PrestoIcebergPartitionSpec& p) { } } // namespace facebook::presto::protocol::iceberg namespace facebook::presto::protocol::iceberg { + +void to_json(json& j, const SortField& p) { + j = json::object(); + to_json_key( + j, + "sourceColumnId", + p.sourceColumnId, + "SortField", + "int", + "sourceColumnId"); + to_json_key( + j, "sortOrder", p.sortOrder, "SortField", "SortOrder", "sortOrder"); +} + +void from_json(const json& j, SortField& p) { + from_json_key( + j, + "sourceColumnId", + p.sourceColumnId, + "SortField", + "int", + "sourceColumnId"); + from_json_key( + j, "sortOrder", p.sortOrder, "SortField", "SortOrder", "sortOrder"); +} +} // namespace facebook::presto::protocol::iceberg +namespace facebook::presto::protocol::iceberg { IcebergInsertTableHandle::IcebergInsertTableHandle() noexcept { _type = "hive-iceberg"; } @@ -728,6 +755,13 @@ void to_json(json& j, const IcebergInsertTableHandle& p) { "IcebergInsertTableHandle", "Map", "storageProperties"); + to_json_key( + j, + "sortOrder", + p.sortOrder, + "IcebergInsertTableHandle", + "List", + "sortOrder"); } void from_json(const json& j, IcebergInsertTableHandle& p) { @@ -795,6 +829,13 @@ void from_json(const json& j, IcebergInsertTableHandle& p) { "IcebergInsertTableHandle", "Map", "storageProperties"); + from_json_key( + j, + "sortOrder", + p.sortOrder, + "IcebergInsertTableHandle", + "List", + "sortOrder"); } } // namespace facebook::presto::protocol::iceberg namespace facebook::presto::protocol::iceberg { @@ -868,6 +909,13 @@ void to_json(json& j, const IcebergOutputTableHandle& p) { "IcebergOutputTableHandle", "Map", "storageProperties"); + to_json_key( + j, + "sortOrder", + p.sortOrder, + "IcebergOutputTableHandle", + "List", + "sortOrder"); } void from_json(const json& j, IcebergOutputTableHandle& p) { @@ -935,6 +983,13 @@ void from_json(const json& j, IcebergOutputTableHandle& p) { "IcebergOutputTableHandle", "Map", "storageProperties"); + from_json_key( + j, + "sortOrder", + p.sortOrder, + "IcebergOutputTableHandle", + "List", + "sortOrder"); } } // namespace facebook::presto::protocol::iceberg namespace facebook::presto::protocol::iceberg { @@ -1151,6 +1206,20 @@ void to_json(json& j, const IcebergTableHandle& p) { "IcebergTableHandle", "List", "equalityFieldIds"); + to_json_key( + j, + "sortOrder", + p.sortOrder, + "IcebergTableHandle", + "List", + "sortOrder"); + to_json_key( + j, + "updatedColumns", + p.updatedColumns, + "IcebergTableHandle", + "List", + "updatedColumns"); } void from_json(const json& j, IcebergTableHandle& p) { @@ -1211,6 +1280,20 @@ void from_json(const json& j, IcebergTableHandle& p) { "IcebergTableHandle", "List", "equalityFieldIds"); + from_json_key( + j, + "sortOrder", + p.sortOrder, + "IcebergTableHandle", + "List", + "sortOrder"); + from_json_key( + j, + "updatedColumns", + p.updatedColumns, + "IcebergTableHandle", + "List", + "updatedColumns"); } } // namespace facebook::presto::protocol::iceberg namespace facebook::presto::protocol::iceberg { diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h index 2d747106ec10f..1436b54b1de31 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h @@ -173,6 +173,14 @@ struct PrestoIcebergPartitionSpec { void to_json(json& j, const PrestoIcebergPartitionSpec& p); void from_json(const json& j, PrestoIcebergPartitionSpec& p); } // namespace facebook::presto::protocol::iceberg +namespace facebook::presto::protocol::iceberg { +struct SortField { + int sourceColumnId = {}; + SortOrder sortOrder = {}; +}; +void to_json(json& j, const SortField& p); +void from_json(const json& j, SortField& p); +} // namespace facebook::presto::protocol::iceberg /* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -201,6 +209,7 @@ struct IcebergInsertTableHandle : public ConnectorInsertTableHandle { FileFormat fileFormat = {}; hive::HiveCompressionCodec compressionCodec = {}; Map storageProperties = {}; + List sortOrder = {}; IcebergInsertTableHandle() noexcept; }; @@ -235,6 +244,7 @@ struct IcebergOutputTableHandle : public ConnectorOutputTableHandle { FileFormat fileFormat = {}; hive::HiveCompressionCodec compressionCodec = {}; Map storageProperties = {}; + List sortOrder = {}; IcebergOutputTableHandle() noexcept; }; @@ -289,6 +299,8 @@ struct IcebergTableHandle : public ConnectorTableHandle { std::shared_ptr tableSchemaJson = {}; std::shared_ptr> partitionFieldIds = {}; std::shared_ptr> equalityFieldIds = {}; + List sortOrder = {}; + List updatedColumns = {}; IcebergTableHandle() noexcept; }; diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.yml b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.yml index f323d42a65f59..197b713b25ae0 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.yml +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.yml @@ -74,3 +74,5 @@ JavaClasses: - presto-iceberg/src/main/java/com/facebook/presto/iceberg/delete/DeleteFile.java - presto-iceberg/src/main/java/com/facebook/presto/iceberg/changelog/ChangelogOperation.java - presto-iceberg/src/main/java/com/facebook/presto/iceberg/changelog/ChangelogSplitInfo.java + - presto-iceberg/src/main/java/com/facebook/presto/iceberg/SortField.java + diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/special/IcebergInsertTableHandle.hpp.inc b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/special/IcebergInsertTableHandle.hpp.inc index 0370c49e87629..91b11fa183a53 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/special/IcebergInsertTableHandle.hpp.inc +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/special/IcebergInsertTableHandle.hpp.inc @@ -26,6 +26,7 @@ struct IcebergInsertTableHandle : public ConnectorInsertTableHandle { FileFormat fileFormat = {}; hive::HiveCompressionCodec compressionCodec = {}; Map storageProperties = {}; + List sortOrder = {}; IcebergInsertTableHandle() noexcept; }; diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/special/IcebergOutputTableHandle.hpp.inc b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/special/IcebergOutputTableHandle.hpp.inc index a887a027e653c..406238e860df3 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/special/IcebergOutputTableHandle.hpp.inc +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/special/IcebergOutputTableHandle.hpp.inc @@ -26,6 +26,7 @@ struct IcebergOutputTableHandle : public ConnectorOutputTableHandle { FileFormat fileFormat = {}; hive::HiveCompressionCodec compressionCodec = {}; Map storageProperties = {}; + List sortOrder = {}; IcebergOutputTableHandle() noexcept; }; diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/ConnectorProtocol.h b/presto-native-execution/presto_cpp/presto_protocol/core/ConnectorProtocol.h index 46cf3495e233d..055dd8fa14784 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/ConnectorProtocol.h +++ b/presto-native-execution/presto_cpp/presto_protocol/core/ConnectorProtocol.h @@ -89,6 +89,13 @@ class ConnectorProtocol { virtual void from_json( const json& j, std::shared_ptr& p) const = 0; + + virtual void to_json( + json& j, + const std::shared_ptr& p) const = 0; + virtual void from_json( + const json& j, + std::shared_ptr& p) const = 0; }; namespace { @@ -104,7 +111,8 @@ template < typename ConnectorSplitType = NotImplemented, typename ConnectorPartitioningHandleType = NotImplemented, typename ConnectorTransactionHandleType = NotImplemented, - typename ConnectorMetadataUpdateHandleType = NotImplemented> + typename ConnectorMetadataUpdateHandleType = NotImplemented, + typename ConnectorDeleteTableHandleType = NotImplemented> class ConnectorProtocolTemplate final : public ConnectorProtocol { public: void to_json(json& j, const std::shared_ptr& p) @@ -186,6 +194,16 @@ class ConnectorProtocolTemplate final : public ConnectorProtocol { from_json_template(j, p); } + void to_json(json& j, const std::shared_ptr& p) + const final { + to_json_template(j, p); + } + void from_json( + const json& j, + std::shared_ptr& p) const final { + from_json_template(j, p); + } + private: template static void to_json_template( diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp index 7a80814f1acee..5968a56e60333 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp @@ -3251,6 +3251,91 @@ void from_json(const json& j, CreateHandle& p) { "schemaTableName"); } } // namespace facebook::presto::protocol +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +namespace facebook::presto::protocol { +void to_json(json& j, const std::shared_ptr& p) { + if (p == nullptr) { + return; + } + String type = p->_type; + getConnectorProtocol(type).to_json(j, p); +} + +void from_json(const json& j, std::shared_ptr& p) { + String type; + try { + type = p->getSubclassKey(j); + } catch (json::parse_error& e) { + throw ParseError( + std::string(e.what()) + + " ConnectorDeleteTableHandle ConnectorDeleteTableHandle"); + } + getConnectorProtocol(type).from_json(j, p); +} +} // namespace facebook::presto::protocol +namespace facebook::presto::protocol { + +void to_json(json& j, const DeleteTableHandle& p) { + j = json::object(); + to_json_key( + j, + "connectorId", + p.connectorId, + "DeleteTableHandle", + "ConnectorId", + "connectorId"); + to_json_key( + j, + "transactionHandle", + p.transactionHandle, + "DeleteTableHandle", + "ConnectorTransactionHandle", + "transactionHandle"); + to_json_key( + j, + "connectorHandle", + p.connectorHandle, + "DeleteTableHandle", + "ConnectorDeleteTableHandle", + "connectorHandle"); +} + +void from_json(const json& j, DeleteTableHandle& p) { + from_json_key( + j, + "connectorId", + p.connectorId, + "DeleteTableHandle", + "ConnectorId", + "connectorId"); + from_json_key( + j, + "transactionHandle", + p.transactionHandle, + "DeleteTableHandle", + "ConnectorTransactionHandle", + "transactionHandle"); + from_json_key( + j, + "connectorHandle", + p.connectorHandle, + "DeleteTableHandle", + "ConnectorDeleteTableHandle", + "connectorHandle"); +} +} // namespace facebook::presto::protocol namespace facebook::presto::protocol { DeleteHandle::DeleteHandle() noexcept { _type = "DeleteHandle"; @@ -3259,7 +3344,8 @@ DeleteHandle::DeleteHandle() noexcept { void to_json(json& j, const DeleteHandle& p) { j = json::object(); j["@type"] = "DeleteHandle"; - to_json_key(j, "handle", p.handle, "DeleteHandle", "TableHandle", "handle"); + to_json_key( + j, "handle", p.handle, "DeleteHandle", "DeleteTableHandle", "handle"); to_json_key( j, "schemaTableName", @@ -3271,7 +3357,8 @@ void to_json(json& j, const DeleteHandle& p) { void from_json(const json& j, DeleteHandle& p) { p._type = j["@type"]; - from_json_key(j, "handle", p.handle, "DeleteHandle", "TableHandle", "handle"); + from_json_key( + j, "handle", p.handle, "DeleteHandle", "DeleteTableHandle", "handle"); from_json_key( j, "schemaTableName", diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h index 908401eefd45e..d77c27a232733 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h @@ -312,6 +312,11 @@ void to_json(json& j, const std::shared_ptr& p); void from_json(const json& j, std::shared_ptr& p); } // namespace facebook::presto::protocol namespace facebook::presto::protocol { +struct ConnectorDeleteTableHandle : public JsonEncodedSubclass {}; +void to_json(json& j, const std::shared_ptr& p); +void from_json(const json& j, std::shared_ptr& p); +} // namespace facebook::presto::protocol +namespace facebook::presto::protocol { struct InputDistribution : public JsonEncodedSubclass {}; void to_json(json& j, const std::shared_ptr& p); void from_json(const json& j, std::shared_ptr& p); @@ -1026,8 +1031,17 @@ void to_json(json& j, const CreateHandle& p); void from_json(const json& j, CreateHandle& p); } // namespace facebook::presto::protocol namespace facebook::presto::protocol { +struct DeleteTableHandle { + ConnectorId connectorId = {}; + std::shared_ptr transactionHandle = {}; + std::shared_ptr connectorHandle = {}; +}; +void to_json(json& j, const DeleteTableHandle& p); +void from_json(const json& j, DeleteTableHandle& p); +} // namespace facebook::presto::protocol +namespace facebook::presto::protocol { struct DeleteHandle : public ExecutionWriterTarget { - TableHandle handle = {}; + DeleteTableHandle handle = {}; SchemaTableName schemaTableName = {}; DeleteHandle() noexcept; diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml index 71fb2ffc1cb63..4dfefa846c4f0 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml @@ -82,6 +82,10 @@ AbstractClasses: - { name: HiveInsertTableHandle, key: hive } - { name: IcebergInsertTableHandle, key: hive-iceberg } + ConnectorDeleteTableHandle: + super: JsonEncodedSubclass + subclasses: + ConnectorTransactionHandle: super: JsonEncodedSubclass subclasses: @@ -288,6 +292,7 @@ JavaClasses: - presto-main/src/main/java/com/facebook/presto/execution/scheduler/ExecutionWriterTarget.java - presto-main/src/main/java/com/facebook/presto/metadata/OutputTableHandle.java - presto-main/src/main/java/com/facebook/presto/metadata/InsertTableHandle.java + - presto-main/src/main/java/com/facebook/presto/metadata/DeleteTableHandle.java - presto-main/src/main/java/com/facebook/presto/execution/TaskInfo.java - presto-main/src/main/java/com/facebook/presto/execution/TaskSource.java - presto-main/src/main/java/com/facebook/presto/execution/TaskState.java diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/special/ConnectorDeleteTableHandle.cpp.inc b/presto-native-execution/presto_cpp/presto_protocol/core/special/ConnectorDeleteTableHandle.cpp.inc new file mode 100644 index 0000000000000..c8112b1847213 --- /dev/null +++ b/presto-native-execution/presto_cpp/presto_protocol/core/special/ConnectorDeleteTableHandle.cpp.inc @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +namespace facebook::presto::protocol { +void to_json(json& j, const std::shared_ptr& p) { + if (p == nullptr) { + return; + } + String type = p->_type; + getConnectorProtocol(type).to_json(j, p); +} + +void from_json(const json& j, std::shared_ptr& p) { + String type; + try { + type = p->getSubclassKey(j); + } catch (json::parse_error& e) { + throw ParseError( + std::string(e.what()) + " ConnectorDeleteTableHandle ConnectorDeleteTableHandle"); + } + getConnectorProtocol(type).from_json(j, p); +} +} // namespace facebook::presto::protocol diff --git a/presto-native-execution/presto_cpp/presto_protocol/java-to-struct-json.py b/presto-native-execution/presto_cpp/presto_protocol/java-to-struct-json.py index bb2811061d22e..f19e2358ddcdf 100755 --- a/presto-native-execution/presto_cpp/presto_protocol/java-to-struct-json.py +++ b/presto-native-execution/presto_cpp/presto_protocol/java-to-struct-json.py @@ -41,7 +41,7 @@ language = { "cpp": { "TypeMap": { - r"([ ,<])(ColumnHandle|PlanNode|RowExpression|ConnectorMetadataUpdateHandle)([ ,>])": r"\1std::shared_ptr<\2>\3", + r"([ ,<])(ColumnHandle|PlanNode|RowExpression|ConnectorMetadataUpdateHandle|ConnectorDeleteTableHandle)([ ,>])": r"\1std::shared_ptr<\2>\3", r"Optional": "Optional>", r"Optional": "Optional>", r"int\[\]": "List", @@ -351,23 +351,25 @@ def main(): classes[abstract_name].comparable = True classes[abstract_name].subclasses = [] - for subclass in abstract_value.subclasses: - subclasses[subclass.name] = util.attrdict( - super=abstract_name, key=subclass.key - ) + if abstract_value.subclasses: + for subclass in abstract_value.subclasses: + subclasses[subclass.name] = util.attrdict( + super=abstract_name, key=subclass.key + ) - classes[abstract_name].subclasses.append( - util.attrdict( - type=subclass.name, - name=member_name(subclass.name), - key=subclass.key, + classes[abstract_name].subclasses.append( + util.attrdict( + type=subclass.name, + name=member_name(subclass.name), + key=subclass.key, + ) + ) + classes[abstract_name].subclasses[-1]._N = len( + classes[abstract_name].subclasses ) - ) - classes[abstract_name].subclasses[-1]._N = len( - classes[abstract_name].subclasses - ) - classes[abstract_name].subclasses[-1]._last = True + if classes[abstract_name].subclasses: + classes[abstract_name].subclasses[-1]._last = True if "source" in abstract_value: file = abstract_value.source diff --git a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml index d1733df0d1ce9..fc0b86ddfc046 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml +++ b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml @@ -81,6 +81,10 @@ AbstractClasses: - { name: HiveInsertTableHandle, key: hive } - { name: IcebergInsertTableHandle, key: hive-iceberg } + ConnectorDeleteTableHandle: + super: JsonEncodedSubclass + subclasses: + ConnectorTransactionHandle: super: JsonEncodedSubclass subclasses: @@ -335,6 +339,7 @@ JavaClasses: - presto-main/src/main/java/com/facebook/presto/execution/scheduler/ExecutionWriterTarget.java - presto-main/src/main/java/com/facebook/presto/metadata/OutputTableHandle.java - presto-main/src/main/java/com/facebook/presto/metadata/InsertTableHandle.java + - presto-main/src/main/java/com/facebook/presto/metadata/DeleteTableHandle.java - presto-hive/src/main/java/com/facebook/presto/hive/TableToPartitionMapping.java - presto-main/src/main/java/com/facebook/presto/execution/TaskInfo.java - presto-main/src/main/java/com/facebook/presto/execution/TaskSource.java From fc33f2552985779510dedc22e6463dc3064becc5 Mon Sep 17 00:00:00 2001 From: Gary Helmling Date: Thu, 20 Mar 2025 22:42:54 -0700 Subject: [PATCH 2/2] Apply missing presto_protocol changes from prior commits --- .../facebook/presto/execution/TaskInfo.java | 50 +--- .../facebook/presto/operator/TaskStats.java | 273 +++++------------- .../iceberg/presto_protocol_iceberg.cpp | 14 + .../iceberg/presto_protocol_iceberg.h | 1 + .../iceberg/special/IcebergSplit.hpp.inc | 1 + .../core/presto_protocol_core.cpp | 88 +++++- .../core/presto_protocol_core.h | 7 +- 7 files changed, 189 insertions(+), 245 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/execution/TaskInfo.java b/presto-main/src/main/java/com/facebook/presto/execution/TaskInfo.java index a714f87250179..57ed6931318d8 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/TaskInfo.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/TaskInfo.java @@ -35,7 +35,6 @@ import static com.facebook.presto.execution.TaskStatus.initialTaskStatus; import static com.facebook.presto.execution.buffer.BufferState.OPEN; import static com.facebook.presto.metadata.MetadataUpdates.DEFAULT_METADATA_UPDATES; -import static com.facebook.presto.util.DateTimeUtils.toTimeStampInMillis; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static java.lang.System.currentTimeMillis; @@ -56,35 +55,12 @@ public class TaskInfo private final MetadataUpdates metadataUpdates; private final String nodeId; - public TaskInfo(TaskId taskId, - TaskStatus taskStatus, - long lastHeartbeatInMillis, - OutputBufferInfo outputBuffers, - Set noMoreSplits, - TaskStats stats, - boolean needsPlan, - MetadataUpdates metadataUpdates, - String nodeId) - { - this.taskId = requireNonNull(taskId, "taskId is null"); - this.taskStatus = requireNonNull(taskStatus, "taskStatus is null"); - checkArgument(lastHeartbeatInMillis >= 0, "lastHeartbeat is negative"); - this.lastHeartbeatInMillis = lastHeartbeatInMillis; - this.outputBuffers = requireNonNull(outputBuffers, "outputBuffers is null"); - this.noMoreSplits = requireNonNull(noMoreSplits, "noMoreSplits is null"); - this.stats = requireNonNull(stats, "stats is null"); - - this.needsPlan = needsPlan; - this.metadataUpdates = metadataUpdates; - this.nodeId = requireNonNull(nodeId, "nodeId is null"); - } - @JsonCreator @ThriftConstructor public TaskInfo( @JsonProperty("taskId") TaskId taskId, @JsonProperty("taskStatus") TaskStatus taskStatus, - @JsonProperty("lastHeartbeat") DateTime lastHeartbeat, + @JsonProperty("lastHeartbeatInMillis") long lastHeartbeatInMillis, @JsonProperty("outputBuffers") OutputBufferInfo outputBuffers, @JsonProperty("noMoreSplits") Set noMoreSplits, @JsonProperty("stats") TaskStats stats, @@ -92,15 +68,17 @@ public TaskInfo( @JsonProperty("metadataUpdates") MetadataUpdates metadataUpdates, @JsonProperty("nodeId") String nodeId) { - this(taskId, - taskStatus, - toTimeStampInMillis(lastHeartbeat), - outputBuffers, - noMoreSplits, - stats, - needsPlan, - metadataUpdates, - nodeId); + this.taskId = requireNonNull(taskId, "taskId is null"); + this.taskStatus = requireNonNull(taskStatus, "taskStatus is null"); + checkArgument(lastHeartbeatInMillis >= 0, "lastHeartbeat is negative"); + this.lastHeartbeatInMillis = lastHeartbeatInMillis; + this.outputBuffers = requireNonNull(outputBuffers, "outputBuffers is null"); + this.noMoreSplits = requireNonNull(noMoreSplits, "noMoreSplits is null"); + this.stats = requireNonNull(stats, "stats is null"); + + this.needsPlan = needsPlan; + this.metadataUpdates = metadataUpdates; + this.nodeId = requireNonNull(nodeId, "nodeId is null"); } @JsonProperty @@ -117,13 +95,13 @@ public TaskStatus getTaskStatus() return taskStatus; } - @JsonProperty - @ThriftField(3) public DateTime getLastHeartbeat() { return new DateTime(lastHeartbeatInMillis); } + @JsonProperty + @ThriftField(3) public long getLastHeartbeatInMillis() { return lastHeartbeatInMillis; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/TaskStats.java b/presto-main/src/main/java/com/facebook/presto/operator/TaskStats.java index 4e275878c4376..71704074c1ed2 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/TaskStats.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/TaskStats.java @@ -23,8 +23,6 @@ import com.google.common.collect.ImmutableSet; import org.joda.time.DateTime; -import javax.annotation.Nullable; - import java.util.List; import java.util.Set; @@ -138,11 +136,11 @@ public TaskStats(long createTimeInMillis, long endTimeInMillis) public TaskStats(DateTime createTime, DateTime endTime) { - this(createTime, - null, - null, - null, - endTime, + this(toTimeStampInMillis(createTime), + 0L, + 0L, + 0L, + toTimeStampInMillis(endTime), 0L, 0L, 0, @@ -181,59 +179,61 @@ public TaskStats(DateTime createTime, DateTime endTime) new RuntimeStats()); } + @JsonCreator + @ThriftConstructor public TaskStats( - long createTimeInMillis, - long firstStartTimeInMillis, - long lastStartTimeInMillis, - long lastEndTimeInMillis, - long endTimeInMillis, - long elapsedTimeInNanos, - long queuedTimeInNanos, - - int totalDrivers, - int queuedDrivers, - int queuedPartitionedDrivers, - long queuedPartitionedSplitsWeight, - int runningDrivers, - int runningPartitionedDrivers, - long runningPartitionedSplitsWeight, - int blockedDrivers, - int completedDrivers, - - double cumulativeUserMemory, - double cumulativeTotalMemory, - long userMemoryReservationInBytes, - long revocableMemoryReservationInBytes, - long systemMemoryReservationInBytes, - - long peakTotalMemoryInBytes, - long peakUserMemoryInBytes, - long peakNodeTotalMemoryInBytes, - - long totalScheduledTimeInNanos, - long totalCpuTimeInNanos, - long totalBlockedTimeInNanos, - boolean fullyBlocked, - Set blockedReasons, - - long totalAllocationInBytes, - - long rawInputDataSizeInBytes, - long rawInputPositions, - - long processedInputDataSizeInBytes, - long processedInputPositions, - - long outputDataSizeInBytes, - long outputPositions, - - long physicalWrittenDataSizeInBytes, - - int fullGcCount, - long fullGcTimeInMillis, - - List pipelines, - RuntimeStats runtimeStats) + @JsonProperty("createTimeInMillis") long createTimeInMillis, + @JsonProperty("firstStartTimeInMillis") long firstStartTimeInMillis, + @JsonProperty("lastStartTimeInMillis") long lastStartTimeInMillis, + @JsonProperty("lastEndTimeInMillis") long lastEndTimeInMillis, + @JsonProperty("endTimeInMillis") long endTimeInMillis, + @JsonProperty("elapsedTimeInNanos") long elapsedTimeInNanos, + @JsonProperty("queuedTimeInNanos") long queuedTimeInNanos, + + @JsonProperty("totalDrivers") int totalDrivers, + @JsonProperty("queuedDrivers") int queuedDrivers, + @JsonProperty("queuedPartitionedDrivers") int queuedPartitionedDrivers, + @JsonProperty("queuedPartitionedSplitsWeight") long queuedPartitionedSplitsWeight, + @JsonProperty("runningDrivers") int runningDrivers, + @JsonProperty("runningPartitionedDrivers") int runningPartitionedDrivers, + @JsonProperty("runningPartitionedSplitsWeight") long runningPartitionedSplitsWeight, + @JsonProperty("blockedDrivers") int blockedDrivers, + @JsonProperty("completedDrivers") int completedDrivers, + + @JsonProperty("cumulativeUserMemory") double cumulativeUserMemory, + @JsonProperty("cumulativeTotalMemory") double cumulativeTotalMemory, + @JsonProperty("userMemoryReservationInBytes") long userMemoryReservationInBytes, + @JsonProperty("revocableMemoryReservationInBytes") long revocableMemoryReservationInBytes, + @JsonProperty("systemMemoryReservationInBytes") long systemMemoryReservationInBytes, + + @JsonProperty("peakTotalMemoryInBytes") long peakTotalMemoryInBytes, + @JsonProperty("peakUserMemoryInBytes") long peakUserMemoryInBytes, + @JsonProperty("peakNodeTotalMemoryInBytes") long peakNodeTotalMemoryInBytes, + + @JsonProperty("totalScheduledTimeInNanos") long totalScheduledTimeInNanos, + @JsonProperty("totalCpuTimeInNanos") long totalCpuTimeInNanos, + @JsonProperty("totalBlockedTimeInNanos") long totalBlockedTimeInNanos, + @JsonProperty("fullyBlocked") boolean fullyBlocked, + @JsonProperty("blockedReasons") Set blockedReasons, + + @JsonProperty("totalAllocationInBytes") long totalAllocationInBytes, + + @JsonProperty("rawInputDataSizeInBytes") long rawInputDataSizeInBytes, + @JsonProperty("rawInputPositions") long rawInputPositions, + + @JsonProperty("processedInputDataSizeInBytes") long processedInputDataSizeInBytes, + @JsonProperty("processedInputPositions") long processedInputPositions, + + @JsonProperty("outputDataSizeInBytes") long outputDataSizeInBytes, + @JsonProperty("outputPositions") long outputPositions, + + @JsonProperty("physicalWrittenDataSizeInBytes") long physicalWrittenDataSizeInBytes, + + @JsonProperty("fullGcCount") int fullGcCount, + @JsonProperty("fullGcTimeInMillis") long fullGcTimeInMillis, + + @JsonProperty("pipelines") List pipelines, + @JsonProperty("runtimeStats") RuntimeStats runtimeStats) { checkArgument(createTimeInMillis >= 0, "createTimeInMillis is negative"); this.createTimeInMillis = createTimeInMillis; @@ -307,176 +307,61 @@ public TaskStats( this.runtimeStats = requireNonNull(runtimeStats, "runtimeStats is null"); } - @JsonCreator - @ThriftConstructor - public TaskStats( - @JsonProperty("createTime") DateTime createTime, - @JsonProperty("firstStartTime") DateTime firstStartTime, - @JsonProperty("lastStartTime") DateTime lastStartTime, - @JsonProperty("lastEndTime") DateTime lastEndTime, - @JsonProperty("endTime") DateTime endTime, - @JsonProperty("elapsedTimeInNanos") long elapsedTimeInNanos, - @JsonProperty("queuedTimeInNanos") long queuedTimeInNanos, - - @JsonProperty("totalDrivers") int totalDrivers, - @JsonProperty("queuedDrivers") int queuedDrivers, - @JsonProperty("queuedPartitionedDrivers") int queuedPartitionedDrivers, - @JsonProperty("queuedPartitionedSplitsWeight") long queuedPartitionedSplitsWeight, - @JsonProperty("runningDrivers") int runningDrivers, - @JsonProperty("runningPartitionedDrivers") int runningPartitionedDrivers, - @JsonProperty("runningPartitionedSplitsWeight") long runningPartitionedSplitsWeight, - @JsonProperty("blockedDrivers") int blockedDrivers, - @JsonProperty("completedDrivers") int completedDrivers, - - @JsonProperty("cumulativeUserMemory") double cumulativeUserMemory, - @JsonProperty("cumulativeTotalMemory") double cumulativeTotalMemory, - @JsonProperty("userMemoryReservationInBytes") long userMemoryReservationInBytes, - @JsonProperty("revocableMemoryReservationInBytes") long revocableMemoryReservationInBytes, - @JsonProperty("systemMemoryReservationInBytes") long systemMemoryReservationInBytes, - - @JsonProperty("peakTotalMemoryInBytes") long peakTotalMemoryInBytes, - @JsonProperty("peakUserMemoryInBytes") long peakUserMemoryInBytes, - @JsonProperty("peakNodeTotalMemoryInBytes") long peakNodeTotalMemoryInBytes, - - @JsonProperty("totalScheduledTimeInNanos") long totalScheduledTimeInNanos, - @JsonProperty("totalCpuTimeInNanos") long totalCpuTimeInNanos, - @JsonProperty("totalBlockedTimeInNanos") long totalBlockedTimeInNanos, - @JsonProperty("fullyBlocked") boolean fullyBlocked, - @JsonProperty("blockedReasons") Set blockedReasons, - - @JsonProperty("totalAllocationInBytes") long totalAllocationInBytes, - - @JsonProperty("rawInputDataSizeInBytes") long rawInputDataSizeInBytes, - @JsonProperty("rawInputPositions") long rawInputPositions, - - @JsonProperty("processedInputDataSizeInBytes") long processedInputDataSizeInBytes, - @JsonProperty("processedInputPositions") long processedInputPositions, - - @JsonProperty("outputDataSizeInBytes") long outputDataSizeInBytes, - @JsonProperty("outputPositions") long outputPositions, - - @JsonProperty("physicalWrittenDataSizeInBytes") long physicalWrittenDataSizeInBytes, - - @JsonProperty("fullGcCount") int fullGcCount, - @JsonProperty("fullGcTimeInMillis") long fullGcTimeInMillis, - - @JsonProperty("pipelines") List pipelines, - @JsonProperty("runtimeStats") RuntimeStats runtimeStats) - { - this(toTimeStampInMillis(createTime), - toTimeStampInMillis(firstStartTime), - toTimeStampInMillis(lastStartTime), - toTimeStampInMillis(lastEndTime), - toTimeStampInMillis(endTime), - - elapsedTimeInNanos, - queuedTimeInNanos, - - totalDrivers, - queuedDrivers, - queuedPartitionedDrivers, - queuedPartitionedSplitsWeight, - runningDrivers, - runningPartitionedDrivers, - runningPartitionedSplitsWeight, - blockedDrivers, - completedDrivers, - - cumulativeUserMemory, - cumulativeTotalMemory, - userMemoryReservationInBytes, - revocableMemoryReservationInBytes, - systemMemoryReservationInBytes, - - peakTotalMemoryInBytes, - peakUserMemoryInBytes, - peakNodeTotalMemoryInBytes, - - totalScheduledTimeInNanos, - totalCpuTimeInNanos, - totalBlockedTimeInNanos, - fullyBlocked, - blockedReasons, - - totalAllocationInBytes, - - rawInputDataSizeInBytes, - rawInputPositions, - - processedInputDataSizeInBytes, - processedInputPositions, - - outputDataSizeInBytes, - outputPositions, - - physicalWrittenDataSizeInBytes, - - fullGcCount, - fullGcTimeInMillis, - - pipelines, - runtimeStats); - } - - @JsonProperty - @ThriftField(1) public DateTime getCreateTime() { return new DateTime(createTimeInMillis); } + @JsonProperty + @ThriftField(1) public long getCreateTimeInMillis() { return createTimeInMillis; } - @Nullable - @JsonProperty - @ThriftField(2) public DateTime getFirstStartTime() { return new DateTime(firstStartTimeInMillis); } + @JsonProperty + @ThriftField(2) public long getFirstStartTimeInMillis() { return firstStartTimeInMillis; } - @Nullable - @JsonProperty - @ThriftField(3) public DateTime getLastStartTime() { return new DateTime(lastStartTimeInMillis); } + @JsonProperty + @ThriftField(3) public long getLastStartTimeInMillis() { return lastStartTimeInMillis; } - @Nullable - @JsonProperty - @ThriftField(4) public DateTime getLastEndTime() { return new DateTime(lastEndTimeInMillis); } + @JsonProperty + @ThriftField(4) public long getLastEndTimeInMillis() { return lastEndTimeInMillis; } - @Nullable - @JsonProperty - @ThriftField(5) public DateTime getEndTime() { return new DateTime(endTimeInMillis); } + @JsonProperty + @ThriftField(5) public long getEndTimeInMillis() { return endTimeInMillis; @@ -737,11 +622,11 @@ public RuntimeStats getRuntimeStats() public TaskStats summarize() { return new TaskStats( - new DateTime(createTimeInMillis), - new DateTime(firstStartTimeInMillis), - new DateTime(lastStartTimeInMillis), - new DateTime(lastEndTimeInMillis), - new DateTime(endTimeInMillis), + createTimeInMillis, + firstStartTimeInMillis, + lastStartTimeInMillis, + lastEndTimeInMillis, + endTimeInMillis, elapsedTimeInNanos, queuedTimeInNanos, totalDrivers, @@ -783,11 +668,11 @@ public TaskStats summarize() public TaskStats summarizeFinal() { return new TaskStats( - new DateTime(createTimeInMillis), - new DateTime(firstStartTimeInMillis), - new DateTime(lastStartTimeInMillis), - new DateTime(lastEndTimeInMillis), - new DateTime(endTimeInMillis), + createTimeInMillis, + firstStartTimeInMillis, + lastStartTimeInMillis, + lastEndTimeInMillis, + endTimeInMillis, elapsedTimeInNanos, queuedTimeInNanos, totalDrivers, diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp index b4f0a998dc2b8..fdfed953d088c 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp @@ -1068,6 +1068,13 @@ void to_json(json& j, const IcebergSplit& p) { "IcebergSplit", "int64_t", "dataSequenceNumber"); + to_json_key( + j, + "affinitySchedulingSectionSize", + p.affinitySchedulingSectionSize, + "IcebergSplit", + "int64_t", + "affinitySchedulingSectionSize"); } void from_json(const json& j, IcebergSplit& p) { @@ -1140,6 +1147,13 @@ void from_json(const json& j, IcebergSplit& p) { "IcebergSplit", "int64_t", "dataSequenceNumber"); + from_json_key( + j, + "affinitySchedulingSectionSize", + p.affinitySchedulingSectionSize, + "IcebergSplit", + "int64_t", + "affinitySchedulingSectionSize"); } } // namespace facebook::presto::protocol::iceberg namespace facebook::presto::protocol::iceberg { diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h index 1436b54b1de31..3f7a41220b4ec 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h @@ -283,6 +283,7 @@ struct IcebergSplit : public ConnectorSplit { List deletes = {}; std::shared_ptr changelogSplitInfo = {}; int64_t dataSequenceNumber = {}; + int64_t affinitySchedulingSectionSize = {}; IcebergSplit() noexcept; }; diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/special/IcebergSplit.hpp.inc b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/special/IcebergSplit.hpp.inc index 2e3e21ffdb708..471beab8d2805 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/special/IcebergSplit.hpp.inc +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/special/IcebergSplit.hpp.inc @@ -30,6 +30,7 @@ struct IcebergSplit : public ConnectorSplit { List deletes = {}; std::shared_ptr changelogSplitInfo = {}; int64_t dataSequenceNumber = {}; + int64_t affinitySchedulingSectionSize = {}; IcebergSplit() noexcept; }; diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp index 5968a56e60333..79978eda2d982 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp @@ -4338,10 +4338,26 @@ void to_json(json& j, const DriverStats& p) { j = json::object(); to_json_key(j, "lifespan", p.lifespan, "DriverStats", "Lifespan", "lifespan"); to_json_key( - j, "createTimeInMillis", p.createTimeInMillis, "DriverStats", "int64_t", "createTimeInMillis"); + j, + "createTimeInMillis", + p.createTimeInMillis, + "DriverStats", + "int64_t", + "createTimeInMillis"); + to_json_key( + j, + "startTimeInMillis", + p.startTimeInMillis, + "DriverStats", + "int64_t", + "startTimeInMillis"); to_json_key( - j, "startTimeInMillis", p.startTimeInMillis, "DriverStats", "int64_t", "startTimeInMillis"); - to_json_key(j, "endTimeInMillis", p.endTimeInMillis, "DriverStats", "int64_t", "endTimeInMillis"); + j, + "endTimeInMillis", + p.endTimeInMillis, + "DriverStats", + "int64_t", + "endTimeInMillis"); to_json_key( j, "queuedTime", p.queuedTime, "DriverStats", "Duration", "queuedTime"); to_json_key( @@ -4478,10 +4494,26 @@ void from_json(const json& j, DriverStats& p) { from_json_key( j, "lifespan", p.lifespan, "DriverStats", "Lifespan", "lifespan"); from_json_key( - j, "createTimeInMillis", p.createTimeInMillis, "DriverStats", "int64_t", "createTimeInMillis"); + j, + "createTimeInMillis", + p.createTimeInMillis, + "DriverStats", + "int64_t", + "createTimeInMillis"); from_json_key( - j, "startTimeInMillis", p.startTimeInMillis, "DriverStats", "int64_t", "startTimeInMillis"); - from_json_key(j, "endTimeInMillis", p.endTimeInMillis, "DriverStats", "int64_t", "endTimeInMillis"); + j, + "startTimeInMillis", + p.startTimeInMillis, + "DriverStats", + "int64_t", + "startTimeInMillis"); + from_json_key( + j, + "endTimeInMillis", + p.endTimeInMillis, + "DriverStats", + "int64_t", + "endTimeInMillis"); from_json_key( j, "queuedTime", p.queuedTime, "DriverStats", "Duration", "queuedTime"); from_json_key( @@ -10014,7 +10046,12 @@ namespace facebook::presto::protocol { void to_json(json& j, const TaskStats& p) { j = json::object(); to_json_key( - j, "createTimeInMillis", p.createTimeInMillis, "TaskStats", "int64_t", "createTimeInMillis"); + j, + "createTimeInMillis", + p.createTimeInMillis, + "TaskStats", + "int64_t", + "createTimeInMillis"); to_json_key( j, "firstStartTimeInMillis", @@ -10030,8 +10067,19 @@ void to_json(json& j, const TaskStats& p) { "int64_t", "lastStartTimeInMillis"); to_json_key( - j, "lastEndTimeInMillis", p.lastEndTimeInMillis, "TaskStats", "int64_t", "lastEndTimeInMillis"); - to_json_key(j, "endTimeInMillis", p.endTimeInMillis, "TaskStats", "int64_t", "endTimeInMillis"); + j, + "lastEndTimeInMillis", + p.lastEndTimeInMillis, + "TaskStats", + "int64_t", + "lastEndTimeInMillis"); + to_json_key( + j, + "endTimeInMillis", + p.endTimeInMillis, + "TaskStats", + "int64_t", + "endTimeInMillis"); to_json_key( j, "elapsedTimeInNanos", @@ -10268,7 +10316,12 @@ void to_json(json& j, const TaskStats& p) { void from_json(const json& j, TaskStats& p) { from_json_key( - j, "createTimeInMillis", p.createTimeInMillis, "TaskStats", "int64_t", "createTimeInMillis"); + j, + "createTimeInMillis", + p.createTimeInMillis, + "TaskStats", + "int64_t", + "createTimeInMillis"); from_json_key( j, "firstStartTimeInMillis", @@ -10284,8 +10337,19 @@ void from_json(const json& j, TaskStats& p) { "int64_t", "lastStartTimeInMillis"); from_json_key( - j, "lastEndTimeInMillis", p.lastEndTimeInMillis, "TaskStats", "int64_t", "lastEndTimeInMillis"); - from_json_key(j, "endTimeInMillis", p.endTimeInMillis, "TaskStats", "int64_t", "endTimeInMillis"); + j, + "lastEndTimeInMillis", + p.lastEndTimeInMillis, + "TaskStats", + "int64_t", + "lastEndTimeInMillis"); + from_json_key( + j, + "endTimeInMillis", + p.endTimeInMillis, + "TaskStats", + "int64_t", + "endTimeInMillis"); from_json_key( j, "elapsedTimeInNanos", diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h index d77c27a232733..00ec0a569cc2d 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h @@ -104,6 +104,7 @@ using Subfield = std::string; using HiveType = std::string; using Type = std::string; +using DateTime = std::string; using Locale = std::string; using TimeZoneKey = long; using URI = std::string; @@ -1191,9 +1192,9 @@ void from_json(const json& j, OperatorStats& p); namespace facebook::presto::protocol { struct DriverStats { Lifespan lifespan = {}; - long createTimeInMillis = {}; - long startTimeInMillis = {}; - long endTimeInMillis = {}; + int64_t createTimeInMillis = {}; + int64_t startTimeInMillis = {}; + int64_t endTimeInMillis = {}; Duration queuedTime = {}; Duration elapsedTime = {}; int64_t userMemoryReservationInBytes = {};