Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions be/src/runtime/primitive_type.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
#include "olap/decimal12.h"
#include "olap/uint24.h"
#include "runtime/define_primitive_type.h"
#include "vec/common/string_view.h"
#include "vec/common/string_container.h"
#include "vec/core/field.h"
#include "vec/core/types.h"
#include "vec/runtime/vdatetime_value.h"
Expand Down Expand Up @@ -634,15 +634,15 @@ struct PrimitiveTypeTraits<TYPE_STRING> {
};
template <>
struct PrimitiveTypeTraits<TYPE_VARBINARY> {
using CppType = doris::StringView;
using CppType = doris::StringContainer;
using StorageFieldType = CppType;
using CppNativeType = doris::StringView;
using ColumnItemType = doris::StringView;
using CppNativeType = doris::StringContainer;
using ColumnItemType = doris::StringContainer;
using DataType = vectorized::DataTypeVarbinary;
using ColumnType = vectorized::ColumnVarbinary;
using NearestFieldType = doris::StringView;
using AvgNearestFieldType = doris::StringView;
using AvgNearestFieldType256 = doris::StringView;
using NearestFieldType = doris::StringContainer;
using AvgNearestFieldType = doris::StringContainer;
using AvgNearestFieldType256 = doris::StringContainer;
static constexpr PrimitiveType NearestPrimitiveType = TYPE_VARBINARY;
static constexpr PrimitiveType AvgNearestPrimitiveType = TYPE_VARBINARY;
static constexpr PrimitiveType AvgNearestPrimitiveType256 = TYPE_VARBINARY;
Expand Down
6 changes: 3 additions & 3 deletions be/src/vec/columns/column_varbinary.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ size_t ColumnVarbinary::filter(const IColumn::Filter& filter) {
} else {
auto val = src_vec.get_data()[i];
const auto* dst = _arena.insert(val.data(), val.size());
_data[pos] = doris::StringView(dst, val.size());
_data[pos] = doris::StringContainer(dst, val.size());
}
pos++;
}
Expand Down Expand Up @@ -144,7 +144,7 @@ MutableColumnPtr ColumnVarbinary::permute(const IColumn::Permutation& perm, size
continue;
}
const auto* dst = const_cast<Arena&>(_arena).insert(val.data(), val.size());
res_data[i] = doris::StringView(dst, val.size());
res_data[i] = doris::StringContainer(dst, val.size());
}

return res;
Expand All @@ -159,7 +159,7 @@ void ColumnVarbinary::replace_column_data(const IColumn& rhs, size_t row, size_t
return;
}
const auto* dst = _arena.insert(val.data(), val.size());
_data[self_row] = doris::StringView(dst, val.size());
_data[self_row] = doris::StringContainer(dst, val.size());
}

#include "common/compile_check_end.h"
Expand Down
18 changes: 9 additions & 9 deletions be/src/vec/columns/column_varbinary.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
#include "vec/columns/column.h"
#include "vec/common/arena.h"
#include "vec/common/assert_cast.h"
#include "vec/common/string_view.h"
#include "vec/common/string_container.h"

namespace doris::vectorized {
#include "common/compile_check_begin.h"
Expand All @@ -36,7 +36,7 @@ class ColumnVarbinary final : public COWHelper<IColumn, ColumnVarbinary> {
friend class COWHelper<IColumn, ColumnVarbinary>;

public:
using Container = PaddedPODArray<doris::StringView>;
using Container = PaddedPODArray<doris::StringContainer>;
ColumnVarbinary() = default;
ColumnVarbinary(const size_t n) : _data(n) {}

Expand Down Expand Up @@ -73,7 +73,7 @@ class ColumnVarbinary final : public COWHelper<IColumn, ColumnVarbinary> {
char* alloc(size_t length) { return _arena.alloc(length); }

void insert(const Field& x) override {
auto value = vectorized::get<const doris::StringView&>(x);
auto value = vectorized::get<const doris::StringContainer&>(x);
insert_data(value.data(), value.size());
}

Expand All @@ -84,24 +84,24 @@ class ColumnVarbinary final : public COWHelper<IColumn, ColumnVarbinary> {
}

void insert_data(const char* pos, size_t length) override {
if (length <= doris::StringView::kInlineSize) {
if (length <= doris::StringContainer::kInlineSize) {
insert_inline_data(pos, length);
} else {
insert_to_buffer(pos, length);
}
}

void insert_inline_data(const char* pos, size_t length) {
DCHECK(length <= doris::StringView::kInlineSize);
_data.push_back(doris::StringView(pos, cast_set<uint32_t>(length)));
DCHECK(length <= doris::StringContainer::kInlineSize);
_data.push_back(doris::StringContainer(pos, cast_set<uint32_t>(length)));
}

void insert_to_buffer(const char* pos, size_t length) {
const char* dst = _arena.insert(pos, length);
_data.push_back(doris::StringView(dst, cast_set<uint32_t>(length)));
_data.push_back(doris::StringContainer(dst, cast_set<uint32_t>(length)));
}

void insert_default() override { _data.push_back(doris::StringView()); }
void insert_default() override { _data.push_back(doris::StringContainer()); }

int compare_at(size_t n, size_t m, const IColumn& rhs_,
int /*nan_direction_hint*/) const override {
Expand Down Expand Up @@ -131,7 +131,7 @@ class ColumnVarbinary final : public COWHelper<IColumn, ColumnVarbinary> {
size_t allocated_bytes() const override { return _data.allocated_bytes() + _arena.size(); }

size_t byte_size() const override {
size_t bytes = _data.size() * sizeof(doris::StringView);
size_t bytes = _data.size() * sizeof(doris::StringContainer);
return bytes + _arena.used_size();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@
// specific language governing permissions and limitations
// under the License.

#include "vec/common/string_view.h"
#include "vec/common/string_container.h"

namespace doris {

bool StringView::operator==(const StringView& other) const {
bool StringContainer::operator==(const StringContainer& other) const {
// Compare lengths and first 4 characters.
if (size_and_prefix_as_int64() != other.size_and_prefix_as_int64()) {
return false;
Expand All @@ -34,7 +34,7 @@ bool StringView::operator==(const StringView& other) const {
return memcmp(value_.data + kPrefixSize, other.value_.data + kPrefixSize,
size_ - kPrefixSize) == 0;
}
int32_t StringView::compare(const StringView& other) const {
int32_t StringContainer::compare(const StringContainer& other) const {
if (prefix_as_int() != other.prefix_as_int()) {
// The result is decided on prefix. The shorter will be less because the
// prefix is padded with zeros.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,22 +35,22 @@ namespace doris {
// exposes a subset of the interface. If the string is 12 characters
// or less, it is inlined and no reference is held. If it is longer, a
// reference to the string is held and the 4 first characters are
// cached in the StringView. This allows failing comparisons early and
// cached in the StringContainer. This allows failing comparisons early and
// reduces the CPU cache working set when dealing with short strings.

class StringView {
class StringContainer {
#include "common/compile_check_begin.h"
public:
using value_type = char;
static constexpr size_t kPrefixSize = 4 * sizeof(char);
static constexpr size_t kInlineSize = 12;

StringView() {
static_assert(sizeof(StringView) == 16);
memset(this, 0, sizeof(StringView));
StringContainer() {
static_assert(sizeof(StringContainer) == 16);
memset(this, 0, sizeof(StringContainer));
}

StringView(const char* data, uint32_t len) : size_(len) {
StringContainer(const char* data, uint32_t len) : size_(len) {
DCHECK_GE(len, 0);
DCHECK(data || len == 0);
if (isInline()) {
Expand All @@ -72,20 +72,20 @@ class StringView {
}
}

StringView(unsigned char* data, uint32_t len)
: StringView(reinterpret_cast<const char*>(data), len) {}
StringContainer(unsigned char* data, uint32_t len)
: StringContainer(reinterpret_cast<const char*>(data), len) {}

bool isInline() const { return isInline(size_); }

ALWAYS_INLINE static constexpr bool isInline(uint32_t size) { return size <= kInlineSize; }

explicit StringView(std::string&& value) = delete;
explicit StringView(const std::string& value)
: StringView(value.data(), cast_set<uint32_t>(value.size())) {}
explicit StringView(std::string_view value)
: StringView(value.data(), cast_set<uint32_t>(value.size())) {}
/* implicit */ StringView(const char* data)
: StringView(data, cast_set<uint32_t>(strlen(data))) {}
explicit StringContainer(std::string&& value) = delete;
explicit StringContainer(const std::string& value)
: StringContainer(value.data(), cast_set<uint32_t>(value.size())) {}
explicit StringContainer(std::string_view value)
: StringContainer(value.data(), cast_set<uint32_t>(value.size())) {}
/* implicit */ StringContainer(const char* data)
: StringContainer(data, cast_set<uint32_t>(strlen(data))) {}
doris::StringRef to_string_ref() const { return {data(), size()}; }

operator std::string_view() && = delete;
Expand All @@ -101,12 +101,12 @@ class StringView {

void set_size(uint32_t size) { size_ = size; }

bool operator==(const StringView& other) const;
friend std::ostream& operator<<(std::ostream& os, const StringView& stringView) {
os.write(stringView.data(), stringView.size());
bool operator==(const StringContainer& other) const;
friend std::ostream& operator<<(std::ostream& os, const StringContainer& StringContainer) {
os.write(StringContainer.data(), StringContainer.size());
return os;
}
auto operator<=>(const StringView& other) const {
auto operator<=>(const StringContainer& other) const {
const auto cmp = compare(other);
return cmp < 0 ? std::strong_ordering::less
: cmp > 0 ? std::strong_ordering::greater
Expand All @@ -116,7 +116,7 @@ class StringView {
// Returns 0, if this == other
// < 0, if this < other
// > 0, if this > other
int32_t compare(const StringView& other) const;
int32_t compare(const StringContainer& other) const;

const char* begin() && = delete;
const char* begin() const& { return data(); }
Expand Down
17 changes: 9 additions & 8 deletions be/src/vec/core/field.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
#include "olap/hll.h"
#include "util/bitmap_value.h"
#include "util/quantile_state.h"
#include "vec/common/string_view.h"
#include "vec/common/string_container.h"
#include "vec/common/uint128.h"
#include "vec/core/types.h"
#include "vec/json/path_in_data.h"
Expand Down Expand Up @@ -388,7 +388,7 @@ class Field {
case PrimitiveType::TYPE_VARCHAR:
return get<String>() <=> rhs.get<String>();
case PrimitiveType::TYPE_VARBINARY:
return get<doris::StringView>() <=> rhs.get<doris::StringView>();
return get<doris::StringContainer>() <=> rhs.get<doris::StringContainer>();
case PrimitiveType::TYPE_DECIMAL32:
return get<Decimal32>() <=> rhs.get<Decimal32>();
case PrimitiveType::TYPE_DECIMAL64:
Expand Down Expand Up @@ -436,7 +436,7 @@ class Field {
f(field.template get<String>());
return;
case PrimitiveType::TYPE_VARBINARY:
f(field.template get<doris::StringView>());
f(field.template get<doris::StringContainer>());
return;
case PrimitiveType::TYPE_JSONB:
f(field.template get<JsonbField>());
Expand Down Expand Up @@ -486,11 +486,12 @@ class Field {
std::string_view as_string_view() const;

private:
std::aligned_union_t<
DBMS_MIN_FIELD_SIZE - sizeof(PrimitiveType), Null, UInt64, UInt128, Int64, Int128, IPv6,
Float64, String, JsonbField, Array, Tuple, Map, VariantMap, DecimalField<Decimal32>,
DecimalField<Decimal64>, DecimalField<Decimal128V2>, DecimalField<Decimal128V3>,
DecimalField<Decimal256>, BitmapValue, HyperLogLog, QuantileState, doris::StringView>
std::aligned_union_t<DBMS_MIN_FIELD_SIZE - sizeof(PrimitiveType), Null, UInt64, UInt128, Int64,
Int128, IPv6, Float64, String, JsonbField, Array, Tuple, Map, VariantMap,
DecimalField<Decimal32>, DecimalField<Decimal64>,
DecimalField<Decimal128V2>, DecimalField<Decimal128V3>,
DecimalField<Decimal256>, BitmapValue, HyperLogLog, QuantileState,
doris::StringContainer>
storage;

PrimitiveType type;
Expand Down
6 changes: 3 additions & 3 deletions be/src/vec/data_types/data_type_varbinary.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,16 @@
#include "vec/columns/column_varbinary.h"
#include "vec/common/assert_cast.h"
#include "vec/common/string_buffer.hpp"
#include "vec/common/string_container.h"
#include "vec/common/string_ref.h"
#include "vec/common/string_view.h"
#include "vec/core/field.h"
#include "vec/core/types.h"

namespace doris::vectorized {
#include "common/compile_check_begin.h"

Field DataTypeVarbinary::get_default() const {
return Field::create_field<TYPE_VARBINARY>(StringView());
return Field::create_field<TYPE_VARBINARY>(StringContainer());
}

MutableColumnPtr DataTypeVarbinary::create_column() const {
Expand Down Expand Up @@ -132,7 +132,7 @@ FieldWithDataType DataTypeVarbinary::get_field_with_data_type(const IColumn& col
const auto& column_data =
assert_cast<const ColumnVarbinary&, TypeCheckOnRelease::DISABLE>(column);
return FieldWithDataType {.field = Field::create_field<TYPE_VARBINARY>(
doris::StringView(column_data.get_data_at(row_num))),
doris::StringContainer(column_data.get_data_at(row_num))),
.base_scalar_type_id = get_primitive_type()};
}

Expand Down
7 changes: 4 additions & 3 deletions be/src/vec/data_types/data_type_varbinary.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
#include "runtime/define_primitive_type.h"
#include "runtime/primitive_type.h"
#include "serde/data_type_string_serde.h"
#include "vec/common/string_view.h"
#include "vec/common/string_container.h"
#include "vec/core/field.h"
#include "vec/data_types/data_type.h"
#include "vec/data_types/serde/data_type_serde.h"
Expand All @@ -40,7 +40,7 @@ class IColumn;
class DataTypeVarbinary : public IDataType {
public:
using ColumnType = ColumnVarbinary;
using FieldType = doris::StringView;
using FieldType = doris::StringContainer;

static constexpr PrimitiveType PType = TYPE_VARBINARY;

Expand Down Expand Up @@ -72,7 +72,8 @@ class DataTypeVarbinary : public IDataType {
Field get_field(const TExprNode& node) const override {
DCHECK_EQ(node.node_type, TExprNodeType::VARBINARY_LITERAL);
DCHECK(node.__isset.varbinary_literal);
return Field::create_field<TYPE_VARBINARY>(doris::StringView(node.varbinary_literal.value));
return Field::create_field<TYPE_VARBINARY>(
doris::StringContainer(node.varbinary_literal.value));
}

FieldWithDataType get_field_with_data_type(const IColumn& column,
Expand Down
8 changes: 4 additions & 4 deletions be/src/vec/functions/function_varbinary.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ class FunctionToBinary : public IFunction {
auto col_res = ColumnVarbinary::create();
const auto& data = col->get_chars();
const auto& offsets = col->get_offsets();
col_res->get_data().assign(input_rows_count, StringView());
col_res->get_data().assign(input_rows_count, StringContainer());

for (int i = 0; i < input_rows_count; ++i) {
const auto* source = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
Expand Down Expand Up @@ -155,7 +155,7 @@ struct VarbinaryLengthImpl {
return {std::make_shared<DataTypeVarbinary>()};
}

static Status vector(const PaddedPODArray<doris::StringView>& data,
static Status vector(const PaddedPODArray<doris::StringContainer>& data,
PaddedPODArray<Int32>& res) {
size_t rows_count = data.size();
res.resize(rows_count);
Expand All @@ -174,7 +174,7 @@ struct ToBase64BinaryImpl {
using ColumnType = ColumnString;
static constexpr auto PrimitiveTypeImpl = PrimitiveType::TYPE_VARBINARY;

static Status vector(const PaddedPODArray<doris::StringView>& data,
static Status vector(const PaddedPODArray<doris::StringContainer>& data,
ColumnString::Chars& dst_data, ColumnString::Offsets& dst_offsets) {
auto rows_count = data.size();
dst_offsets.resize(rows_count);
Expand Down Expand Up @@ -221,7 +221,7 @@ struct FromBase64BinaryImpl {
static Status vector(const ColumnString::Chars& data, const ColumnString::Offsets& offsets,
ColumnVarbinary* res, NullMap& null_map) {
auto rows_count = offsets.size();
res->get_data().assign(rows_count, StringView());
res->get_data().assign(rows_count, StringContainer());

for (size_t i = 0; i < rows_count; i++) {
const auto* source = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
Expand Down
8 changes: 4 additions & 4 deletions be/src/vec/utils/varbinaryop_subbinary.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,17 @@ namespace doris::vectorized {
constexpr auto SIZE_OF_UINT = sizeof(uint32_t);

struct VarBinaryOP {
static void check_and_insert_data(doris::StringView& sView, const char* data, uint32_t len,
static void check_and_insert_data(doris::StringContainer& sView, const char* data, uint32_t len,
bool before_is_inline) {
if (before_is_inline) {
sView.set_size(len);
} else {
sView = doris::StringView(data, len);
sView = doris::StringContainer(data, len);
}
}

static std::pair<bool, char*> alloc(ColumnVarbinary* res_col, size_t index, uint32_t len) {
bool is_inline = StringView::isInline(len);
bool is_inline = StringContainer::isInline(len);
char* dst = nullptr;
if (is_inline) {
dst = reinterpret_cast<char*>(&(res_col->get_data()[index])) + SIZE_OF_UINT;
Expand Down Expand Up @@ -90,7 +90,7 @@ struct SubBinaryUtil {
res->get_data().reserve(size);

for (size_t i = 0; i < size; ++i) {
doris::StringView binary = binarys->get_data()[index_check_const<binary_const>(i)];
doris::StringContainer binary = binarys->get_data()[index_check_const<binary_const>(i)];
int binary_size = static_cast<int>(binary.size());

int start_value = start->get_data()[index_check_const<start_const>(i)];
Expand Down
Loading
Loading