diff --git a/tensorflow/compiler/mlir/lite/schema/schema.fbs b/tensorflow/compiler/mlir/lite/schema/schema.fbs index 074b924af6b..d74477be913 100644 --- a/tensorflow/compiler/mlir/lite/schema/schema.fbs +++ b/tensorflow/compiler/mlir/lite/schema/schema.fbs @@ -70,6 +70,19 @@ table CustomQuantization { // Represents a specific quantization technique's parameters. union QuantizationDetails { CustomQuantization, + BlockwiseQuantization, +} + +// Parameters for blockwise quantization. +table BlockwiseQuantization { + // index to the scale tensor, the tensor can be found in tensors array in + // subgraph. + scales: int; + // index to the zero point tensor. If zero_points is -1, the zero point is + // assumed to be 0, following the convention of optional tensors in tflite. + zero_points: int; + // The block size of the tensor. + block_size: int; } // Parameters for converting a quantized tensor back to float. diff --git a/tensorflow/lite/core/c/c_api_types.h b/tensorflow/lite/core/c/c_api_types.h index 75112bdedc8..b8f3b47f4dd 100644 --- a/tensorflow/lite/core/c/c_api_types.h +++ b/tensorflow/lite/core/c/c_api_types.h @@ -56,7 +56,7 @@ extern "C" { #define TFL_CAPI_EXPORT #elif defined(TFL_STATIC_LIBRARY_BUILD) #define TFL_CAPI_EXPORT -#else // not definded TFL_STATIC_LIBRARY_BUILD +#else // not defined TFL_STATIC_LIBRARY_BUILD #if defined(_WIN32) #ifdef TFL_COMPILE_LIBRARY #define TFL_CAPI_EXPORT __declspec(dllexport) diff --git a/tensorflow/lite/core/c/common.h b/tensorflow/lite/core/c/common.h index 70fd2a9be09..1131adb669a 100644 --- a/tensorflow/lite/core/c/common.h +++ b/tensorflow/lite/core/c/common.h @@ -80,7 +80,9 @@ typedef enum TfLiteExternalContextType { kTfLiteGemmLowpContext = 1, /// include gemm_support.h to use. kTfLiteEdgeTpuContext = 2, /// Placeholder for Edge TPU support. kTfLiteCpuBackendContext = 3, /// include cpu_backend_context.h to use. - kTfLiteMaxExternalContexts = 4 + kTfLiteLiteRtBufferContext = + 4, /// include external_litert_buffer_context.h to use. + kTfLiteMaxExternalContexts = 5 } TfLiteExternalContextType; // Forward declare so dependent structs and methods can reference these types diff --git a/tensorflow/lite/kernels/internal/reference/comparisons.h b/tensorflow/lite/kernels/internal/reference/comparisons.h index a9f1e42c0a6..e40e4045cc7 100644 --- a/tensorflow/lite/kernels/internal/reference/comparisons.h +++ b/tensorflow/lite/kernels/internal/reference/comparisons.h @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_COMPARISONS_H_ #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_COMPARISONS_H_ +#include + #include "tensorflow/lite/core/c/common.h" #include "tensorflow/lite/core/macros.h" #include "tensorflow/lite/kernels/internal/common.h" diff --git a/tensorflow/lite/python/schema_py_generated.py b/tensorflow/lite/python/schema_py_generated.py index 5e22506214f..5fb12737d43 100755 --- a/tensorflow/lite/python/schema_py_generated.py +++ b/tensorflow/lite/python/schema_py_generated.py @@ -32,6 +32,7 @@ class TensorType(object): class QuantizationDetails(object): NONE = 0 CustomQuantization = 1 + BlockwiseQuantization = 2 def QuantizationDetailsCreator(unionType, table): from flatbuffers.table import Table @@ -39,6 +40,8 @@ def QuantizationDetailsCreator(unionType, table): return None if unionType == QuantizationDetails().CustomQuantization: return CustomQuantizationT.InitFromBuf(table.Bytes, table.Pos) + if unionType == QuantizationDetails().BlockwiseQuantization: + return BlockwiseQuantizationT.InitFromBuf(table.Bytes, table.Pos) return None @@ -949,6 +952,109 @@ def Pack(self, builder): return customQuantization +class BlockwiseQuantization(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAs(cls, buf, offset=0): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = BlockwiseQuantization() + x.Init(buf, n + offset) + return x + + @classmethod + def GetRootAsBlockwiseQuantization(cls, buf, offset=0): + """This method is deprecated. Please switch to GetRootAs.""" + return cls.GetRootAs(buf, offset) + @classmethod + def BlockwiseQuantizationBufferHasIdentifier(cls, buf, offset, size_prefixed=False): + return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x54\x46\x4C\x33", size_prefixed=size_prefixed) + + # BlockwiseQuantization + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # BlockwiseQuantization + def Scales(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Int32Flags, o + self._tab.Pos) + return 0 + + # BlockwiseQuantization + def ZeroPoints(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Int32Flags, o + self._tab.Pos) + return 0 + + # BlockwiseQuantization + def BlockSize(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Int32Flags, o + self._tab.Pos) + return 0 + +def BlockwiseQuantizationStart(builder): + builder.StartObject(3) + +def BlockwiseQuantizationAddScales(builder, scales): + builder.PrependInt32Slot(0, scales, 0) + +def BlockwiseQuantizationAddZeroPoints(builder, zeroPoints): + builder.PrependInt32Slot(1, zeroPoints, 0) + +def BlockwiseQuantizationAddBlockSize(builder, blockSize): + builder.PrependInt32Slot(2, blockSize, 0) + +def BlockwiseQuantizationEnd(builder): + return builder.EndObject() + + + +class BlockwiseQuantizationT(object): + + # BlockwiseQuantizationT + def __init__(self): + self.scales = 0 # type: int + self.zeroPoints = 0 # type: int + self.blockSize = 0 # type: int + + @classmethod + def InitFromBuf(cls, buf, pos): + blockwiseQuantization = BlockwiseQuantization() + blockwiseQuantization.Init(buf, pos) + return cls.InitFromObj(blockwiseQuantization) + + @classmethod + def InitFromPackedBuf(cls, buf, pos=0): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, pos) + return cls.InitFromBuf(buf, pos+n) + + @classmethod + def InitFromObj(cls, blockwiseQuantization): + x = BlockwiseQuantizationT() + x._UnPack(blockwiseQuantization) + return x + + # BlockwiseQuantizationT + def _UnPack(self, blockwiseQuantization): + if blockwiseQuantization is None: + return + self.scales = blockwiseQuantization.Scales() + self.zeroPoints = blockwiseQuantization.ZeroPoints() + self.blockSize = blockwiseQuantization.BlockSize() + + # BlockwiseQuantizationT + def Pack(self, builder): + BlockwiseQuantizationStart(builder) + BlockwiseQuantizationAddScales(builder, self.scales) + BlockwiseQuantizationAddZeroPoints(builder, self.zeroPoints) + BlockwiseQuantizationAddBlockSize(builder, self.blockSize) + blockwiseQuantization = BlockwiseQuantizationEnd(builder) + return blockwiseQuantization + + class QuantizationParameters(object): __slots__ = ['_tab'] @@ -1157,7 +1263,7 @@ def __init__(self): self.scale = None # type: List[float] self.zeroPoint = None # type: List[int] self.detailsType = 0 # type: int - self.details = None # type: Union[None, CustomQuantizationT] + self.details = None # type: Union[None, CustomQuantizationT, BlockwiseQuantizationT] self.quantizedDimension = 0 # type: int @classmethod diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h index f35845595af..1f055d2045f 100755 --- a/tensorflow/lite/schema/schema_generated.h +++ b/tensorflow/lite/schema/schema_generated.h @@ -19,6 +19,10 @@ struct CustomQuantization; struct CustomQuantizationBuilder; struct CustomQuantizationT; +struct BlockwiseQuantization; +struct BlockwiseQuantizationBuilder; +struct BlockwiseQuantizationT; + struct QuantizationParameters; struct QuantizationParametersBuilder; struct QuantizationParametersT; @@ -763,29 +767,32 @@ inline const char *EnumNameTensorType(TensorType e) { enum QuantizationDetails : uint8_t { QuantizationDetails_NONE = 0, QuantizationDetails_CustomQuantization = 1, + QuantizationDetails_BlockwiseQuantization = 2, QuantizationDetails_MIN = QuantizationDetails_NONE, - QuantizationDetails_MAX = QuantizationDetails_CustomQuantization + QuantizationDetails_MAX = QuantizationDetails_BlockwiseQuantization }; -inline const QuantizationDetails (&EnumValuesQuantizationDetails())[2] { +inline const QuantizationDetails (&EnumValuesQuantizationDetails())[3] { static const QuantizationDetails values[] = { QuantizationDetails_NONE, - QuantizationDetails_CustomQuantization + QuantizationDetails_CustomQuantization, + QuantizationDetails_BlockwiseQuantization }; return values; } inline const char * const *EnumNamesQuantizationDetails() { - static const char * const names[3] = { + static const char * const names[4] = { "NONE", "CustomQuantization", + "BlockwiseQuantization", nullptr }; return names; } inline const char *EnumNameQuantizationDetails(QuantizationDetails e) { - if (::flatbuffers::IsOutRange(e, QuantizationDetails_NONE, QuantizationDetails_CustomQuantization)) return ""; + if (::flatbuffers::IsOutRange(e, QuantizationDetails_NONE, QuantizationDetails_BlockwiseQuantization)) return ""; const size_t index = static_cast(e); return EnumNamesQuantizationDetails()[index]; } @@ -798,6 +805,10 @@ template<> struct QuantizationDetailsTraits { static const QuantizationDetails enum_value = QuantizationDetails_CustomQuantization; }; +template<> struct QuantizationDetailsTraits { + static const QuantizationDetails enum_value = QuantizationDetails_BlockwiseQuantization; +}; + template struct QuantizationDetailsUnionTraits { static const QuantizationDetails enum_value = QuantizationDetails_NONE; }; @@ -806,6 +817,10 @@ template<> struct QuantizationDetailsUnionTraits { static const QuantizationDetails enum_value = QuantizationDetails_CustomQuantization; }; +template<> struct QuantizationDetailsUnionTraits { + static const QuantizationDetails enum_value = QuantizationDetails_BlockwiseQuantization; +}; + struct QuantizationDetailsUnion { QuantizationDetails type; void *value; @@ -844,6 +859,14 @@ struct QuantizationDetailsUnion { return type == QuantizationDetails_CustomQuantization ? reinterpret_cast(value) : nullptr; } + tflite::BlockwiseQuantizationT *AsBlockwiseQuantization() { + return type == QuantizationDetails_BlockwiseQuantization ? + reinterpret_cast(value) : nullptr; + } + const tflite::BlockwiseQuantizationT *AsBlockwiseQuantization() const { + return type == QuantizationDetails_BlockwiseQuantization ? + reinterpret_cast(value) : nullptr; + } }; bool VerifyQuantizationDetails(::flatbuffers::Verifier &verifier, const void *obj, QuantizationDetails type); @@ -5141,6 +5164,80 @@ inline ::flatbuffers::Offset CreateCustomQuantizationDirect( ::flatbuffers::Offset CreateCustomQuantization(::flatbuffers::FlatBufferBuilder &_fbb, const CustomQuantizationT *_o, const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); +struct BlockwiseQuantizationT : public ::flatbuffers::NativeTable { + typedef BlockwiseQuantization TableType; + int32_t scales = 0; + int32_t zero_points = 0; + int32_t block_size = 0; +}; + +struct BlockwiseQuantization FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { + typedef BlockwiseQuantizationT NativeTableType; + typedef BlockwiseQuantizationBuilder Builder; + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_SCALES = 4, + VT_ZERO_POINTS = 6, + VT_BLOCK_SIZE = 8 + }; + int32_t scales() const { + return GetField(VT_SCALES, 0); + } + int32_t zero_points() const { + return GetField(VT_ZERO_POINTS, 0); + } + int32_t block_size() const { + return GetField(VT_BLOCK_SIZE, 0); + } + bool Verify(::flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyField(verifier, VT_SCALES, 4) && + VerifyField(verifier, VT_ZERO_POINTS, 4) && + VerifyField(verifier, VT_BLOCK_SIZE, 4) && + verifier.EndTable(); + } + BlockwiseQuantizationT *UnPack(const ::flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(BlockwiseQuantizationT *_o, const ::flatbuffers::resolver_function_t *_resolver = nullptr) const; + static ::flatbuffers::Offset Pack(::flatbuffers::FlatBufferBuilder &_fbb, const BlockwiseQuantizationT* _o, const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct BlockwiseQuantizationBuilder { + typedef BlockwiseQuantization Table; + ::flatbuffers::FlatBufferBuilder &fbb_; + ::flatbuffers::uoffset_t start_; + void add_scales(int32_t scales) { + fbb_.AddElement(BlockwiseQuantization::VT_SCALES, scales, 0); + } + void add_zero_points(int32_t zero_points) { + fbb_.AddElement(BlockwiseQuantization::VT_ZERO_POINTS, zero_points, 0); + } + void add_block_size(int32_t block_size) { + fbb_.AddElement(BlockwiseQuantization::VT_BLOCK_SIZE, block_size, 0); + } + explicit BlockwiseQuantizationBuilder(::flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + ::flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = ::flatbuffers::Offset(end); + return o; + } +}; + +inline ::flatbuffers::Offset CreateBlockwiseQuantization( + ::flatbuffers::FlatBufferBuilder &_fbb, + int32_t scales = 0, + int32_t zero_points = 0, + int32_t block_size = 0) { + BlockwiseQuantizationBuilder builder_(_fbb); + builder_.add_block_size(block_size); + builder_.add_zero_points(zero_points); + builder_.add_scales(scales); + return builder_.Finish(); +} + +::flatbuffers::Offset CreateBlockwiseQuantization(::flatbuffers::FlatBufferBuilder &_fbb, const BlockwiseQuantizationT *_o, const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); + struct QuantizationParametersT : public ::flatbuffers::NativeTable { typedef QuantizationParameters TableType; std::vector min{}; @@ -5185,6 +5282,9 @@ struct QuantizationParameters FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::T const tflite::CustomQuantization *details_as_CustomQuantization() const { return details_type() == tflite::QuantizationDetails_CustomQuantization ? static_cast(details()) : nullptr; } + const tflite::BlockwiseQuantization *details_as_BlockwiseQuantization() const { + return details_type() == tflite::QuantizationDetails_BlockwiseQuantization ? static_cast(details()) : nullptr; + } int32_t quantized_dimension() const { return GetField(VT_QUANTIZED_DIMENSION, 0); } @@ -5213,6 +5313,10 @@ template<> inline const tflite::CustomQuantization *QuantizationParameters::deta return details_as_CustomQuantization(); } +template<> inline const tflite::BlockwiseQuantization *QuantizationParameters::details_as() const { + return details_as_BlockwiseQuantization(); +} + struct QuantizationParametersBuilder { typedef QuantizationParameters Table; ::flatbuffers::FlatBufferBuilder &fbb_; @@ -16872,6 +16976,38 @@ inline ::flatbuffers::Offset CreateCustomQuantization(::flat _custom); } +inline BlockwiseQuantizationT *BlockwiseQuantization::UnPack(const ::flatbuffers::resolver_function_t *_resolver) const { + auto _o = std::unique_ptr(new BlockwiseQuantizationT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); +} + +inline void BlockwiseQuantization::UnPackTo(BlockwiseQuantizationT *_o, const ::flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; + { auto _e = scales(); _o->scales = _e; } + { auto _e = zero_points(); _o->zero_points = _e; } + { auto _e = block_size(); _o->block_size = _e; } +} + +inline ::flatbuffers::Offset BlockwiseQuantization::Pack(::flatbuffers::FlatBufferBuilder &_fbb, const BlockwiseQuantizationT* _o, const ::flatbuffers::rehasher_function_t *_rehasher) { + return CreateBlockwiseQuantization(_fbb, _o, _rehasher); +} + +inline ::flatbuffers::Offset CreateBlockwiseQuantization(::flatbuffers::FlatBufferBuilder &_fbb, const BlockwiseQuantizationT *_o, const ::flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { ::flatbuffers::FlatBufferBuilder *__fbb; const BlockwiseQuantizationT* __o; const ::flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + auto _scales = _o->scales; + auto _zero_points = _o->zero_points; + auto _block_size = _o->block_size; + return tflite::CreateBlockwiseQuantization( + _fbb, + _scales, + _zero_points, + _block_size); +} + inline QuantizationParametersT *QuantizationParameters::UnPack(const ::flatbuffers::resolver_function_t *_resolver) const { auto _o = std::unique_ptr(new QuantizationParametersT()); UnPackTo(_o.get(), _resolver); @@ -21681,6 +21817,10 @@ inline bool VerifyQuantizationDetails(::flatbuffers::Verifier &verifier, const v auto ptr = reinterpret_cast(obj); return verifier.VerifyTable(ptr); } + case QuantizationDetails_BlockwiseQuantization: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } default: return true; } } @@ -21704,6 +21844,10 @@ inline void *QuantizationDetailsUnion::UnPack(const void *obj, QuantizationDetai auto ptr = reinterpret_cast(obj); return ptr->UnPack(resolver); } + case QuantizationDetails_BlockwiseQuantization: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } default: return nullptr; } } @@ -21715,6 +21859,10 @@ inline ::flatbuffers::Offset QuantizationDetailsUnion::Pack(::flatbuffers: auto ptr = reinterpret_cast(value); return CreateCustomQuantization(_fbb, ptr, _rehasher).Union(); } + case QuantizationDetails_BlockwiseQuantization: { + auto ptr = reinterpret_cast(value); + return CreateBlockwiseQuantization(_fbb, ptr, _rehasher).Union(); + } default: return 0; } } @@ -21725,6 +21873,10 @@ inline QuantizationDetailsUnion::QuantizationDetailsUnion(const QuantizationDeta value = new tflite::CustomQuantizationT(*reinterpret_cast(u.value)); break; } + case QuantizationDetails_BlockwiseQuantization: { + value = new tflite::BlockwiseQuantizationT(*reinterpret_cast(u.value)); + break; + } default: break; } @@ -21737,6 +21889,11 @@ inline void QuantizationDetailsUnion::Reset() { delete ptr; break; } + case QuantizationDetails_BlockwiseQuantization: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } default: break; } value = nullptr;