cc72042a94
The TOSA-v1.0 specification moves the "zero point" parameters of the convolution operators CONV2D, CONV3D, DEPTHWISE_CONV2D, and TRANSPOSE_CONV2D from attributes to inputs. Make the zero points of the convolutions in the MLIR TOSA dialect inputs and update any transformations, materializations and lit tests appropriately. Rename the "filter" argument of `tosa.transpose_conv2d` to weight to align with the TOSA specification. Remove the quantization_info attribute on the convolution operations. Co-authored-by: TatWai Chong <tatwai.chong@arm.com>
391 lines
13 KiB
C++
391 lines
13 KiB
C++
//===- QuantUtils.cpp -----------------------------------------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file contains TOSA numerical support functions and quantization
|
|
// attribute builders.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "mlir/Dialect/Tosa/Utils/QuantUtils.h"
|
|
|
|
using namespace mlir;
|
|
using namespace mlir::tosa;
|
|
|
|
/// From a scale value, generates multiplier and shift values where
|
|
/// mantissa is in [-1.0,-0.5] or [0.5, 1.0] such that
|
|
/// multiplier = mantissa*2^shift for 16-bit scaling.
|
|
static void computeMultiplierAndShiftTosaScale16(double scale,
|
|
int32_t &multiplier,
|
|
int32_t &shift) {
|
|
|
|
const double mantissa = std::frexp(scale, &shift);
|
|
auto shiftedM = std::round(mantissa * (int64_t(1) << 15));
|
|
|
|
// Can't be greater than 1.0.
|
|
assert(shiftedM <= (int64_t(1) << 15) &&
|
|
"Shifted mantissa exceeds 16 signed bits");
|
|
|
|
if (shiftedM == (int64_t(1) << 15)) {
|
|
shiftedM /= 2;
|
|
shift++;
|
|
}
|
|
|
|
// TOSA expects right shift to be positive and embed (1 << 15) into right
|
|
// shift bits.
|
|
shift = (-shift) + 15;
|
|
|
|
assert(shiftedM <= std::numeric_limits<int32_t>::max() &&
|
|
"Shifted mantissa exceeds 32-bit signed output type");
|
|
|
|
multiplier = static_cast<int32_t>(shiftedM);
|
|
|
|
// Shifting tops out at 62 bits. Right shift to make 62 bits the max.
|
|
// The limit of 62 on shift allows the shift to be decomposed as
|
|
// two right shifts of 31.
|
|
if (shift > 62) {
|
|
// Shifting the multiplier by more than 31-bits is unnecessary.
|
|
multiplier = multiplier >> std::min<int32_t>(31, shift - 62);
|
|
shift = 62;
|
|
}
|
|
}
|
|
|
|
/// From a scale value, generates multiplier and shift values where
|
|
/// mantissa is in [-1.0,-0.5] or [0.5, 1.0] such that
|
|
/// multiplier = mantissa*2^shift for 32-bit scaling.
|
|
static void computeMultiplierAndShiftTosaScale32(double scale,
|
|
int32_t &multiplier,
|
|
int32_t &shift) {
|
|
|
|
const double mantissa = std::frexp(scale, &shift);
|
|
auto shiftedM = std::round(mantissa * (int64_t(1) << 31));
|
|
|
|
// Can't be greater than 1.0.
|
|
assert(shiftedM <= (int64_t(1) << 31) &&
|
|
"Shifted mantissa exceeds 32 signed bits");
|
|
if (shiftedM == (int64_t(1) << 31)) {
|
|
shiftedM /= 2;
|
|
shift++;
|
|
}
|
|
|
|
// TOSA expects right shift to be positive, and embed (1 << 31) into right
|
|
// shift bits.
|
|
shift = (-shift) + 31;
|
|
|
|
assert(shiftedM <= std::numeric_limits<int32_t>::max() &&
|
|
"Shifted mantissa exceeds 32-bit signed output type");
|
|
|
|
multiplier = static_cast<int32_t>(shiftedM);
|
|
|
|
// Shifting tops out at 62 bits. Right shift to make 62 bits the max.
|
|
// The limit of 62 on shift allows the shift to be decomposed as
|
|
// two right shifts of 31.
|
|
if (shift > 62) {
|
|
// Shifting the multiplier by more than 32-bits is unnecessary.
|
|
multiplier = multiplier >> std::min<int32_t>(31, shift - 62);
|
|
shift = 62;
|
|
}
|
|
}
|
|
|
|
/// Generates a quantized multiplier/shift from double.
|
|
void mlir::tosa::computeMultiplierAndShift(double scale, int32_t &multiplier,
|
|
int32_t &shift, int32_t scaleWidth) {
|
|
|
|
switch (scaleWidth) {
|
|
case 16:
|
|
computeMultiplierAndShiftTosaScale16(scale, multiplier, shift);
|
|
return;
|
|
case 32:
|
|
computeMultiplierAndShiftTosaScale32(scale, multiplier, shift);
|
|
return;
|
|
default:
|
|
assert(0 && "Unsupported Tosa quantized_scale regime specified!");
|
|
}
|
|
}
|
|
|
|
#define GET_UQTYPE(inputType) \
|
|
(llvm::dyn_cast<quant::UniformQuantizedType>((inputType).getElementType()))
|
|
#define GET_QTYPE(inputType) \
|
|
(llvm::dyn_cast<quant::QuantizedType>((inputType).getElementType()))
|
|
|
|
static std::optional<std::pair<std::int64_t, std::int64_t>>
|
|
getConvZeroPoints(Value input, Value weight) {
|
|
|
|
auto inputType = dyn_cast<ShapedType>(input.getType());
|
|
auto weightType = dyn_cast<ShapedType>(weight.getType());
|
|
|
|
if (!inputType || !weightType)
|
|
return std::nullopt;
|
|
|
|
auto inputQType = GET_UQTYPE(inputType);
|
|
auto weightPerTensorQType = GET_UQTYPE(weightType);
|
|
auto weightPerAxisQType =
|
|
dyn_cast<quant::UniformQuantizedPerAxisType>(weightType.getElementType());
|
|
|
|
// Weights must be either per-tensor quantized or per-axis quantized.
|
|
assert(!((bool)weightPerTensorQType && (bool)weightPerAxisQType) &&
|
|
"Weights must be either per-tensor or per-axis quantized");
|
|
|
|
// Either all quantized or all not quantized.
|
|
assert(!((bool)inputQType ^
|
|
((bool)weightPerTensorQType || (bool)weightPerAxisQType)) &&
|
|
"Inputs and weights must be all quantized or all not quantized");
|
|
|
|
if (inputQType) {
|
|
int64_t inputZp = inputQType.getZeroPoint();
|
|
int64_t weightZp = 0;
|
|
|
|
if (weightPerTensorQType) {
|
|
weightZp = weightPerTensorQType.getZeroPoint();
|
|
} else if (weightPerAxisQType) {
|
|
weightZp = weightPerAxisQType.getZeroPoints().front();
|
|
}
|
|
|
|
return std::make_pair(inputZp, weightZp);
|
|
}
|
|
|
|
return std::nullopt;
|
|
}
|
|
|
|
std::pair<Value, Value>
|
|
mlir::tosa::createZPsAsConst(OpBuilder &builder, Value input, Value weight) {
|
|
std::int64_t inputZp, weightZp;
|
|
|
|
auto inputEType = getElementTypeOrSelf(input.getType());
|
|
auto weightEType = getElementTypeOrSelf(weight.getType());
|
|
|
|
if (mlir::isa<FloatType>(inputEType) && mlir::isa<FloatType>(weightEType)) {
|
|
inputZp = 0;
|
|
weightZp = 0;
|
|
} else {
|
|
auto maybeZps = getConvZeroPoints(input, weight);
|
|
if (!maybeZps.has_value())
|
|
return {};
|
|
|
|
inputZp = maybeZps->first;
|
|
weightZp = maybeZps->second;
|
|
}
|
|
|
|
auto maybeInputZpValue =
|
|
createZeroPointTensor(builder, input.getLoc(), inputEType, inputZp);
|
|
if (!maybeInputZpValue.has_value())
|
|
return {};
|
|
|
|
auto maybeWeightZpValue =
|
|
createZeroPointTensor(builder, weight.getLoc(), weightEType, weightZp);
|
|
if (!maybeWeightZpValue.has_value())
|
|
return {};
|
|
|
|
return std::make_pair(*maybeInputZpValue, *maybeWeightZpValue);
|
|
}
|
|
|
|
/// Method to build ConvOpQuantizationAttr, called from
|
|
/// ConvOpQuantInfoBuilder/TransConvOpQuantInfoBuilder:
|
|
/// input_zp: input zeropoint
|
|
/// weight_zp: weight zeropoint.
|
|
ConvOpQuantizationAttr
|
|
mlir::tosa::buildConvOpQuantizationAttr(OpBuilder &builder, Value input,
|
|
Value weight) {
|
|
|
|
auto maybeZps = getConvZeroPoints(input, weight);
|
|
if (!maybeZps.has_value())
|
|
return nullptr;
|
|
|
|
return builder.getAttr<tosa::ConvOpQuantizationAttr>(maybeZps->first,
|
|
maybeZps->second);
|
|
}
|
|
|
|
/// Builds MatMulOpQuantizationAttr, called from
|
|
/// MatMulOpQuantInfoBuilder:
|
|
/// aZp: input a zeropoint
|
|
/// bZp: input b zeropoint.
|
|
MatMulOpQuantizationAttr
|
|
mlir::tosa::buildMatMulOpQuantizationAttr(OpBuilder &builder, Value a,
|
|
Value b) {
|
|
|
|
auto aType = dyn_cast<ShapedType>(a.getType());
|
|
auto bType = dyn_cast<ShapedType>(b.getType());
|
|
|
|
if (!aType || !bType)
|
|
return nullptr;
|
|
|
|
auto aQType = GET_UQTYPE(aType);
|
|
auto bQType = GET_UQTYPE(bType);
|
|
|
|
// A and B are either all quantized or all not quantized.
|
|
assert(!((bool)aQType ^ (bool)bQType) &&
|
|
"Matmul operands must be all quantized or all not quantized");
|
|
|
|
if (aQType) {
|
|
return builder.getAttr<tosa::MatMulOpQuantizationAttr>(
|
|
aQType.getZeroPoint(), bQType.getZeroPoint());
|
|
}
|
|
|
|
return nullptr;
|
|
}
|
|
|
|
/// Builds UnaryOpQuantizationAttr
|
|
/// UnaryOpQuantInfoBuilder:
|
|
/// inputZp: input zeropoint
|
|
/// outputZp: output zeropoint.
|
|
UnaryOpQuantizationAttr
|
|
mlir::tosa::buildUnaryOpQuantizationAttr(OpBuilder &builder, Value input,
|
|
Type outputRawType) {
|
|
|
|
auto inputType = dyn_cast<ShapedType>(input.getType());
|
|
auto outputType = dyn_cast<ShapedType>(outputRawType);
|
|
|
|
if (!inputType || !outputType)
|
|
return nullptr;
|
|
|
|
auto inputQType = GET_UQTYPE(inputType);
|
|
auto outputQType = GET_UQTYPE(outputType);
|
|
|
|
// Either all quantized or all not quantized.
|
|
assert(!((bool)inputQType ^ (bool)outputQType) &&
|
|
"Unary inputs/outputs must be all quantized or all not quantized");
|
|
|
|
if (inputQType) {
|
|
return builder.getAttr<UnaryOpQuantizationAttr>(inputQType.getZeroPoint(),
|
|
outputQType.getZeroPoint());
|
|
}
|
|
|
|
return nullptr;
|
|
}
|
|
|
|
/// Builds PadOpQuantizationAttr, called from PadOpQuantInfoBuilder:
|
|
/// inputZp: input zeropoint.
|
|
PadOpQuantizationAttr mlir::tosa::buildPadOpQuantizationAttr(OpBuilder &builder,
|
|
Value input) {
|
|
|
|
auto inputType = dyn_cast<ShapedType>(input.getType());
|
|
|
|
if (!inputType)
|
|
return nullptr;
|
|
|
|
auto inputQType = GET_UQTYPE(inputType);
|
|
|
|
if (inputQType) {
|
|
return builder.getAttr<tosa::PadOpQuantizationAttr>(
|
|
inputQType.getZeroPoint());
|
|
}
|
|
|
|
return nullptr;
|
|
}
|
|
|
|
/// Builds output type for a quantized ConvOp with the right bitwidth.
|
|
/// This is called by the builder when dealing with quantized content.
|
|
Type mlir::tosa::buildConvOpResultTypeInfo(OpBuilder &builder, Type outputType,
|
|
Value input, Value weight) {
|
|
|
|
auto inputType = dyn_cast<ShapedType>(input.getType());
|
|
auto weightType = dyn_cast<ShapedType>(weight.getType());
|
|
|
|
assert(inputType && weightType &&
|
|
"Could not extract input or weight tensors from Conv op");
|
|
|
|
auto inputQType = GET_QTYPE(inputType);
|
|
auto weightQType = GET_QTYPE(weightType);
|
|
|
|
assert(inputQType && weightQType &&
|
|
"Could not extract input or weight tensor types from Conv op");
|
|
|
|
unsigned inputBits = inputQType.getStorageTypeIntegralWidth();
|
|
unsigned weightBits = weightQType.getStorageTypeIntegralWidth();
|
|
|
|
auto outputShapedType = dyn_cast<ShapedType>(outputType);
|
|
assert(outputShapedType &&
|
|
"Could not extract output shape type from Conv op");
|
|
|
|
IntegerType accElementType;
|
|
if (inputBits == 16 && weightBits == 8)
|
|
accElementType = builder.getIntegerType(48);
|
|
else
|
|
accElementType = builder.getI32Type();
|
|
auto accType = outputShapedType.clone(accElementType);
|
|
return accType;
|
|
}
|
|
|
|
/// Builds Tosa quantization attributes from min/max values.
|
|
Type mlir::tosa::buildQTypeFromMinMax(OpBuilder builder, Type inputDType,
|
|
Attribute minAttr, Attribute maxAttr,
|
|
IntegerAttr quantBits, int filterQuantDim,
|
|
bool isSigned, BoolAttr narrowRange) {
|
|
|
|
quant::QuantizedType retType;
|
|
|
|
auto convfunc =
|
|
quant::ExpressedToQuantizedConverter::forInputType(inputDType);
|
|
|
|
auto minElems = dyn_cast<DenseFPElementsAttr>(minAttr);
|
|
auto maxElems = dyn_cast<DenseFPElementsAttr>(maxAttr);
|
|
|
|
SmallVector<double, 2> min, max;
|
|
|
|
// At least one is per-axis quantized elementsattr.
|
|
if (minElems || maxElems) {
|
|
// Must have the same number of elements.
|
|
if (minElems.getNumElements() != maxElems.getNumElements())
|
|
return {};
|
|
min.reserve(minElems.getNumElements());
|
|
max.reserve(maxElems.getNumElements());
|
|
for (auto i : minElems)
|
|
min.push_back(FloatAttr::getValueAsDouble(i));
|
|
for (auto i : maxElems)
|
|
max.push_back(FloatAttr::getValueAsDouble(i));
|
|
} else { // Just a single FP value.
|
|
auto minVal = dyn_cast<FloatAttr>(minAttr);
|
|
if (minVal)
|
|
min.push_back(minVal.getValueAsDouble());
|
|
else
|
|
return {};
|
|
auto maxVal = dyn_cast<FloatAttr>(maxAttr);
|
|
if (maxVal)
|
|
max.push_back(maxVal.getValueAsDouble());
|
|
else
|
|
return {};
|
|
}
|
|
|
|
if (min.size() == max.size()) {
|
|
if (min.size() == 1) { // Per-tensor quantization with one min/max pair.
|
|
retType = quant::fakeQuantAttrsToType(
|
|
builder.getUnknownLoc(), quantBits.getInt(), min[0], max[0],
|
|
narrowRange.getValue(), convfunc.expressedType, isSigned);
|
|
} else if (min.size() > 1) { // Per-axis quant on filterQuantDim.
|
|
auto shape = dyn_cast<ShapedType>(inputDType);
|
|
if (!shape)
|
|
return {};
|
|
if ((filterQuantDim) >= 0 && (shape.getRank() > filterQuantDim)) {
|
|
retType = quant::fakeQuantAttrsToType(
|
|
builder.getUnknownLoc(), quantBits.getInt(), filterQuantDim, min[0],
|
|
max[0], narrowRange.getValue(), convfunc.expressedType, isSigned);
|
|
}
|
|
} else {
|
|
return {};
|
|
}
|
|
} else {
|
|
return {};
|
|
}
|
|
|
|
if (!retType)
|
|
return {};
|
|
|
|
return convfunc.convert(retType);
|
|
}
|
|
|
|
/// Builds Tosa quantization attributes from min/max values.
|
|
TypeAttr
|
|
mlir::tosa::buildQTypeAttrFromMinMax(OpBuilder builder, Type inputDtype,
|
|
Attribute minAttr, Attribute maxAttr,
|
|
IntegerAttr quantBits, int filterQuantDim,
|
|
bool isSigned, BoolAttr narrowRange) {
|
|
|
|
return TypeAttr::get(buildQTypeFromMinMax(builder, inputDtype, minAttr,
|
|
maxAttr, quantBits, filterQuantDim,
|
|
isSigned, narrowRange));
|
|
}
|