1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
|
#ifndef QUANT_DECODE_OP_H_
#define QUANT_DECODE_OP_H_
#include <c10/util/irange.h>
#include <c10/util/typeid.h>
#include "caffe2/core/context.h"
#include "caffe2/core/operator.h"
#include "caffe2/core/tensor.h"
namespace caffe2 {
namespace {
template <class CodebookT, class CodeT>
void Decode(
const Tensor& codebook,
const Tensor& codes,
/* optional */ const Tensor* const decoded_grad,
Tensor* const output,
bool resizeOnly) {
CAFFE_ENFORCE(codebook.IsType<CodebookT>());
auto* cb_ptr = codebook.data<CodebookT>();
int cb_size = codebook.numel();
CAFFE_ENFORCE(codes.IsType<CodeT>());
auto* code_ptr = codes.data<CodeT>();
if (decoded_grad == nullptr) {
// Forward pass: decode and store codebook values in output.
output->ResizeLike(codes);
auto* out_ptr = output->template mutable_data<CodebookT>();
if (resizeOnly) {
return;
}
int sz = output->numel();
for (C10_UNUSED const auto i : c10::irange(sz)) {
TORCH_DCHECK_LE(*code_ptr, cb_size);
*out_ptr++ = cb_ptr[*code_ptr++];
}
} else {
// Backward pass: decode and accumulate gradient w.r.t. codebook values.
CAFFE_ENFORCE_EQ(codes.numel(), decoded_grad->numel());
auto* gradient_ptr = decoded_grad->data<CodebookT>();
auto* const gradient_end = gradient_ptr + decoded_grad->numel();
CAFFE_ENFORCE_EQ(cb_size, output->numel());
auto* out_ptr = output->template mutable_data<CodebookT>();
while (gradient_ptr < gradient_end) {
TORCH_DCHECK_LE(*code_ptr, cb_size);
out_ptr[*code_ptr++] += *gradient_ptr++;
}
}
}
#define REGISTER_DECODER(codebookType, codesType) \
{ \
{TypeMeta::Id<codebookType>(), TypeMeta::Id<codesType>()}, \
[](const Tensor& codebook_, \
const Tensor& codes_, \
const Tensor* gradient_, \
Tensor* outDecoded_, \
bool resizeOnly_) { \
Decode<codebookType, codesType>( \
codebook_, codes_, gradient_, outDecoded_, resizeOnly_); \
} \
}
inline void DecodeGeneral(
const Tensor& codebook,
const Tensor& codes,
const Tensor* gradient,
Tensor* outDecoded,
bool resizeOnly) {
const static std::map<
std::pair<TypeIdentifier, TypeIdentifier>,
std::function<void(
const Tensor& codebook,
const Tensor& codes,
const Tensor* gradient,
Tensor* outDecoded,
bool resizeOnly)>>
gDecoderMapper = {REGISTER_DECODER(float, uint8_t),
REGISTER_DECODER(float, uint16_t),
REGISTER_DECODER(float, int32_t)};
gDecoderMapper.at({codebook.dtype().id(), codes.dtype().id()})(
codebook, codes, gradient, outDecoded, resizeOnly);
}
} // namespace
// Decode tensors based on given codebook,
// The codebook is generated by model_quantize.py
enum class QuantDecodeRunTy {
RUN_ALWAYS,
RUN_ONCE,
};
template <QuantDecodeRunTy QuantDecodeRun>
class QuantDecodeOp final : public Operator<CPUContext> {
public:
USE_OPERATOR_FUNCTIONS(CPUContext);
template <class... Args>
explicit QuantDecodeOp(Args&&... args)
: Operator<CPUContext>(std::forward<Args>(args)...) {}
~QuantDecodeOp() {}
bool RunOnDevice() override {
CAFFE_ENFORCE_GT(InputSize(), 1);
// first input is the codebook
CAFFE_ENFORCE_EQ(InputSize(), OutputSize() + 1);
const auto& codebook = Input(0);
CAFFE_ENFORCE(codebook.template IsType<float>(), codebook.dtype().name());
for (const auto i : c10::irange(OutputSize())) {
auto& ci = Input(i + 1);
auto* co = Output(i);
DecodeGeneral(
codebook,
ci,
nullptr,
co,
/*resizeOnly=*/QuantDecodeRun == QuantDecodeRunTy::RUN_ONCE &&
hasRun_);
}
hasRun_ = true;
return true;
}
private:
bool hasRun_{false};
};
class QuantDecodeGradientOp final : public Operator<CPUContext> {
public:
USE_OPERATOR_FUNCTIONS(CPUContext);
template <class... Args>
explicit QuantDecodeGradientOp(Args&&... args)
: Operator<CPUContext>(std::forward<Args>(args)...) {}
~QuantDecodeGradientOp() {}
bool RunOnDevice() override {
// Inputs: 1 codebook, n tensors of codes, and n corresponding gradients.
CAFFE_ENFORCE(InputSize() >= 3 && InputSize() % 2 == 1);
const int num_code_tensors = (InputSize() - 1) / 2;
CAFFE_ENFORCE_EQ(OutputSize(), 1);
const auto& codebook = Input(0);
CAFFE_ENFORCE(codebook.template IsType<float>(), codebook.dtype().name());
auto* gradient = Output(0, codebook.sizes(), at::dtype<float>());
auto* gradient_ptr = gradient->template mutable_data<float>();
std::fill(gradient_ptr, gradient_ptr + gradient->numel(), 0);
for (const auto i : c10::irange(num_code_tensors)) {
auto& codes_i = Input(i + 1);
auto& output_gradient_i = Input(i + num_code_tensors + 1);
DecodeGeneral(codebook, codes_i, &output_gradient_i, gradient, false);
}
return true;
}
};
} // namespace caffe2
#endif // QUANT_DECODE_OP_H_
|