@@ -39,6 +39,15 @@ using llvm::cast;
39
39
using ArgumentDictionaryTy =
40
40
std::unordered_map<std::string, const caffe2::Argument *>;
41
41
42
+ // / For the quantized Caffe2 ops, the activations are quantized to uint_8.
43
+ // / In Glow, the activations are quantized to int_8. Therefore, for the offset
44
+ // / read from quantized caffe2 model, we need to subtract 128(i.e. INT8_MIN) to
45
+ // / make the activations becomes int8_t.
46
+ // / For Glow: -127 <= orig_fp32/scale_1 + offset_1 < 128
47
+ // / For Caffe2: 0 <= orig_fp32/scale_2 + offset_2 < 255
48
+ // / Therefore, we can make scale_1 == scale_2, and offset_1 = offset2 - 128
49
+ const int32_t OFFSETSHIFT = 128 ;
50
+
42
51
// / Translates the protocol buffer node \p op into a random access map.
43
52
static ArgumentDictionaryTy loadArgumentMap (const caffe2::OperatorDef &op) {
44
53
ArgumentDictionaryTy dict;
@@ -147,7 +156,8 @@ void Caffe2ModelLoader::loadOperator(const caffe2::OperatorDef &op) {
147
156
148
157
const std::string &opName = loadOperatorName (op);
149
158
150
- if (typeName == " Conv" ) {
159
+ if (typeName == " Conv" || typeName == " Int8Conv" ||
160
+ typeName == " Int8ConvRelu" ) {
151
161
// Load the inputs:
152
162
std::vector<unsigned_t > strides = getSizeHW (dict, " stride" , 1 );
153
163
std::vector<unsigned_t > pads = getPads (dict);
@@ -159,34 +169,22 @@ void Caffe2ModelLoader::loadOperator(const caffe2::OperatorDef &op) {
159
169
Tensor *w = getTensorByName (op.input (1 ));
160
170
161
171
// Transpose the weights to the right format. Glow expects to read the
162
- // weights in the format CRSK. Caffe2 stores the operators as KCRS.
172
+ // weights in the format CRSK.
163
173
// C - output_depth, R - filter_height, S - filter_width, K - input_depth.
174
+ // Caffe2 "Conv" op always stores the weight as CKRS, while for "Int8Conv",
175
+ // and "Int8ConvRelu", the weights always follows the "order" arg.
164
176
Tensor wtag;
165
- w->transpose (&wtag, NCHW2NHWC);
177
+ if (typeName != " Conv" && order == " NHWC" ) {
178
+ wtag.assign (w);
179
+ } else {
180
+ w->transpose (&wtag, NCHW2NHWC);
181
+ }
166
182
167
183
// The structure of the conv weigts is: NHWC. We take the C, which is the
168
184
// number of filters. We use this value to calculate the size of the bias
169
185
// if it is not specified.
170
186
size_t depth = wtag.dims ()[0 ];
171
187
172
- // Construct the Filter field.
173
- auto *filter = G_.getParent ()->createConstant (" conv.filter" , wtag);
174
-
175
- // Construct the Bias field.
176
- Tensor biasTensor (ElemKind::FloatTy, {depth});
177
- biasTensor.zero ();
178
-
179
- // Check if we have a serialized bias vector.
180
- if (op.input_size () > 2 ) {
181
- auto &biasTensorName = op.input (2 );
182
- if (tensors_.count (biasTensorName)) {
183
- // Load the serialized bias vector.
184
- Tensor *b = getTensorByName (biasTensorName);
185
- biasTensor.assign (b);
186
- }
187
- }
188
- auto *bias = G_.getParent ()->createConstant (" conv.bias" , biasTensor);
189
-
190
188
// We expect the input to be NHWC.
191
189
Node *tr;
192
190
if (order == " NCHW" ) {
@@ -201,7 +199,60 @@ void Caffe2ModelLoader::loadOperator(const caffe2::OperatorDef &op) {
201
199
calculateConvPoolOutputDims (idim.h , idim.w , kernels, strides, pads);
202
200
std::array<size_t , 4 > outDims = {
203
201
{idim.n , outSz.first , outSz.second , depth}};
204
- auto outTy = G_.getParent ()->uniqueType (ElemKind::FloatTy, outDims);
202
+
203
+ TypeRef outTy;
204
+ Constant *filter;
205
+ Constant *bias;
206
+ if (typeName == " Conv" ) {
207
+ // Construct the Bias field.
208
+ Tensor biasTensor (ElemKind::FloatTy, {depth});
209
+ biasTensor.zero ();
210
+
211
+ // Check if we have a serialized bias vector.
212
+ if (op.input_size () > 2 ) {
213
+ const auto &biasTensorName = op.input (2 );
214
+ if (tensors_.count (biasTensorName)) {
215
+ // Load the serialized bias vector.
216
+ Tensor *b = getTensorByName (biasTensorName);
217
+ biasTensor.assign (b);
218
+ }
219
+ }
220
+ outTy = G_.getParent ()->uniqueType (ElemKind::FloatTy, outDims);
221
+ filter = G_.getParent ()->createConstant (" conv.filter" , wtag);
222
+ bias = G_.getParent ()->createConstant (" conv.bias" , biasTensor);
223
+ } else {
224
+ assert (dict.count (" Y_zero_point" ) &&
225
+ " missing zero point for quantized output type" );
226
+ assert (dict.count (" Y_scale" ) &&
227
+ " missing Y_scale for quantized output type" );
228
+ // Construct the Bias field.
229
+ Tensor biasTensor (ElemKind::Int32QTy, {depth}, 1.0 , 0 );
230
+ biasTensor.zero ();
231
+ // Check if we have a serialized bias vector.
232
+ if (op.input_size () > 2 ) {
233
+ const auto &biasTensorName = op.input (2 );
234
+ if (tensors_.count (biasTensorName)) {
235
+ // Load the serialized bias vector.
236
+ Tensor *b = getTensorByName (biasTensorName);
237
+ biasTensor.assign (b);
238
+ }
239
+ }
240
+ float scale = loadFloat (dict[" Y_scale" ]);
241
+ int32_t offset = loadInt (dict[" Y_zero_point" ]);
242
+ outTy = G_.getParent ()->uniqueType (ElemKind::Int8QTy, outDims, scale,
243
+ offset - OFFSETSHIFT);
244
+
245
+ // Construct the quantized Filter and bias field.
246
+ filter = G_.getParent ()->createConstant (
247
+ ElemKind::Int8QTy, wtag.dims (), wtag.getType ().getScale (),
248
+ wtag.getType ().getOffset (), " conv.filter" );
249
+ filter->assign (&wtag);
250
+ bias = G_.getParent ()->createConstant (
251
+ ElemKind::Int32QTy, biasTensor.dims (),
252
+ biasTensor.getType ().getScale (), biasTensor.getType ().getOffset (),
253
+ " conv.bias" );
254
+ bias->assign (&biasTensor);
255
+ }
205
256
206
257
Node *node = G_.createConv (opName, tr, filter, bias, outTy, kernels,
207
258
strides, pads, group);
@@ -214,7 +265,47 @@ void Caffe2ModelLoader::loadOperator(const caffe2::OperatorDef &op) {
214
265
return ;
215
266
}
216
267
217
- if (typeName == " MaxPool" || typeName == " AveragePool" ) {
268
+ if (typeName == " Int8SumRelu" ) {
269
+ assert (op.input_size () == 2 && " Only Sum of 2 inputs is supported." );
270
+ assert (dict.count (" Y_zero_point" ) &&
271
+ " missing zero point for quantized outout type" );
272
+ assert (dict.count (" Y_scale" ) &&
273
+ " missing Y_scale for quantized output type" );
274
+ auto in0 = getNodeValueOrCreateConstantByName (op.input (0 ));
275
+ auto in1 = getNodeValueOrCreateConstantByName (op.input (1 ));
276
+ auto outDims = in0.getType ()->dims ();
277
+ auto outTy = G_.getParent ()->uniqueType (
278
+ ElemKind::Int8QTy, outDims, loadFloat (dict[" Y_scale" ]),
279
+ loadInt (dict[" Y_zero_point" ]) - OFFSETSHIFT);
280
+ auto *node = G_.createAdd (opName, outTy, in0, in1);
281
+ addNodeAsOutput (op, node);
282
+ return ;
283
+ }
284
+
285
+ if (typeName == " Int8Quantize" ) {
286
+ assert (dict.count (" Y_zero_point" ) &&
287
+ " missing zero point for quantized output type" );
288
+ assert (dict.count (" Y_scale" ) &&
289
+ " missing Y_scale for quantized output type" );
290
+ auto in = getNodeValueOrCreateConstantByName (op.input (0 ));
291
+ auto outDims = in.getType ()->dims ();
292
+ auto outTy = G_.getParent ()->uniqueType (
293
+ ElemKind::Int8QTy, outDims, loadFloat (dict[" Y_scale" ]),
294
+ loadInt (dict[" Y_zero_point" ]) - OFFSETSHIFT);
295
+ Node *N = G_.createQuantize (opName, in, outTy);
296
+ addNodeAsOutput (op, N);
297
+ return ;
298
+ }
299
+
300
+ if (typeName == " Int8Dequantize" ) {
301
+ auto in = getNodeValueOrCreateConstantByName (op.input (0 ));
302
+ auto *node = G_.createDequantize (opName, in);
303
+ addNodeAsOutput (op, node);
304
+ return ;
305
+ }
306
+
307
+ if (typeName == " MaxPool" || typeName == " AveragePool" ||
308
+ typeName == " Int8MaxPool" || typeName == " Int8AveragePool" ) {
218
309
// Load the inputs:
219
310
auto in = getNodeValueOrCreateConstantByName (op.input (0 ));
220
311
std::vector<unsigned_t > strides = getSizeHW (dict, " stride" , 1 );
@@ -238,7 +329,29 @@ void Caffe2ModelLoader::loadOperator(const caffe2::OperatorDef &op) {
238
329
}
239
330
240
331
Node *node = nullptr ;
241
- if (typeName == " MaxPool" ) {
332
+
333
+ if (typeName == " Int8MaxPool" || typeName == " Int8AveragePool" ) {
334
+ // Create the node with quantized type.
335
+ assert (dict.count (" Y_zero_point" ) &&
336
+ " missing zero point for quantized output type" );
337
+ assert (dict.count (" Y_scale" ) &&
338
+ " missing Y_scale for quantized output type" );
339
+ ShapeNHWC idim = ShapeNHWC (tr->getType (0 )->dims ());
340
+ auto outSz =
341
+ calculateConvPoolOutputDims (idim.h , idim.w , kernels, strides, pads);
342
+ std::array<size_t , 4 > outDims = {
343
+ {idim.n , outSz.first , outSz.second , idim.c }};
344
+ if (typeName == " Int8MaxPool" ) {
345
+ // Int8Maxpool output quantization should be same as the input, so just
346
+ // ignore the given params.
347
+ node = G_.createMaxPool (opName, tr, kernels, strides, pads);
348
+ } else {
349
+ auto outTy = G_.getParent ()->uniqueType (
350
+ ElemKind::Int8QTy, outDims, loadFloat (dict[" Y_scale" ]),
351
+ loadInt (dict[" Y_zero_point" ]) - OFFSETSHIFT);
352
+ node = G_.createAvgPool (opName, tr, outTy, kernels, strides, pads);
353
+ }
354
+ } else if (typeName == " MaxPool" ) {
242
355
node = G_.createMaxPool (opName, tr, kernels, strides, pads);
243
356
} else {
244
357
node = G_.createAvgPool (opName, tr, kernels, strides, pads);
@@ -309,7 +422,7 @@ void Caffe2ModelLoader::loadOperator(const caffe2::OperatorDef &op) {
309
422
return ;
310
423
}
311
424
312
- if (typeName == " FC" || typeName == " FCTransposed" ) {
425
+ if (typeName == " FC" || typeName == " FCTransposed" || typeName == " Int8FC " ) {
313
426
// Load the inputs:
314
427
auto in = getNodeValueOrCreateConstantByName (op.input (0 ));
315
428
if (in.getType ()->dims ().size () > 2 ) {
@@ -327,12 +440,18 @@ void Caffe2ModelLoader::loadOperator(const caffe2::OperatorDef &op) {
327
440
Tensor tmp;
328
441
if (w->dims ().size () > 2 ) {
329
442
auto wDims = flattenCdr (w->dims (), axis_w);
330
- tmp.reset (ElemKind::FloatTy, {wDims.first , wDims.second });
443
+ if (typeName == " FC" || typeName == " FCTransposed" ) {
444
+ tmp.reset (ElemKind::FloatTy, {wDims.first , wDims.second });
445
+ } else {
446
+ tmp.reset (ElemKind::Int8QTy, {wDims.first , wDims.second },
447
+ w->getType ().getScale (), w->getType ().getOffset ());
448
+ }
331
449
tmp.copyRawFrom (w);
332
450
w = &tmp;
333
451
}
452
+
334
453
Tensor wtag;
335
- if (typeName == " FC" ) {
454
+ if (typeName == " FC" || typeName == " Int8FC " ) {
336
455
w->transpose (&wtag, {1 , 0 });
337
456
} else {
338
457
wtag.assign (w);
@@ -341,7 +460,22 @@ void Caffe2ModelLoader::loadOperator(const caffe2::OperatorDef &op) {
341
460
auto W =
342
461
G_.getParent ()->addConstant (new Constant (" weights" , std::move (wtag)));
343
462
auto B = G_.getParent ()->addConstant (new Constant (" biases" , std::move (*b)));
344
- auto *node = G_.createFullyConnected (opName, in, W, B);
463
+
464
+ Node *node = nullptr ;
465
+ if (typeName == " Int8FC" ) {
466
+ // Create the node with quantized type.
467
+ assert (dict.count (" Y_zero_point" ) &&
468
+ " missing zero point for quantized output type" );
469
+ assert (dict.count (" Y_scale" ) &&
470
+ " missing Y_scale for quantized output type" );
471
+ auto outTy = G_.getParent ()->uniqueType (
472
+ ElemKind::Int8QTy, {in.getType ()->dims ()[0 ], B->getType ()->dims ()[0 ]},
473
+ loadFloat (dict[" Y_scale" ]),
474
+ loadInt (dict[" Y_zero_point" ]) - OFFSETSHIFT);
475
+ node = G_.createFullyConnected (opName, in, W, B, outTy);
476
+ } else {
477
+ node = G_.createFullyConnected (opName, in, W, B);
478
+ }
345
479
346
480
// Save the outputs:
347
481
addNodeAsOutput (op, node);
@@ -602,6 +736,73 @@ void Caffe2ModelLoader::loadWeight(const caffe2::OperatorDef &op) {
602
736
return ;
603
737
}
604
738
739
+ // Load quantized tensors:
740
+ if (typeName == " Int8GivenTensorFill" ||
741
+ typeName == " Int8GivenIntTensorFill" ) {
742
+ /*
743
+ output: "conv1_w"
744
+ name: ""
745
+ type: "Int8GivenTensorFill"
746
+ arg {
747
+ name: "shape"
748
+ ints: 96
749
+ ints: 3
750
+ ints: 11
751
+ ints: 11
752
+ }
753
+ arg {
754
+ name: "values"
755
+ s: "\x7f\x80\x80\x7"
756
+ }
757
+ arg {
758
+ name: "Y_scale"
759
+ f: 0.00044428
760
+ }
761
+ arg {
762
+ name: "Y_zero_point"
763
+ i: 127
764
+ }
765
+ */
766
+ auto *T = new Tensor ();
767
+ for (auto &o : op.output ()) {
768
+ if (tensors_.count (o))
769
+ continue ;
770
+ tensors_[o] = T;
771
+ }
772
+
773
+ auto dim = getShape (dict[" shape" ]);
774
+
775
+ assert (dict.count (" Y_zero_point" ) &&
776
+ " missing zero point for quantized output type" );
777
+ assert (dict.count (" Y_scale" ) &&
778
+ " missing Y_scale for quantized output type" );
779
+
780
+ float scale = loadFloat (dict[" Y_scale" ]);
781
+ int32_t offset = loadInt (dict[" Y_zero_point" ]);
782
+ size_t i = 0 ;
783
+ if (typeName == " Int8GivenTensorFill" ) {
784
+ // Although in Caffe2 quantized model, the weights is int8 quantized,
785
+ // the weights is stored in uint8_t format due to that Caffe2 requires the
786
+ // type of input and weights must be the same. Therefore, we need to
787
+ // convert it to int8 by subtracting 128.
788
+ T->reset (ElemKind::Int8QTy, dim, scale, offset - OFFSETSHIFT);
789
+ auto TH = T->getHandle <int8_t >();
790
+ std::string str = dict[" values" ]->s ();
791
+ for (; i < str.size (); i++) {
792
+ TH.raw (i) = ((uint8_t )(str.c_str ()[i]) - OFFSETSHIFT);
793
+ }
794
+ } else {
795
+ T->reset (ElemKind::Int32QTy, dim, scale, offset);
796
+ auto TH = T->getHandle <int32_t >();
797
+ for (auto num : dict[" values" ]->ints ()) {
798
+ TH.raw (i++) = num;
799
+ }
800
+ }
801
+ assert (i == T->size () && " The number of serialized values does not "
802
+ " match the size of the tensor." );
803
+ return ;
804
+ }
805
+
605
806
// Load tensors with constant fill:
606
807
if (typeName == " ConstantFill" ) {
607
808
/*
0 commit comments