diff --git a/onnxscript/rewriter/rules/common/_fuse_conv_affine.py b/onnxscript/rewriter/rules/common/_fuse_conv_affine.py index 6d95b79a03..9730d48fd8 100644 --- a/onnxscript/rewriter/rules/common/_fuse_conv_affine.py +++ b/onnxscript/rewriter/rules/common/_fuse_conv_affine.py @@ -27,14 +27,26 @@ def check( conv_out: ir.Value, ) -> MatchResult: check_result = MatchResult() - if get_const_value(w) is None: - return check_result.fail("The weight of Conv should be constant") - if get_const_value(b) is None: - return check_result.fail("The bias of Conv should be constant") if get_singleton_value(scale) is None: return check_result.fail("Operand for Mul should be constant scalar value") if get_singleton_value(offset) is None: return check_result.fail("Operand for Add should be constant scalar value") + w_val = get_const_value(w) + if w_val is None: + return check_result.fail("The weight of Conv should be constant") + b_val = get_const_value(b) + if b_val is None: + return check_result.fail("The bias of Conv should be constant") + w_shape = w_val.shape + b_shape = b_val.shape + if len(w_shape) < 3: + return check_result.fail("Conv weight tensor must have rank >= 3") + if len(b_shape) != 1: + return check_result.fail("Conv bias tensor must have rank 1") + if w_shape[0] != b_shape[0]: + return check_result.fail( + f"Conv weight output channels ({w_shape[0]}) does not match bias channels ({b_shape[0]})" + ) return check_result @@ -46,11 +58,54 @@ def pattern(self, op, x, w, b, scale, offset): x * scale + offset, w, b, - pads=[0, 0, 0, 0], _allow_other_attributes=True, _outputs=["conv_out"], ) + def check( + self, + context, + x: ir.Value, + w: ir.Value, + b: ir.Value, + scale: ir.Value, + offset: ir.Value, + conv_out: ir.Value, + ) -> MatchResult: + check_result = MatchResult() + conv_node = conv_out.producer() + if conv_node is None: + return check_result.fail("Conv node not found") + pads_attr = conv_node.attributes.get("pads") + if pads_attr is not None: + if hasattr(pads_attr, "is_ref") and pads_attr.is_ref(): + return check_result.fail( + f"Conv 'pads' refers to reference attribute {pads_attr.ref_attr_name!r}" + ) + try: + pads = list(pads_attr.as_ints()) + if any(p != 0 for p in pads): + return check_result.fail(f"Conv has non-zero pads: {pads}") + except (TypeError, ValueError, AttributeError): + return check_result.fail("Conv 'pads' attribute could not be read as integers") + auto_pad_attr = conv_node.attributes.get("auto_pad") + if auto_pad_attr is not None: + if hasattr(auto_pad_attr, "is_ref") and auto_pad_attr.is_ref(): + return check_result.fail( + f"Conv 'auto_pad' refers to reference attribute {auto_pad_attr.ref_attr_name!r}" + ) + try: + auto_pad = auto_pad_attr.as_string() + if auto_pad not in {"NOTSET", "VALID"}: + return check_result.fail( + f"Conv 'auto_pad' {auto_pad!r} introduces padding" + ) + except (TypeError, ValueError, AttributeError): + return check_result.fail( + "Conv 'auto_pad' attribute could not be read as string" + ) + return super().check(context, x, w, b, scale, offset, conv_out) + def rewrite( self, op, @@ -65,9 +120,15 @@ def rewrite( offset_value = offset.const_value.numpy() w_value = w.const_value.numpy() b_value = b.const_value.numpy() - scaled_w_value = op.initializer(ir.tensor(w_value * scale_value), w.name + "_scaled") + scaled_w_value = op.initializer( + ir.tensor((w_value * scale_value).astype(w_value.dtype)), + w.name + "_scaled", + ) + axes = tuple(range(1, w_value.ndim)) offset_bias = ir.tensor( - b_value + np.sum(w_value * offset_value, axis=(1, 2, 3), keepdims=False) + (b_value + np.sum(w_value * offset_value, axis=axes, keepdims=False)).astype( + b_value.dtype + ) ) offset_bias = op.initializer(offset_bias, b.name + "_offset") conv_attributes = conv_out.producer().attributes @@ -97,8 +158,11 @@ def rewrite( offset_value = offset.const_value.numpy() w_value = w.const_value.numpy() b_value = b.const_value.numpy() - scaled_w_weight = op.initializer(ir.tensor(w_value * scale_value), w.name + "_scaled") - offset_bias = ir.tensor(b_value * scale_value + offset_value) + scaled_w_weight = op.initializer( + ir.tensor((w_value * scale_value).astype(w_value.dtype)), + w.name + "_scaled", + ) + offset_bias = ir.tensor((b_value * scale_value + offset_value).astype(b_value.dtype)) offset_bias = op.initializer(offset_bias, b.name + "_offset") conv_attributes = conv_out.producer().attributes return op.Conv(x, scaled_w_weight, offset_bias, **conv_attributes) diff --git a/onnxscript/rewriter/rules/common/_fuse_conv_affine_extended_test.py b/onnxscript/rewriter/rules/common/_fuse_conv_affine_extended_test.py index 2d301b2808..fa6ff7158b 100644 --- a/onnxscript/rewriter/rules/common/_fuse_conv_affine_extended_test.py +++ b/onnxscript/rewriter/rules/common/_fuse_conv_affine_extended_test.py @@ -150,6 +150,211 @@ def model_fn(x: FLOAT[1, 3, 32, 32]) -> FLOAT[1, 3, 32, 32]: inputs = [rng.random((1, 3, 32, 32), dtype=np.float32)] testing.assert_numerically_equal(model, rewritten, inputs) + def test_affine_conv_auto_pad_same_upper_rejected(self): + """Conv with auto_pad='SAME_UPPER' introduces padding and must be rejected.""" + + @script() + def model_fn(x: FLOAT[1, 3, 32, 32]) -> FLOAT[1, 3, 32, 32]: + w = op.Constant(value=_W_ONES) + b = op.Constant(value=_B_ONES) + scale = op.Constant(value_float=2.0) + offset = op.Constant(value_float=3.0) + affine_x = (x * scale) + offset + return op.Conv(affine_x, w, b, auto_pad="SAME_UPPER") + + model_proto = model_fn.to_model_proto() + model = ir.serde.deserialize_model(model_proto) + rewritten = self._clone(model) + rewritten = rewrite(rewritten, pattern_rewrite_rules=[affine_conv_fusion_rule]) + self.assertEqual(model.graph.num_nodes(), rewritten.graph.num_nodes()) + + def test_affine_conv_auto_pad_valid_accepted(self): + """Conv with auto_pad='VALID' has zero padding and should fuse into Conv.""" + + @script() + def model_fn(x: FLOAT[1, 3, 32, 32]) -> FLOAT[1, 3, 30, 30]: + w = op.Constant(value=_W_ONES) + b = op.Constant(value=_B_ONES) + scale = op.Constant(value_float=2.0) + offset = op.Constant(value_float=3.0) + affine_x = (x * scale) + offset + return op.Conv(affine_x, w, b, auto_pad="VALID") + + model_proto = model_fn.to_model_proto() + model = ir.serde.deserialize_model(model_proto) + rewritten = self._clone(model) + rewritten = rewrite(rewritten, pattern_rewrite_rules=[affine_conv_fusion_rule]) + rewritten_ops = [n.op_type for n in rewritten.graph] + self.assertNotIn("Mul", rewritten_ops) + self.assertNotIn("Add", rewritten_ops) + self.assertIn("Conv", rewritten_ops) + + rng = np.random.default_rng(42) + inputs = [rng.random((1, 3, 32, 32), dtype=np.float32)] + testing.assert_numerically_equal(model, rewritten, inputs) + + def test_conv_affine_mixed_precision_scale_offset_preserves_dtypes(self): + """Mixed precision scalar operands (e.g. float64) must not alter Conv weight/bias float32 dtypes.""" + tape = ir.tape.Tape() + x = ir.val("x", dtype=ir.DataType.FLOAT, shape=ir.Shape([1, 3, 32, 32])) + w = tape.initializer(ir.tensor(np.ones((3, 3, 3, 3), dtype=np.float32), name="w")) + b = tape.initializer(ir.tensor(np.ones((3,), dtype=np.float32), name="b")) + scale = tape.initializer(ir.tensor(np.array(2.0, dtype=np.float64), name="scale")) + offset = tape.initializer(ir.tensor(np.array(3.0, dtype=np.float64), name="offset")) + + conv_out = tape.op("Conv", [x, w, b], attributes={"pads": [1, 1, 1, 1]}) + mul_out = tape.op("Mul", [conv_out, scale]) + z = tape.op( + "Add", + [mul_out, offset], + output=ir.val( + "z", + dtype=ir.DataType.FLOAT, + shape=ir.Shape([1, 3, 32, 32]), + ), + ) + model = ir.Model( + ir.Graph( + inputs=[x], + outputs=[z], + nodes=tape.nodes, + initializers=tape.initializers, + opset_imports={"": 17}, + name="test_graph", + ), + ir_version=8, + ) + rewritten = self._clone(model) + rewritten = rewrite(rewritten, pattern_rewrite_rules=[conv_affine_fusion_rule]) + conv_node = next(n for n in rewritten.graph if n.op_type == "Conv") + self.assertEqual(conv_node.inputs[1].const_value.dtype, ir.DataType.FLOAT) + self.assertEqual(conv_node.inputs[2].const_value.dtype, ir.DataType.FLOAT) + + import onnx + + proto = ir.serde.serialize_model(rewritten) + onnx.checker.check_model(proto, full_check=True) + + def test_conv_affine_reference_attribute_pads_rejected(self): + """Conv with reference attribute for pads must be safely rejected.""" + tape = ir.tape.Tape() + x = ir.val("x", dtype=ir.DataType.FLOAT, shape=ir.Shape([1, 3, 32, 32])) + w = tape.initializer(ir.tensor(np.ones((3, 3, 3, 3), dtype=np.float32), name="w")) + b = tape.initializer(ir.tensor(np.ones((3,), dtype=np.float32), name="b")) + scale = tape.initializer(ir.tensor(np.array(2.0, dtype=np.float32), name="scale")) + offset = tape.initializer(ir.tensor(np.array(3.0, dtype=np.float32), name="offset")) + + mul_out = tape.op("Mul", [x, scale]) + affine_x = tape.op("Add", [mul_out, offset]) + ref_attr = ir.Attr( + name="pads", + type=ir.AttributeType.INTS, + value=None, + ref_attr_name="pads_val", + ) + conv_out = tape.op("Conv", [affine_x, w, b], attributes={"pads": ref_attr}) + + model = ir.Model( + ir.Graph( + inputs=[x], + outputs=[conv_out], + nodes=tape.nodes, + initializers=tape.initializers, + opset_imports={"": 17}, + name="test_graph", + ), + ir_version=8, + ) + count = affine_conv_fusion_rule.apply_to_model(model) + self.assertEqual(count, 0) + self.assertIn("Mul", [n.op_type for n in model.graph]) + + def test_conv_affine_external_weights_nonconstant_scale_rejected_without_filenotfound( + self, + ): + """ConvAffineFusion with external weights and non-constant scale must reject without FileNotFoundError.""" + tape = ir.tape.Tape() + x = ir.val("x", dtype=ir.DataType.FLOAT, shape=ir.Shape([1, 3, 32, 32])) + scale = ir.val("scale", dtype=ir.DataType.FLOAT, shape=ir.Shape([1])) + offset = tape.initializer(ir.tensor(np.array(3.0, dtype=np.float32), name="offset")) + b = tape.initializer(ir.tensor(np.ones((3,), dtype=np.float32), name="b")) + w = tape.initializer( + ir.ExternalTensor( + location="nonexistent_weights.bin", + offset=0, + length=100, + dtype=ir.DataType.FLOAT, + shape=ir.Shape([3, 3, 3, 3]), + name="w", + ) + ) + + conv_out = tape.op("Conv", [x, w, b], attributes={"pads": [0, 0, 0, 0]}) + mul_out = tape.op("Mul", [conv_out, scale]) + z = tape.op( + "Add", + [mul_out, offset], + output=ir.val( + "z", + dtype=ir.DataType.FLOAT, + shape=ir.Shape([1, 3, 32, 32]), + ), + ) + + model = ir.Model( + ir.Graph( + inputs=[x, scale], + outputs=[z], + nodes=tape.nodes, + initializers=tape.initializers, + opset_imports={"": 17}, + name="test_graph", + ), + ir_version=8, + ) + count = conv_affine_fusion_rule.apply_to_model(model) + self.assertEqual(count, 0) + self.assertIn("Mul", [n.op_type for n in model.graph]) + + def test_affine_conv_external_weights_with_padding_rejected_without_filenotfound( + self, + ): + """AffineConvFusion with external weights and padding must reject without FileNotFoundError.""" + tape = ir.tape.Tape() + x = ir.val("x", dtype=ir.DataType.FLOAT, shape=ir.Shape([1, 3, 32, 32])) + scale = tape.initializer(ir.tensor(np.array(2.0, dtype=np.float32), name="scale")) + offset = tape.initializer(ir.tensor(np.array(3.0, dtype=np.float32), name="offset")) + b = tape.initializer(ir.tensor(np.ones((3,), dtype=np.float32), name="b")) + w = tape.initializer( + ir.ExternalTensor( + location="nonexistent_weights.bin", + offset=0, + length=100, + dtype=ir.DataType.FLOAT, + shape=ir.Shape([3, 3, 3, 3]), + name="w", + ) + ) + + mul_out = tape.op("Mul", [x, scale]) + affine_x = tape.op("Add", [mul_out, offset]) + conv_out = tape.op("Conv", [affine_x, w, b], attributes={"pads": [1, 1, 1, 1]}) + + model = ir.Model( + ir.Graph( + inputs=[x], + outputs=[conv_out], + nodes=tape.nodes, + initializers=tape.initializers, + opset_imports={"": 17}, + name="test_graph", + ), + ir_version=8, + ) + count = affine_conv_fusion_rule.apply_to_model(model) + self.assertEqual(count, 0) + self.assertIn("Mul", [n.op_type for n in model.graph]) + if __name__ == "__main__": unittest.main() diff --git a/onnxscript/rewriter/rules/common/_fuse_conv_affine_test.py b/onnxscript/rewriter/rules/common/_fuse_conv_affine_test.py index cbe8a8bf41..496f12e444 100644 --- a/onnxscript/rewriter/rules/common/_fuse_conv_affine_test.py +++ b/onnxscript/rewriter/rules/common/_fuse_conv_affine_test.py @@ -106,6 +106,51 @@ def test_affine_conv_fusion_without_pad(self): ] testing.assert_numerically_equal(model, rewritten_model, inputs) + def test_affine_conv_fusion_omitted_pads(self): + """Conv with omitted pads attribute defaults to zero-padding and should be fused.""" + tape = ir.tape.Tape() + x = ir.val("x", dtype=ir.DataType.FLOAT, shape=ir.Shape([1, 3, 32, 32])) + w = tape.initializer(ir.tensor(np.ones((3, 3, 3, 3), dtype=np.float32), name="w")) + b = tape.initializer(ir.tensor(np.ones((3,), dtype=np.float32), name="b")) + scale = tape.initializer(ir.tensor(np.array([2.0], dtype=np.float32), name="scale")) + offset = tape.initializer(ir.tensor(np.array([3.0], dtype=np.float32), name="offset")) + + mul_out = tape.op("Mul", [x, scale]) + z = tape.op( + "Add", + [mul_out, offset], + output=ir.val( + "z", + dtype=ir.DataType.FLOAT, + shape=ir.Shape([1, 3, 32, 32]), + ), + ) + conv_out = tape.op("Conv", [z, w, b]) # Omitted pads attribute + + model = ir.Model( + ir.Graph( + inputs=[x], + outputs=[conv_out], + nodes=tape.nodes, + initializers=tape.initializers, + opset_imports={"": 17}, + ), + ir_version=8, + ) + rewritten_model = self.clone_model(model) + rewritten_model = rewrite( + rewritten_model, + pattern_rewrite_rules=[affine_conv_fusion_rule], + ) + self.assertEqual(model.graph.num_nodes() - 2, rewritten_model.graph.num_nodes()) + self.assertEqual([node.op_type for node in rewritten_model.graph], ["Conv"]) + + rng = np.random.default_rng(42) + inputs = [ + rng.random((1, 3, 32, 32), dtype=np.float32), + ] + testing.assert_numerically_equal(model, rewritten_model, inputs) + if __name__ == "__main__": unittest.main()