Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 73 additions & 9 deletions onnxscript/rewriter/rules/common/_fuse_conv_affine.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,26 @@ def check(
conv_out: ir.Value,
) -> MatchResult:
check_result = MatchResult()
if get_const_value(w) is None:
return check_result.fail("The weight of Conv should be constant")
if get_const_value(b) is None:
return check_result.fail("The bias of Conv should be constant")
if get_singleton_value(scale) is None:
return check_result.fail("Operand for Mul should be constant scalar value")
if get_singleton_value(offset) is None:
return check_result.fail("Operand for Add should be constant scalar value")
w_val = get_const_value(w)
if w_val is None:
return check_result.fail("The weight of Conv should be constant")
b_val = get_const_value(b)
if b_val is None:
return check_result.fail("The bias of Conv should be constant")
w_shape = w_val.shape
b_shape = b_val.shape
if len(w_shape) < 3:
return check_result.fail("Conv weight tensor must have rank >= 3")
if len(b_shape) != 1:
return check_result.fail("Conv bias tensor must have rank 1")
if w_shape[0] != b_shape[0]:
return check_result.fail(
f"Conv weight output channels ({w_shape[0]}) does not match bias channels ({b_shape[0]})"
)
return check_result


Expand All @@ -46,11 +58,54 @@ def pattern(self, op, x, w, b, scale, offset):
x * scale + offset,
w,
b,
pads=[0, 0, 0, 0],
_allow_other_attributes=True,
_outputs=["conv_out"],
)

def check(
self,
context,
x: ir.Value,
w: ir.Value,
b: ir.Value,
scale: ir.Value,
offset: ir.Value,
conv_out: ir.Value,
) -> MatchResult:
check_result = MatchResult()
conv_node = conv_out.producer()
if conv_node is None:
return check_result.fail("Conv node not found")
pads_attr = conv_node.attributes.get("pads")
if pads_attr is not None:
if hasattr(pads_attr, "is_ref") and pads_attr.is_ref():
return check_result.fail(
f"Conv 'pads' refers to reference attribute {pads_attr.ref_attr_name!r}"
)
try:
pads = list(pads_attr.as_ints())
if any(p != 0 for p in pads):
return check_result.fail(f"Conv has non-zero pads: {pads}")
except (TypeError, ValueError, AttributeError):
return check_result.fail("Conv 'pads' attribute could not be read as integers")
auto_pad_attr = conv_node.attributes.get("auto_pad")
if auto_pad_attr is not None:
if hasattr(auto_pad_attr, "is_ref") and auto_pad_attr.is_ref():
return check_result.fail(
f"Conv 'auto_pad' refers to reference attribute {auto_pad_attr.ref_attr_name!r}"
)
try:
auto_pad = auto_pad_attr.as_string()
if auto_pad not in {"NOTSET", "VALID"}:
return check_result.fail(
f"Conv 'auto_pad' {auto_pad!r} introduces padding"
)
except (TypeError, ValueError, AttributeError):
return check_result.fail(
"Conv 'auto_pad' attribute could not be read as string"
)
return super().check(context, x, w, b, scale, offset, conv_out)

def rewrite(
self,
op,
Expand All @@ -65,9 +120,15 @@ def rewrite(
offset_value = offset.const_value.numpy()
w_value = w.const_value.numpy()
b_value = b.const_value.numpy()
scaled_w_value = op.initializer(ir.tensor(w_value * scale_value), w.name + "_scaled")
scaled_w_value = op.initializer(
ir.tensor((w_value * scale_value).astype(w_value.dtype)),
w.name + "_scaled",
)
axes = tuple(range(1, w_value.ndim))
offset_bias = ir.tensor(
b_value + np.sum(w_value * offset_value, axis=(1, 2, 3), keepdims=False)
(b_value + np.sum(w_value * offset_value, axis=axes, keepdims=False)).astype(
b_value.dtype
)
)
offset_bias = op.initializer(offset_bias, b.name + "_offset")
conv_attributes = conv_out.producer().attributes
Expand Down Expand Up @@ -97,8 +158,11 @@ def rewrite(
offset_value = offset.const_value.numpy()
w_value = w.const_value.numpy()
b_value = b.const_value.numpy()
scaled_w_weight = op.initializer(ir.tensor(w_value * scale_value), w.name + "_scaled")
offset_bias = ir.tensor(b_value * scale_value + offset_value)
scaled_w_weight = op.initializer(
ir.tensor((w_value * scale_value).astype(w_value.dtype)),
w.name + "_scaled",
)
offset_bias = ir.tensor((b_value * scale_value + offset_value).astype(b_value.dtype))
offset_bias = op.initializer(offset_bias, b.name + "_offset")
conv_attributes = conv_out.producer().attributes
return op.Conv(x, scaled_w_weight, offset_bias, **conv_attributes)
Expand Down
205 changes: 205 additions & 0 deletions onnxscript/rewriter/rules/common/_fuse_conv_affine_extended_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,211 @@ def model_fn(x: FLOAT[1, 3, 32, 32]) -> FLOAT[1, 3, 32, 32]:
inputs = [rng.random((1, 3, 32, 32), dtype=np.float32)]
testing.assert_numerically_equal(model, rewritten, inputs)

def test_affine_conv_auto_pad_same_upper_rejected(self):
"""Conv with auto_pad='SAME_UPPER' introduces padding and must be rejected."""

@script()
def model_fn(x: FLOAT[1, 3, 32, 32]) -> FLOAT[1, 3, 32, 32]:
w = op.Constant(value=_W_ONES)
b = op.Constant(value=_B_ONES)
scale = op.Constant(value_float=2.0)
offset = op.Constant(value_float=3.0)
affine_x = (x * scale) + offset
return op.Conv(affine_x, w, b, auto_pad="SAME_UPPER")

model_proto = model_fn.to_model_proto()
model = ir.serde.deserialize_model(model_proto)
rewritten = self._clone(model)
rewritten = rewrite(rewritten, pattern_rewrite_rules=[affine_conv_fusion_rule])
self.assertEqual(model.graph.num_nodes(), rewritten.graph.num_nodes())

def test_affine_conv_auto_pad_valid_accepted(self):
"""Conv with auto_pad='VALID' has zero padding and should fuse into Conv."""

@script()
def model_fn(x: FLOAT[1, 3, 32, 32]) -> FLOAT[1, 3, 30, 30]:
w = op.Constant(value=_W_ONES)
b = op.Constant(value=_B_ONES)
scale = op.Constant(value_float=2.0)
offset = op.Constant(value_float=3.0)
affine_x = (x * scale) + offset
return op.Conv(affine_x, w, b, auto_pad="VALID")

model_proto = model_fn.to_model_proto()
model = ir.serde.deserialize_model(model_proto)
rewritten = self._clone(model)
rewritten = rewrite(rewritten, pattern_rewrite_rules=[affine_conv_fusion_rule])
rewritten_ops = [n.op_type for n in rewritten.graph]
self.assertNotIn("Mul", rewritten_ops)
self.assertNotIn("Add", rewritten_ops)
self.assertIn("Conv", rewritten_ops)

rng = np.random.default_rng(42)
inputs = [rng.random((1, 3, 32, 32), dtype=np.float32)]
testing.assert_numerically_equal(model, rewritten, inputs)

def test_conv_affine_mixed_precision_scale_offset_preserves_dtypes(self):
"""Mixed precision scalar operands (e.g. float64) must not alter Conv weight/bias float32 dtypes."""
tape = ir.tape.Tape()
x = ir.val("x", dtype=ir.DataType.FLOAT, shape=ir.Shape([1, 3, 32, 32]))
w = tape.initializer(ir.tensor(np.ones((3, 3, 3, 3), dtype=np.float32), name="w"))
b = tape.initializer(ir.tensor(np.ones((3,), dtype=np.float32), name="b"))
scale = tape.initializer(ir.tensor(np.array(2.0, dtype=np.float64), name="scale"))
offset = tape.initializer(ir.tensor(np.array(3.0, dtype=np.float64), name="offset"))

conv_out = tape.op("Conv", [x, w, b], attributes={"pads": [1, 1, 1, 1]})
mul_out = tape.op("Mul", [conv_out, scale])
z = tape.op(
"Add",
[mul_out, offset],
output=ir.val(
"z",
dtype=ir.DataType.FLOAT,
shape=ir.Shape([1, 3, 32, 32]),
),
)
model = ir.Model(
ir.Graph(
inputs=[x],
outputs=[z],
nodes=tape.nodes,
initializers=tape.initializers,
opset_imports={"": 17},
name="test_graph",
),
ir_version=8,
)
rewritten = self._clone(model)
rewritten = rewrite(rewritten, pattern_rewrite_rules=[conv_affine_fusion_rule])
conv_node = next(n for n in rewritten.graph if n.op_type == "Conv")
self.assertEqual(conv_node.inputs[1].const_value.dtype, ir.DataType.FLOAT)
self.assertEqual(conv_node.inputs[2].const_value.dtype, ir.DataType.FLOAT)

import onnx

proto = ir.serde.serialize_model(rewritten)
onnx.checker.check_model(proto, full_check=True)

def test_conv_affine_reference_attribute_pads_rejected(self):
"""Conv with reference attribute for pads must be safely rejected."""
tape = ir.tape.Tape()
x = ir.val("x", dtype=ir.DataType.FLOAT, shape=ir.Shape([1, 3, 32, 32]))
w = tape.initializer(ir.tensor(np.ones((3, 3, 3, 3), dtype=np.float32), name="w"))
b = tape.initializer(ir.tensor(np.ones((3,), dtype=np.float32), name="b"))
scale = tape.initializer(ir.tensor(np.array(2.0, dtype=np.float32), name="scale"))
offset = tape.initializer(ir.tensor(np.array(3.0, dtype=np.float32), name="offset"))

mul_out = tape.op("Mul", [x, scale])
affine_x = tape.op("Add", [mul_out, offset])
ref_attr = ir.Attr(
name="pads",
type=ir.AttributeType.INTS,
value=None,
ref_attr_name="pads_val",
)
conv_out = tape.op("Conv", [affine_x, w, b], attributes={"pads": ref_attr})

model = ir.Model(
ir.Graph(
inputs=[x],
outputs=[conv_out],
nodes=tape.nodes,
initializers=tape.initializers,
opset_imports={"": 17},
name="test_graph",
),
ir_version=8,
)
count = affine_conv_fusion_rule.apply_to_model(model)
self.assertEqual(count, 0)
self.assertIn("Mul", [n.op_type for n in model.graph])

def test_conv_affine_external_weights_nonconstant_scale_rejected_without_filenotfound(
self,
):
"""ConvAffineFusion with external weights and non-constant scale must reject without FileNotFoundError."""
tape = ir.tape.Tape()
x = ir.val("x", dtype=ir.DataType.FLOAT, shape=ir.Shape([1, 3, 32, 32]))
scale = ir.val("scale", dtype=ir.DataType.FLOAT, shape=ir.Shape([1]))
offset = tape.initializer(ir.tensor(np.array(3.0, dtype=np.float32), name="offset"))
b = tape.initializer(ir.tensor(np.ones((3,), dtype=np.float32), name="b"))
w = tape.initializer(
ir.ExternalTensor(
location="nonexistent_weights.bin",
offset=0,
length=100,
dtype=ir.DataType.FLOAT,
shape=ir.Shape([3, 3, 3, 3]),
name="w",
)
)

conv_out = tape.op("Conv", [x, w, b], attributes={"pads": [0, 0, 0, 0]})
mul_out = tape.op("Mul", [conv_out, scale])
z = tape.op(
"Add",
[mul_out, offset],
output=ir.val(
"z",
dtype=ir.DataType.FLOAT,
shape=ir.Shape([1, 3, 32, 32]),
),
)

model = ir.Model(
ir.Graph(
inputs=[x, scale],
outputs=[z],
nodes=tape.nodes,
initializers=tape.initializers,
opset_imports={"": 17},
name="test_graph",
),
ir_version=8,
)
count = conv_affine_fusion_rule.apply_to_model(model)
self.assertEqual(count, 0)
self.assertIn("Mul", [n.op_type for n in model.graph])

def test_affine_conv_external_weights_with_padding_rejected_without_filenotfound(
self,
):
"""AffineConvFusion with external weights and padding must reject without FileNotFoundError."""
tape = ir.tape.Tape()
x = ir.val("x", dtype=ir.DataType.FLOAT, shape=ir.Shape([1, 3, 32, 32]))
scale = tape.initializer(ir.tensor(np.array(2.0, dtype=np.float32), name="scale"))
offset = tape.initializer(ir.tensor(np.array(3.0, dtype=np.float32), name="offset"))
b = tape.initializer(ir.tensor(np.ones((3,), dtype=np.float32), name="b"))
w = tape.initializer(
ir.ExternalTensor(
location="nonexistent_weights.bin",
offset=0,
length=100,
dtype=ir.DataType.FLOAT,
shape=ir.Shape([3, 3, 3, 3]),
name="w",
)
)

mul_out = tape.op("Mul", [x, scale])
affine_x = tape.op("Add", [mul_out, offset])
conv_out = tape.op("Conv", [affine_x, w, b], attributes={"pads": [1, 1, 1, 1]})

model = ir.Model(
ir.Graph(
inputs=[x],
outputs=[conv_out],
nodes=tape.nodes,
initializers=tape.initializers,
opset_imports={"": 17},
name="test_graph",
),
ir_version=8,
)
count = affine_conv_fusion_rule.apply_to_model(model)
self.assertEqual(count, 0)
self.assertIn("Mul", [n.op_type for n in model.graph])


if __name__ == "__main__":
unittest.main()
45 changes: 45 additions & 0 deletions onnxscript/rewriter/rules/common/_fuse_conv_affine_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,51 @@ def test_affine_conv_fusion_without_pad(self):
]
testing.assert_numerically_equal(model, rewritten_model, inputs)

def test_affine_conv_fusion_omitted_pads(self):
"""Conv with omitted pads attribute defaults to zero-padding and should be fused."""
tape = ir.tape.Tape()
x = ir.val("x", dtype=ir.DataType.FLOAT, shape=ir.Shape([1, 3, 32, 32]))
w = tape.initializer(ir.tensor(np.ones((3, 3, 3, 3), dtype=np.float32), name="w"))
b = tape.initializer(ir.tensor(np.ones((3,), dtype=np.float32), name="b"))
scale = tape.initializer(ir.tensor(np.array([2.0], dtype=np.float32), name="scale"))
offset = tape.initializer(ir.tensor(np.array([3.0], dtype=np.float32), name="offset"))

mul_out = tape.op("Mul", [x, scale])
z = tape.op(
"Add",
[mul_out, offset],
output=ir.val(
"z",
dtype=ir.DataType.FLOAT,
shape=ir.Shape([1, 3, 32, 32]),
),
)
conv_out = tape.op("Conv", [z, w, b]) # Omitted pads attribute

model = ir.Model(
ir.Graph(
inputs=[x],
outputs=[conv_out],
nodes=tape.nodes,
initializers=tape.initializers,
opset_imports={"": 17},
),
ir_version=8,
)
rewritten_model = self.clone_model(model)
rewritten_model = rewrite(
rewritten_model,
pattern_rewrite_rules=[affine_conv_fusion_rule],
)
self.assertEqual(model.graph.num_nodes() - 2, rewritten_model.graph.num_nodes())
self.assertEqual([node.op_type for node in rewritten_model.graph], ["Conv"])

rng = np.random.default_rng(42)
inputs = [
rng.random((1, 3, 32, 32), dtype=np.float32),
]
testing.assert_numerically_equal(model, rewritten_model, inputs)


if __name__ == "__main__":
unittest.main()