Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions examples/tutorial/tensor_ops_tutorial.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,9 +101,12 @@ def check(name, ark_tensor, expected, atol=1e-5):
# Activations
check("relu", a_relu, np.maximum(x_np, 0))

# GELU: approximate check (ARK uses erff-based GELU)

gelu_ref = 0.5 * x_np * (1 + np.vectorize(math.erf)(x_np / np.sqrt(2)))
# GELU: ARK uses the tanh approximation
gelu_ref = (
0.5
* x_np
* (1 + np.tanh(np.sqrt(2 / np.pi) * (x_np + 0.044715 * x_np**3)))
)
check("gelu", a_gelu, gelu_ref, atol=1e-4)

sig_ref = 1.0 / (1.0 + np.exp(-x_np))
Expand Down
2 changes: 1 addition & 1 deletion python/ark/ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -673,7 +673,7 @@ def all_reduce_packet(
rank: int,
world_size: int,
output: Tensor = NullTensor,
name: str = "",
name: str = "all_reduce_packet",
) -> Tensor:
"""
Packet-based intra-node all-reduce — single-shot reduce-scatter + allgather using
Expand Down
8 changes: 6 additions & 2 deletions python/unittest/ops/test_math.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ def test_exp(dtype):
assert torch.allclose(ark.exp(a).eval(), torch.exp(a), atol=atol, rtol=0)


@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@pytest.mark.parametrize(
"dtype", [torch.float32, torch.float16, torch.bfloat16]
)
def test_gelu(dtype):
a = torch.randn(4, 2, 1024, dtype=dtype, device=DEVICE)
atol = 1e-5 if dtype == torch.float32 else 1e-2
Expand All @@ -39,7 +41,9 @@ def test_relu(dtype):
assert torch.allclose(ark.relu(a).eval(), F.relu(a), atol=0, rtol=0)


@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@pytest.mark.parametrize(
"dtype", [torch.float32, torch.float16, torch.bfloat16]
)
def test_sigmoid(dtype):
a = torch.randn(4, 2, 1024, dtype=dtype, device=DEVICE)
atol = 1e-5 if dtype == torch.float32 else 1e-2
Expand Down
7 changes: 7 additions & 0 deletions python/unittest/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,10 @@
from test_tensor import *
from test_conversion import *
from test_eval import *
from test_data_type_edges import *
from test_model_edges import *
from test_module import *
from test_ops_edges import *
from test_planner_edges import *
from test_serialize import *
from test_tensor_edges import *
2 changes: 1 addition & 1 deletion python/unittest/test_data_type_edges.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def test_data_type_from_torch_unknown():
ark.DataType.from_torch(torch.float64)


@pytest_ark()
@pytest_ark(need_torch=True)
def test_data_type_bf16_torch_type():
"""bf16 to_torch returns bfloat16."""
import torch
Expand Down
Loading