Summary
Failed on commit: 5c2e86b
System MacOS.
____________________________________________________________________ test_character_level_conditioning[unpacked / no_image_encoder] _____________________________________________________________________
model_configuration = (ImageLatentTransformerForCausalLM(
(image_encoder): None
(bytes_encoder): BertForMaskedLM(
(bert): BertModel(...48,
"processor_class": "TextImageProcessor"
}
, functools.partial(<function collate_fn at 0x12fc33a60>, pad_value=0))
@parameterization
def test_character_level_conditioning(model_configuration):
"""Test 1: Character-level conditioning (a b vs a a, b a vs b b)"""
model, processor, collator = model_configuration
print("\n=== Test 1: Character-level conditioning ===")
test_texts_char = ["a b", "b a", "a a", "b b"]
losses, predictions = predict_dataset(test_texts_char, model, processor, collator)
# Check conditioning: trained sequences should have lower loss
assert losses['a b'] < losses['a a'], \
f"'a b' should have lower loss than 'a a': {losses['a b']:.4f} vs {losses['a a']:.4f}"
> assert losses['b a'] < losses['b b'], \
f"'b a' should have lower loss than 'b b': {losses['b a']:.4f} vs {losses['b b']:.4f}"
E AssertionError: 'b a' should have lower loss than 'b b': 0.4799 vs 0.4407
E assert 0.4799067974090576 < 0.4406731923421224
tests/test_model_overfitting.py:132: AssertionError
----------------------------------------------------------------------------------------- Captured stdout setup -----------------------------------------------------------------------------------------
[Configuration: Model without image encoder]
----------------------------------------------------------------------------------------- Captured stdout call ------------------------------------------------------------------------------------------
=== Test 1: Character-level conditioning ===
Loss for 'a b': 0.3614
Loss for 'b a': 0.4799
Loss for 'a a': 0.9456
Loss for 'b b': 0.4407
FAILED tests/test_model_overfitting.py::test_character_level_conditioning[unpacked / no_image_encoder] - AssertionError: 'b a' should have lower loss than 'b b': 0.4799 vs 0.4407
Reproduce
Summary
Failed on commit: 5c2e86b
System MacOS.
____________________________________________________________________ test_character_level_conditioning[unpacked / no_image_encoder] _____________________________________________________________________ model_configuration = (ImageLatentTransformerForCausalLM( (image_encoder): None (bytes_encoder): BertForMaskedLM( (bert): BertModel(...48, "processor_class": "TextImageProcessor" } , functools.partial(<function collate_fn at 0x12fc33a60>, pad_value=0)) @parameterization def test_character_level_conditioning(model_configuration): """Test 1: Character-level conditioning (a b vs a a, b a vs b b)""" model, processor, collator = model_configuration print("\n=== Test 1: Character-level conditioning ===") test_texts_char = ["a b", "b a", "a a", "b b"] losses, predictions = predict_dataset(test_texts_char, model, processor, collator) # Check conditioning: trained sequences should have lower loss assert losses['a b'] < losses['a a'], \ f"'a b' should have lower loss than 'a a': {losses['a b']:.4f} vs {losses['a a']:.4f}" > assert losses['b a'] < losses['b b'], \ f"'b a' should have lower loss than 'b b': {losses['b a']:.4f} vs {losses['b b']:.4f}" E AssertionError: 'b a' should have lower loss than 'b b': 0.4799 vs 0.4407 E assert 0.4799067974090576 < 0.4406731923421224 tests/test_model_overfitting.py:132: AssertionError ----------------------------------------------------------------------------------------- Captured stdout setup ----------------------------------------------------------------------------------------- [Configuration: Model without image encoder] ----------------------------------------------------------------------------------------- Captured stdout call ------------------------------------------------------------------------------------------ === Test 1: Character-level conditioning === Loss for 'a b': 0.3614 Loss for 'b a': 0.4799 Loss for 'a a': 0.9456 Loss for 'b b': 0.4407Reproduce