diff --git a/brainscore_language/models/falcon40b/__init__.py b/brainscore_language/models/falcon40b/__init__.py new file mode 100644 index 00000000..9ae312c8 --- /dev/null +++ b/brainscore_language/models/falcon40b/__init__.py @@ -0,0 +1,12 @@ +from brainscore_language import model_registry +from brainscore_language import ArtificialSubject +from brainscore_language.model_helpers.huggingface import HuggingfaceSubject + +# Falcon-40B: 60 transformer layers, hidden size 8192. +# Layer 59 (last) chosen as default mapping pending benchmark-driven selection. +model_registry['falcon-40b'] = lambda: HuggingfaceSubject( + model_id='tiiuae/falcon-40b', + region_layer_mapping={ + ArtificialSubject.RecordingTarget.language_system: 'transformer.h.59' + }, +) diff --git a/brainscore_language/models/falcon40b/metadata.yml b/brainscore_language/models/falcon40b/metadata.yml new file mode 100644 index 00000000..d2739763 --- /dev/null +++ b/brainscore_language/models/falcon40b/metadata.yml @@ -0,0 +1,15 @@ +models: + falcon40b: + architecture: DCNN + model_family: falcon40b + total_parameter_count: 1234567 + trainable_parameter_count: 1234567 + total_layers: 55 + trainable_layers: 40 + model_size_mb: 1202 + training_dataset: null + task_specialization: null + brainscore_link: https://github.com/brain-score/language/tree/master/brainscore_language/models/falcon40b + huggingface_link: null + extra_notes: Temporary hardcoded metadata - will be replaced with actual generation + runnable: true diff --git a/brainscore_language/models/falcon40b/test.py b/brainscore_language/models/falcon40b/test.py new file mode 100644 index 00000000..38f7999b --- /dev/null +++ b/brainscore_language/models/falcon40b/test.py @@ -0,0 +1,37 @@ +import numpy as np +import pytest + +from brainscore_language import load_model +from brainscore_language.artificial_subject import ArtificialSubject + + +@pytest.mark.memory_intense +def test_load_model(): + model = load_model('falcon-40b') + assert model is not None + + +@pytest.mark.memory_intense +def test_neural(): + model = load_model('falcon-40b') + text = ['the quick brown fox', 'jumps over', 'the lazy dog'] + model.start_neural_recording( + recording_target=ArtificialSubject.RecordingTarget.language_system, + recording_type=ArtificialSubject.RecordingType.fMRI, + ) + representations = model.digest_text(text)['neural'] + assert len(representations['presentation']) == 3 + np.testing.assert_array_equal(representations['stimulus'], text) + assert len(representations['neuroid']) == 8192 + + +@pytest.mark.memory_intense +def test_next_word(): + model = load_model('falcon-40b') + text = ['the quick brown fox', 'jumps over', 'the lazy'] + model.start_behavioral_task(task=ArtificialSubject.Task.next_word) + next_words = model.digest_text(text)['behavior'] + assert len(next_words) == 3 + for word in next_words.values: + assert isinstance(word, str) + assert len(word.strip()) > 0 diff --git a/brainscore_language/models/mistral_small_24b/__init__.py b/brainscore_language/models/mistral_small_24b/__init__.py new file mode 100644 index 00000000..66d05f9f --- /dev/null +++ b/brainscore_language/models/mistral_small_24b/__init__.py @@ -0,0 +1,12 @@ +from brainscore_language import model_registry +from brainscore_language import ArtificialSubject +from brainscore_language.model_helpers.huggingface import HuggingfaceSubject + +# Mistral-Small-24B: 40 transformer layers, hidden size 5120. +# Layer 39 (last) chosen as default mapping pending benchmark-driven selection. +model_registry['mistral-small-24b'] = lambda: HuggingfaceSubject( + model_id='mistralai/Mistral-Small-24B', + region_layer_mapping={ + ArtificialSubject.RecordingTarget.language_system: 'model.layers.39' + }, +) diff --git a/brainscore_language/models/mistral_small_24b/metadata.yml b/brainscore_language/models/mistral_small_24b/metadata.yml new file mode 100644 index 00000000..c83fd00b --- /dev/null +++ b/brainscore_language/models/mistral_small_24b/metadata.yml @@ -0,0 +1,15 @@ +models: + mistral_small_24b: + architecture: DCNN + model_family: mistral_small_24b + total_parameter_count: 1234567 + trainable_parameter_count: 1234567 + total_layers: 55 + trainable_layers: 40 + model_size_mb: 1202 + training_dataset: null + task_specialization: null + brainscore_link: https://github.com/brain-score/language/tree/master/brainscore_language/models/mistral_small_24b + huggingface_link: null + extra_notes: Temporary hardcoded metadata - will be replaced with actual generation + runnable: true diff --git a/brainscore_language/models/mistral_small_24b/test.py b/brainscore_language/models/mistral_small_24b/test.py new file mode 100644 index 00000000..3d08f641 --- /dev/null +++ b/brainscore_language/models/mistral_small_24b/test.py @@ -0,0 +1,37 @@ +import numpy as np +import pytest + +from brainscore_language import load_model +from brainscore_language.artificial_subject import ArtificialSubject + + +@pytest.mark.memory_intense +def test_load_model(): + model = load_model('mistral-small-24b') + assert model is not None + + +@pytest.mark.memory_intense +def test_neural(): + model = load_model('mistral-small-24b') + text = ['the quick brown fox', 'jumps over', 'the lazy dog'] + model.start_neural_recording( + recording_target=ArtificialSubject.RecordingTarget.language_system, + recording_type=ArtificialSubject.RecordingType.fMRI, + ) + representations = model.digest_text(text)['neural'] + assert len(representations['presentation']) == 3 + np.testing.assert_array_equal(representations['stimulus'], text) + assert len(representations['neuroid']) == 5120 + + +@pytest.mark.memory_intense +def test_next_word(): + model = load_model('mistral-small-24b') + text = ['the quick brown fox', 'jumps over', 'the lazy'] + model.start_behavioral_task(task=ArtificialSubject.Task.next_word) + next_words = model.digest_text(text)['behavior'] + assert len(next_words) == 3 + for word in next_words.values: + assert isinstance(word, str) + assert len(word.strip()) > 0 diff --git a/brainscore_language/models/mpt30b/__init__.py b/brainscore_language/models/mpt30b/__init__.py new file mode 100644 index 00000000..343272fd --- /dev/null +++ b/brainscore_language/models/mpt30b/__init__.py @@ -0,0 +1,12 @@ +from brainscore_language import model_registry +from brainscore_language import ArtificialSubject +from brainscore_language.model_helpers.huggingface import HuggingfaceSubject + +# MPT-30B: 48 transformer layers, hidden size 7168. +# Layer 47 (last) chosen as default mapping pending benchmark-driven selection. +model_registry['mpt-30b'] = lambda: HuggingfaceSubject( + model_id='mosaicml/mpt-30b', + region_layer_mapping={ + ArtificialSubject.RecordingTarget.language_system: 'transformer.blocks.47' + }, +) diff --git a/brainscore_language/models/mpt30b/metadata.yml b/brainscore_language/models/mpt30b/metadata.yml new file mode 100644 index 00000000..6b34024a --- /dev/null +++ b/brainscore_language/models/mpt30b/metadata.yml @@ -0,0 +1,15 @@ +models: + mpt30b: + architecture: DCNN + model_family: mpt30b + total_parameter_count: 1234567 + trainable_parameter_count: 1234567 + total_layers: 55 + trainable_layers: 40 + model_size_mb: 1202 + training_dataset: null + task_specialization: null + brainscore_link: https://github.com/brain-score/language/tree/master/brainscore_language/models/mpt30b + huggingface_link: null + extra_notes: Temporary hardcoded metadata - will be replaced with actual generation + runnable: true diff --git a/brainscore_language/models/mpt30b/test.py b/brainscore_language/models/mpt30b/test.py new file mode 100644 index 00000000..1fc2de84 --- /dev/null +++ b/brainscore_language/models/mpt30b/test.py @@ -0,0 +1,37 @@ +import numpy as np +import pytest + +from brainscore_language import load_model +from brainscore_language.artificial_subject import ArtificialSubject + + +@pytest.mark.memory_intense +def test_load_model(): + model = load_model('mpt-30b') + assert model is not None + + +@pytest.mark.memory_intense +def test_neural(): + model = load_model('mpt-30b') + text = ['the quick brown fox', 'jumps over', 'the lazy dog'] + model.start_neural_recording( + recording_target=ArtificialSubject.RecordingTarget.language_system, + recording_type=ArtificialSubject.RecordingType.fMRI, + ) + representations = model.digest_text(text)['neural'] + assert len(representations['presentation']) == 3 + np.testing.assert_array_equal(representations['stimulus'], text) + assert len(representations['neuroid']) == 7168 + + +@pytest.mark.memory_intense +def test_next_word(): + model = load_model('mpt-30b') + text = ['the quick brown fox', 'jumps over', 'the lazy'] + model.start_behavioral_task(task=ArtificialSubject.Task.next_word) + next_words = model.digest_text(text)['behavior'] + assert len(next_words) == 3 + for word in next_words.values: + assert isinstance(word, str) + assert len(word.strip()) > 0 diff --git a/brainscore_language/models/opt30b/__init__.py b/brainscore_language/models/opt30b/__init__.py new file mode 100644 index 00000000..a0e2a665 --- /dev/null +++ b/brainscore_language/models/opt30b/__init__.py @@ -0,0 +1,12 @@ +from brainscore_language import model_registry +from brainscore_language import ArtificialSubject +from brainscore_language.model_helpers.huggingface import HuggingfaceSubject + +# OPT-30B: 48 transformer layers, hidden size 7168. +# Layer 47 (last) chosen as default mapping pending benchmark-driven selection. +model_registry['opt-30b'] = lambda: HuggingfaceSubject( + model_id='facebook/opt-30b', + region_layer_mapping={ + ArtificialSubject.RecordingTarget.language_system: 'model.decoder.layers.47' + }, +) diff --git a/brainscore_language/models/opt30b/metadata.yml b/brainscore_language/models/opt30b/metadata.yml new file mode 100644 index 00000000..5cb43b75 --- /dev/null +++ b/brainscore_language/models/opt30b/metadata.yml @@ -0,0 +1,15 @@ +models: + opt30b: + architecture: DCNN + model_family: opt30b + total_parameter_count: 1234567 + trainable_parameter_count: 1234567 + total_layers: 55 + trainable_layers: 40 + model_size_mb: 1202 + training_dataset: null + task_specialization: null + brainscore_link: https://github.com/brain-score/language/tree/master/brainscore_language/models/opt30b + huggingface_link: null + extra_notes: Temporary hardcoded metadata - will be replaced with actual generation + runnable: true diff --git a/brainscore_language/models/opt30b/test.py b/brainscore_language/models/opt30b/test.py new file mode 100644 index 00000000..bf0b2968 --- /dev/null +++ b/brainscore_language/models/opt30b/test.py @@ -0,0 +1,37 @@ +import numpy as np +import pytest + +from brainscore_language import load_model +from brainscore_language.artificial_subject import ArtificialSubject + + +@pytest.mark.memory_intense +def test_load_model(): + model = load_model('opt-30b') + assert model is not None + + +@pytest.mark.memory_intense +def test_neural(): + model = load_model('opt-30b') + text = ['the quick brown fox', 'jumps over', 'the lazy dog'] + model.start_neural_recording( + recording_target=ArtificialSubject.RecordingTarget.language_system, + recording_type=ArtificialSubject.RecordingType.fMRI, + ) + representations = model.digest_text(text)['neural'] + assert len(representations['presentation']) == 3 + np.testing.assert_array_equal(representations['stimulus'], text) + assert len(representations['neuroid']) == 7168 + + +@pytest.mark.memory_intense +def test_next_word(): + model = load_model('opt-30b') + text = ['the quick brown fox', 'jumps over', 'the lazy'] + model.start_behavioral_task(task=ArtificialSubject.Task.next_word) + next_words = model.digest_text(text)['behavior'] + assert len(next_words) == 3 + for word in next_words.values: + assert isinstance(word, str) + assert len(word.strip()) > 0 diff --git a/brainscore_language/models/qwen2_5_14b/__init__.py b/brainscore_language/models/qwen2_5_14b/__init__.py new file mode 100644 index 00000000..06baf861 --- /dev/null +++ b/brainscore_language/models/qwen2_5_14b/__init__.py @@ -0,0 +1,12 @@ +from brainscore_language import model_registry +from brainscore_language import ArtificialSubject +from brainscore_language.model_helpers.huggingface import HuggingfaceSubject + +# Qwen2.5-14B: 48 transformer layers, hidden size 5120. +# Layer 47 (last) chosen as default mapping pending benchmark-driven selection. +model_registry['qwen2.5-14b'] = lambda: HuggingfaceSubject( + model_id='Qwen/Qwen2.5-14B', + region_layer_mapping={ + ArtificialSubject.RecordingTarget.language_system: 'model.layers.47' + }, +) diff --git a/brainscore_language/models/qwen2_5_14b/metadata.yml b/brainscore_language/models/qwen2_5_14b/metadata.yml new file mode 100644 index 00000000..ad845523 --- /dev/null +++ b/brainscore_language/models/qwen2_5_14b/metadata.yml @@ -0,0 +1,15 @@ +models: + qwen2_5_14b: + architecture: DCNN + model_family: qwen2_5_14b + total_parameter_count: 1234567 + trainable_parameter_count: 1234567 + total_layers: 55 + trainable_layers: 40 + model_size_mb: 1202 + training_dataset: null + task_specialization: null + brainscore_link: https://github.com/brain-score/language/tree/master/brainscore_language/models/qwen2_5_14b + huggingface_link: null + extra_notes: Temporary hardcoded metadata - will be replaced with actual generation + runnable: true diff --git a/brainscore_language/models/qwen2_5_14b/test.py b/brainscore_language/models/qwen2_5_14b/test.py new file mode 100644 index 00000000..eff15947 --- /dev/null +++ b/brainscore_language/models/qwen2_5_14b/test.py @@ -0,0 +1,37 @@ +import numpy as np +import pytest + +from brainscore_language import load_model +from brainscore_language.artificial_subject import ArtificialSubject + + +@pytest.mark.memory_intense +def test_load_model(): + model = load_model('qwen2.5-14b') + assert model is not None + + +@pytest.mark.memory_intense +def test_neural(): + model = load_model('qwen2.5-14b') + text = ['the quick brown fox', 'jumps over', 'the lazy dog'] + model.start_neural_recording( + recording_target=ArtificialSubject.RecordingTarget.language_system, + recording_type=ArtificialSubject.RecordingType.fMRI, + ) + representations = model.digest_text(text)['neural'] + assert len(representations['presentation']) == 3 + np.testing.assert_array_equal(representations['stimulus'], text) + assert len(representations['neuroid']) == 5120 + + +@pytest.mark.memory_intense +def test_next_word(): + model = load_model('qwen2.5-14b') + text = ['the quick brown fox', 'jumps over', 'the lazy'] + model.start_behavioral_task(task=ArtificialSubject.Task.next_word) + next_words = model.digest_text(text)['behavior'] + assert len(next_words) == 3 + for word in next_words.values: + assert isinstance(word, str) + assert len(word.strip()) > 0