diff --git a/docs/CHANGELOG.rst b/docs/CHANGELOG.rst index e394e1bd..ae539842 100644 --- a/docs/CHANGELOG.rst +++ b/docs/CHANGELOG.rst @@ -3,7 +3,7 @@ Changelog This is a log of the latest changes and improvements to KLibs. -0.8.0a1 +0.7.8a1 ------- (Unreleased) @@ -18,6 +18,10 @@ Runtime Changes: *all* remaining trials, which could unexpectedly affect the even distribution of trial factors within blocks. * Practice blocks are no longer added when `P.run_practice_blocks` is False. +* During runtime, `P.trials_per_block` is now updated at the start of each + block to reflect the actual number of trials in the current block. +* Added new parameter `P.max_trials_per_block` for temporarily limiting the + number of trials to run per block for testing and development purposes. API Changes: @@ -36,6 +40,17 @@ API Changes: * Added a new method :method:`~klibs.KLExperiment.write_trials_txt` for exporting the full sequence of generated trials and blocks to a human readable text file. +* Added a new flexible API for specifying custom block structures using the new + :class:`~klibs.KLStructure.Block` class. If a variable named `structure` + exists in a project's `_independent_variables.py` file and is defined as a + list of :class:`~klibs.KLStructure.Block` objects, the KLibs runtime will + use the specified structure to define the block/trial sequence for the + experiment. +* Added a new exception type :class:`~klibs.KLExceptions.TerminateBlock` that + can be raised to end a block early, allowing for blocks with flexible lengths + based on participant performance (e.g. practice blocks that continue until a + minimum performance threshold is met). + Fixed Bugs: @@ -48,6 +63,7 @@ Fixed Bugs: and 1000 ms already elapsed on the clock, making stimuli appear sooner than expected. + 0.7.7b1 ------- diff --git a/klibs/KLExceptions.py b/klibs/KLExceptions.py index fbbb1bac..f0cd22f2 100755 --- a/klibs/KLExceptions.py +++ b/klibs/KLExceptions.py @@ -27,6 +27,21 @@ def __init__(self, msg): def __str__(self): return self.message +class TerminateBlock(Exception): + """Exception that ends a block of trials early when raised. + + Intended for studies using titration or other situations where block length is + based on participant performance and should end when a threshold is met. + + .. note:: When TermiateBlock is raised the block will end immediately, meaning that + if raised within a trial any data from that trial will not be saved. + To ensure that data for the final trial is recorded, you can check for the + termination criteria and raise the exception within `self.trial_prep()`. + + """ + def __init__(self, msg=""): + self.message = msg + class EyeTrackerError(Exception): """Raised when a problem relating to an :obj:`~klibs.KLEyeTracking.KLEyeTracker.EyeTracker` object or the misuse of eye event inspect/report types is encountered. diff --git a/klibs/KLExperiment.py b/klibs/KLExperiment.py index 43fb6738..deaa9886 100755 --- a/klibs/KLExperiment.py +++ b/klibs/KLExperiment.py @@ -8,7 +8,7 @@ from klibs import P from klibs.KLEnvironment import EnvAgent -from klibs.KLExceptions import TrialException +from klibs.KLExceptions import TrialException, TerminateBlock from klibs.KLInternal import full_trace, iterable from klibs.KLInternal import colored_stdout as cso @@ -37,9 +37,8 @@ def __init__(self): self._evm = EventManager() self._exp_factors = self._get_exp_factors() + self._exp_structure = self._get_exp_structure() self.trial_factory = TrialFactory(self._exp_factors) - if P.manual_trial_generation is False: - self.trial_factory.generate() def _get_exp_factors(self): @@ -56,6 +55,22 @@ def _get_exp_factors(self): return factors + def _get_exp_structure(self): + # Reads in the block structure for the study (if provided) + from klibs.KLTrialFactory import _load_structure + + # Load experiment structure from the project's _independent_variables.py file(s) + structure = _load_structure(P.ind_vars_file_path) + if structure and os.path.exists(P.ind_vars_file_local_path): + if not P.dm_ignore_local_overrides: + # If structure exists in overrides, use that instead + local = _load_structure(P.ind_vars_file_local_path) + if local: + structure = local + + return structure + + def __execute_experiment__(self, *args, **kwargs): """For internal use, actually runs the blocks/trials of the experiment in sequence. @@ -69,11 +84,14 @@ def __execute_experiment__(self, *args, **kwargs): for block in self.blocks: P.recycle_count = 0 P.block_number += 1 + P.trials_per_block = len(block) P.practicing = block.practice self.block_label = block.label self.block() P.trial_number = 1 remaining = list(block.trials) + if P.max_trials_per_block != False: + remaining = remaining[:P.max_trials_per_block] while len(remaining): trial = remaining.pop(0) try: @@ -83,6 +101,8 @@ def __execute_experiment__(self, *args, **kwargs): except TrialException: remaining = self._recycle_trial(remaining, trial) P.recycle_count += 1 + except TerminateBlock: + remaining = [] self.rc.reset() self.clean_up() @@ -119,13 +139,13 @@ def __trial__(self, trial): self.evm.start_clock() # Actually run the trial and log the data to the database - recycle = None + exc = None try: P.in_trial = True self.__log_trial__(self.trial()) P.in_trial = False - except TrialException as e: - recycle = e + except (TrialException, TerminateBlock) as e: + exc = e # Clean up after the trial self.evm.stop_clock() @@ -135,9 +155,9 @@ def __trial__(self, trial): hide_cursor() self.trial_clean_up() - # Recycle trial if TrialException encountered - if recycle: - raise recycle + # Raise TerminateBlock or TrialException if encountered + if exc: + raise exc def __log_trial__(self, trial_data): @@ -285,10 +305,11 @@ def insert_practice_block(self, block_nums, trial_counts=None, factor_mask=None) # keyword, however. if self.blocks: - # If setup has passed and trial execution has started, blocks have already been exported - # from trial_factory so this function will no longer work. If it is called after it is no - # longer useful, we throw a TrialException - raise RuntimeError("Cannot insert practice blocks after setup() is complete.") + if self._exp_structure: + e = "in setup() when using a custom block structure." + else: + e = "after setup() is complete." + raise RuntimeError("Cannot insert practice blocks " + e) if not trial_counts: trial_counts = P.trials_per_block @@ -401,6 +422,7 @@ def run(self, *args, **kwargs): """ from klibs.KLGraphics.KLDraw import Ellipse + from klibs.KLTrialFactory import _parse_structure if P.eye_tracking: RED = (255, 0, 0) @@ -408,6 +430,12 @@ def run(self, *args, **kwargs): self.tracker_dot = Ellipse(8, stroke=[2, WHITE], fill=RED).render() if not P.manual_eyelink_setup: self.el.setup() + + # Generate blocks of trials (from either custom structure or trial factory) + if self._exp_structure: + self.blocks = _parse_structure(self._exp_structure, self.exp_factors) + elif P.manual_trial_generation is False: + self.trial_factory.generate() self.setup() try: diff --git a/klibs/KLParams.py b/klibs/KLParams.py index 4eb05eb7..634d278d 100755 --- a/klibs/KLParams.py +++ b/klibs/KLParams.py @@ -54,6 +54,7 @@ conditions = [] default_condition = None table_defaults = {} # default column values for db tables when using EntryTemplate +max_trials_per_block = False run_practice_blocks = True color_output = False # whether cso() outputs colorized text or not diff --git a/klibs/KLStructure.py b/klibs/KLStructure.py index 59d203a4..a4bda06e 100644 --- a/klibs/KLStructure.py +++ b/klibs/KLStructure.py @@ -1,7 +1,9 @@ +import random import itertools from copy import deepcopy from collections import OrderedDict +import klibs.KLParams as P from klibs.KLInternal import iterable @@ -124,3 +126,95 @@ def names(self): def set_length(self): """int: The number of trials required for the full factor set.""" return len(self._get_combinations()) + + + +class Block(object): + """Defines a custom block of trials. + + This class allows you to specify custom block structures for studies that + have different blocks types within the same session. For example, if your + study involves exposure, training, and test blocks, each with slightly + different factors levels and trial lengths, you can use this class to + define the block sequence accordingly:: + + exp_factors = FactorSet({ + 'sequence_type': ['practiced', 'unpracticed'], + }) + practiced_only = exp_factors.override({'sequence_type': ['practiced']}) + + structure = [ + Block(exp_factors, label='exposure', trials=18), + Block(practiced_only, label='training', trials=90), + Block(exp_factors, label='test', trials=60), + ] + + Different blocks can have different factor levels, but all blocks must + contain the same factors. If a factor is unneeded for a given block, it + should be given a dummy level (e.g. `None`). + + If provided, the label for the block will be set as `self.block_label` within + the Experiment object during runtime. Labels are meant to allow for easy + handling of different block types within the experiment code:: + + # If first block or block type changed, show instructions + if self.last_block_label != self.block_label: + if self.block_label == "exo": + self.exo_instructions() + elif self.block_label == "endo": + self.endo_instructions() + + self.last_block_label = self.block_label + + If you identify a block as a practice block, the runtime parameter + `P.practicing` will be set to True during that block. + + Args: + factors (:obj:`FactorSet` or dict): The factor set to use for the block. + label (str, optional): The label for the block. Defaults to None. + trials (int, optional): The trial count for the block. If not specified, + defaults to `P.trials_per_block`. + practice (bool, optional): Indicates whether the block is a practice + block. Defaults to False. + + """ + def __init__(self, factors, label=None, trials=None, practice=False): + self.practice = practice + self.label = label + if not isinstance(factors, FactorSet): + factors = FactorSet(factors) + self._factors = factors + if trials: + self.trialcount = trials + elif P.trials_per_block > 0: + self.trialcount = P.trials_per_block + else: + self.trialcount = self._factors.set_length + + def get_trials(self): + """Generates a shuffled set of trials from the block. + + Each complete set of factors is generated and shuffled sequentially + to prevent the possibility of all trials of the same type ending up + together (e.g. all trials with invalidly cued targets happening + consecutively). + + Returns: + list: A list of dicts containing the trial factors for each trial. + + """ + trials = [] + while len(trials) < self.trialcount: + new = self._factors._get_combinations() + remaining = self.trialcount - len(trials) + random.shuffle(new) + if remaining < len(new): + new = new[:remaining] + trials += new + + return trials + + @property + def factors(self): + """list: The names of all trial factors used in the block.""" + return self._factors.names diff --git a/klibs/KLTrialFactory.py b/klibs/KLTrialFactory.py index e035a1f5..452d652e 100755 --- a/klibs/KLTrialFactory.py +++ b/klibs/KLTrialFactory.py @@ -7,7 +7,7 @@ from klibs import P from klibs.KLInternal import load_source -from klibs.KLStructure import FactorSet +from klibs.KLStructure import FactorSet, Block def _load_factors(path): @@ -31,6 +31,59 @@ def _load_factors(path): return factors +def _load_structure(path): + # Imports a custom task structure from a file, returning an empty list if + # a structure is not specified. + + ind_vars = load_source(path) + if not 'structure' in ind_vars.keys(): + return [] + + structure = ind_vars['structure'] + if structure: + try: + structure = list(structure) + except Exception: + e = "If specified, task structure must be a list of Blocks." + raise TypeError(e) + + return structure + + +def _parse_structure(structure, exp_factors): + # Parses/validates a custom task structure and returns a list of TrialSets + + count = 0 + blocks = [] + for block in structure: + + if not isinstance(block, Block): + raise TypeError("Task structure must be made of Blocks") + + # Ensure all blocks have same factor levels + count += 1 + extra = set(block.factors) - set(exp_factors) + missing = set(exp_factors) - set(block.factors) + if len(extra): + e = "Extra factors" if len(extra) > 1 else "Extra factor" + e += " in block {0} not present in exp_factors: {1}" + raise RuntimeError(e.format(count, str(list(extra)))) + if len(missing): + e = "Missing the following factors in block {0}: {1}" + raise RuntimeError(e.format(count, str(list(missing)))) + + # Skip practice blocks if disabled + if block.practice and not P.run_practice_blocks: + continue + + # Generate trials and add block to block sequence + trials = block.get_trials() + b = TrialSet(trials, block.practice, block.label) + blocks.append(b) + + return blocks + + def _generate_blocks(factors, block_count, trial_count): # Generates a list of blocks (which are lists of trials, which are dicts of # trial factors) based on a given factor set, trial count, & block count. diff --git a/klibs/resources/template/independent_variables.py b/klibs/resources/template/independent_variables.py index 92d9b977..0921cf23 100644 --- a/klibs/resources/template/independent_variables.py +++ b/klibs/resources/template/independent_variables.py @@ -1,4 +1,4 @@ -from klibs.KLStructure import FactorSet +from klibs.KLStructure import FactorSet, Block """ ##### FactorSet Tutorial ##### @@ -50,3 +50,65 @@ exp_factors = FactorSet({ # Insert trial factors here }) + + +""" ##### Structure Tutorial ##### + +In addition to specifying trial factors and their levels, you can also use this file to +specify a custom block structure for the experiment. + +By default, klibs will generate blocks of trials using the factors defined above and +the number of blocks (and trials per block) specified in the project's params.py file. +For experiments that require more complex block structures, such as blocks with +different lengths or different factors, you can define your own sequences of blocks +using Block objects: + +structure = [ + Block(exp_factors, trials=24, practice=True), + Block(exp_factors, trials=96), + Block(exp_factors, trials=96), +] + +This would specify a simple 3-block structure: a 24 trial practice block followed by +two 96 trial test blocks. + +### Block Factors ### + +In many experiments, you might want to have different block types with different factor +levels (e.g. blocks with different difficulties, stimulus types, or task demands). As +such, the Block class makes it easy to use custom factor sets for different blocks. + +For example, to extend the FactorSet example above and add blocks of trials that have +50% cue validity, you could do the following: + +endo_factors = exp_factors +exo_factors = exp_factors.override({ + 'cue_validity' : ['valid', 'invalid'] +}) + +exo = Block(exo_factors, label='exo') +endo = Block(endo_factors, label='endo') + +structure = [ + exo, endo, exo, endo +] + +### Block Labels & Practice Blocks ### + +If you specify a label for a block, the value of this label can be accessed from within +the experiment runtime through the attribute `self.block_label`. For example, at the +start of each block you can show different instructions depending on the block label: + + self.block(self): + # Show instructions based on block type + if self.block_label == "endo": + self.show_endo_instructions() + elif self.block_label == "exo": + self.show_exo_instructions() + +Similarly, setting `practice` to True for a block will flag it as a practice block, +meaning that `P.practicing` will be set to True while within the block. + +""" + +structure = [] diff --git a/klibs/resources/template/params.py b/klibs/resources/template/params.py index 6c25a61d..f0ca4ed8 100755 --- a/klibs/resources/template/params.py +++ b/klibs/resources/template/params.py @@ -6,6 +6,7 @@ collect_demographics = True manual_demographics_collection = False manual_trial_generation = False +max_trials_per_block = False run_practice_blocks = True multi_user = False view_distance = 57 # in centimeters, 57cm = 1 deg of visual angle per cm of screen diff --git a/klibs/tests/conftest.py b/klibs/tests/conftest.py index 715f0b91..ace0ae29 100644 --- a/klibs/tests/conftest.py +++ b/klibs/tests/conftest.py @@ -27,6 +27,16 @@ def get_resource_path(resource): return os.path.join(klibs_root, 'resources', resource) +def create_tempfile(content, prefix="klibs", suffix=".py"): + tmp = tempfile.NamedTemporaryFile( + prefix=prefix, suffix=suffix, delete=False + ) + if isinstance(content, list): + content = "\n".join(content) + tmp.write(content.encode('utf-8')) + return tmp.name + + @pytest.fixture(scope='module') def with_sdl(): sdl2.SDL_ClearError() diff --git a/klibs/tests/test_KLExperiment.py b/klibs/tests/test_KLExperiment.py index af884a19..1c0acb55 100755 --- a/klibs/tests/test_KLExperiment.py +++ b/klibs/tests/test_KLExperiment.py @@ -7,7 +7,7 @@ import klibs from klibs.KLJSON_Object import AttributeDict from klibs.KLTrialFactory import TrialIterator, TrialSet -from klibs.KLExceptions import TrialException +from klibs.KLExceptions import TrialException, TerminateBlock from klibs.KLExperiment import Experiment from conftest import get_resource_path @@ -50,6 +50,7 @@ def setup(self): self.total_trials = 0 self.was_recycled = False self.database = AttributeDict({'tables': []}) + self.test_type = "default" self.blocks = [] def block(self): @@ -60,6 +61,9 @@ def block(self): # Check block label getting set as expected expected = 'test' if P.block_number == 1 else None assert self.block_label == expected + # Check trials_per_block is updated based on trial count + if not P.max_trials_per_block: + assert P.trials_per_block == 4 def __trial__(self, trial): # Check trial id increments correctly @@ -78,6 +82,10 @@ def __trial__(self, trial): assert 'fac2' in list(trial.keys()) # Test that practice getting set correctly assert P.practicing == (P.block_number == 1) + # Test block termination + if self.test_type == "terminate" and P.block_number == 2: + if trial_num == 3: + raise TerminateBlock() # Test trial recycling if self.was_recycled: assert P.recycle_count == 1 @@ -93,19 +101,42 @@ def __trial__(self, trial): {'fac1': True, 'fac2': 400}, {'fac1': False, 'fac2': 400}, ] + blocks = [ + TrialSet(trials, label='test', practice=True), + TrialIterator(trials), # alias for backwards compat + ] # Test with blocks as list + P.trials_per_block = 30 tst = TestExperiment() tst.setup() - tst.blocks = [ - TrialSet(trials, label='test', practice=True), - TrialIterator(trials), # alias for backwards compat - ] + tst.blocks = blocks tst.__execute_experiment__() assert tst.last_block == 2 assert tst.last_trial == 5 # 4 + 1 recycled assert tst.total_trials == 10 + # Test setting max trials per block + P.max_trials_per_block = 2 + tst = TestExperiment() + tst.setup() + tst.blocks = blocks + tst.__execute_experiment__() + P.max_trials_per_block = False + assert tst.last_block == 2 + assert tst.last_trial == 2 + assert tst.total_trials == 4 + + # Test terminating blocks early + tst = TestExperiment() + tst.setup() + tst.test_type = "terminate" + tst.blocks = blocks + tst.__execute_experiment__() + assert tst.last_block == 2 + assert tst.last_trial == 3 + assert tst.total_trials == 8 # Last two trials skipped + def test_insert_practice_block(experiment): from klibs import P diff --git a/klibs/tests/test_KLStructure.py b/klibs/tests/test_KLStructure.py index 9b1ebb56..35f13ed9 100644 --- a/klibs/tests/test_KLStructure.py +++ b/klibs/tests/test_KLStructure.py @@ -4,8 +4,11 @@ import random from collections import Counter -from klibs.KLStructure import FactorSet -from klibs.KLTrialFactory import _generate_blocks +import klibs.KLParams as P +from klibs.KLStructure import FactorSet, Block +from klibs.KLTrialFactory import _generate_blocks, _load_structure, _parse_structure + +from conftest import create_tempfile class TestFactorSet(object): @@ -135,3 +138,145 @@ def test_generate_blocks(): assert block[0]['soa'] == 200 and block[0]['cue_loc'] == 'none' assert block[1]['soa'] == 0 and block[1]['easy_trial'] == True assert block[2]['soa'] == 800 and block[2]['cue_loc'] == 'right' + + + +class TestBlock(object): + + def test_init(self): + + factors = FactorSet({ + 'cue_loc': ['left', 'right', 'none'], + 'easy_trial': [True, False], + }) + + # Test simple initialization + P.trials_per_block = 0 + tst = Block(factors) + assert tst.trialcount == 6 + assert tst.practice == False + assert tst.label == None + + # Test initialization with dict + tst = Block({'soa': [200, 800], 'target_loc': ['L', 'R']}) + assert tst.trialcount == 4 + assert isinstance(tst._factors, FactorSet) + + # Test defaulting trial count to P.trials_per_block + P.trials_per_block = 30 + tst = Block(factors) + assert tst.trialcount == 30 + + # Test custom trial counts: + tst = Block(factors, trials=36) + assert tst.trialcount == 36 + + # Test labels and practice flags + tst = Block(factors, label='endo', practice=True) + assert tst.label == 'endo' + assert tst.practice == True + + + def test_get_trials(self): + + factors = FactorSet({ + 'cue_loc': ['left', 'right', 'none'], + 'easy_trial': [True, False], + }) + + # Test generating trials with specified trial count + tst = Block(factors, trials=30) + trials = tst.get_trials() + assert len(trials) == 30 + assert isinstance(trials[0], dict) + assert 'cue_loc' in list(trials[0].keys()) + + # Test partial shuffling + random.seed(308053045) + trials = tst.get_trials() + easy_count = 0 + left_count = 0 + for trial in trials[:6]: + easy_count += int(trial['easy_trial'] == True) + left_count += int(trial['cue_loc'] == 'left') + assert easy_count == 3 + assert left_count == 2 + + + def test_factors(self): + + tst = Block({'a': [1, 2], 'b': [3, 4], 'c': [5, 6]}) + assert tst.factors == ['a', 'b', 'c'] + + +def test_load_structure(): + # NOTE: Move to KLTrialFactory tests once created + header = "from klibs.KLStructure import FactorSet, Block" + + # Test loading structure + tmp = create_tempfile([ + header, "", + "structure = [", + " Block({}, label='a', trials=10),", + " Block({}, label='B', trials=20)", + "]" + ]) + tst = _load_structure(tmp) + assert len(tst) == 2 + assert isinstance(tst[0], Block) + + # Test loading missing structure + tmp = create_tempfile([ + header, "", + "exp_factors = FactorSet({})" + ]) + tst = _load_structure(tmp) + assert not tst + + # Test loading empty structure + tmp = create_tempfile([ + header, "", + "exp_factors = FactorSet({})", + "", + "structure = []", + ]) + tst = _load_structure(tmp) + assert not tst + + +def test_parse_structure(): + # NOTE: Move to KLTrialFactory tests once created + exp_factors = {'fac1': ['a', 'b', 'c'], 'fac2': [True, False]} + tst = [ + Block(exp_factors, label='a', trials=10, practice=True), + Block(exp_factors, label='b', trials=20) + ] + blocks = _parse_structure(tst, exp_factors) + assert len(blocks) == 2 + + # Test exception if structure not made of Blocks + with pytest.raises(TypeError): + _parse_structure([exp_factors], exp_factors) + + # Test exception on missing factor level + fac_missing = {'fac1': ['a', 'b', 'c']} + tst_missing = [ + Block(fac_missing, trials=10), + Block(exp_factors, trials=10) + ] + with pytest.raises(RuntimeError): + _parse_structure(tst_missing, exp_factors) + + # Test exception on extra factor level + fac_extra = exp_factors.copy() + fac_extra['fac3'] = [200, 800] + tst_extra = [ + Block(exp_factors, trials=10), + Block(fac_extra, trials=10) + ] + with pytest.raises(RuntimeError): + _parse_structure(tst_extra, exp_factors) + + # Test exception when factors given but exp_factors is empty + with pytest.raises(RuntimeError): + _parse_structure(tst, {}) diff --git a/setup.py b/setup.py index c6f1c6d9..dee38d1f 100755 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ setup( name='KLibs', - version='0.7.7b1', + version='0.7.8a1', description='A framework for building psychological experiments in Python', author='Jonathan Mulle & Austin Hurst', author_email='mynameisaustinhurst@gmail.com',