From 3c0905d5cf51fc943ff0d9cda7dc2f953a3f40f4 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 10 Apr 2026 10:05:34 -0400 Subject: [PATCH 01/22] feat: MyRecord.record(using=False, ...) --- elasticsearch_metrics/imps/elastic8.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/elasticsearch_metrics/imps/elastic8.py b/elasticsearch_metrics/imps/elastic8.py index d03aacb..fa209b3 100644 --- a/elasticsearch_metrics/imps/elastic8.py +++ b/elasticsearch_metrics/imps/elastic8.py @@ -160,11 +160,21 @@ class Meta: @classmethod def record( - cls, *, using: str | None = None, **kwargs: typing.Any + cls, + *, + using: str | typing.Literal[False] | None = None, + **kwargs: typing.Any, ) -> "typing.Self": # typing.Self added in py 3.11 -- str annotation until 3.10 eol - """Persist a record in Elasticsearch.""" + """Construct a record instance and save it in Elasticsearch. + + Keyword args: + using -- name of the djelme backend or elasticsearch8.dsl connection + to use to save, or `False` to skip saving (e.g. for use in a bulk operation) + all other kwargs passed thru to the class constructor + """ _instance = cls(**kwargs) - _instance.save(using=using) + if using is not False: + _instance.save(using=using) return _instance @classmethod From b280fb695ddeb49166b390bf3c8444664a07d794 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 10 Apr 2026 10:07:36 -0400 Subject: [PATCH 02/22] wip: less eager autosetup --- elasticsearch_metrics/apps.py | 42 +++++++++++++++++++++++++------ elasticsearch_metrics/registry.py | 8 +++--- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/elasticsearch_metrics/apps.py b/elasticsearch_metrics/apps.py index e44d24c..6179b7e 100644 --- a/elasticsearch_metrics/apps.py +++ b/elasticsearch_metrics/apps.py @@ -2,6 +2,8 @@ from django.apps import AppConfig from django.conf import settings +from django.core import checks +from django.db.models.signals import post_migrate from django.utils.module_loading import autodiscover_modules from elasticsearch_metrics.registry import djelme_registry @@ -9,10 +11,41 @@ AUTOSETUP_SETTING = "DJELME_AUTOSETUP" +def do_djelme_setup(*, app_config, stdout, **kwargs): + _types_by_backend = djelme_registry.recordtypes_by_backend(app_label=app_config.label) + breakpoint() + for _backend_name, _recordtypes in _types_by_backend.items(): + _backend = djelme_registry.get_backend(_backend_name) + _backend.djelme_setup(_recordtypes) + + +def check_djelme_setup(app_configs, **kwargs): + _errors = [] + for _app_config in app_configs: + for _recordtype in djelme_registry.each_recordtype(app_label=_app_config.label): + if not _recordtype.check_djelme_setup(): + _errors.append( + checks.Error( + "elasticsearch_metrics requires setup!", + hint="run `manage.py djelme_backend_setup`", + obj=_recordtype, + id="elasticsearch_metrics.E001", + ) + ) + return _errors + + class ElasticsearchMetricsConfig(AppConfig): name = "elasticsearch_metrics" def ready(self) -> None: + # register django system checks + checks.register(check_djelme_setup) + # connect django signals + if getattr(settings, AUTOSETUP_SETTING, False) is True: + post_migrate.connect(do_djelme_setup) + # discover any `foo.metrics` in installed apps + autodiscover_modules("metrics") # load backends settings _backend_names_by_module = collections.defaultdict(list) for ( @@ -21,9 +54,7 @@ def ready(self) -> None: _, ) in djelme_registry.each_backend_settings(): _backend_names_by_module[_imp_module_name].append(_backend_name) - # discover any `foo.metrics` in installed apps - autodiscover_modules("metrics") - # call `djelme_when_ready` for each imp module (only once) + # call `djelme_when_ready` once for each imp module for _imp_module_name, _backend_names in _backend_names_by_module.items(): _imp_module = djelme_registry.get_imp_module(_imp_module_name) _imp_module.djelme_when_ready( @@ -31,8 +62,3 @@ def ready(self) -> None: djelme_registry.get_backend(_name) for _name in _backend_names ] ) - # autosetup? (default no) - if getattr(settings, AUTOSETUP_SETTING, False) is True: - _types_by_backend = djelme_registry.recordtypes_by_backend() - for _backend_name, _recordtypes in _types_by_backend.items(): - djelme_registry.get_backend(_backend_name).djelme_setup(_recordtypes) diff --git a/elasticsearch_metrics/registry.py b/elasticsearch_metrics/registry.py index a473b38..07e56c0 100644 --- a/elasticsearch_metrics/registry.py +++ b/elasticsearch_metrics/registry.py @@ -218,15 +218,13 @@ def recordtypes_by_backend( self, app_label: str = "" ) -> dict[str, collections.abc.Iterable[type[ProtoDjelmeRecord]]]: apps.check_apps_ready() # ensure django setup done - _by_backend_name: dict[str, list[type[ProtoDjelmeRecord]]] = ( - collections.defaultdict(list) - ) + _by_backend_name: dict[str, list[type[ProtoDjelmeRecord]]] = {} _app_labels = [app_label] if app_label else self._all_recordtypes.keys() for _app_label in _app_labels: for _recordtype in self._get_recordtypes_for_app(_app_label).values(): _backend_name = self.get_backend_name_for_recordtype(_recordtype) - _by_backend_name[_backend_name].append(_recordtype) - return dict(_by_backend_name.items()) + _by_backend_name.setdefault(_backend_name, []).append(_recordtype) + return _by_backend_name ### # private methods From 648817ac44ca39417a0ca124b8c31c8a376d4f27 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 10 Apr 2026 12:12:03 -0400 Subject: [PATCH 03/22] check templates on first save (not with django check) --- elasticsearch_metrics/apps.py | 23 +---- elasticsearch_metrics/imps/elastic6.py | 21 +++-- elasticsearch_metrics/imps/elastic8.py | 54 ++++++++---- elasticsearch_metrics/protocols.py | 3 +- elasticsearch_metrics/registry.py | 2 - elasticsearch_metrics/tests/_test_util.py | 83 ++++++++++++------- .../tests/dummy8app/metrics.py | 12 +-- .../tests/test_imps_elastic6.py | 8 +- .../tests/test_imps_elastic8.py | 47 +++++++++-- 9 files changed, 162 insertions(+), 91 deletions(-) diff --git a/elasticsearch_metrics/apps.py b/elasticsearch_metrics/apps.py index 6179b7e..81d4440 100644 --- a/elasticsearch_metrics/apps.py +++ b/elasticsearch_metrics/apps.py @@ -2,7 +2,6 @@ from django.apps import AppConfig from django.conf import settings -from django.core import checks from django.db.models.signals import post_migrate from django.utils.module_loading import autodiscover_modules @@ -12,35 +11,19 @@ def do_djelme_setup(*, app_config, stdout, **kwargs): - _types_by_backend = djelme_registry.recordtypes_by_backend(app_label=app_config.label) + _types_by_backend = djelme_registry.recordtypes_by_backend( + app_label=app_config.label + ) breakpoint() for _backend_name, _recordtypes in _types_by_backend.items(): _backend = djelme_registry.get_backend(_backend_name) _backend.djelme_setup(_recordtypes) -def check_djelme_setup(app_configs, **kwargs): - _errors = [] - for _app_config in app_configs: - for _recordtype in djelme_registry.each_recordtype(app_label=_app_config.label): - if not _recordtype.check_djelme_setup(): - _errors.append( - checks.Error( - "elasticsearch_metrics requires setup!", - hint="run `manage.py djelme_backend_setup`", - obj=_recordtype, - id="elasticsearch_metrics.E001", - ) - ) - return _errors - - class ElasticsearchMetricsConfig(AppConfig): name = "elasticsearch_metrics" def ready(self) -> None: - # register django system checks - checks.register(check_djelme_setup) # connect django signals if getattr(settings, AUTOSETUP_SETTING, False) is True: post_migrate.connect(do_djelme_setup) diff --git a/elasticsearch_metrics/imps/elastic6.py b/elasticsearch_metrics/imps/elastic6.py index ae4cd17..e8f5386 100644 --- a/elasticsearch_metrics/imps/elastic6.py +++ b/elasticsearch_metrics/imps/elastic6.py @@ -8,6 +8,7 @@ from collections.abc import Iterator import dataclasses import datetime +import functools import logging from django.apps import apps @@ -188,7 +189,7 @@ def sync_index_template(cls, using=None): return index_template @classmethod - def check_index_template(cls, using: str | None = None) -> bool: + def check_index_template(cls, using: str | None = None) -> None: """Check if class is in sync with index template in Elasticsearch. :raise: IndexTemplateNotFoundError if index template does not exist. @@ -204,7 +205,7 @@ def check_index_template(cls, using: str | None = None) -> bool: template_name = cls._template_name metric_name = cls.__name__ raise exceptions.IndexTemplateNotFoundError( - "{template_name} does not exist for {metric_name}".format(**locals()), + f"Index template {template_name!r} does not exist for {metric_name}", client_error=client_error, ) from client_error else: @@ -241,18 +242,21 @@ def check_index_template(cls, using: str | None = None) -> bool: [key for key, value in word_map.items() if not value] ) raise exceptions.IndexTemplateOutOfSyncError( - "{template_name} is out of sync with {metric_name} ({out_of_sync})".format( - **locals() - ), + f"Index template {template_name!r} is out of sync with {metric_name} ({out_of_sync})", mappings_in_sync=mappings_in_sync, patterns_in_sync=patterns_in_sync, settings_in_sync=settings_in_sync, ) - return True @classmethod - def check_djelme_setup(cls, using: str | None = None) -> bool: - return cls.check_index_template(using) + def check_djelme_setup(cls, using: str | None = None) -> None: + cls.check_index_template(using) + + @classmethod + @functools.cache + def require_been_setup(cls, using: str | None = None) -> None: + """check setup once -- raise on failure, remember success""" + cls.check_djelme_setup(using) @classmethod def get_timeseries_index_template(cls): @@ -302,6 +306,7 @@ def save(self, using=None, index=None, validate=True, **kwargs): """Same as `Document.save`, except will save into the index determined by the metric's timestamp field. """ + self.require_been_setup(using=using) # prevent automapped indexes self.timestamp = self.timestamp or timezone.now() if not index: index = self.get_index_name(date=self.timestamp) diff --git a/elasticsearch_metrics/imps/elastic8.py b/elasticsearch_metrics/imps/elastic8.py index fa209b3..2c27a14 100644 --- a/elasticsearch_metrics/imps/elastic8.py +++ b/elasticsearch_metrics/imps/elastic8.py @@ -22,6 +22,7 @@ import collections import dataclasses import datetime +import functools import logging import typing @@ -68,14 +69,22 @@ def utcnow() -> datetime.datetime: class _DjelmeRecordtypeMetaclass(IndexMeta): """Metaclass for the base `DjelmeRecordtype` class. - overrides behavior in elasticsearch-py's `IndexMeta` to allow - additional config in a type's `class Meta` + extend elasticsearch-py's `IndexMeta` to: + - belong to a django app (identified by `app_label` property) + - register concrete types with elasticsearch_metrics.registry.djelme_registry + - allow abstract record types (similar to django's abstract models, with `Meta.abstract`) """ Meta: type - # override IndexMeta.__new__, to do a few things differently def __new__(mcls, name, bases, attrs): # noqa: B902 + """create a new DjelmeRecordtype subclass + + extend elasticsearch-py's `IndexMeta.__new__` to: + - register concrete types with elasticsearch_metrics.registry.djelme_registry + - preserve a type's `class Meta` so it can hold additional custom config + - inherit field defaults (bug workaround?) + """ # save `class Meta` to un-remove it later _cls_meta = attrs.get("Meta") or type("Meta", (), {}) # call IndexMeta.__new__ @@ -88,7 +97,7 @@ def __new__(mcls, name, bases, attrs): # noqa: B902 setattr(_cls.Meta, _attrname, _attrval) else: # guarantee non-inherited Meta _cls.Meta = _cls_meta - # workaround elasticsearch.dsl inheriting only fields, not defaults + # workaround elasticsearch8.dsl inheriting only fields, not defaults for _b in bases: for _fieldname, _default in getattr(_b, "_defaults", {}).items(): _cls._defaults.setdefault(_fieldname, _default) @@ -153,7 +162,6 @@ class DjelmeRecordtype(esdsl.Document, metaclass=_DjelmeRecordtypeMetaclass): """ UNIQUE_TOGETHER_FIELDS: typing.ClassVar[collections.abc.Iterable[str]] = () - unique_id: str = esdsl.mapped_field(esdsl.Keyword(), default="") # filled on save class Meta: abstract = True @@ -178,9 +186,18 @@ def record( return _instance @classmethod - def check_djelme_setup(cls, using: str | None = None) -> bool: + def check_djelme_setup(cls, using: str | None = None) -> None: # this base class has only a single index -- does it exist? - return bool(cls._index.get(using=using)) + if not cls._index.get(using=using): + raise exceptions.TimeseriesSetupError( + f"Index {cls._index._name} does not exist" + ) + + @classmethod + @functools.cache + def require_been_setup(cls, using: str | None = None) -> None: + """check setup once -- raise on failure, remember success""" + cls.check_djelme_setup(using) @classmethod def _djelme_teardown(cls, es_client): @@ -193,9 +210,9 @@ def _get_using( ) -> str | Elastic8Client: """get the elasticsearch8 connection name to use - overrides elasticsearch8.Document._get_using to allow - getting connection name from a djelme backend and default - to the first configured backend that uses this imp module + extend `elasticsearch8.dsl.Document._get_using` to: + - recognize djelme backend names in `using` (given or configured) + - if no `using` given or configured, find a djelme backend by imp module """ _backend: ProtoDjelmeBackend | None = None if using in (None, "default"): @@ -218,8 +235,11 @@ def save( ) -> typing.Any: """save the record - overrides `save` to populate document_id and send pre_save/post_save signals + extend `elasticsearch8.dsl.Document.save` to: + - populate document id based on UNIQUE_TOGETHER_FIELDS + - send pre_save/post_save signals """ + self.require_been_setup(using=using) # prevent automapped indexes self._populate_unique_id() signals.pre_save.send(self.__class__, instance=self, using=using, index=index) _saved = super().save( @@ -231,7 +251,6 @@ def save( def _populate_unique_id(self) -> None: _unique_id = self._get_unique_id() assert _unique_id or not self.UNIQUE_TOGETHER_FIELDS - self.unique_id = _unique_id or "" # make it unique by setting doc id in elasticsearch if _unique_id: self.meta.id = _unique_id @@ -267,8 +286,7 @@ class Meta: def init(cls, index=None, using=None) -> None: """Create an index template with mappings for timeseries indexes - overrides elasticsearch.Document.init - (but doesn't call super().init(), which would create a "now" index) + override `elasticsearch8.dsl.Document.init` to create a template instead of an index """ assert not cls.is_abstract cls.sync_index_template(using=using) @@ -419,16 +437,17 @@ def sync_index_template(cls, using=None): # -> ComposableIndexTemplate: return _template @classmethod - def check_djelme_setup(cls, using: str | Elastic8Client | None = None) -> bool: + def check_djelme_setup(cls, using: str | Elastic8Client | None = None) -> None: """Check if class is in sync with index template in Elasticsearch. + override `DjelmeRecordtype.check_djelme_setup` to check for a template instead of an index + :raise: IndexTemplateNotFoundError if index template does not exist. :raise: IndexTemplateOutOfSyncError if mappings, settings, or index patterns are out of sync. :return: True if index template exsits and mappings, settings, and index patterns are in sync. """ - super().check_djelme_setup() client = cls._get_connection(using) try: _template_response = client.indices.get_index_template( @@ -475,7 +494,6 @@ def check_djelme_setup(cls, using: str | Elastic8Client | None = None) -> bool: patterns_in_sync=patterns_in_sync, settings_in_sync=settings_in_sync, ) - return True ### # instance methods @@ -516,7 +534,7 @@ def save( ) -> typing.Any: """save the record to a timeseries index - overrides `Document.save` to choose a specific timeseries index + extend `elasticsearch8.dsl.Document.save` to choose a specific timeseries index """ if index is None: index = self.djelme_index_name() diff --git a/elasticsearch_metrics/protocols.py b/elasticsearch_metrics/protocols.py index 632525f..2379bc5 100644 --- a/elasticsearch_metrics/protocols.py +++ b/elasticsearch_metrics/protocols.py @@ -48,7 +48,8 @@ def record( ... @classmethod - def check_djelme_setup(cls, using: str | None = None) -> bool: ... + def check_djelme_setup(cls, using: str | None = None) -> None: + """Check backend setup for this record type; raise helpful exceptions""" @classmethod def search_timeseries_range( diff --git a/elasticsearch_metrics/registry.py b/elasticsearch_metrics/registry.py index 07e56c0..b4e3a8d 100644 --- a/elasticsearch_metrics/registry.py +++ b/elasticsearch_metrics/registry.py @@ -162,10 +162,8 @@ def get_backend_name_for_recordtype( try: (_backend_name,) = _each_backend_name except StopIteration as _e: # no backends - breakpoint() raise LookupError(f"no backends for recordtype {recordtype!r}") from _e except ValueError as _e: # too many backends - breakpoint() raise LookupError( f"more than one backend for recordtype {recordtype!r}, must be set explicitly" ) from _e diff --git a/elasticsearch_metrics/tests/_test_util.py b/elasticsearch_metrics/tests/_test_util.py index ab5d3c1..1e48fdf 100644 --- a/elasticsearch_metrics/tests/_test_util.py +++ b/elasticsearch_metrics/tests/_test_util.py @@ -1,7 +1,8 @@ +import contextlib from io import StringIO -from unittest import mock import types import typing +from unittest import mock import uuid from django.core.management import call_command @@ -14,6 +15,10 @@ class SimpleDjelmeTestCase(SimpleTestCase): """SimpleDjelmeTestCase: base test case with djelme-specific conveniences""" + def setUp(self): + super().setUp() + clear_setup_check_caches() + def enterContext(self, context_manager): # unittest.TestCase.enterContext added in python3.11 -- implementing here until 3.10 eol result = context_manager.__enter__() @@ -52,46 +57,68 @@ def __init_subclass__( cls.__autoteardown_backends = autoteardown_djelme_backends def setUp(self): + self.enterContext(prefixed_index_names()) super().setUp() - _name_prefix = f"{uuid.uuid4().hex}_" - self.enterContext( - mock.patch( - "elasticsearch_metrics.imps.elastic8.TimeseriesRecord.get_timeseries_name_prefix", - return_value=_name_prefix, - ) - ) - self.enterContext( - mock.patch( - "elasticsearch_metrics.imps.elastic6.BaseMetric.get_timeseries_name_prefix", - return_value=_name_prefix, - ), - ) if self.__autosetup_backends: - self.setup_backends() + setup_backends() def tearDown(self): if self.__autoteardown_backends: - self.teardown_backends() + teardown_backends() super().tearDown() - def setup_backends(self): - # backends based on settings in django.conf.settings.DJELME_BACKENDS - _types_by_backend = djelme_registry.recordtypes_by_backend() - for _backend_name, _recordtypes in _types_by_backend.items(): - djelme_registry.get_backend(_backend_name).djelme_setup(_recordtypes) - - def teardown_backends(self): - # backends based on settings in django.conf.settings.DJELME_BACKENDS - _types_by_backend = djelme_registry.recordtypes_by_backend() - for _backend_name, _recordtypes in _types_by_backend.items(): - djelme_registry.get_backend(_backend_name).djelme_teardown(_recordtypes) - class MockSaveTestCase(SimpleDjelmeTestCase): def setUp(self): + super().setUp() self.mocked_es6_save = self.enterContext( mock.patch("elasticsearch_metrics.imps.elastic6.Document.save"), ) self.mocked_es8_save = self.enterContext( mock.patch("elasticsearch_metrics.imps.elastic8.esdsl.Document.save"), ) + self.mocked_es6_require_been_setup = self.enterContext( + mock.patch("elasticsearch_metrics.imps.elastic6.Metric.require_been_setup"), + ) + self.mocked_es8_require_been_setup = self.enterContext( + mock.patch( + "elasticsearch_metrics.imps.elastic8.DjelmeRecordtype.require_been_setup" + ), + ) + + +@contextlib.contextmanager +def prefixed_index_names(prefix: str = ""): + _name_prefix = prefix or f"{uuid.uuid4().hex}_" + with ( + mock.patch( + "elasticsearch_metrics.imps.elastic8.TimeseriesRecord.get_timeseries_name_prefix", + return_value=_name_prefix, + ), + mock.patch( + "elasticsearch_metrics.imps.elastic6.BaseMetric.get_timeseries_name_prefix", + return_value=_name_prefix, + ), + ): + yield + + +def clear_setup_check_caches(): + from elasticsearch_metrics.imps import elastic6, elastic8 + + elastic6.Metric.require_been_setup.cache_clear() + elastic8.DjelmeRecordtype.require_been_setup.cache_clear() + + +def setup_backends(): + # backends based on settings in django.conf.settings.DJELME_BACKENDS + _types_by_backend = djelme_registry.recordtypes_by_backend() + for _backend_name, _recordtypes in _types_by_backend.items(): + djelme_registry.get_backend(_backend_name).djelme_setup(_recordtypes) + + +def teardown_backends(): + # backends based on settings in django.conf.settings.DJELME_BACKENDS + _types_by_backend = djelme_registry.recordtypes_by_backend() + for _backend_name, _recordtypes in _types_by_backend.items(): + djelme_registry.get_backend(_backend_name).djelme_teardown(_recordtypes) diff --git a/elasticsearch_metrics/tests/dummy8app/metrics.py b/elasticsearch_metrics/tests/dummy8app/metrics.py index ec3035e..cb664ff 100644 --- a/elasticsearch_metrics/tests/dummy8app/metrics.py +++ b/elasticsearch_metrics/tests/dummy8app/metrics.py @@ -1,9 +1,9 @@ -from elasticsearch8.dsl import Text, mapped_field, analyzer, tokenizer +from elasticsearch8 import dsl as esdsl from elasticsearch_metrics.imps import elastic8 as djelme -dot_path_analyzer = analyzer( +dot_path_analyzer = esdsl.analyzer( "dot_path_analyzer", - tokenizer=tokenizer("dot_path_tokenizer", "path_hierarchy", delimiter="."), + tokenizer=esdsl.tokenizer("dot_path_tokenizer", "path_hierarchy", delimiter="."), ) @@ -29,8 +29,10 @@ class Index: class ThingHappened(djelme.EventRecord): thing_id: str = "" happen_code: str | None = None - dot_path: str | None = mapped_field(Text(analyzer=dot_path_analyzer), default=None) - commentary: str | None = mapped_field(Text(), default=None) + dot_path: str | None = esdsl.mapped_field( + esdsl.Text(analyzer=dot_path_analyzer), default=None + ) + commentary: str | None = esdsl.mapped_field(esdsl.Text(), default=None) class Index: settings = {"refresh_interval": "-1"} diff --git a/elasticsearch_metrics/tests/test_imps_elastic6.py b/elasticsearch_metrics/tests/test_imps_elastic6.py index e4db776..7ebd7a3 100644 --- a/elasticsearch_metrics/tests/test_imps_elastic6.py +++ b/elasticsearch_metrics/tests/test_imps_elastic6.py @@ -296,20 +296,20 @@ def test_init(self): def test_check_djelme_setup(self): with self.assertRaises(IndexTemplateNotFoundError): - assert PreprintView.check_djelme_setup() is False + PreprintView.check_djelme_setup() PreprintView.sync_index_template() - assert PreprintView.check_djelme_setup() is True + assert PreprintView.check_djelme_setup() is None # When settings change, template is out of sync PreprintView._index.settings( **{"refresh_interval": "1s", "number_of_shards": 1, "number_of_replicas": 2} ) with self.assertRaises(IndexTemplateOutOfSyncError) as excinfo: - assert PreprintView.check_djelme_setup() is False + PreprintView.check_djelme_setup() error = excinfo.exception assert error.settings_in_sync is False assert error.mappings_in_sync is True assert error.patterns_in_sync is True PreprintView.sync_index_template() - assert PreprintView.check_djelme_setup() is True + assert PreprintView.check_djelme_setup() is None diff --git a/elasticsearch_metrics/tests/test_imps_elastic8.py b/elasticsearch_metrics/tests/test_imps_elastic8.py index bd3b273..57ab2fc 100644 --- a/elasticsearch_metrics/tests/test_imps_elastic8.py +++ b/elasticsearch_metrics/tests/test_imps_elastic8.py @@ -385,7 +385,44 @@ def test_record(self): assert properties["happen_code"] == {"type": "keyword"} -class TestInit(RealElasticTestCase, autosetup_djelme_backends=False): +class TestWithoutAutosetup(RealElasticTestCase, autosetup_djelme_backends=False): + def test_cannot_save_without_template(self): + _event = Dummy8Event(intensity=2) + with self.assertRaises(IndexTemplateNotFoundError): + _event.save() + + def test_cannot_save_with_wrong_template_pattern(self): + with unittest.mock.patch.object( + Dummy8Event, + "format_timeseries_index_pattern", + return_value="wrong_pattern_haha_*", + ): + Dummy8Event.init() + with self.assertRaises(IndexTemplateOutOfSyncError): + Dummy8Event.record(intensity=2) + + def test_cannot_save_with_extra_property_mapping(self): + # create template with extra property mapping + _template = Dummy8Event.get_timeseries_template().to_dict() + _template["template"]["mappings"]["properties"]["foo"] = {"type": "keyword"} + _es8_client().indices.put_index_template( + name=Dummy8Event.get_timeseries_template_name(), + **_template, + ) + with self.assertRaises(IndexTemplateOutOfSyncError): + Dummy8Event.record(intensity=2) + + def test_cannot_save_with_missing_property_mapping(self): + # create template with missing property mapping + _template = Dummy8Event.get_timeseries_template().to_dict() + del _template["template"]["mappings"]["properties"]["intensity"] + _es8_client().indices.put_index_template( + name=Dummy8Event.get_timeseries_template_name(), + **_template, + ) + with self.assertRaises(IndexTemplateOutOfSyncError): + Dummy8Event.record(intensity=2) + def test_init(self): ThingHappened.init() _client = _es8_client() @@ -407,23 +444,23 @@ def test_init(self): def test_check_djelme_setup(self): with self.assertRaises(IndexTemplateNotFoundError): - assert ThingHappened.check_djelme_setup() is False + ThingHappened.check_djelme_setup() ThingHappened.sync_index_template() - assert ThingHappened.check_djelme_setup() is True + assert ThingHappened.check_djelme_setup() is None # When settings change, template is out of sync ThingHappened._index.settings( **{"refresh_interval": "1s", "number_of_shards": 1, "number_of_replicas": 2} ) with self.assertRaises(IndexTemplateOutOfSyncError) as excinfo: - assert ThingHappened.check_djelme_setup() is False + ThingHappened.check_djelme_setup() error = excinfo.exception assert error.settings_in_sync is False assert error.mappings_in_sync is True assert error.patterns_in_sync is True ThingHappened.sync_index_template() - assert ThingHappened.check_djelme_setup() is True + assert ThingHappened.check_djelme_setup() is None class TestDailyIndexes(RealElasticTestCase, autosetup_djelme_backends=True): From d35e1fd62e3d616096a7152ebfeefab227aa51e0 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 10 Apr 2026 13:02:38 -0400 Subject: [PATCH 04/22] update github action steps --- .github/workflows/test_djelme.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_djelme.yml b/.github/workflows/test_djelme.yml index 22191ae..2a12a1b 100644 --- a/.github/workflows/test_djelme.yml +++ b/.github/workflows/test_djelme.yml @@ -12,11 +12,11 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 id: setup-py with: - python-version: '3.11' + python-version: '3.13' - run: pip install poetry - run: poetry install --no-root --with=lint - run: poetry run python -m elasticsearch_metrics.tests --lint @@ -27,7 +27,6 @@ jobs: matrix: python: ['3.10', '3.11', '3.12', '3.13', '3.14'] django: ['4.2', '5.1', '5.2'] - # TODO: elasticsearch: ['6', '7', '8', '9'] services: elasticsearch6: image: elasticsearch:6.8.23 @@ -42,8 +41,8 @@ jobs: node.name: singlenode cluster.initial_master_nodes: singlenode steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 id: setup-py with: python-version: ${{ matrix.python }} From c43abd63c623cdfbfaf87da6194d2a6f74ac2dd5 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 10 Apr 2026 13:10:14 -0400 Subject: [PATCH 05/22] remove DJELME_AUTOSETUP --- README.md | 14 ++--------- elasticsearch_metrics/apps.py | 32 ++++--------------------- elasticsearch_metrics/tests/settings.py | 1 - 3 files changed, 7 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 8ee185d..844ba8c 100644 --- a/README.md +++ b/README.md @@ -61,15 +61,9 @@ class UsageRecord(EventRecord): using = "my-es8-backend" # backend name -- required if multiple backends use the same imp ``` -Either enable autosetup... -```python -# ... in your django project settings file ... -DJELME_AUTOSETUP = True -``` - -...or be sure to run the `djelme_backend_setup` management command before trying to store anything. +and be sure to run the `djelme_backend_setup` management command before trying to store anything: ```shell -# This will create an index template for usagerecord timeseries indexes +# This will create an index template for each timeseries record type python manage.py djelme_backend_setup ``` @@ -192,10 +186,6 @@ class UsageRecord(MyBaseMetric): } ``` -* `DJELME_AUTOSETUP`: Optional feature, default `False` -- - set `True` to run backend setup automatically when your django app starts - (like creating index templates in elasticsearch, if they don't already exist) - * `DJELME_DEFAULT_TIMEDEPTH`: Set the granularity of timeseries indexes by the number of "time parts" in index names ``` DJELME_DEFAULT_TIMEDEPTH = 1 # yearly indexes; YYYY diff --git a/elasticsearch_metrics/apps.py b/elasticsearch_metrics/apps.py index 81d4440..caa6ff1 100644 --- a/elasticsearch_metrics/apps.py +++ b/elasticsearch_metrics/apps.py @@ -1,44 +1,22 @@ import collections from django.apps import AppConfig -from django.conf import settings -from django.db.models.signals import post_migrate from django.utils.module_loading import autodiscover_modules from elasticsearch_metrics.registry import djelme_registry -AUTOSETUP_SETTING = "DJELME_AUTOSETUP" - - -def do_djelme_setup(*, app_config, stdout, **kwargs): - _types_by_backend = djelme_registry.recordtypes_by_backend( - app_label=app_config.label - ) - breakpoint() - for _backend_name, _recordtypes in _types_by_backend.items(): - _backend = djelme_registry.get_backend(_backend_name) - _backend.djelme_setup(_recordtypes) - class ElasticsearchMetricsConfig(AppConfig): name = "elasticsearch_metrics" def ready(self) -> None: - # connect django signals - if getattr(settings, AUTOSETUP_SETTING, False) is True: - post_migrate.connect(do_djelme_setup) # discover any `foo.metrics` in installed apps autodiscover_modules("metrics") - # load backends settings - _backend_names_by_module = collections.defaultdict(list) - for ( - _backend_name, - _imp_module_name, - _, - ) in djelme_registry.each_backend_settings(): - _backend_names_by_module[_imp_module_name].append(_backend_name) - # call `djelme_when_ready` once for each imp module - for _imp_module_name, _backend_names in _backend_names_by_module.items(): + # call `djelme_when_ready` once for each djelme imp module used by a backend + _backends_by_imp: dict[str, list[str]] = collections.defaultdict(list) + for _backend_name, _imp_name, _ in djelme_registry.each_backend_settings(): + _backends_by_imp[_imp_name].append(_backend_name) + for _imp_module_name, _backend_names in _backends_by_imp.items(): _imp_module = djelme_registry.get_imp_module(_imp_module_name) _imp_module.djelme_when_ready( backends=[ diff --git a/elasticsearch_metrics/tests/settings.py b/elasticsearch_metrics/tests/settings.py index 4b77c82..e7b3c91 100644 --- a/elasticsearch_metrics/tests/settings.py +++ b/elasticsearch_metrics/tests/settings.py @@ -13,7 +13,6 @@ MIDDLEWARE_CLASSES = [] DATABASES = {"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": "test_djelme"}} -DJELME_AUTOSETUP = True DJELME_BACKENDS = { "my_elastic6": { "elasticsearch_metrics.imps.elastic6": { From 4beb4b1aaad40c7fba78f317830b3da717ce221b Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Tue, 14 Apr 2026 15:56:49 -0400 Subject: [PATCH 06/22] fix: getting CyclicRecord --- elasticsearch_metrics/imps/elastic8.py | 13 ++-- .../tests/dummy8app/metrics.py | 2 +- .../tests/test_imps_elastic8.py | 66 +++++++++++++++++++ 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/elasticsearch_metrics/imps/elastic8.py b/elasticsearch_metrics/imps/elastic8.py index 2c27a14..2c1fbd9 100644 --- a/elasticsearch_metrics/imps/elastic8.py +++ b/elasticsearch_metrics/imps/elastic8.py @@ -308,8 +308,7 @@ def search( def search_timeseries_range( cls, from_when: tuple[int, ...] | datetime.date, - until_when: tuple[int, ...] | datetime.date | None, - **kwargs: typing.Any, + until_when: tuple[int, ...] | datetime.date, ) -> typing.Any: _index_pattern = cls.format_timeseries_index_pattern_for_range( from_when, until_when @@ -498,11 +497,6 @@ def check_djelme_setup(cls, using: str | Elastic8Client | None = None) -> None: ### # instance methods - def __init__(self, *args, **kwargs): - assert not self.__class__.is_abstract - super().__init__(*args, **kwargs) - self.timeseries_timeparts = self.get_timeseries_timeparts() - def get_timeseries_timeparts(self) -> str: """semverlike string of timeparts, used to choose a timeseries index""" raise NotImplementedError( @@ -514,11 +508,10 @@ def djelme_index_name(self) -> str: for ProtoDjelmeRecord """ - assert self.timeseries_timeparts _index_name = timeseries_naming.format_index_name( app_label=self.__class__.app_label, recordtype=self.get_timeseries_recordtype_name(), - timeparts=self.timeseries_timeparts, + timeparts=self.timeseries_timeparts or self.get_timeseries_timeparts(), max_timedepth=self.get_timedepth(), ) return "".join((self.get_timeseries_name_prefix(), _index_name)) @@ -536,6 +529,8 @@ def save( extend `elasticsearch8.dsl.Document.save` to choose a specific timeseries index """ + assert not self.__class__.is_abstract + self.timeseries_timeparts = self.get_timeseries_timeparts() if index is None: index = self.djelme_index_name() ret = super().save(using=using, index=index, **kwargs) diff --git a/elasticsearch_metrics/tests/dummy8app/metrics.py b/elasticsearch_metrics/tests/dummy8app/metrics.py index cb664ff..415f9ee 100644 --- a/elasticsearch_metrics/tests/dummy8app/metrics.py +++ b/elasticsearch_metrics/tests/dummy8app/metrics.py @@ -43,9 +43,9 @@ class Meta: timedepth = 1 # yearly timeseries indexes -# TODO: tests using ThingHappeningsReport class ThingHappeningsReport(djelme.CyclicRecord): CYCLE_TIMEDEPTH = 2 + UNIQUE_TOGETHER_FIELDS = ("cycle_coverage", "thing_id") thing_id: str happen_count: int diff --git a/elasticsearch_metrics/tests/test_imps_elastic8.py b/elasticsearch_metrics/tests/test_imps_elastic8.py index 57ab2fc..fde51a6 100644 --- a/elasticsearch_metrics/tests/test_imps_elastic8.py +++ b/elasticsearch_metrics/tests/test_imps_elastic8.py @@ -645,6 +645,72 @@ def _assert_things(hits, expected_thing_ids): ) +class TestCyclicRecord(RealElasticTestCase, autosetup_djelme_backends=True): + def setUp(self): + super().setUp() + ThingHappeningsReport.record( + cycle_coverage="2000.1", thing_id="a", happen_count=2 + ) + ThingHappeningsReport.record( + cycle_coverage="2000.1", thing_id="b", happen_count=3 + ) + ThingHappeningsReport.record( + cycle_coverage="2000.1", thing_id="c", happen_count=4 + ) + ThingHappeningsReport.record( + cycle_coverage="2000.2", thing_id="a", happen_count=5 + ) + ThingHappeningsReport.record( # this duplicate gets overwritten + cycle_coverage="2000.2", thing_id="b", happen_count=67 + ) + ThingHappeningsReport.record( # this duplicate overwrites + cycle_coverage="2000.2", thing_id="b", happen_count=6 + ) + ThingHappeningsReport.record( + cycle_coverage="2000.2", thing_id="c", happen_count=7 + ) + ThingHappeningsReport.refresh_timeseries_indexes() + + def test_indexes(self): + _index_names = { + _strip_test_prefix(_name) + for _name, _ in ThingHappeningsReport.each_timeseries_index() + } + self.assertEqual( + _index_names, + { + "dummy8app_thinghappeningsreport_2000.1.", + "dummy8app_thinghappeningsreport_2000.2.", + }, + ) + + def test_search(self): + _b_search = ( + ThingHappeningsReport.search() + .query({"term": {"thing_id": "b"}}) + .sort("cycle_coverage") + ) + (_actual_1, _actual_2) = _b_search + self.assertEqual(_actual_1.cycle_coverage, "2000.1") + self.assertEqual(_actual_1.thing_id, "b") + self.assertEqual(_actual_1.happen_count, 3) + self.assertEqual(_actual_1.timeseries_timeparts, "2000.1") + self.assertEqual(_actual_2.cycle_coverage, "2000.2") + self.assertEqual(_actual_2.thing_id, "b") + self.assertEqual(_actual_2.happen_count, 6) + self.assertEqual(_actual_2.timeseries_timeparts, "2000.2") + + def test_search_range(self): + _b_search = ThingHappeningsReport.search_timeseries_range( + (2000, 2), (2001,) + ).query({"term": {"thing_id": "b"}}) + (_actual_2,) = _b_search + self.assertEqual(_actual_2.cycle_coverage, "2000.2") + self.assertEqual(_actual_2.thing_id, "b") + self.assertEqual(_actual_2.happen_count, 6) + self.assertEqual(_actual_2.timeseries_timeparts, "2000.2") + + def _strip_test_prefix(index_name: str) -> str: # strip uuid test prefix (_, _, _without_prefix) = index_name.partition("_") From 1ae23d706957e69b69369435676908daa7c7ea21 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Tue, 14 Apr 2026 16:28:52 -0400 Subject: [PATCH 07/22] usable elasticsearch_metrics.tests.util --- .../tests/test_autodiscovery.py | 2 +- .../tests/test_imps_elastic6.py | 7 +-- .../tests/test_imps_elastic8.py | 15 ++++--- .../test_djelme_check.py | 2 +- .../test_djelme_setup.py | 2 +- .../test_djelme_types.py | 2 +- .../tests/{_test_util.py => util.py} | 45 ++++++++++++------- 7 files changed, 44 insertions(+), 31 deletions(-) rename elasticsearch_metrics/tests/{_test_util.py => util.py} (82%) diff --git a/elasticsearch_metrics/tests/test_autodiscovery.py b/elasticsearch_metrics/tests/test_autodiscovery.py index fae22f4..b5aa6de 100644 --- a/elasticsearch_metrics/tests/test_autodiscovery.py +++ b/elasticsearch_metrics/tests/test_autodiscovery.py @@ -1,5 +1,5 @@ from elasticsearch_metrics.registry import djelme_registry -from elasticsearch_metrics.tests._test_util import SimpleDjelmeTestCase +from elasticsearch_metrics.tests.util import SimpleDjelmeTestCase class TestAutodiscovery(SimpleDjelmeTestCase): diff --git a/elasticsearch_metrics/tests/test_imps_elastic6.py b/elasticsearch_metrics/tests/test_imps_elastic6.py index 7ebd7a3..e9209f1 100644 --- a/elasticsearch_metrics/tests/test_imps_elastic6.py +++ b/elasticsearch_metrics/tests/test_imps_elastic6.py @@ -17,10 +17,11 @@ IndexTemplateNotFoundError, IndexTemplateOutOfSyncError, ) -from elasticsearch_metrics.tests._test_util import ( +from elasticsearch_metrics.tests.util import ( SimpleDjelmeTestCase, MockSaveTestCase, RealElasticTestCase, + NoSetupRealElasticTestCase, ) from elasticsearch_metrics.tests.dummy6app.metrics import ( Dummy6Metric, @@ -253,7 +254,7 @@ def test_save_sends_signals(self): assert post_save_kwargs["sender"] is PreprintView -class TestIntegration(RealElasticTestCase, autosetup_djelme_backends=True): +class TestIntegration(RealElasticTestCase): @property def es6_client(self): return connections.get_connection("my_elastic6") @@ -279,7 +280,7 @@ def test_create_document(self): assert properties["preprint_id"] == {"type": "keyword"} -class TestIntegrationSetup(RealElasticTestCase, autosetup_djelme_backends=False): +class TestIntegrationSetup(NoSetupRealElasticTestCase): @property def es6_client(self): return connections.get_connection("my_elastic6") diff --git a/elasticsearch_metrics/tests/test_imps_elastic8.py b/elasticsearch_metrics/tests/test_imps_elastic8.py index fde51a6..ce0add8 100644 --- a/elasticsearch_metrics/tests/test_imps_elastic8.py +++ b/elasticsearch_metrics/tests/test_imps_elastic8.py @@ -14,10 +14,11 @@ IndexTemplateOutOfSyncError, ) from elasticsearch_metrics.registry import djelme_registry -from elasticsearch_metrics.tests._test_util import ( +from elasticsearch_metrics.tests.util import ( SimpleDjelmeTestCase, MockSaveTestCase, RealElasticTestCase, + NoSetupRealElasticTestCase, ) from elasticsearch_metrics.tests.dummy8app.metrics import ( Dummy8Event, @@ -338,7 +339,7 @@ def test_save_sends_signals(self): assert post_save_kwargs["sender"] is ThingHappened -class TestRealCreate(RealElasticTestCase, autosetup_djelme_backends=True): +class TestRealCreate(RealElasticTestCase): def test_save(self): _thing_id = "12345" _happen_code = "zyxwv" @@ -385,7 +386,7 @@ def test_record(self): assert properties["happen_code"] == {"type": "keyword"} -class TestWithoutAutosetup(RealElasticTestCase, autosetup_djelme_backends=False): +class TestWithoutAutosetup(NoSetupRealElasticTestCase): def test_cannot_save_without_template(self): _event = Dummy8Event(intensity=2) with self.assertRaises(IndexTemplateNotFoundError): @@ -463,7 +464,7 @@ def test_check_djelme_setup(self): assert ThingHappened.check_djelme_setup() is None -class TestDailyIndexes(RealElasticTestCase, autosetup_djelme_backends=True): +class TestDailyIndexes(RealElasticTestCase): def setUp(self): super().setUp() Dummy8Event.record(timestamp=dt.datetime(1234, 5, 6), intensity=1) @@ -525,7 +526,7 @@ def _assert_intens(hits, expected_intensities): ) -class TestMonthlyIndexes(RealElasticTestCase, autosetup_djelme_backends=True): +class TestMonthlyIndexes(RealElasticTestCase): def setUp(self): super().setUp() Monthly8Event.record(timestamp=dt.datetime(1234, 5, 6), intenzity=1) @@ -586,7 +587,7 @@ def _assert_intenz(hits, expected_intenzities): ) -class TestYearlyIndexes(RealElasticTestCase, autosetup_djelme_backends=True): +class TestYearlyIndexes(RealElasticTestCase): def setUp(self): super().setUp() ThingHappened.record(timestamp=dt.datetime(1234, 5, 6), thing_id="a") @@ -645,7 +646,7 @@ def _assert_things(hits, expected_thing_ids): ) -class TestCyclicRecord(RealElasticTestCase, autosetup_djelme_backends=True): +class TestCyclicRecord(RealElasticTestCase): def setUp(self): super().setUp() ThingHappeningsReport.record( diff --git a/elasticsearch_metrics/tests/test_management_commands/test_djelme_check.py b/elasticsearch_metrics/tests/test_management_commands/test_djelme_check.py index 3872e29..dc60897 100644 --- a/elasticsearch_metrics/tests/test_management_commands/test_djelme_check.py +++ b/elasticsearch_metrics/tests/test_management_commands/test_djelme_check.py @@ -3,7 +3,7 @@ from elasticsearch_metrics import exceptions from elasticsearch_metrics.management.commands import djelme_backend_check from elasticsearch_metrics.registry import registry -from elasticsearch_metrics.tests._test_util import SimpleDjelmeTestCase +from elasticsearch_metrics.tests.util import SimpleDjelmeTestCase class TestCheckRecordtypes(SimpleDjelmeTestCase): diff --git a/elasticsearch_metrics/tests/test_management_commands/test_djelme_setup.py b/elasticsearch_metrics/tests/test_management_commands/test_djelme_setup.py index 51b105b..2459d7a 100644 --- a/elasticsearch_metrics/tests/test_management_commands/test_djelme_setup.py +++ b/elasticsearch_metrics/tests/test_management_commands/test_djelme_setup.py @@ -5,7 +5,7 @@ from elasticsearch_metrics.management.commands import djelme_backend_setup from elasticsearch_metrics.imps import elastic6 from elasticsearch_metrics.registry import djelme_registry -from elasticsearch_metrics.tests._test_util import SimpleDjelmeTestCase +from elasticsearch_metrics.tests.util import SimpleDjelmeTestCase class TestDjelmeSetup(SimpleDjelmeTestCase): diff --git a/elasticsearch_metrics/tests/test_management_commands/test_djelme_types.py b/elasticsearch_metrics/tests/test_management_commands/test_djelme_types.py index f94b88c..e1ab535 100644 --- a/elasticsearch_metrics/tests/test_management_commands/test_djelme_types.py +++ b/elasticsearch_metrics/tests/test_management_commands/test_djelme_types.py @@ -1,5 +1,5 @@ from elasticsearch_metrics.management.commands import djelme_backend_types -from elasticsearch_metrics.tests._test_util import SimpleDjelmeTestCase +from elasticsearch_metrics.tests.util import SimpleDjelmeTestCase class TestDjelmeTypesCommand(SimpleDjelmeTestCase): diff --git a/elasticsearch_metrics/tests/_test_util.py b/elasticsearch_metrics/tests/util.py similarity index 82% rename from elasticsearch_metrics/tests/_test_util.py rename to elasticsearch_metrics/tests/util.py index 1e48fdf..b15a76b 100644 --- a/elasticsearch_metrics/tests/_test_util.py +++ b/elasticsearch_metrics/tests/util.py @@ -1,7 +1,6 @@ import contextlib from io import StringIO import types -import typing from unittest import mock import uuid @@ -12,6 +11,23 @@ from elasticsearch_metrics.registry import djelme_registry +@contextlib.contextmanager +def djelme_test_backends(): + """context manager to wrap tests that use djelme with actual elasticsearch + + gives index names a unique prefix + + sets up and tears down all backends configured in django.conf.settings.DJELME_BACKENDS + """ + clear_setup_check_caches() + with prefixed_index_names(): + setup_backends() + try: + yield + finally: + teardown_backends() + + class SimpleDjelmeTestCase(SimpleTestCase): """SimpleDjelmeTestCase: base test case with djelme-specific conveniences""" @@ -42,35 +58,30 @@ def run_mgmt_command( class RealElasticTestCase(SimpleDjelmeTestCase): """RealElasticTestCase: base test case with actual elasticsearch running""" - __autosetup_backends: bool - __autoteardown_backends: bool + def setUp(self): + self.enterContext(djelme_test_backends()) - def __init_subclass__( - cls, - /, # kwargs on class creation e.g. `Foo(RealElasticTestCase, autosetup_djelme_backends=False) - autosetup_djelme_backends: bool, # required - autoteardown_djelme_backends: bool = True, - **kwargs: typing.Any, - ): - super().__init_subclass__(**kwargs) - cls.__autosetup_backends = autosetup_djelme_backends - cls.__autoteardown_backends = autoteardown_djelme_backends + +class NoSetupRealElasticTestCase(SimpleDjelmeTestCase): + """NoSetupRealElasticTestCase: base test case with actual elasticsearch running + + same as RealElasticTestCase but skip djelme backend setup + """ def setUp(self): + clear_setup_check_caches() self.enterContext(prefixed_index_names()) super().setUp() - if self.__autosetup_backends: - setup_backends() def tearDown(self): - if self.__autoteardown_backends: - teardown_backends() + teardown_backends() super().tearDown() class MockSaveTestCase(SimpleDjelmeTestCase): def setUp(self): super().setUp() + clear_setup_check_caches() self.mocked_es6_save = self.enterContext( mock.patch("elasticsearch_metrics.imps.elastic6.Document.save"), ) From 8025d58e23b4e0c562e1d59c98b10ec936eb56e6 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Tue, 14 Apr 2026 16:29:58 -0400 Subject: [PATCH 08/22] fix: sessionhour_id --- elasticsearch_metrics/imps/elastic8.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch_metrics/imps/elastic8.py b/elasticsearch_metrics/imps/elastic8.py index 2c1fbd9..f8b1434 100644 --- a/elasticsearch_metrics/imps/elastic8.py +++ b/elasticsearch_metrics/imps/elastic8.py @@ -597,7 +597,7 @@ def record( if (request_host or (django_request is None)) else django_request.get_host() ) - _sessionhour_id = kwargs.pop("sessionhour_id", None) or opaque_sessionhour_id( + _sessionhour_id = sessionhour_id or opaque_sessionhour_id( client_session_id=session_id, user_id=user_id, request_host=_useragent, From e18f029c406d743d407f18fda8a133b261f9c4d2 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Wed, 15 Apr 2026 15:59:46 -0400 Subject: [PATCH 09/22] non-timeseries djelme record --- elasticsearch_metrics/exceptions.py | 22 ++- elasticsearch_metrics/imps/elastic6.py | 6 +- elasticsearch_metrics/imps/elastic8.py | 131 +++++++++++++----- .../commands/djelme_backend_check.py | 10 +- elasticsearch_metrics/registry.py | 4 +- .../tests/dummy8app/metrics.py | 13 +- .../tests/test_imps_elastic8.py | 17 +++ .../test_djelme_check.py | 27 +++- .../test_djelme_setup.py | 34 +++-- elasticsearch_metrics/tests/util.py | 8 +- elasticsearch_metrics/util/django.py | 12 +- 11 files changed, 195 insertions(+), 89 deletions(-) diff --git a/elasticsearch_metrics/exceptions.py b/elasticsearch_metrics/exceptions.py index ca80ff6..8b97f97 100644 --- a/elasticsearch_metrics/exceptions.py +++ b/elasticsearch_metrics/exceptions.py @@ -2,17 +2,33 @@ class DjelmeError(Exception): """Base class from which all django-elasticsearch-metrics -related exceptions inherit.""" -class TimeseriesSetupError(DjelmeError): +class DjelmeSetupError(DjelmeError): """for errors that might be solved by `djelme_backend_setup`""" -class IndexTemplateNotFoundError(TimeseriesSetupError): +class IndexNotFoundError(DjelmeSetupError): + """specific index not found""" + + def __init__(self, message, client_error): + self.client_error = client_error + super().__init__(message, client_error) + + +class IndexOutOfSyncError(DjelmeSetupError): + """specific index has different mappings than expected""" + + +class IndexTemplateNotFoundError(DjelmeSetupError): + """index template not found""" + def __init__(self, message, client_error): self.client_error = client_error super().__init__(message, client_error) -class IndexTemplateOutOfSyncError(TimeseriesSetupError): +class IndexTemplateOutOfSyncError(DjelmeSetupError): + """index template has different mappings, settings, or patterns than expected""" + def __init__(self, message, mappings_in_sync, patterns_in_sync, settings_in_sync): self.mappings_in_sync = mappings_in_sync self.patterns_in_sync = patterns_in_sync diff --git a/elasticsearch_metrics/imps/elastic6.py b/elasticsearch_metrics/imps/elastic6.py index e8f5386..6852e4c 100644 --- a/elasticsearch_metrics/imps/elastic6.py +++ b/elasticsearch_metrics/imps/elastic6.py @@ -138,12 +138,12 @@ def construct_index(cls, opts, bases): @property def _template_name(self): - _prefix = self.get_timeseries_name_prefix() + _prefix = self.get_index_name_prefix() return f"{_prefix}{self.__template_name}" @property def _template_pattern(self): - _prefix = self.get_timeseries_name_prefix() + _prefix = self.get_index_name_prefix() return f"{_prefix}{self.__template_pattern}" @@ -266,7 +266,7 @@ def get_timeseries_index_template(cls): ) @classmethod - def get_timeseries_name_prefix(cls) -> str: + def get_index_name_prefix(cls) -> str: return "" @classmethod diff --git a/elasticsearch_metrics/imps/elastic8.py b/elasticsearch_metrics/imps/elastic8.py index f8b1434..fb049bc 100644 --- a/elasticsearch_metrics/imps/elastic8.py +++ b/elasticsearch_metrics/imps/elastic8.py @@ -43,6 +43,7 @@ ProtoDjelmeRecord, ) from elasticsearch_metrics.util import timeseries_naming +from elasticsearch_metrics.util.django import find_app_label_for_module from elasticsearch_metrics.util.timeparts import format_full_timeparts, format_timeparts from elasticsearch_metrics.util.unique_together import get_unique_id from elasticsearch_metrics.util.anon_enough import opaque_sessionhour_id @@ -66,8 +67,8 @@ def utcnow() -> datetime.datetime: # changes to document metaclass behavior -class _DjelmeRecordtypeMetaclass(IndexMeta): - """Metaclass for the base `DjelmeRecordtype` class. +class _DjelmeRecordMetaclass(IndexMeta): + """Metaclass for the base `BaseDjelmeRecord` class. extend elasticsearch-py's `IndexMeta` to: - belong to a django app (identified by `app_label` property) @@ -78,7 +79,7 @@ class _DjelmeRecordtypeMetaclass(IndexMeta): Meta: type def __new__(mcls, name, bases, attrs): # noqa: B902 - """create a new DjelmeRecordtype subclass + """create a new BaseDjelmeRecord subclass extend elasticsearch-py's `IndexMeta.__new__` to: - register concrete types with elasticsearch_metrics.registry.djelme_registry @@ -89,7 +90,7 @@ def __new__(mcls, name, bases, attrs): # noqa: B902 _cls_meta = attrs.get("Meta") or type("Meta", (), {}) # call IndexMeta.__new__ _cls = super().__new__(mcls, name, bases, attrs) - assert isinstance(_cls, _DjelmeRecordtypeMetaclass) + assert isinstance(_cls, _DjelmeRecordMetaclass) # un-remove `class Meta` for later use if "Meta" in _cls.__dict__: for _attrname, _attrval in _cls_meta.__dict__.items(): @@ -103,7 +104,7 @@ def __new__(mcls, name, bases, attrs): # noqa: B902 _cls._defaults.setdefault(_fieldname, _default) # and register concrete record types with the djelme registry if not _cls.is_abstract: - assert issubclass(_cls, DjelmeRecordtype) + assert issubclass(_cls, BaseDjelmeRecord) _given_using = _cls._index._using _default_backend = ( _given_using @@ -134,10 +135,10 @@ def app_label(self) -> str: return _app_label -class DjelmeRecordtype(esdsl.Document, metaclass=_DjelmeRecordtypeMetaclass): +class BaseDjelmeRecord(esdsl.Document, metaclass=_DjelmeRecordMetaclass): """a subclass of elasticsearch8.dsl.Document, with conveniences - >>> class MyAbstractRecord(DjelmeRecordtype): + >>> class MyAbstractRecord(BaseDjelmeRecord): ... foo: int ... class Meta: ... abstract = True @@ -166,6 +167,14 @@ class DjelmeRecordtype(esdsl.Document, metaclass=_DjelmeRecordtypeMetaclass): class Meta: abstract = True + @classmethod + def check_djelme_setup(cls, using: str | None = None) -> None: + raise NotImplementedError # expected on subclasses + + @classmethod + def do_teardown(cls, es_client): + raise NotImplementedError # expected on subclasses + @classmethod def record( cls, @@ -185,14 +194,6 @@ def record( _instance.save(using=using) return _instance - @classmethod - def check_djelme_setup(cls, using: str | None = None) -> None: - # this base class has only a single index -- does it exist? - if not cls._index.get(using=using): - raise exceptions.TimeseriesSetupError( - f"Index {cls._index._name} does not exist" - ) - @classmethod @functools.cache def require_been_setup(cls, using: str | None = None) -> None: @@ -200,9 +201,10 @@ def require_been_setup(cls, using: str | None = None) -> None: cls.check_djelme_setup(using) @classmethod - def _djelme_teardown(cls, es_client): - # this base class has only a single index -- delete it - cls._index.delete(using=es_client) + def get_index_name_prefix(cls) -> str: + _name_prefix = cls._get_meta_attr("index_name_prefix") or "" + assert isinstance(_name_prefix, str) + return _name_prefix @classmethod def _get_using( @@ -269,7 +271,72 @@ def _get_unique_id(self) -> str | None: ) -class TimeseriesRecord(DjelmeRecordtype): +class _SimpleRecordMetaclass(_DjelmeRecordMetaclass): + def __new__(mcls, name, bases, attrs): # noqa: B902 + """ + extend _DjelmeRecordMetaclass.__new__ to set index name by app label and type name + """ + _index = attrs.get("Index") + if not _index: + _index = type("Index", (), {}) + attrs["Index"] = _index + if not getattr(_index, "name", None): + _app_label = find_app_label_for_module(attrs["__module__"]) + _index.name = f"{_app_label.lower()}_{name.lower()}" + return super().__new__(mcls, name, bases, attrs) + + +class SimpleRecord(BaseDjelmeRecord, metaclass=_SimpleRecordMetaclass): + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + if not cls.is_abstract: + _prefix = cls.get_index_name_prefix() + _name = cls._index._name + if not _name.startswith(_prefix): + cls._index._name = f"{_prefix}{_name}" + + @classmethod + def djelme_index_name(cls) -> str: + """the index name for this record type + + for ProtoDjelmeRecord + """ + return cls._index._name + + @classmethod + def check_djelme_setup(cls, using: str | None = None) -> None: + """Check if class is in sync with index mappings in Elasticsearch. + + :raise: IndexNotFoundError if index does not exist. + :raise: IndexOutOfSyncError if mappings are out of sync. + :return: if index exists and mappings are in sync. + """ + _client = cls._get_connection(using) + _index_name = cls.djelme_index_name() + try: + _index_response = _client.indices.get(index=_index_name) + except NotFoundError as client_error: + raise exceptions.IndexNotFoundError( + f"Index {_index_name} does not exist for {cls.__name__}", + client_error=client_error, + ) from client_error + else: + _current_mappings = _index_response[_index_name]["mappings"] + _expected_mappings = cls._index.to_dict()["mappings"] + if _current_mappings != _expected_mappings: + raise exceptions.IndexOutOfSyncError( + f"Index {_index_name} has mappings out of sync with {cls.__name__}", + ) + + @classmethod + def do_teardown(cls, es_client): + cls._index.delete(using=es_client, ignore_unavailable=True) + + class Meta: + abstract = True + + +class TimeseriesRecord(BaseDjelmeRecord): # the 'version' field type allows range queries on semver-like strings # that fit perfectly with "timeparts" representation of a UTC datetime # as a sequence of integers -- helps to avoid time zones and date math @@ -340,7 +407,7 @@ def each_timeseries_index( yield _index_name, _index_info @classmethod - def _djelme_teardown(cls, es8_client: Elastic8Client) -> None: + def do_teardown(cls, es8_client: Elastic8Client) -> None: _indexname_wildcard = cls.format_timeseries_index_pattern() _indices = es8_client.indices.get(index=_indexname_wildcard, features=",") for _index_name in _indices.keys(): @@ -363,7 +430,7 @@ def get_timeseries_template_name(cls) -> str: _template_name = timeseries_naming.format_template_name( cls.app_label, cls.get_timeseries_recordtype_name() ) - return "".join((cls.get_timeseries_name_prefix(), _template_name)) + return "".join((cls.get_index_name_prefix(), _template_name)) @classmethod def get_timeseries_recordtype_name(cls) -> str: @@ -373,12 +440,6 @@ def get_timeseries_recordtype_name(cls) -> str: assert isinstance(_recordtype_name, str) return _recordtype_name - @classmethod - def get_timeseries_name_prefix(cls) -> str: - _name_prefix = cls._get_meta_attr("timeseries_name_prefix") or "" - assert isinstance(_name_prefix, str) - return _name_prefix - @classmethod def format_timeseries_index_pattern(cls, timeparts: tuple[int, ...] = ()) -> str: _pattern = timeseries_naming.format_index_pattern( @@ -387,7 +448,7 @@ def format_timeseries_index_pattern(cls, timeparts: tuple[int, ...] = ()) -> str timeparts=timeparts, max_timedepth=cls.get_timedepth(), ) - return "".join((cls.get_timeseries_name_prefix(), _pattern)) + return "".join((cls.get_index_name_prefix(), _pattern)) @classmethod def format_timeseries_index_pattern_for_range( @@ -402,7 +463,7 @@ def format_timeseries_index_pattern_for_range( until_when or utcnow(), timedepth=cls.get_timedepth(), ) - return "".join((cls.get_timeseries_name_prefix(), _pattern)) + return "".join((cls.get_index_name_prefix(), _pattern)) @classmethod def get_timedepth(cls) -> int: @@ -439,7 +500,7 @@ def sync_index_template(cls, using=None): # -> ComposableIndexTemplate: def check_djelme_setup(cls, using: str | Elastic8Client | None = None) -> None: """Check if class is in sync with index template in Elasticsearch. - override `DjelmeRecordtype.check_djelme_setup` to check for a template instead of an index + override `BaseDjelmeRecord.check_djelme_setup` to check for a template instead of an index :raise: IndexTemplateNotFoundError if index template does not exist. :raise: IndexTemplateOutOfSyncError if mappings, settings, or index patterns @@ -514,7 +575,7 @@ def djelme_index_name(self) -> str: timeparts=self.timeseries_timeparts or self.get_timeseries_timeparts(), max_timedepth=self.get_timedepth(), ) - return "".join((self.get_timeseries_name_prefix(), _index_name)) + return "".join((self.get_index_name_prefix(), _index_name)) def save( self, @@ -676,21 +737,21 @@ def djelme_setup(self, recordtypes: collections.abc.Iterable[type]) -> None: # for ProtoDjelmeBackend for _recordtype in recordtypes: # TODO: logger.info - assert issubclass(_recordtype, DjelmeRecordtype) + assert issubclass(_recordtype, BaseDjelmeRecord) _recordtype.init(using=self._elastic8dsl_connection_name) def djelme_teardown(self, recordtypes: collections.abc.Iterable[type]) -> None: # for ProtoDjelmeBackend for _recordtype in recordtypes: - assert issubclass(_recordtype, DjelmeRecordtype) - _recordtype._djelme_teardown(self.elastic8_client) + assert issubclass(_recordtype, BaseDjelmeRecord) + _recordtype.do_teardown(self.elastic8_client) if __debug__ and typing.TYPE_CHECKING: # for static type-checking; verify intent _: type[ProtoCountedUsage] = CountedUsageRecord __: type[ProtoDjelmeBackend] = DjelmeElastic8Backend - ___: type[ProtoDjelmeRecord] = DjelmeRecordtype + ___: type[ProtoDjelmeRecord] = BaseDjelmeRecord ### # names expected by ProtoDjelmeImp diff --git a/elasticsearch_metrics/management/commands/djelme_backend_check.py b/elasticsearch_metrics/management/commands/djelme_backend_check.py index 9df68f8..ed039e9 100644 --- a/elasticsearch_metrics/management/commands/djelme_backend_check.py +++ b/elasticsearch_metrics/management/commands/djelme_backend_check.py @@ -1,6 +1,5 @@ -import sys import logging -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand, CommandError from django.utils.termcolors import colorize @@ -31,10 +30,7 @@ def handle(self, *args, **options): for _recordtype in djelme_registry.each_recordtype(app_label=_app_label): try: _recordtype.check_djelme_setup() - except ( - exceptions.IndexTemplateNotFoundError, - exceptions.IndexTemplateOutOfSyncError, - ) as error: + except exceptions.DjelmeSetupError as error: self.stdout.write(" " + error.args[0]) out_of_sync_count += 1 @@ -45,6 +41,6 @@ def handle(self, *args, **options): ) cmd = colorize("python manage.py djelme_backend_setup", opts=("bold",)) self.stdout.write("Run {cmd} to set up index templates.".format(cmd=cmd)) - sys.exit(1) + raise CommandError(1) else: self.stdout.write("All djelme recordtypes set up.", style.SUCCESS) diff --git a/elasticsearch_metrics/registry.py b/elasticsearch_metrics/registry.py index b4e3a8d..6104148 100644 --- a/elasticsearch_metrics/registry.py +++ b/elasticsearch_metrics/registry.py @@ -14,7 +14,7 @@ ProtoDjelmeRecord, ProtoDjelmeImp, ) -from elasticsearch_metrics.util.django import find_app_label_for_type +from elasticsearch_metrics.util.django import find_app_label_for_module from elasticsearch_metrics.util.timeseries_naming import format_namepart __all__ = ("djelme_registry",) @@ -73,7 +73,7 @@ def register_recordtype( default_backend: str = "", ) -> None: """Add a record type to the registry.""" - _app_label = app_label or find_app_label_for_type(recordtype) + _app_label = app_label or find_app_label_for_module(recordtype.__module__) app_recordtypes = self._all_recordtypes[_app_label] recordtype_name = format_namepart(recordtype.__name__) if recordtype_name in app_recordtypes: diff --git a/elasticsearch_metrics/tests/dummy8app/metrics.py b/elasticsearch_metrics/tests/dummy8app/metrics.py index 415f9ee..b3e291e 100644 --- a/elasticsearch_metrics/tests/dummy8app/metrics.py +++ b/elasticsearch_metrics/tests/dummy8app/metrics.py @@ -18,7 +18,7 @@ class Monthly8Event(djelme.EventRecord): intenzity: int class Meta: - timeseries_name_prefix = "dummy8evenz" + index_name_prefix = "dummy8evenz" timeseries_recordtype_name = "eventlog" timedepth = 2 @@ -54,5 +54,14 @@ class Index: using = "my_elastic8_reports" class Meta: - timeseries_name_prefix = "blarg_" + index_name_prefix = "blarg_" timedepth = 2 # monthly timeseries indexes + + +class SimpleKV(djelme.SimpleRecord): + UNIQUE_TOGETHER_FIELDS = ("key",) + key: str + val: int + + class Index: + using = "my_elastic8_reports" diff --git a/elasticsearch_metrics/tests/test_imps_elastic8.py b/elasticsearch_metrics/tests/test_imps_elastic8.py index ce0add8..8f4e4cb 100644 --- a/elasticsearch_metrics/tests/test_imps_elastic8.py +++ b/elasticsearch_metrics/tests/test_imps_elastic8.py @@ -25,6 +25,7 @@ Monthly8Event, ThingHappened, ThingHappeningsReport, + SimpleKV, ) @@ -712,6 +713,22 @@ def test_search_range(self): self.assertEqual(_actual_2.timeseries_timeparts, "2000.2") +class TestSingleIndex(RealElasticTestCase): + def setUp(self): + super().setUp() + SimpleKV.record(key="hello", val=2) + SimpleKV.record(key="goodbye", val=-2) + SimpleKV._index.refresh() + + def test_search(self): + (_hello,) = SimpleKV.search().query({"term": {"key": "hello"}}).execute() + self.assertEqual(_hello.key, "hello") + self.assertEqual(_hello.val, 2) + (_goodbye,) = SimpleKV.search().query({"term": {"key": "goodbye"}}).execute() + self.assertEqual(_goodbye.key, "goodbye") + self.assertEqual(_goodbye.val, -2) + + def _strip_test_prefix(index_name: str) -> str: # strip uuid test prefix (_, _, _without_prefix) = index_name.partition("_") diff --git a/elasticsearch_metrics/tests/test_management_commands/test_djelme_check.py b/elasticsearch_metrics/tests/test_management_commands/test_djelme_check.py index dc60897..06c93be 100644 --- a/elasticsearch_metrics/tests/test_management_commands/test_djelme_check.py +++ b/elasticsearch_metrics/tests/test_management_commands/test_djelme_check.py @@ -1,5 +1,7 @@ from unittest import mock +from django.core.management.base import CommandError + from elasticsearch_metrics import exceptions from elasticsearch_metrics.management.commands import djelme_backend_check from elasticsearch_metrics.registry import registry @@ -11,11 +13,16 @@ def setUp(self): self.mock6_check_djelme_setup = self.enterContext( mock.patch("elasticsearch_metrics.imps.elastic6.Metric.check_djelme_setup"), ) - self.mock8_check_djelme_setup = self.enterContext( + self.mock8_timeseries_check_djelme_setup = self.enterContext( mock.patch( "elasticsearch_metrics.imps.elastic8.TimeseriesRecord.check_djelme_setup" ), ) + self.mock8_simple_check_djelme_setup = self.enterContext( + mock.patch( + "elasticsearch_metrics.imps.elastic8.SimpleRecord.check_djelme_setup" + ), + ) def test_exits_with_error_if_out_of_sync_6(self): self.mock6_check_djelme_setup.side_effect = ( @@ -23,24 +30,30 @@ def test_exits_with_error_if_out_of_sync_6(self): "Index template does not exist", client_error=None ) ) - with self.assertRaises(SystemExit): + with self.assertRaises(CommandError): self.run_mgmt_command(djelme_backend_check) def test_exits_with_error_if_out_of_sync_8(self): - self.mock8_check_djelme_setup.side_effect = ( + self.mock8_timeseries_check_djelme_setup.side_effect = ( exceptions.IndexTemplateNotFoundError( "Index template does not exist", client_error=None ) ) - with self.assertRaises(SystemExit): + with self.assertRaises(CommandError): + self.run_mgmt_command(djelme_backend_check) + + def test_exits_with_error_if_out_of_sync_8_simplerec(self): + self.mock8_simple_check_djelme_setup.side_effect = ( + exceptions.IndexNotFoundError("Index does not exist", client_error=None) + ) + with self.assertRaises(CommandError): self.run_mgmt_command(djelme_backend_check) def test_exits_with_success(self): - self.mock6_check_djelme_setup.return_value = True - self.mock8_check_djelme_setup.return_value = True self.run_mgmt_command(djelme_backend_check) _call_count = ( self.mock6_check_djelme_setup.call_count - + self.mock8_check_djelme_setup.call_count + + self.mock8_timeseries_check_djelme_setup.call_count + + self.mock8_simple_check_djelme_setup.call_count ) assert _call_count == len(list(registry.each_recordtype())) diff --git a/elasticsearch_metrics/tests/test_management_commands/test_djelme_setup.py b/elasticsearch_metrics/tests/test_management_commands/test_djelme_setup.py index 2459d7a..dcd1729 100644 --- a/elasticsearch_metrics/tests/test_management_commands/test_djelme_setup.py +++ b/elasticsearch_metrics/tests/test_management_commands/test_djelme_setup.py @@ -9,27 +9,28 @@ class TestDjelmeSetup(SimpleDjelmeTestCase): - mock6_sync_index_template: unittest.mock.Mock - mock8_sync_index_template: unittest.mock.Mock + mock_inits: list[unittest.mock.Mock] def setUp(self): - self.mock6_sync_index_template = self.enterContext( - unittest.mock.patch( - "elasticsearch_metrics.imps.elastic6.Metric.sync_index_template" + self.mock_inits = [ + self.enterContext( + unittest.mock.patch("elasticsearch_metrics.imps.elastic6.Metric.init"), ), - ) - self.mock8_sync_index_template = self.enterContext( - unittest.mock.patch( - "elasticsearch_metrics.imps.elastic8.TimeseriesRecord.sync_index_template" + self.enterContext( + unittest.mock.patch( + "elasticsearch_metrics.imps.elastic8.TimeseriesRecord.init" + ), ), - ) + self.enterContext( + unittest.mock.patch( + "elasticsearch_metrics.imps.elastic8.SimpleRecord.init" + ), + ), + ] def test_without_args(self): out, err = self.run_mgmt_command(djelme_backend_setup) - _call_count = ( - self.mock6_sync_index_template.call_count - + self.mock8_sync_index_template.call_count - ) + _call_count = sum(_mock.call_count for _mock in self.mock_inits) assert _call_count == len(list(djelme_registry.each_recordtype())) assert "Synchronized recordtypes." in out @@ -44,8 +45,5 @@ class Meta: app_label = "dummyapp2" out, err = self.run_mgmt_command(djelme_backend_setup, "dummyapp2") - _call_count = ( - self.mock6_sync_index_template.call_count - + self.mock8_sync_index_template.call_count - ) + _call_count = sum(_mock.call_count for _mock in self.mock_inits) assert _call_count == 1 diff --git a/elasticsearch_metrics/tests/util.py b/elasticsearch_metrics/tests/util.py index b15a76b..6cc7bb1 100644 --- a/elasticsearch_metrics/tests/util.py +++ b/elasticsearch_metrics/tests/util.py @@ -93,7 +93,7 @@ def setUp(self): ) self.mocked_es8_require_been_setup = self.enterContext( mock.patch( - "elasticsearch_metrics.imps.elastic8.DjelmeRecordtype.require_been_setup" + "elasticsearch_metrics.imps.elastic8.BaseDjelmeRecord.require_been_setup" ), ) @@ -103,11 +103,11 @@ def prefixed_index_names(prefix: str = ""): _name_prefix = prefix or f"{uuid.uuid4().hex}_" with ( mock.patch( - "elasticsearch_metrics.imps.elastic8.TimeseriesRecord.get_timeseries_name_prefix", + "elasticsearch_metrics.imps.elastic8.BaseDjelmeRecord.get_index_name_prefix", return_value=_name_prefix, ), mock.patch( - "elasticsearch_metrics.imps.elastic6.BaseMetric.get_timeseries_name_prefix", + "elasticsearch_metrics.imps.elastic6.BaseMetric.get_index_name_prefix", return_value=_name_prefix, ), ): @@ -118,7 +118,7 @@ def clear_setup_check_caches(): from elasticsearch_metrics.imps import elastic6, elastic8 elastic6.Metric.require_been_setup.cache_clear() - elastic8.DjelmeRecordtype.require_been_setup.cache_clear() + elastic8.BaseDjelmeRecord.require_been_setup.cache_clear() def setup_backends(): diff --git a/elasticsearch_metrics/util/django.py b/elasticsearch_metrics/util/django.py index 1f213d3..78ac57c 100644 --- a/elasticsearch_metrics/util/django.py +++ b/elasticsearch_metrics/util/django.py @@ -3,17 +3,15 @@ from django.apps import apps from django.core import exceptions -__all__ = ("find_app_label_for_type",) +__all__ = ("find_app_label_for_module",) -def find_app_label_for_type(given_type: type) -> str: - # look for an installed django app the given type is defined within - _given_module_name = given_type.__module__ +def find_app_label_for_module(module_name: str) -> str: _containing_app_configs = ( _app_config for _app_config in apps.get_app_configs() if (_app_config.module is not None) - and _given_module_name.startswith(_app_config.module.__name__) + and module_name.startswith(_app_config.module.__name__) ) _nearest_containing_app_config = max( _containing_app_configs, @@ -24,9 +22,7 @@ def find_app_label_for_type(given_type: type) -> str: ) if _nearest_containing_app_config is None: raise exceptions.ImproperlyConfigured( - f"type {given_type.__module__}.{given_type.__qualname__} " - "doesn't declare an explicit app_label and isn't in an " - "application in INSTALLED_APPS." + f"module {module_name} isn't in an application in INSTALLED_APPS." ) _label = _nearest_containing_app_config.label if not (_label and isinstance(_label, str)): From d7e0483972a58b940bec843679c2a8c9b8bcb75c Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Thu, 16 Apr 2026 08:58:29 -0400 Subject: [PATCH 10/22] fix: index settings inherited, not mappings --- elasticsearch_metrics/imps/elastic8.py | 75 ++++++++++++++----- .../tests/dummy8app/metrics.py | 13 ---- elasticsearch_metrics/tests/settings.py | 7 +- .../tests/test_imps_elastic8.py | 19 ++--- 4 files changed, 64 insertions(+), 50 deletions(-) diff --git a/elasticsearch_metrics/imps/elastic8.py b/elasticsearch_metrics/imps/elastic8.py index fb049bc..234ecd2 100644 --- a/elasticsearch_metrics/imps/elastic8.py +++ b/elasticsearch_metrics/imps/elastic8.py @@ -120,6 +120,32 @@ def __new__(mcls, name, bases, attrs): # noqa: B902 ) return _cls + @classmethod + def construct_index(cls, opts, bases): + """ + Override IndexMeta.construct_index so a new Index is created for each class + and Index.settings, Index.analyzers, and Index.using are inherited + (but not Index.name or Index.aliases) + """ + _base_index_configs = collections.ChainMap( + *(base._index.to_dict() for base in bases if hasattr(base, "_index")) + ) + _index_opts = opts or type("Index", (), {}) + if not hasattr(_index_opts, "settings") and ( + _inherited_settings := _base_index_configs.get("settings") + ): + _index_opts.settings = _inherited_settings + if not hasattr(_index_opts, "analyzers") and ( + _inherited_analyzers := _base_index_configs.get("analyzers") + ): + _index_opts.analyzers = _inherited_analyzers + if not hasattr(_index_opts, "using"): + _inherited_using = _base_index_configs.get("using") + if _inherited_using and (_inherited_using != "default"): + _index_opts.using = _inherited_using + assert _index_opts is not None # guarantee separate index per class + return super().construct_index(_index_opts, bases) + def _get_meta_attr(self, attr_name: str, default: typing.Any = None) -> typing.Any: _meta = getattr(self, "Meta", None) return getattr(_meta, attr_name, default) @@ -189,6 +215,7 @@ def record( to use to save, or `False` to skip saving (e.g. for use in a bulk operation) all other kwargs passed thru to the class constructor """ + assert not cls.is_abstract _instance = cls(**kwargs) if using is not False: _instance.save(using=using) @@ -241,6 +268,7 @@ def save( - populate document id based on UNIQUE_TOGETHER_FIELDS - send pre_save/post_save signals """ + assert not self.__class__.is_abstract self.require_been_setup(using=using) # prevent automapped indexes self._populate_unique_id() signals.pre_save.send(self.__class__, instance=self, using=using, index=index) @@ -276,24 +304,26 @@ def __new__(mcls, name, bases, attrs): # noqa: B902 """ extend _DjelmeRecordMetaclass.__new__ to set index name by app label and type name """ - _index = attrs.get("Index") - if not _index: - _index = type("Index", (), {}) - attrs["Index"] = _index - if not getattr(_index, "name", None): - _app_label = find_app_label_for_module(attrs["__module__"]) - _index.name = f"{_app_label.lower()}_{name.lower()}" - return super().__new__(mcls, name, bases, attrs) + _cls = super().__new__(mcls, name, bases, attrs) + if not _cls.is_abstract: + # set `_index._name`, if not already specified + _index_name = _cls._index._name + if not _index_name or (_index_name in ("*", "default")): + _app_label = find_app_label_for_module(attrs["__module__"]) + _index_name = f"{_app_label.lower()}_{_cls.__name__.lower()}" + _prefix = _cls.get_index_name_prefix() + if not _index_name.startswith(_prefix): + _index_name = f"{_prefix}{_index_name}" + _cls._index._name = _index_name + # set `_index._using`, so `_index` can be used normally + _backend = djelme_registry.get_backend_for_recordtype(_cls) + _cls._index._using = _backend._elastic8dsl_connection_name + return _cls class SimpleRecord(BaseDjelmeRecord, metaclass=_SimpleRecordMetaclass): - def __init_subclass__(cls, **kwargs): - super().__init_subclass__(**kwargs) - if not cls.is_abstract: - _prefix = cls.get_index_name_prefix() - _name = cls._index._name - if not _name.startswith(_prefix): - cls._index._name = f"{_prefix}{_name}" + class Meta: + abstract = True @classmethod def djelme_index_name(cls) -> str: @@ -301,6 +331,7 @@ def djelme_index_name(cls) -> str: for ProtoDjelmeRecord """ + assert not cls.is_abstract return cls._index._name @classmethod @@ -311,6 +342,7 @@ def check_djelme_setup(cls, using: str | None = None) -> None: :raise: IndexOutOfSyncError if mappings are out of sync. :return: if index exists and mappings are in sync. """ + assert not cls.is_abstract _client = cls._get_connection(using) _index_name = cls.djelme_index_name() try: @@ -330,10 +362,13 @@ def check_djelme_setup(cls, using: str | None = None) -> None: @classmethod def do_teardown(cls, es_client): + assert not cls.is_abstract cls._index.delete(using=es_client, ignore_unavailable=True) - class Meta: - abstract = True + @classmethod + def refresh(cls, using: str | None = None) -> None: + assert not cls.is_abstract + cls._index.refresh(using=using) class TimeseriesRecord(BaseDjelmeRecord): @@ -366,6 +401,7 @@ def search( ) -> esdsl.Search[ "typing.Self" ]: # typing.Self added in py 3.11 -- str annotation until 3.10 eol + assert not cls.is_abstract return super().search( using=using, index=(index or cls.format_timeseries_index_pattern()), @@ -389,7 +425,8 @@ def search_timeseries_range( return cls.search(index=_index_pattern).filter(_timeseries_q) @classmethod - def refresh_timeseries_indexes(cls, using: str | None = None) -> None: + def refresh(cls, using: str | None = None) -> None: + assert not cls.is_abstract cls._get_connection(using).indices.refresh( index=cls.format_timeseries_index_pattern() ) @@ -408,6 +445,7 @@ def each_timeseries_index( @classmethod def do_teardown(cls, es8_client: Elastic8Client) -> None: + assert not cls.is_abstract _indexname_wildcard = cls.format_timeseries_index_pattern() _indices = es8_client.indices.get(index=_indexname_wildcard, features=",") for _index_name in _indices.keys(): @@ -420,6 +458,7 @@ def do_teardown(cls, es8_client: Elastic8Client) -> None: @classmethod def get_timeseries_template(cls) -> esdsl.ComposableIndexTemplate: + assert not cls.is_abstract return cls._index.as_composable_template( template_name=cls.get_timeseries_template_name(), pattern=cls.format_timeseries_index_pattern(), diff --git a/elasticsearch_metrics/tests/dummy8app/metrics.py b/elasticsearch_metrics/tests/dummy8app/metrics.py index b3e291e..a05fdbd 100644 --- a/elasticsearch_metrics/tests/dummy8app/metrics.py +++ b/elasticsearch_metrics/tests/dummy8app/metrics.py @@ -10,9 +10,6 @@ class Dummy8Event(djelme.EventRecord): intensity: int - class Index: - using = "my_elastic8_events" - class Monthly8Event(djelme.EventRecord): intenzity: int @@ -22,9 +19,6 @@ class Meta: timeseries_recordtype_name = "eventlog" timedepth = 2 - class Index: - using = "my_elastic8_events" - class ThingHappened(djelme.EventRecord): thing_id: str = "" @@ -36,7 +30,6 @@ class ThingHappened(djelme.EventRecord): class Index: settings = {"refresh_interval": "-1"} - using = "my_elastic8_events" class Meta: timeseries_recordtype_name = "happen" @@ -50,9 +43,6 @@ class ThingHappeningsReport(djelme.CyclicRecord): thing_id: str happen_count: int - class Index: - using = "my_elastic8_reports" - class Meta: index_name_prefix = "blarg_" timedepth = 2 # monthly timeseries indexes @@ -62,6 +52,3 @@ class SimpleKV(djelme.SimpleRecord): UNIQUE_TOGETHER_FIELDS = ("key",) key: str val: int - - class Index: - using = "my_elastic8_reports" diff --git a/elasticsearch_metrics/tests/settings.py b/elasticsearch_metrics/tests/settings.py index e7b3c91..8de980e 100644 --- a/elasticsearch_metrics/tests/settings.py +++ b/elasticsearch_metrics/tests/settings.py @@ -19,12 +19,7 @@ "hosts": os.environ.get("ELASTICSEARCH6_URL", ""), }, }, - "my_elastic8_events": { - "elasticsearch_metrics.imps.elastic8": { - "hosts": os.environ.get("ELASTICSEARCH8_URL", ""), - }, - }, - "my_elastic8_reports": { + "my_elastic8": { "elasticsearch_metrics.imps.elastic8": { "hosts": os.environ.get("ELASTICSEARCH8_URL", ""), }, diff --git a/elasticsearch_metrics/tests/test_imps_elastic8.py b/elasticsearch_metrics/tests/test_imps_elastic8.py index 8f4e4cb..c6ebdee 100644 --- a/elasticsearch_metrics/tests/test_imps_elastic8.py +++ b/elasticsearch_metrics/tests/test_imps_elastic8.py @@ -30,7 +30,7 @@ def _es8_client( - backend_name: str = "my_elastic8_events", + backend_name: str = "my_elastic8", ) -> elasticsearch8.Elasticsearch: _backend = djelme_registry.get_backend(backend_name) assert isinstance(_backend, djelme.DjelmeElastic8Backend) @@ -224,9 +224,6 @@ def test_get_index_template_default_template_name(self): def test_get_index_template_uses_app_label_in_class_meta(self): class MyRecord(djelme.TimeseriesRecord): - class Index: - using = "my_elastic8_events" - class Meta: app_label = "myapp" @@ -248,7 +245,6 @@ class MyBaseRecord(djelme.TimeseriesRecord): class Index: settings = {"number_of_shards": 2} - using = "my_elastic8_events" class Meta: abstract = True @@ -263,9 +259,6 @@ class Meta: def test_source_may_be_enabled(self): class MyRecord(djelme.TimeseriesRecord): - class Index: - using = "my_elastic8_events" - class Meta: app_label = "dummy8app" timeseries_recordtype_name = "myrecord" @@ -475,7 +468,7 @@ def setUp(self): Dummy8Event.record(timestamp=dt.datetime(1235, 5, 6), intensity=111) Dummy8Event.record(timestamp=dt.datetime(2345, 6, 9), intensity=11) Dummy8Event.record(timestamp=dt.datetime(2345, 7, 9), intensity=13) - Dummy8Event.refresh_timeseries_indexes() + Dummy8Event.refresh() def test_indexes(self): _index_names = { @@ -537,7 +530,7 @@ def setUp(self): Monthly8Event.record(timestamp=dt.datetime(1235, 5, 6), intenzity=111) Monthly8Event.record(timestamp=dt.datetime(2345, 6, 9), intenzity=11) Monthly8Event.record(timestamp=dt.datetime(2345, 7, 9), intenzity=13) - Monthly8Event.refresh_timeseries_indexes() + Monthly8Event.refresh() def test_indexes(self): _index_names = { @@ -598,7 +591,7 @@ def setUp(self): ThingHappened.record(timestamp=dt.datetime(1235, 5, 6), thing_id="e") ThingHappened.record(timestamp=dt.datetime(2345, 6, 9), thing_id="f") ThingHappened.record(timestamp=dt.datetime(2345, 7, 9), thing_id="g") - ThingHappened.refresh_timeseries_indexes() + ThingHappened.refresh() def test_indexes(self): _index_names = { @@ -671,7 +664,7 @@ def setUp(self): ThingHappeningsReport.record( cycle_coverage="2000.2", thing_id="c", happen_count=7 ) - ThingHappeningsReport.refresh_timeseries_indexes() + ThingHappeningsReport.refresh() def test_indexes(self): _index_names = { @@ -718,7 +711,7 @@ def setUp(self): super().setUp() SimpleKV.record(key="hello", val=2) SimpleKV.record(key="goodbye", val=-2) - SimpleKV._index.refresh() + SimpleKV.refresh() def test_search(self): (_hello,) = SimpleKV.search().query({"term": {"key": "hello"}}).execute() From 87bb66a86893956d323acfc0f73af3afa6a9863a Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 17 Apr 2026 13:40:32 -0400 Subject: [PATCH 11/22] fix: autofill on clean, not save --- elasticsearch_metrics/imps/elastic8.py | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/elasticsearch_metrics/imps/elastic8.py b/elasticsearch_metrics/imps/elastic8.py index 234ecd2..38028a5 100644 --- a/elasticsearch_metrics/imps/elastic8.py +++ b/elasticsearch_metrics/imps/elastic8.py @@ -253,13 +253,28 @@ def _get_using( return _backend._elastic8dsl_connection_name return super()._get_using(using) + def _get_index(self, index: str | None = None, required: bool = True) -> str | None: + """get the index name for this record + + extend elasticsearch8.dsl's DocumentBase._get_index to default to djelme_index_name() + """ + return super()._get_index(index or self.djelme_index_name(), required) + + def clean(self) -> None: + """fill fields based on other fields, before saving + + extend elasticsearch8.dsl's DocumentBase.clean to use the djelme index name + + subclasses that need to autofill or transform fields should further extend this method + """ + self._populate_unique_id() + super().clean() + def save( self, + *, using: str | Elastic8Client | None = None, index: str | None = None, - validate: bool = True, - skip_empty: bool = True, - return_doc_meta: bool = False, **kwargs: typing.Any, ) -> typing.Any: """save the record @@ -270,11 +285,8 @@ def save( """ assert not self.__class__.is_abstract self.require_been_setup(using=using) # prevent automapped indexes - self._populate_unique_id() signals.pre_save.send(self.__class__, instance=self, using=using, index=index) - _saved = super().save( - using=using, index=index, validate=validate, skip_empty=skip_empty, **kwargs - ) + _saved = super().save(using=using, index=index, **kwargs) signals.post_save.send(self.__class__, instance=self, using=using, index=index) return _saved @@ -616,25 +628,13 @@ def djelme_index_name(self) -> str: ) return "".join((self.get_index_name_prefix(), _index_name)) - def save( - self, - using: str | Elastic8Client | None = None, - index: str | None = None, - validate: bool = True, - skip_empty: bool = True, - return_doc_meta: bool = False, - **kwargs: typing.Any, - ) -> typing.Any: + def clean(self) -> None: """save the record to a timeseries index extend `elasticsearch8.dsl.Document.save` to choose a specific timeseries index """ - assert not self.__class__.is_abstract + super().clean() self.timeseries_timeparts = self.get_timeseries_timeparts() - if index is None: - index = self.djelme_index_name() - ret = super().save(using=using, index=index, **kwargs) - return ret # TODO: EventRecord expiration From 1776a82cf600073cec690fe5e89d32074ade11df Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 17 Apr 2026 13:41:45 -0400 Subject: [PATCH 12/22] clearer CountedUsageRecord params and use given timestamp --- elasticsearch_metrics/imps/elastic8.py | 11 ++++++----- elasticsearch_metrics/util/anon_enough.py | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/elasticsearch_metrics/imps/elastic8.py b/elasticsearch_metrics/imps/elastic8.py index 38028a5..06a6240 100644 --- a/elasticsearch_metrics/imps/elastic8.py +++ b/elasticsearch_metrics/imps/elastic8.py @@ -678,10 +678,10 @@ def record( sessionhour_id: str = "", # ...but when saving new data, give either the dirty identifying strings: user_id: str = "", - session_id: str = "", + client_session_id: str = "", request_host: str = "", request_useragent: str = "", - # ...or a django request to infer from + # ...or a django request to infer user/host/useragent from django_request: django.http.HttpRequest | None = None, # additional kwargs presumed to give field values **kwargs: typing.Any, @@ -698,10 +698,11 @@ def record( else django_request.get_host() ) _sessionhour_id = sessionhour_id or opaque_sessionhour_id( - client_session_id=session_id, + client_session_id=client_session_id, user_id=user_id, - request_host=_useragent, - request_useragent=_host, + request_host=_host, + request_useragent=_useragent, + timestamp=kwargs.get("timestamp"), ) _new_record = super().record(**kwargs, sessionhour_id=_sessionhour_id) assert isinstance(_new_record, cls) diff --git a/elasticsearch_metrics/util/anon_enough.py b/elasticsearch_metrics/util/anon_enough.py index 1deb22b..c99f84a 100644 --- a/elasticsearch_metrics/util/anon_enough.py +++ b/elasticsearch_metrics/util/anon_enough.py @@ -33,13 +33,14 @@ def opaque_sessionhour_id( user_id: str = "", request_host: str = "", request_useragent: str = "", + timestamp: datetime.datetime or None = None, ) -> str: """opaque_sessionhour_id: get a hashed id for a "user session" compatible with COUNTER code of practice: https://cop5.projectcounter.org/en/5.0.2/07-processing/03-counting-unique-items.html """ - _now = timezone.now().astimezone(datetime.timezone.utc) + _now = timestamp or timezone.now().astimezone(datetime.timezone.utc) _today_str = _now.date().isoformat() # "A user session is defined any of the following ways: ..." (quotes out of order) From a878fe251752365d148d72641c0a9591acb03604 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 17 Apr 2026 13:42:50 -0400 Subject: [PATCH 13/22] mock connection, not save() --- .../tests/test_imps_elastic6.py | 12 +++--- .../tests/test_imps_elastic8.py | 13 +++---- elasticsearch_metrics/tests/util.py | 38 ++++++++++++------- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/elasticsearch_metrics/tests/test_imps_elastic6.py b/elasticsearch_metrics/tests/test_imps_elastic6.py index e9209f1..096f154 100644 --- a/elasticsearch_metrics/tests/test_imps_elastic6.py +++ b/elasticsearch_metrics/tests/test_imps_elastic6.py @@ -19,7 +19,7 @@ ) from elasticsearch_metrics.tests.util import ( SimpleDjelmeTestCase, - MockSaveTestCase, + MockConnectionTestCase, RealElasticTestCase, NoSetupRealElasticTestCase, ) @@ -189,11 +189,11 @@ class Meta: assert doc["_source"]["enabled"] is True -class TestRecord(MockSaveTestCase): - def test_calls_save(self): +class TestRecord(MockConnectionTestCase): + def test_calls_index(self): timestamp = dt.datetime(2017, 8, 21) p = PreprintView.record(timestamp=timestamp, provider_id="abc12") - assert self.mocked_es6_save.call_count == 1 + assert self.mock_es6_connection.index.call_count == 1 assert p.timestamp == timestamp assert p.provider_id == "abc12" @@ -203,11 +203,11 @@ def test_defaults_timestamp_to_now(self, mock_now): mock_now.return_value = fake_now p = PreprintView.record(provider_id="abc12") - assert self.mocked_es6_save.call_count == 1 + assert self.mock_es6_connection.index.call_count == 1 assert p.timestamp == fake_now -class TestSignals(MockSaveTestCase): +class TestSignals(MockConnectionTestCase): @unittest.mock.patch.object(PreprintView, "get_timeseries_index_template") def test_create_metric_sends_signals(self, mock_get_index_template): mock_pre_index_template_listener = unittest.mock.Mock() diff --git a/elasticsearch_metrics/tests/test_imps_elastic8.py b/elasticsearch_metrics/tests/test_imps_elastic8.py index c6ebdee..1ac129a 100644 --- a/elasticsearch_metrics/tests/test_imps_elastic8.py +++ b/elasticsearch_metrics/tests/test_imps_elastic8.py @@ -16,7 +16,7 @@ from elasticsearch_metrics.registry import djelme_registry from elasticsearch_metrics.tests.util import ( SimpleDjelmeTestCase, - MockSaveTestCase, + MockConnectionTestCase, RealElasticTestCase, NoSetupRealElasticTestCase, ) @@ -270,11 +270,11 @@ class Meta: assert doc["_source"]["enabled"] is True -class TestRecord(MockSaveTestCase): +class TestRecord(MockConnectionTestCase): def test_calls_save(self): timestamp = dt.datetime(2017, 8, 21) p = ThingHappened.record(timestamp=timestamp, thing_id="abc12") - assert self.mocked_es8_save.call_count == 1 + assert self.mock_es8_connection.index.call_count == 1 assert p.timestamp == timestamp assert p.timeseries_timeparts == "2017.8.21.0.0.0" assert p.thing_id == "abc12" @@ -285,13 +285,12 @@ def test_defaults_timestamp_to_now(self): "elasticsearch_metrics.imps.elastic8.utcnow", return_value=_fake_now ): p = ThingHappened.record(thing_id="abc12") - assert self.mocked_es8_save.call_count == 1 + assert self.mock_es8_connection.index.call_count == 1 assert p.timestamp == _fake_now -class TestSignals(MockSaveTestCase): - @unittest.mock.patch.object(ThingHappened, "get_timeseries_template") - def test_create_record_sends_signals(self, mock_timeseries_template): +class TestSignals(MockConnectionTestCase): + def test_create_record_sends_signals(self): mock_pre_index_template_listener = unittest.mock.Mock() mock_post_index_template_listener = unittest.mock.Mock() signals.pre_index_template_create.connect(mock_pre_index_template_listener) diff --git a/elasticsearch_metrics/tests/util.py b/elasticsearch_metrics/tests/util.py index 6cc7bb1..402148f 100644 --- a/elasticsearch_metrics/tests/util.py +++ b/elasticsearch_metrics/tests/util.py @@ -1,7 +1,7 @@ import contextlib from io import StringIO import types -from unittest import mock +import unittest import uuid from django.core.management import call_command @@ -78,21 +78,33 @@ def tearDown(self): super().tearDown() -class MockSaveTestCase(SimpleDjelmeTestCase): +class MockConnectionTestCase(SimpleDjelmeTestCase): def setUp(self): super().setUp() clear_setup_check_caches() - self.mocked_es6_save = self.enterContext( - mock.patch("elasticsearch_metrics.imps.elastic6.Document.save"), + self.mock_es6_connection = unittest.mock.Mock() + self.mock_es8_connection = unittest.mock.Mock() + self.mock_es6_connection.index.return_value = {"result": "created"} + self.mock_es8_connection.index.return_value = {"result": "created"} + self.enterContext( + unittest.mock.patch( + "elasticsearch_metrics.imps.elastic6.Document._get_connection", + return_value=self.mock_es6_connection, + ), ) - self.mocked_es8_save = self.enterContext( - mock.patch("elasticsearch_metrics.imps.elastic8.esdsl.Document.save"), + self.enterContext( + unittest.mock.patch( + "elasticsearch_metrics.imps.elastic8.esdsl.Document._get_connection", + return_value=self.mock_es8_connection, + ), ) - self.mocked_es6_require_been_setup = self.enterContext( - mock.patch("elasticsearch_metrics.imps.elastic6.Metric.require_been_setup"), + self.mock_es6_require_been_setup = self.enterContext( + unittest.mock.patch( + "elasticsearch_metrics.imps.elastic6.Metric.require_been_setup" + ), ) - self.mocked_es8_require_been_setup = self.enterContext( - mock.patch( + self.mock_es8_require_been_setup = self.enterContext( + unittest.mock.patch( "elasticsearch_metrics.imps.elastic8.BaseDjelmeRecord.require_been_setup" ), ) @@ -100,13 +112,13 @@ def setUp(self): @contextlib.contextmanager def prefixed_index_names(prefix: str = ""): - _name_prefix = prefix or f"{uuid.uuid4().hex}_" + _name_prefix = prefix or f"testrun{uuid.uuid4().hex}_" with ( - mock.patch( + unittest.mock.patch( "elasticsearch_metrics.imps.elastic8.BaseDjelmeRecord.get_index_name_prefix", return_value=_name_prefix, ), - mock.patch( + unittest.mock.patch( "elasticsearch_metrics.imps.elastic6.BaseMetric.get_index_name_prefix", return_value=_name_prefix, ), From 445fcea0aa6b5d07523cd67e959cb14088f15bb0 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 17 Apr 2026 13:55:22 -0400 Subject: [PATCH 14/22] fix: importing unittest.mock having learned something about python imports today... turns out this gives an AttributeError: ``` import unittest unittest.mock ``` ...unless something somewhere (maybe another module) has already done `from unittest import mock` (or `import unittest.mock`), after which `unittest`, however and wherever it's imported, does have a `mock` attr (fair enough within a module, but boo to the inter-module side effect) --- .../tests/test_imps_elastic6.py | 13 ++++++------ .../tests/test_imps_elastic8.py | 14 ++++++------- elasticsearch_metrics/tests/util.py | 20 +++++++++---------- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/elasticsearch_metrics/tests/test_imps_elastic6.py b/elasticsearch_metrics/tests/test_imps_elastic6.py index 096f154..f7de7aa 100644 --- a/elasticsearch_metrics/tests/test_imps_elastic6.py +++ b/elasticsearch_metrics/tests/test_imps_elastic6.py @@ -1,4 +1,5 @@ import unittest +from unittest import mock import datetime as dt from django.utils import timezone @@ -197,7 +198,7 @@ def test_calls_index(self): assert p.timestamp == timestamp assert p.provider_id == "abc12" - @unittest.mock.patch.object(timezone, "now") + @mock.patch.object(timezone, "now") def test_defaults_timestamp_to_now(self, mock_now): fake_now = dt.datetime(2016, 8, 21) mock_now.return_value = fake_now @@ -208,10 +209,10 @@ def test_defaults_timestamp_to_now(self, mock_now): class TestSignals(MockConnectionTestCase): - @unittest.mock.patch.object(PreprintView, "get_timeseries_index_template") + @mock.patch.object(PreprintView, "get_timeseries_index_template") def test_create_metric_sends_signals(self, mock_get_index_template): - mock_pre_index_template_listener = unittest.mock.Mock() - mock_post_index_template_listener = unittest.mock.Mock() + mock_pre_index_template_listener = mock.Mock() + mock_post_index_template_listener = mock.Mock() signals.pre_index_template_create.connect(mock_pre_index_template_listener) signals.post_index_template_create.connect(mock_post_index_template_listener) PreprintView.sync_index_template() @@ -226,8 +227,8 @@ def test_create_metric_sends_signals(self, mock_get_index_template): assert "using" in post_call_kwargs def test_save_sends_signals(self): - mock_pre_save_listener = unittest.mock.Mock() - mock_post_save_listener = unittest.mock.Mock() + mock_pre_save_listener = mock.Mock() + mock_post_save_listener = mock.Mock() signals.pre_save.connect(mock_pre_save_listener, sender=PreprintView) signals.post_save.connect(mock_post_save_listener, sender=PreprintView) diff --git a/elasticsearch_metrics/tests/test_imps_elastic8.py b/elasticsearch_metrics/tests/test_imps_elastic8.py index 1ac129a..943b8d7 100644 --- a/elasticsearch_metrics/tests/test_imps_elastic8.py +++ b/elasticsearch_metrics/tests/test_imps_elastic8.py @@ -1,4 +1,4 @@ -import unittest +from unittest import mock import datetime as dt import elasticsearch8 @@ -281,7 +281,7 @@ def test_calls_save(self): def test_defaults_timestamp_to_now(self): _fake_now = dt.datetime(2016, 8, 21, tzinfo=dt.timezone.utc) - with unittest.mock.patch( + with mock.patch( "elasticsearch_metrics.imps.elastic8.utcnow", return_value=_fake_now ): p = ThingHappened.record(thing_id="abc12") @@ -291,8 +291,8 @@ def test_defaults_timestamp_to_now(self): class TestSignals(MockConnectionTestCase): def test_create_record_sends_signals(self): - mock_pre_index_template_listener = unittest.mock.Mock() - mock_post_index_template_listener = unittest.mock.Mock() + mock_pre_index_template_listener = mock.Mock() + mock_post_index_template_listener = mock.Mock() signals.pre_index_template_create.connect(mock_pre_index_template_listener) signals.post_index_template_create.connect(mock_post_index_template_listener) ThingHappened.sync_index_template() @@ -307,8 +307,8 @@ def test_create_record_sends_signals(self): assert "using" in post_call_kwargs def test_save_sends_signals(self): - mock_pre_save_listener = unittest.mock.Mock() - mock_post_save_listener = unittest.mock.Mock() + mock_pre_save_listener = mock.Mock() + mock_post_save_listener = mock.Mock() signals.pre_save.connect(mock_pre_save_listener, sender=ThingHappened) signals.post_save.connect(mock_post_save_listener, sender=ThingHappened) @@ -386,7 +386,7 @@ def test_cannot_save_without_template(self): _event.save() def test_cannot_save_with_wrong_template_pattern(self): - with unittest.mock.patch.object( + with mock.patch.object( Dummy8Event, "format_timeseries_index_pattern", return_value="wrong_pattern_haha_*", diff --git a/elasticsearch_metrics/tests/util.py b/elasticsearch_metrics/tests/util.py index 402148f..4845e2e 100644 --- a/elasticsearch_metrics/tests/util.py +++ b/elasticsearch_metrics/tests/util.py @@ -1,7 +1,7 @@ import contextlib from io import StringIO import types -import unittest +from unittest import mock import uuid from django.core.management import call_command @@ -82,29 +82,27 @@ class MockConnectionTestCase(SimpleDjelmeTestCase): def setUp(self): super().setUp() clear_setup_check_caches() - self.mock_es6_connection = unittest.mock.Mock() - self.mock_es8_connection = unittest.mock.Mock() + self.mock_es6_connection = mock.Mock() + self.mock_es8_connection = mock.Mock() self.mock_es6_connection.index.return_value = {"result": "created"} self.mock_es8_connection.index.return_value = {"result": "created"} self.enterContext( - unittest.mock.patch( + mock.patch( "elasticsearch_metrics.imps.elastic6.Document._get_connection", return_value=self.mock_es6_connection, ), ) self.enterContext( - unittest.mock.patch( + mock.patch( "elasticsearch_metrics.imps.elastic8.esdsl.Document._get_connection", return_value=self.mock_es8_connection, ), ) self.mock_es6_require_been_setup = self.enterContext( - unittest.mock.patch( - "elasticsearch_metrics.imps.elastic6.Metric.require_been_setup" - ), + mock.patch("elasticsearch_metrics.imps.elastic6.Metric.require_been_setup"), ) self.mock_es8_require_been_setup = self.enterContext( - unittest.mock.patch( + mock.patch( "elasticsearch_metrics.imps.elastic8.BaseDjelmeRecord.require_been_setup" ), ) @@ -114,11 +112,11 @@ def setUp(self): def prefixed_index_names(prefix: str = ""): _name_prefix = prefix or f"testrun{uuid.uuid4().hex}_" with ( - unittest.mock.patch( + mock.patch( "elasticsearch_metrics.imps.elastic8.BaseDjelmeRecord.get_index_name_prefix", return_value=_name_prefix, ), - unittest.mock.patch( + mock.patch( "elasticsearch_metrics.imps.elastic6.BaseMetric.get_index_name_prefix", return_value=_name_prefix, ), From d1ee8e6d5401fcfc77eece6ca6507aeefa4bdafd Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 17 Apr 2026 14:47:59 -0400 Subject: [PATCH 15/22] easier unique_together overriding --- elasticsearch_metrics/imps/elastic8.py | 28 +++++++++---------- elasticsearch_metrics/util/anon_enough.py | 3 ++ elasticsearch_metrics/util/unique_together.py | 16 ----------- 3 files changed, 16 insertions(+), 31 deletions(-) delete mode 100644 elasticsearch_metrics/util/unique_together.py diff --git a/elasticsearch_metrics/imps/elastic8.py b/elasticsearch_metrics/imps/elastic8.py index 06a6240..84275f4 100644 --- a/elasticsearch_metrics/imps/elastic8.py +++ b/elasticsearch_metrics/imps/elastic8.py @@ -45,8 +45,7 @@ from elasticsearch_metrics.util import timeseries_naming from elasticsearch_metrics.util.django import find_app_label_for_module from elasticsearch_metrics.util.timeparts import format_full_timeparts, format_timeparts -from elasticsearch_metrics.util.unique_together import get_unique_id -from elasticsearch_metrics.util.anon_enough import opaque_sessionhour_id +from elasticsearch_metrics.util.anon_enough import opaque_key, opaque_sessionhour_id logger = logging.getLogger(__name__) @@ -291,24 +290,23 @@ def save( return _saved def _populate_unique_id(self) -> None: - _unique_id = self._get_unique_id() - assert _unique_id or not self.UNIQUE_TOGETHER_FIELDS - # make it unique by setting doc id in elasticsearch - if _unique_id: - self.meta.id = _unique_id - - def _get_unique_id(self) -> str | None: """ - Get a unique document id by hashing values of "unique together" + Set a unique document id by hashing values of "unique together" fields for "ON CONFLICT UPDATE" behavior -- if the document already exists, it will be replaced rather than duplicated. Cannot detect/avoid conflicts this way, but that's ok. """ - if not self.UNIQUE_TOGETHER_FIELDS: - return None - return get_unique_id( - (getattr(self, _field_name) for _field_name in self.UNIQUE_TOGETHER_FIELDS) - ) + _unique_together = self._get_unique_together_values() + assert _unique_together or not self.UNIQUE_TOGETHER_FIELDS + if _unique_together: + # make it unique by setting doc id in elasticsearch + self.meta.id = opaque_key(_unique_together) + + def _get_unique_together_values(self) -> list: + return [ + getattr(self, _field_name) + for _field_name in (self.UNIQUE_TOGETHER_FIELDS or ()) + ] class _SimpleRecordMetaclass(_DjelmeRecordMetaclass): diff --git a/elasticsearch_metrics/util/anon_enough.py b/elasticsearch_metrics/util/anon_enough.py index c99f84a..0215ddf 100644 --- a/elasticsearch_metrics/util/anon_enough.py +++ b/elasticsearch_metrics/util/anon_enough.py @@ -21,6 +21,9 @@ def opaque_key( >>> opaque_key(['hello', 'hello', 'hello']) '*58u_=`?3#G!N(%j!3kqU7#Npt>Xvj=|3<75BRoi$0j;F-*3V+Cc?P1FvcVW76T_`5^NaI*_3787SsBn' """ + _parts = [str(_part) for _part in key_parts] + if not _parts: + raise ValueError("opaque_key expects at least one key part") _plain_key = json.dumps([str(_part) for _part in key_parts]) return base64.b85encode( hashlib.blake2b(bytes(_plain_key, encoding="utf")).digest() diff --git a/elasticsearch_metrics/util/unique_together.py b/elasticsearch_metrics/util/unique_together.py deleted file mode 100644 index 0906d76..0000000 --- a/elasticsearch_metrics/util/unique_together.py +++ /dev/null @@ -1,16 +0,0 @@ -import collections - -from elasticsearch_metrics.util.anon_enough import opaque_key - - -def get_unique_id(unique_together_field_values: collections.abc.Iterable[str]) -> str: - # Set the document id to a hash of "unique together" fields - # for "ON CONFLICT UPDATE" behavior -- if the document - # already exists, it will be updated rather than duplicated. - # Cannot detect/avoid conflicts this way, but that's ok. - _key_values = [] - for _field_value in unique_together_field_values: - if not isinstance(_field_value, str): - raise ValueError(f"expected str, got {_field_value!r}") - _key_values.append(_field_value) - return opaque_key(_key_values) From a1e00e468830a40758caa8afa4b838821471f5c1 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 17 Apr 2026 15:04:58 -0400 Subject: [PATCH 16/22] less noisy test/lint error --- elasticsearch_metrics/tests/__main__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/elasticsearch_metrics/tests/__main__.py b/elasticsearch_metrics/tests/__main__.py index 08be2c8..7d6715b 100644 --- a/elasticsearch_metrics/tests/__main__.py +++ b/elasticsearch_metrics/tests/__main__.py @@ -7,6 +7,7 @@ import collections import os import subprocess +import sys _parser = argparse.ArgumentParser() _parser.add_argument("--lint", action="store_true") # _args.lint @@ -40,7 +41,11 @@ def print_header(*args: str) -> None: def _run(*args: str, header: str = "") -> None: print_header(header) if header else print_header(*args) - subprocess.run(args, check=True) # stop on error + try: + subprocess.run(args, check=True) # stop on error + except subprocess.CalledProcessError as _e: + print(f"\n\n^^ errored ({_e.returncode}) ^^") + sys.exit() def run_lint() -> None: From 34c7b180e6d595b3374534cd50efb00f5a809582 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Tue, 21 Apr 2026 09:44:57 -0400 Subject: [PATCH 17/22] timeseries_index_timedepth (for clarity) --- README.md | 14 +++++++------- elasticsearch_metrics/imps/elastic8.py | 12 +++++++----- elasticsearch_metrics/tests/dummy8app/metrics.py | 6 +++--- pyproject.toml | 2 +- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 844ba8c..49091c3 100644 --- a/README.md +++ b/README.md @@ -94,18 +94,18 @@ UsageRecord.search_timeseries_range(datetime.date(2030, 1, 1), datetime.date(203 By default, behind the scenes, a new elasticsearch index is created for each record type for each day in which a record is saved (using UTC timezone). You can change this for a record type by setting -`Meta.timedepth`, or change the default timedepth with the setting `DJELME_DEFAULT_TIMEDEPTH` (see below). +`Meta.timeseries_index_timedepth`, or change the default timedepth with the setting `DJELME_DEFAULT_TIMEDEPTH` (see below). ```python class MyEventWithMonthlyIndexes(EventRecord): class Meta: - timedepth = 2 # year and month + timeseries_index_timedepth = 2 # year and month ``` -- index per year: `timedepth = 1` -- index per month: `timedepth = 2` -- index per day: `timedepth = 3` (default) -- index per hour: `timedepth = 4` +- index per year: `timeseries_index_timedepth = 1` +- index per month: `timeseries_index_timedepth = 2` +- index per day: `timeseries_index_timedepth = 3` (default) +- index per hour: `timeseries_index_timedepth = 4` ## Index settings @@ -193,7 +193,7 @@ class UsageRecord(MyBaseMetric): DJELME_DEFAULT_TIMEDEPTH = 3 # daily indexes; YYYY.MM.DD (this is the default) DJELME_DEFAULT_TIMEDEPTH = 4 # hourly indexes; YYYY.MM.DD.HH ``` - you can also set `Meta.timedepth` on a specific record type; this will take precedence + you can also set `Meta.timeseries_index_timedepth` on a specific record type; this will take precedence ## Management commands diff --git a/elasticsearch_metrics/imps/elastic8.py b/elasticsearch_metrics/imps/elastic8.py index 84275f4..84ec166 100644 --- a/elasticsearch_metrics/imps/elastic8.py +++ b/elasticsearch_metrics/imps/elastic8.py @@ -495,7 +495,7 @@ def format_timeseries_index_pattern(cls, timeparts: tuple[int, ...] = ()) -> str app_label=cls.app_label, recordtype=cls.get_timeseries_recordtype_name(), timeparts=timeparts, - max_timedepth=cls.get_timedepth(), + max_timedepth=cls.get_timeseries_index_timedepth(), ) return "".join((cls.get_index_name_prefix(), _pattern)) @@ -510,16 +510,18 @@ def format_timeseries_index_pattern_for_range( cls.get_timeseries_recordtype_name(), from_when, until_when or utcnow(), - timedepth=cls.get_timedepth(), + timedepth=cls.get_timeseries_index_timedepth(), ) return "".join((cls.get_index_name_prefix(), _pattern)) @classmethod - def get_timedepth(cls) -> int: + def get_timeseries_index_timedepth(cls) -> int: _default_timedepth = getattr( settings, _DEFAULT_TIMEDEPTH_SETTING, _DEFAULT_TIMEDEPTH ) - _timedepth = cls._get_meta_attr("timedepth", _default_timedepth) + _timedepth = cls._get_meta_attr( + "timeseries_index_timedepth", _default_timedepth + ) assert isinstance(_timedepth, int) return _timedepth @@ -622,7 +624,7 @@ def djelme_index_name(self) -> str: app_label=self.__class__.app_label, recordtype=self.get_timeseries_recordtype_name(), timeparts=self.timeseries_timeparts or self.get_timeseries_timeparts(), - max_timedepth=self.get_timedepth(), + max_timedepth=self.get_timeseries_index_timedepth(), ) return "".join((self.get_index_name_prefix(), _index_name)) diff --git a/elasticsearch_metrics/tests/dummy8app/metrics.py b/elasticsearch_metrics/tests/dummy8app/metrics.py index a05fdbd..c3f4e76 100644 --- a/elasticsearch_metrics/tests/dummy8app/metrics.py +++ b/elasticsearch_metrics/tests/dummy8app/metrics.py @@ -17,7 +17,7 @@ class Monthly8Event(djelme.EventRecord): class Meta: index_name_prefix = "dummy8evenz" timeseries_recordtype_name = "eventlog" - timedepth = 2 + timeseries_index_timedepth = 2 class ThingHappened(djelme.EventRecord): @@ -33,7 +33,7 @@ class Index: class Meta: timeseries_recordtype_name = "happen" - timedepth = 1 # yearly timeseries indexes + timeseries_index_timedepth = 1 # yearly timeseries indexes class ThingHappeningsReport(djelme.CyclicRecord): @@ -45,7 +45,7 @@ class ThingHappeningsReport(djelme.CyclicRecord): class Meta: index_name_prefix = "blarg_" - timedepth = 2 # monthly timeseries indexes + timeseries_index_timedepth = 2 # monthly timeseries indexes class SimpleKV(djelme.SimpleRecord): diff --git a/pyproject.toml b/pyproject.toml index d317014..b729aaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "django-elasticsearch-metrics" -version = "2026.0.3" +version = "2026.0.4" description = "Django app for storing time-series metrics in Elasticsearch." authors = [ {name = "CenterForOpenScience", email = "support@cos.io"} From 222f03e92ec45a86f76db7a0461ae4fc483b2810 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Tue, 21 Apr 2026 16:53:22 -0400 Subject: [PATCH 18/22] fixes: lazy index name and utc --- elasticsearch_metrics/imps/elastic8.py | 51 +++++++++++-------- .../tests/test_imps_elastic8.py | 18 +++++-- elasticsearch_metrics/util/anon_enough.py | 14 ++--- 3 files changed, 52 insertions(+), 31 deletions(-) diff --git a/elasticsearch_metrics/imps/elastic8.py b/elasticsearch_metrics/imps/elastic8.py index 84ec166..846e03e 100644 --- a/elasticsearch_metrics/imps/elastic8.py +++ b/elasticsearch_metrics/imps/elastic8.py @@ -63,6 +63,11 @@ def utcnow() -> datetime.datetime: # change default mapping for `str` annotations from Text to Keyword: esdsl.document_base.DocumentOptions.type_annotation_map[str] = (esdsl.Keyword, {}) +# change default timezone for `datetime` annotations from "local" to UTC: +esdsl.document_base.DocumentOptions.type_annotation_map[datetime.datetime] = ( + esdsl.Date, + {"default_timezone": "UTC"}, +) # changes to document metaclass behavior @@ -310,25 +315,26 @@ def _get_unique_together_values(self) -> list: class _SimpleRecordMetaclass(_DjelmeRecordMetaclass): - def __new__(mcls, name, bases, attrs): # noqa: B902 - """ - extend _DjelmeRecordMetaclass.__new__ to set index name by app label and type name - """ - _cls = super().__new__(mcls, name, bases, attrs) - if not _cls.is_abstract: - # set `_index._name`, if not already specified - _index_name = _cls._index._name - if not _index_name or (_index_name in ("*", "default")): - _app_label = find_app_label_for_module(attrs["__module__"]) - _index_name = f"{_app_label.lower()}_{_cls.__name__.lower()}" - _prefix = _cls.get_index_name_prefix() - if not _index_name.startswith(_prefix): - _index_name = f"{_prefix}{_index_name}" - _cls._index._name = _index_name - # set `_index._using`, so `_index` can be used normally - _backend = djelme_registry.get_backend_for_recordtype(_cls) - _cls._index._using = _backend._elastic8dsl_connection_name - return _cls + @property + def _index(self): + if self.is_abstract: + return self.__index + # return a copy with `name` and `using` freshly computed + try: + _backend = djelme_registry.get_backend_for_recordtype(self) + except LookupError: + _using = None # may not be registered yet, is ok + else: + _using = _backend._elastic8dsl_connection_name + return self.__index.clone( + name=self.djelme_index_name(), + using=_using, + ) + + @_index.setter + def _index(self, val): + # stash in a private attr so only the property getter can reach it + self.__index = val class SimpleRecord(BaseDjelmeRecord, metaclass=_SimpleRecordMetaclass): @@ -342,7 +348,12 @@ def djelme_index_name(cls) -> str: for ProtoDjelmeRecord """ assert not cls.is_abstract - return cls._index._name + _app_label = find_app_label_for_module(cls.__module__) + _index_name = f"{_app_label.lower()}_{cls.__name__.lower()}" + _prefix = cls.get_index_name_prefix() + if not _index_name.startswith(_prefix): + _index_name = f"{_prefix}{_index_name}" + return _index_name @classmethod def check_djelme_setup(cls, using: str | None = None) -> None: diff --git a/elasticsearch_metrics/tests/test_imps_elastic8.py b/elasticsearch_metrics/tests/test_imps_elastic8.py index 943b8d7..bb0106d 100644 --- a/elasticsearch_metrics/tests/test_imps_elastic8.py +++ b/elasticsearch_metrics/tests/test_imps_elastic8.py @@ -65,6 +65,11 @@ def test_index_name(self): _thingreport.djelme_index_name(), "blarg_dummy8app_thinghappeningsreport_1999.3.", ) + _kv = SimpleKV(key="wha", val=0) + self.assertEqual( + _kv.djelme_index_name(), + "dummy8app_simplekv", + ) def test_format_index_name_respects_date_format_setting(self): _stamp = dt.datetime(2020, 2, 14) @@ -272,7 +277,7 @@ class Meta: class TestRecord(MockConnectionTestCase): def test_calls_save(self): - timestamp = dt.datetime(2017, 8, 21) + timestamp = dt.datetime(2017, 8, 21, tzinfo=dt.timezone.utc) p = ThingHappened.record(timestamp=timestamp, thing_id="abc12") assert self.mock_es8_connection.index.call_count == 1 assert p.timestamp == timestamp @@ -706,13 +711,16 @@ def test_search_range(self): class TestSingleIndex(RealElasticTestCase): - def setUp(self): - super().setUp() + def test_index_name(self): + self.assertEqual( + _strip_test_prefix(SimpleKV._index._name), + "dummy8app_simplekv", + ) + + def test_search(self): SimpleKV.record(key="hello", val=2) SimpleKV.record(key="goodbye", val=-2) SimpleKV.refresh() - - def test_search(self): (_hello,) = SimpleKV.search().query({"term": {"key": "hello"}}).execute() self.assertEqual(_hello.key, "hello") self.assertEqual(_hello.val, 2) diff --git a/elasticsearch_metrics/util/anon_enough.py b/elasticsearch_metrics/util/anon_enough.py index 0215ddf..2c97684 100644 --- a/elasticsearch_metrics/util/anon_enough.py +++ b/elasticsearch_metrics/util/anon_enough.py @@ -47,13 +47,13 @@ def opaque_sessionhour_id( _today_str = _now.date().isoformat() # "A user session is defined any of the following ways: ..." (quotes out of order) - if client_session_id: - # "...by a logged user cookie + transaction date + hour of day..." - _session_id_parts = [client_session_id, _today_str, _now.hour] - elif user_id: + if user_id: # "...by a logged user ID (if users log in with personal accounts) # + transaction date + hour of day (day is divided into 24 one-hour slices) ..." _session_id_parts = [user_id, _today_str, _now.hour] + elif client_session_id: + # "...by a logged user cookie + transaction date + hour of day..." + _session_id_parts = [client_session_id, _today_str, _now.hour] elif request_host and request_useragent: # "...or by a combination of IP address + user agent + transaction date + hour of day." _session_id_parts = [request_host, request_useragent, _today_str, _now.hour] @@ -69,8 +69,10 @@ def opaque_sessionhour_id( >>> _now_patcher.start() and None >>> opaque_sessionhour_id(client_session_id='foo') 'R26L*vmd?|G}S5AZ}ONXq>^B*T-!TLCE`uboEXF-LpK8Hysi$nUve^2aG~PTWiX<6BDv}wQtowotKSdV' ->>> opaque_sessionhour_id(client_session_id='feh', user_id='blah') -'JfeGFBfil1y$8fnmhi)8LU4}9vUBX6VfHmDPiVfiB~0nT&%3tKWsTTF_z2wynPj}`EF=}Y6=?}e5nDK0' +>>> opaque_sessionhour_id(user_id='blah') +'vToyLf@So{-(dZ*?<`d{f{|w+2j^OaNw{yYKEJ6}q4#288|8^lai=Hy@F-c?19rafSrA{mb&$z*p6AQ>' +>>> opaque_sessionhour_id(user_id='blah', client_session_id='feh') +'vToyLf@So{-(dZ*?<`d{f{|w+2j^OaNw{yYKEJ6}q4#288|8^lai=Hy@F-c?19rafSrA{mb&$z*p6AQ>' >>> opaque_sessionhour_id(request_host='999.999.999.999', request_useragent='hehe') 'Q^-x^v~@WQRHrWsbbji+pNmz)1`sp3SywCJ4n`W_aoY0tfbL6byxqUpw#DXoqU3>DtZC*^D@qjc7EmO=' >>> _now_patcher.stop() or None From 65c888c4dd6d200b3cda54390b28ab223b63a95e Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Wed, 22 Apr 2026 08:23:18 -0400 Subject: [PATCH 19/22] fix(ci): error on error --- elasticsearch_metrics/tests/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch_metrics/tests/__main__.py b/elasticsearch_metrics/tests/__main__.py index 7d6715b..f211166 100644 --- a/elasticsearch_metrics/tests/__main__.py +++ b/elasticsearch_metrics/tests/__main__.py @@ -45,7 +45,7 @@ def _run(*args: str, header: str = "") -> None: subprocess.run(args, check=True) # stop on error except subprocess.CalledProcessError as _e: print(f"\n\n^^ errored ({_e.returncode}) ^^") - sys.exit() + sys.exit(1) def run_lint() -> None: From 4e833670178beb682bb0d64e4f33db012cf8f014 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Wed, 22 Apr 2026 15:39:19 -0400 Subject: [PATCH 20/22] default_index_name_prefix --- elasticsearch_metrics/imps/elastic8.py | 61 ++++++++++++++------------ 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/elasticsearch_metrics/imps/elastic8.py b/elasticsearch_metrics/imps/elastic8.py index 846e03e..e31aa84 100644 --- a/elasticsearch_metrics/imps/elastic8.py +++ b/elasticsearch_metrics/imps/elastic8.py @@ -26,7 +26,6 @@ import logging import typing -import django from django.core.exceptions import ImproperlyConfigured from django.conf import settings from elasticsearch8.exceptions import NotFoundError @@ -127,7 +126,7 @@ def __new__(mcls, name, bases, attrs): # noqa: B902 @classmethod def construct_index(cls, opts, bases): """ - Override IndexMeta.construct_index so a new Index is created for each class + Extend IndexMeta.construct_index so a new Index is created for each class and Index.settings, Index.analyzers, and Index.using are inherited (but not Index.name or Index.aliases) """ @@ -233,10 +232,20 @@ def require_been_setup(cls, using: str | None = None) -> None: @classmethod def get_index_name_prefix(cls) -> str: - _name_prefix = cls._get_meta_attr("index_name_prefix") or "" + _name_prefix = ( + cls._get_meta_attr("index_name_prefix") + or cls._get_djelme_backend().default_index_name_prefix + or "" + ) assert isinstance(_name_prefix, str) return _name_prefix + @classmethod + def _get_djelme_backend(cls) -> "DjelmeElastic8Backend": + _backend = djelme_registry.get_backend_for_recordtype(cls) + assert isinstance(_backend, DjelmeElastic8Backend) + return _backend + @classmethod def _get_using( cls, using: str | Elastic8Client | None = None @@ -249,7 +258,7 @@ def _get_using( """ _backend: ProtoDjelmeBackend | None = None if using in (None, "default"): - _backend = djelme_registry.get_backend_for_recordtype(cls) + _backend = cls._get_djelme_backend() elif isinstance(using, str) and (using in djelme_registry.all_backends): _backend = djelme_registry.get_backend(using) if _backend is not None: @@ -321,15 +330,14 @@ def _index(self): return self.__index # return a copy with `name` and `using` freshly computed try: - _backend = djelme_registry.get_backend_for_recordtype(self) + _backend = self._get_djelme_backend() except LookupError: _using = None # may not be registered yet, is ok + _index_name = "" else: _using = _backend._elastic8dsl_connection_name - return self.__index.clone( - name=self.djelme_index_name(), - using=_using, - ) + _index_name = self.djelme_index_name() + return self.__index.clone(name=_index_name, using=_using) @_index.setter def _index(self, val): @@ -562,8 +570,6 @@ def sync_index_template(cls, using=None): # -> ComposableIndexTemplate: def check_djelme_setup(cls, using: str | Elastic8Client | None = None) -> None: """Check if class is in sync with index template in Elasticsearch. - override `BaseDjelmeRecord.check_djelme_setup` to check for a template instead of an index - :raise: IndexTemplateNotFoundError if index template does not exist. :raise: IndexTemplateOutOfSyncError if mappings, settings, or index patterns are out of sync. @@ -687,32 +693,21 @@ def record( *, # each usage record needs a sessionhour_id -- for migrating old data, can set explicitly... sessionhour_id: str = "", - # ...but when saving new data, give either the dirty identifying strings: + # ...but when saving new data, give the dirty identifying strings + # (which won't be stored, but used to create an opaque sessionhour_id) user_id: str = "", client_session_id: str = "", request_host: str = "", request_useragent: str = "", - # ...or a django request to infer user/host/useragent from - django_request: django.http.HttpRequest | None = None, # additional kwargs presumed to give field values **kwargs: typing.Any, ) -> "typing.Self": # typing.Self added in py 3.11 -- str annotation until 3.10 eol """CountedUsageRecord.record(...): construct and save a record""" - _useragent = ( - request_useragent - if (request_useragent or (django_request is None)) - else django_request.META.get("HTTP_USER_AGENT", "") - ) - _host = ( - request_host - if (request_host or (django_request is None)) - else django_request.get_host() - ) _sessionhour_id = sessionhour_id or opaque_sessionhour_id( client_session_id=client_session_id, user_id=user_id, - request_host=_host, - request_useragent=_useragent, + request_host=request_host, + request_useragent=request_useragent, timestamp=kwargs.get("timestamp"), ) _new_record = super().record(**kwargs, sessionhour_id=_sessionhour_id) @@ -762,6 +757,10 @@ def get_timeseries_timeparts(self) -> str: class DjelmeElastic8Backend: """DjelmeElastic8Backend: elastic8 backend for djelme (for use by generic djelme code)""" + _NON_PASSTHRU_KWARGS: typing.ClassVar[collections.abc.Collection[str]] = { + "djelme_default_index_name_prefix", + } + backend_name: str imp_kwargs: dict[str, str] # pass-thru to elasticsearch connection kwargs @@ -771,6 +770,10 @@ def djelme_backend_name(self) -> str: # for ProtoDjelmeBackend def djelme_imp_kwargs(self) -> dict[str, str]: # for ProtoDjelmeBackend return self.imp_kwargs + @property + def default_index_name_prefix(self) -> str: + return self.imp_kwargs.get("djelme_default_index_name_prefix", "") + @property def elastic8_client(self) -> Elastic8Client: # assumes `connections.configure` was already called @@ -782,7 +785,11 @@ def _elastic8dsl_connection_name(self) -> str: @property def _elastic8dsl_connection_kwargs(self) -> dict[str, typing.Any]: - return self.imp_kwargs + return { + _key: _val + for _key, _val in self.imp_kwargs.items() + if _key not in self._NON_PASSTHRU_KWARGS + } def djelme_setup(self, recordtypes: collections.abc.Iterable[type]) -> None: # for ProtoDjelmeBackend From f2b92e5509389bb6c33f5a90c9ca4fe4e68187e2 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 24 Apr 2026 08:40:55 -0400 Subject: [PATCH 21/22] unofficial ducktype elastic_client --- elasticsearch_metrics/imps/elastic6.py | 7 ++++--- elasticsearch_metrics/imps/elastic8.py | 4 ++-- elasticsearch_metrics/tests/test_imps_elastic8.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/elasticsearch_metrics/imps/elastic6.py b/elasticsearch_metrics/imps/elastic6.py index 6852e4c..5933fc0 100644 --- a/elasticsearch_metrics/imps/elastic6.py +++ b/elasticsearch_metrics/imps/elastic6.py @@ -356,7 +356,7 @@ class DjelmeElastic6Backend: imp_kwargs: dict[str, str] @property - def elastic6_client(self): + def elastic_client(self): # assumes `connections.configure` was already called return connections.get_connection(self.backend_name) @@ -378,9 +378,10 @@ def djelme_teardown(self, recordtypes: collections.abc.Iterable[type]) -> None: logger.info("tearing down %r", _metric_type) _indexname_wildcard = _metric_type._template_pattern _templatename = _metric_type._template_name - self.elastic6_client.indices.delete(index=_indexname_wildcard) + _client = self.elastic_client + _client.indices.delete(index=_indexname_wildcard) try: - self.elastic6_client.indices.delete_template(_templatename) + _client.indices.delete_template(_templatename) except NotFoundError: pass diff --git a/elasticsearch_metrics/imps/elastic8.py b/elasticsearch_metrics/imps/elastic8.py index e31aa84..799078b 100644 --- a/elasticsearch_metrics/imps/elastic8.py +++ b/elasticsearch_metrics/imps/elastic8.py @@ -775,7 +775,7 @@ def default_index_name_prefix(self) -> str: return self.imp_kwargs.get("djelme_default_index_name_prefix", "") @property - def elastic8_client(self) -> Elastic8Client: + def elastic_client(self) -> Elastic8Client: # assumes `connections.configure` was already called return esdsl.connections.get_connection(self._elastic8dsl_connection_name) @@ -802,7 +802,7 @@ def djelme_teardown(self, recordtypes: collections.abc.Iterable[type]) -> None: # for ProtoDjelmeBackend for _recordtype in recordtypes: assert issubclass(_recordtype, BaseDjelmeRecord) - _recordtype.do_teardown(self.elastic8_client) + _recordtype.do_teardown(self.elastic_client) if __debug__ and typing.TYPE_CHECKING: diff --git a/elasticsearch_metrics/tests/test_imps_elastic8.py b/elasticsearch_metrics/tests/test_imps_elastic8.py index bb0106d..410d7a2 100644 --- a/elasticsearch_metrics/tests/test_imps_elastic8.py +++ b/elasticsearch_metrics/tests/test_imps_elastic8.py @@ -34,7 +34,7 @@ def _es8_client( ) -> elasticsearch8.Elasticsearch: _backend = djelme_registry.get_backend(backend_name) assert isinstance(_backend, djelme.DjelmeElastic8Backend) - return _backend.elastic8_client + return _backend.elastic_client class TestNamesAndPatterns(SimpleDjelmeTestCase): From 709ff1d5c869d5696212b9109ed79e5d9766c60c Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Thu, 7 May 2026 09:45:43 -0400 Subject: [PATCH 22/22] more usable do_teardown --- elasticsearch_metrics/imps/elastic8.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/elasticsearch_metrics/imps/elastic8.py b/elasticsearch_metrics/imps/elastic8.py index 799078b..f9b78c7 100644 --- a/elasticsearch_metrics/imps/elastic8.py +++ b/elasticsearch_metrics/imps/elastic8.py @@ -473,17 +473,24 @@ def each_timeseries_index( yield _index_name, _index_info @classmethod - def do_teardown(cls, es8_client: Elastic8Client) -> None: + def do_teardown( + cls, + using: str | Elastic8Client | None = None, + *, + keep_templates: bool = False, + ) -> None: assert not cls.is_abstract + _client = cls._get_connection(using) _indexname_wildcard = cls.format_timeseries_index_pattern() - _indices = es8_client.indices.get(index=_indexname_wildcard, features=",") + _indices = _client.indices.get(index=_indexname_wildcard, features=",") for _index_name in _indices.keys(): - es8_client.indices.delete(index=_index_name) - _templatename = cls.get_timeseries_template_name() - try: - es8_client.indices.delete_index_template(name=_templatename) - except NotFoundError: - pass + _client.indices.delete(index=_index_name) + if not keep_templates: + _templatename = cls.get_timeseries_template_name() + try: + _client.indices.delete_index_template(name=_templatename) + except NotFoundError: + pass @classmethod def get_timeseries_template(cls) -> esdsl.ComposableIndexTemplate: