- view progress of the osf-metrics migration (fixup) from elastic6 to elastic8 (or start it) -
- -diff --git a/api/base/settings/defaults.py b/api/base/settings/defaults.py index 017aacd9caf..3b8f090ef23 100644 --- a/api/base/settings/defaults.py +++ b/api/base/settings/defaults.py @@ -322,12 +322,6 @@ # django-elasticsearch-metrics DJELME_BACKENDS = { - 'osfmetrics_es6': { - 'elasticsearch_metrics.imps.elastic6': { - 'hosts': osf_settings.ELASTIC6_URI, - 'retry_on_timeout': True, - }, - }, 'osfmetrics_es8': { 'elasticsearch_metrics.imps.elastic8': { # passthru kwargs to elasticsearch8 connection constructor @@ -345,7 +339,6 @@ }, } OSF_USAGEEVENT_EXPIRATION_DAYS = 90 -ELASTICSEARCH_METRICS_DATE_FORMAT = '%Y' MONTHLY_USAGE_REPORT_EPOCH = '2026-05' # cannot create monthly usage reports before this point WAFFLE_CACHE_NAME = 'waffle_cache' diff --git a/docker-compose.yml b/docker-compose.yml index 02ac3a78ce6..c62541d6596 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,8 +13,6 @@ volumes: external: false elasticsearch8_data_vol: external: false - elasticsearch6_data_vol: - external: false rabbitmq_vol: external: false preprints_dist_vol: @@ -85,22 +83,6 @@ services: retries: 30 stdin_open: true - # Temporary: Remove when done with es6 - elasticsearch6: - image: docker.elastic.co/elasticsearch/elasticsearch:6.3.1 - environment: - - ES_JAVA_OPTS=-Xms512m -Xmx512m # reduce memory usage - ports: - - 9201:9200 - volumes: - - elasticsearch6_data_vol:/usr/share/elasticsearch/data - healthcheck: - start_period: 15s - test: curl -s http://localhost:9200/_cluster/health | grep -vq '"status":"red"' - interval: 10s - retries: 30 - stdin_open: true - postgres: image: postgres:15.4 command: diff --git a/osf/management/commands/migrate_osfmetrics_fix_6to8.py b/osf/management/commands/migrate_osfmetrics_fix_6to8.py deleted file mode 100644 index 45a87de9242..00000000000 --- a/osf/management/commands/migrate_osfmetrics_fix_6to8.py +++ /dev/null @@ -1,525 +0,0 @@ -import collections -import heapq -import itertools -import logging - -from django.conf import settings as api_settings -from django.core.management.base import BaseCommand -from django.db import OperationalError as DjangoOperationalError -from elasticsearch6.exceptions import ConnectionError as Elastic6ConnectionError -from elasticsearch8.exceptions import TransportError as Elastic8TransportError -from psycopg2 import OperationalError as PostgresOperationalError - -from framework.celery_tasks import app as celery_app -from framework.sentry import log_exception -from osf.metrics import es6_metrics -from osf.metrics.monthly_reports import MonthlyPublicItemUsageReport -from osf.metrics.utils import ( - YearMonth, - get_database_iri, - get_item_type, - iter_composite_bucket_keys, -) -from osf import models as osfdb -from osf.models.base import osfid_iri -from website import settings as website_settings - - -### -# constants - -_EPOCH_YEARMONTH = YearMonth.from_str(api_settings.MONTHLY_USAGE_REPORT_EPOCH) - -_MAX_CARDINALITY_PRECISION = 40000 # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-cardinality-aggregation.html#_precision_control - -_COMPOSITE_CHUNK_SIZE = 500 - -_RETRY_ERRORS = ( - DjangoOperationalError, - Elastic6ConnectionError, - Elastic8TransportError, - PostgresOperationalError, -) -_TASK_KWARGS = dict( - autoretry_for=_RETRY_ERRORS, - retry_backoff=True, # exponential backoff, with jitter - max_retries=20, -) - - -### -# fix usage report migration - -@celery_app.task(**_TASK_KWARGS) -def schedule_fix_usage_reports(after_osfid: str | None = None): - _until_when = _EPOCH_YEARMONTH.month_end() - _last_osfid = None - try: - for _osfid in _merge_sorted_osfids( - _each_countedusage_osfid(_until_when, after_osfid), - _each_preprintview_osfid(_until_when, after_osfid), - _each_preprintdownload_osfid(_until_when, after_osfid), - ): - add_fixed_usage_reports.delay(_osfid) - _last_osfid = _osfid - except _RETRY_ERRORS as _error: - if _last_osfid is None: - raise # didn't even get started - # schedule another task to continue scheduling - schedule_fix_usage_reports.delay( - after_osfid=( # avoid infinite loop from _merge_sorted_osfids removing "_v1" - f'{_last_osfid}_v1' - if '_' not in _last_osfid - else _last_osfid - ), - ) - # let this task succeed but log the error - log_exception(_error) - - -@celery_app.task(**_TASK_KWARGS) -def add_fixed_usage_reports(osfid: str): - # from PublicItemUsageReport to MonthlyPublicItemUsageReportEs8 - _osfobj, _ = osfdb.Guid.load_referent(osfid) - if isinstance(_osfobj, osfdb.Preprint) and '_' not in osfid: - osfid = f'{osfid}_v1' - _osfobj, _ = osfdb.Guid.load_referent(osfid) - if _osfobj: - _months = _months_with_usage(osfid, _osfobj, _EPOCH_YEARMONTH.month_end()) - _months.add(_EPOCH_YEARMONTH) # guarantee at least the one - for _ym in _months: - _usage_report = _make_usage_report(osfid, _osfobj, _ym) - _usage_report.save() - else: - raise RuntimeError('osfid does not exist! skipping...', osfid) - - -### -# various helper functions - -def _yearmonth_range_to_epoch(start): - _ym = max(YearMonth.from_any(start), YearMonth(2026, 1)) - while _ym <= _EPOCH_YEARMONTH: - yield _ym - _ym = _ym.next() - - -def _months_with_usage(osfid, osf_obj, until_when) -> set[YearMonth]: - _months = set() - if isinstance(osf_obj, osfdb.Preprint): - _searches = [ - _es6_osfid_preprint_event_search(es6_metrics.PreprintView, osfid, until_when), - _es6_osfid_preprint_event_search(es6_metrics.PreprintDownload, osfid, until_when), - ] - else: - _searches = [ - _es6_osfid_nonpreprint_view_search(osfid, until_when), - _es6_osfid_nonpreprint_download_search(osfid, until_when), - ] - for _search in _searches: - _search.aggs.bucket( - 'agg_months', - 'date_histogram', - field='timestamp', - interval='month', - format='yyyy-MM', - min_doc_count=1, - ) - _response = _search.execute() - if 'agg_months' in _response.aggregations: - _months.update( - YearMonth.from_str(_bucket.key_as_string) - for _bucket in _response.aggregations.agg_months.buckets - ) - return _months - - -def _semverish_from_yearmonth(given_yearmonth): - _ym = YearMonth.from_any(given_yearmonth) - return f'{_ym.year}.{_ym.month}' - - -def _make_usage_report(osfid: str, osf_obj, yearmonth: YearMonth): - # add a report for the given month with cumulative counts up to that point - _is_preprint = isinstance(osf_obj, osfdb.Preprint) - # cumulative counts - _c_views, _c_view_sess, _c_downloads, _c_download_sess = _get_cumulative_usage( - osfid=osfid, - until_when=yearmonth.month_end().isoformat(), - is_preprint=_is_preprint, - ) - # counts for that month only - _views, _view_sess, _downloads, _download_sess = _get_cumulative_usage( - osfid=osfid, - until_when=yearmonth.month_end().isoformat(), - from_when=yearmonth.month_start().isoformat(), - is_preprint=_is_preprint, - ) - return MonthlyPublicItemUsageReport( - cycle_coverage=_semverish_from_yearmonth(yearmonth), - item_iri=osfid_iri(osfid), - item_osfids=[osfid], - item_types=[get_item_type(osf_obj)], - provider_ids=[_get_provider_id(osf_obj)], - database_iris=[get_database_iri(osf_obj)], - platform_iris=[website_settings.DOMAIN], - view_count=_views, - view_session_count=_view_sess, - cumulative_view_count=_c_views, - cumulative_view_session_count=_c_view_sess, - download_count=_downloads, - download_session_count=_download_sess, - cumulative_download_count=_c_downloads, - cumulative_download_session_count=_c_download_sess, - ) - - -def _get_provider_id(osfid_referent): - _provider = getattr(osfid_referent, 'provider', None) - if _provider is None: - return 'osf' # quacks like Node, Comment, WikiPage - elif isinstance(_provider, str): - return _provider # quacks like BaseFileNode - else: - return _provider._id # quacks like Registration, Preprint, Collection - - -def _get_cumulative_usage(osfid: str, *, until_when, from_when='', is_preprint: bool): - if is_preprint: - _views = _cumulative_preprint_count( - es6_metrics.PreprintView, - osfid, - until_when=until_when, - from_when=from_when, - ) - _downloads = _cumulative_preprint_count( - es6_metrics.PreprintDownload, - osfid, - until_when=until_when, - from_when=from_when, - ) - _view_sess, _download_sess = _views, _downloads # no session info on preprints (yet) - else: - _views, _view_sess = _cumulative_countedusage_views( - osfid, - until_when=until_when, - from_when=from_when, - ) - _downloads, _download_sess = _cumulative_countedusage_downloads( - osfid, - until_when=until_when, - from_when=from_when, - ) - return (_views, _view_sess, _downloads, _download_sess) - - -def _cumulative_countedusage_views( - osfid: str, *, until_when: str, from_when: str = '' -) -> tuple[int, int]: - # copied/adapted from osf.metrics.reporters.public_item_usage - _search = ( - _es6_osfid_nonpreprint_view_search(osfid, until_when) - .extra(size=0) # only aggregations, no hits - ) - if from_when: - _search = _search.filter('range', timestamp={'gte': from_when}) - _search.aggs.metric( - 'agg_session_count', - 'cardinality', - field='session_id', - precision_threshold=_MAX_CARDINALITY_PRECISION, - ) - _response = _search.execute() - _view_count = _response.hits.total - _view_session_count = ( - _response.aggregations.agg_session_count.value - if 'agg_session_count' in _response.aggregations - else 0 - ) - return (_view_count, _view_session_count) - - -def _es6_preprint_event_search(preprint_metric_cls, until_when, from_when=None): - _search = ( - preprint_metric_cls.search() - .filter('range', timestamp={'lt': until_when}) - .extra(size=0) # no hits; only aggs - ) - if from_when: - _search = _search.filter('range', timestamp={'gte': from_when}) - return _search - - -def _es6_osfid_preprint_event_search(preprint_metric_cls, osfid, until_when, from_when=None): - return ( - _es6_preprint_event_search(preprint_metric_cls, until_when, from_when) - .filter('terms', preprint_id=_synonymous_osfids(osfid)) - ) - -def _es6_nonpreprint_event_search(until_when, from_when='', *, action_labels=None): - _search = ( - es6_metrics.CountedAuthUsage.search() - .filter('term', item_public=True) - .filter('terms', action_labels=(action_labels or ['view', 'download'])) - .filter('range', timestamp={'lt': until_when}) - .extra(size=0) # only aggregations, no hits - ) - if from_when: - _search = _search.filter('range', timestamp={'gte': from_when}) - return _search - - -def _es6_nonpreprint_download_search(until_when, from_when=''): - return _es6_nonpreprint_event_search(until_when, from_when, action_labels=['download']) - - -def _es6_osfid_nonpreprint_download_search(osfid, until_when, from_when=''): - return ( - _es6_nonpreprint_download_search(until_when, from_when) - .filter('term', item_guid=osfid) - ) - - -def _es6_nonpreprint_view_search(until_when, from_when=''): - return _es6_nonpreprint_event_search(until_when, from_when, action_labels=['view']) - - -def _es6_osfid_nonpreprint_view_search(osfid, until_when, from_when=''): - return ( - _es6_nonpreprint_view_search(until_when, from_when) - .filter( - 'bool', - should=[ - {'term': {'item_guid': osfid}}, - {'term': {'surrounding_guids': osfid}}, - ], - minimum_should_match=1, - ) - ) - - -def _cumulative_countedusage_downloads(osfid, *, until_when, from_when) -> tuple[int, int]: - '''aggregate downloads on each osfid (not including components/files)''' - # copied/adapted from osf.metrics.reporters.public_item_usage - _search = _es6_osfid_nonpreprint_download_search(osfid, until_when, from_when) - _search.aggs.metric( - 'agg_session_count', - 'cardinality', - field='session_id', - precision_threshold=_MAX_CARDINALITY_PRECISION, - ) - _response = _search.execute() - _download_count = _response.hits.total - _download_session_count = ( - _response.aggregations.agg_session_count.value - if 'agg_session_count' in _response.aggregations - else 0 - ) - return (_download_count, _download_session_count) - - -def _cumulative_preprint_count( - preprint_metric_cls, osfid: str, *, until_when: str, from_when: str = '' -) -> int: - '''aggregate counts on given preprint''' - # copied/adapted from osf.metrics.preprint_metrics - _search = _es6_osfid_preprint_event_search(preprint_metric_cls, osfid, until_when, from_when) - _search.aggs.metric('agg_count', 'sum', field='count') - _response = _search.execute() - return ( - int(_response.aggregations.agg_count.value) - if hasattr(_response.aggregations, 'agg_count') - else 0 - ) - - -def _synonymous_osfids(osfid: str) -> list[str]: - _synonyms = [osfid] - if osfid.endswith('_v1'): - # include pre-versioned-guid counts for v1 - _synonyms.append(osfid.removesuffix('_v1')) - elif '_' not in osfid: - # include v1 (if it exists) with unversioned guid - _synonyms.append(f'{osfid}_v1') - return _synonyms - - -def _each_countedusage_osfid(until_when, after_osfid=None) -> collections.abc.Iterator[str]: - _search = _es6_nonpreprint_event_search(until_when) - _search.aggs.bucket( - 'agg_osfid', - 'composite', - sources=[{'osfid': {'terms': {'field': 'item_guid'}}}], - size=_COMPOSITE_CHUNK_SIZE, - ) - return iter_composite_bucket_keys(_search, 'agg_osfid', 'osfid', after=after_osfid) - - -def _each_preprintview_osfid(until_when, after_osfid=None) -> collections.abc.Iterator[str]: - _search = _es6_preprint_event_search(es6_metrics.PreprintView, until_when) - _search.aggs.bucket( - 'agg_osfid', - 'composite', - sources=[{'osfid': {'terms': {'field': 'preprint_id'}}}], - size=_COMPOSITE_CHUNK_SIZE, - ) - return iter_composite_bucket_keys(_search, 'agg_osfid', 'osfid', after=after_osfid) - - -def _each_preprintdownload_osfid(until_when, after_osfid=None) -> collections.abc.Iterator[str]: - _search = _es6_preprint_event_search(es6_metrics.PreprintDownload, until_when) - _search.aggs.bucket( - 'agg_osfid', - 'composite', - sources=[{'osfid': {'terms': {'field': 'preprint_id'}}}], - size=_COMPOSITE_CHUNK_SIZE, - ) - return iter_composite_bucket_keys(_search, 'agg_osfid', 'osfid', after=after_osfid) - - -def _merge_sorted_osfids(*osfid_iterables): - def _osfids_group_key(osfid: str): - return ( # v1 same as unversioned - osfid.removesuffix('_v1') - if osfid.endswith('_v1') - else osfid - ) - for _k, _g in itertools.groupby( - heapq.merge(*osfid_iterables), - key=_osfids_group_key, - ): - yield _k - - -def _es8_usage_report_count(yearmonth: YearMonth) -> int: - _search = ( - MonthlyPublicItemUsageReport.search() - .filter('term', cycle_coverage=_semverish_from_yearmonth(yearmonth)) - .extra(track_total_hits=True) - ) - _response = _search.execute() - _total_count = _response.hits.total.value - return _total_count - - -def _es8_usage_report_osfid_count() -> int: - _search = ( - MonthlyPublicItemUsageReport.search() - .filter('range', cycle_coverage={'lte': _semverish_from_yearmonth(_EPOCH_YEARMONTH)}) - .extra(size=0) # only aggs, no hits - ) - _search.aggs.metric( - 'agg_osfid_count', - 'cardinality', - field='item_osfids', - precision_threshold=_MAX_CARDINALITY_PRECISION, - ) - _response = _search.execute() - return ( - _response.aggregations.agg_osfid_count.value - if 'agg_osfid_count' in _response.aggregations - else 0 - ) - - -def _es6_preprint_osfid_count(preprint_metric_cls) -> int: - _search = _es6_preprint_event_search(preprint_metric_cls, _EPOCH_YEARMONTH.month_end()) - _search.aggs.metric( - 'agg_osfid_count', - 'cardinality', - field='preprint_id', - precision_threshold=_MAX_CARDINALITY_PRECISION, - ) - _response = _search.execute() - return ( - _response.aggregations.agg_osfid_count.value - if 'agg_osfid_count' in _response.aggregations - else 0 - ) - - -def _es6_cu_osfid_count() -> int: - _search = _es6_nonpreprint_event_search(_EPOCH_YEARMONTH.month_end()) - _search.aggs.metric( - 'agg_osfid_count', - 'cardinality', - field='item_guid', - precision_threshold=_MAX_CARDINALITY_PRECISION, - ) - _response = _search.execute() - return ( - _response.aggregations.agg_osfid_count.value - if 'agg_osfid_count' in _response.aggregations - else 0 - ) - - -### -# the command itself - -class Command(BaseCommand): - def add_arguments(self, parser): - parser.add_argument( - '--no-counts', - action='store_true', - ) - parser.add_argument( - '--start', - action='store_true', - ) - parser.add_argument( - '--delete-es8-usage-reports', - action='store_true', - ) - - def handle(self, *, start, no_counts, delete_es8_usage_reports, **kwargs): - self._quiet_chatty_loggers() - if delete_es8_usage_reports: - self._delete_es8_usage_reports() - if not no_counts: - # display counts of reports and distinct items - _prior_ym = _EPOCH_YEARMONTH.prior() - self.stdout.write( - f'total osfids with preprint views thru {_EPOCH_YEARMONTH} in es6' - f': {_es6_preprint_osfid_count(es6_metrics.PreprintView)}' - ) - self.stdout.write( - f'total osfids with preprint downloads thru {_EPOCH_YEARMONTH} in es6' - f': {_es6_preprint_osfid_count(es6_metrics.PreprintDownload)}' - ) - self.stdout.write( - f'total osfids with with counted usage thru {_EPOCH_YEARMONTH} in es6' - f': {_es6_cu_osfid_count()}' - ) - self.stdout.write( - f'total osfids with usage reports in es8' - f': {_es8_usage_report_osfid_count()}\t<== compare this' - ) - self.stdout.write( - f'total usage reports for {_prior_ym} in es8' - f': {_es8_usage_report_count(_prior_ym)}' - ) - self.stdout.write( - f'total usage reports for {_EPOCH_YEARMONTH} in es8' - f': {_es8_usage_report_count(_EPOCH_YEARMONTH)}\t<== to this' - ) - # (if --start) schedule task per item (by composite agg on es6 usage reports and events) - # each item-task iter thru reports oldest to newest, adding cumulative counts - if start: - self.stdout.write( - f'starting per-item tasks to add a corrected usage report for {_EPOCH_YEARMONTH}' - ) - schedule_fix_usage_reports.delay() - - def _quiet_chatty_loggers(self): - _chatty_loggers = [ - 'elasticsearch', - 'elastic_transport', - 'elasticsearch_metrics', - ] - for logger_name in _chatty_loggers: - logging.getLogger(logger_name).setLevel(logging.ERROR) - - def _delete_es8_usage_reports(self): - self.stdout.write('deleting monthly usage reports in es8', self.style.NOTICE) - MonthlyPublicItemUsageReport.do_teardown(keep_templates=True) diff --git a/osf/metrics/__init__.py b/osf/metrics/__init__.py index 08c3f938950..7d124c501b7 100644 --- a/osf/metrics/__init__.py +++ b/osf/metrics/__init__.py @@ -2,7 +2,6 @@ events, daily_reports, monthly_reports, - es6_metrics, ) @@ -10,5 +9,4 @@ 'events', 'daily_reports', 'monthly_reports', - 'es6_metrics', ) diff --git a/osf/metrics/es6_metrics.py b/osf/metrics/es6_metrics.py deleted file mode 100644 index 4b29d80a01a..00000000000 --- a/osf/metrics/es6_metrics.py +++ /dev/null @@ -1,143 +0,0 @@ -import enum -import logging - -from elasticsearch6.exceptions import NotFoundError -from elasticsearch6_dsl import InnerDoc, analyzer, tokenizer -import elasticsearch_metrics.imps.elastic6 as metrics - - -logger = logging.getLogger(__name__) - -### -# preprint views and downloads - -class BasePreprintMetric(metrics.Metric): - count = metrics.Integer(doc_values=True, index=True, required=True) - provider_id = metrics.Keyword(index=True, doc_values=True, required=True) - user_id = metrics.Keyword(index=True, doc_values=True, required=False) - preprint_id = metrics.Keyword(index=True, doc_values=True, required=True) - version = metrics.Keyword(index=True, doc_values=True) - path = metrics.Text(index=True) - - # TODO: locale - - class Index: - settings = { - 'number_of_shards': 1, - 'number_of_replicas': 1, - 'refresh_interval': '1s', - } - - class Meta: - abstract = True - source = metrics.MetaField(enabled=True) - - @classmethod - def get_count_for_preprint(cls, preprint, after=None, before=None, index=None) -> int: - if preprint.version == 1: - search = cls.search(index=index).filter('terms', preprint_id=[preprint.get_guid()._id, preprint._id]) - else: - search = cls.search(index=index).filter('term', preprint_id=preprint._id) - timestamp = {} - if after: - timestamp['gte'] = after - if before: - timestamp['lt'] = before - if timestamp: - search = search.filter('range', timestamp=timestamp) - search.aggs.metric('sum_count', 'sum', field='count') - # Optimization: set size to 0 so that hits aren't returned (we only care about the aggregation) - search = search.extra(size=0) - try: - response = search.execute() - except NotFoundError: - # _get_relevant_indices returned 1 or more indices - # that doesn't exist. Fall back to unoptimized query - search = search.index().index(cls._default_index()) - response = search.execute() - # No indexed data - if not hasattr(response.aggregations, 'sum_count'): - return 0 - return int(response.aggregations.sum_count.value) - - def save(self, *args, **kwargs): - raise RuntimeError('no saving to es6') - - -class PreprintView(BasePreprintMetric): - pass - - -class PreprintDownload(BasePreprintMetric): - pass - - -### -# counted-usage - -route_prefix_analyzer = analyzer( - 'route_prefix_analyzer', - tokenizer=tokenizer('route_prefix_tokenizer', 'path_hierarchy', delimiter='.'), -) - -class PageviewInfo(InnerDoc): - """PageviewInfo - - for CountedAuthUsage generated by viewing a web page - """ - # fields that should be provided - referer_url = metrics.Keyword() - page_url = metrics.Keyword() - page_title = metrics.Keyword() - route_name = metrics.Keyword( - fields={ - 'by_prefix': metrics.Text(analyzer=route_prefix_analyzer), - }, - ) - - # fields autofilled from the above (see `_autofill_fields`) - page_path = metrics.Keyword() - referer_domain = metrics.Keyword() - hour_of_day = metrics.Integer() - - -class CountedAuthUsage(metrics.Metric): - """CountedAuthUsage - - Something was used! Let's quickly take note of that and - move on, then come back later to query/analyze/investigate. - - Aim to support a COUNTER-style reporting api - (see https://cop5.projectcounter.org/en/5.0.2/) - """ - - # where noted, fields correspond to defined terms from COUNTER - # https://cop5.projectcounter.org/en/5.0.2/appendices/a-glossary-of-terms.html - platform_iri = metrics.Keyword() # counter:Platform - provider_id = metrics.Keyword() # counter:Database(?) - session_id = metrics.Keyword() # counter:Session - item_guid = metrics.Keyword() # counter:Item - item_type = metrics.Keyword() # counter:Data-Type - surrounding_guids = metrics.Keyword(multi=True) # counter:Title - item_public = metrics.Boolean() # counter:Access-Type(?) - user_is_authenticated = metrics.Boolean() - - action_labels = metrics.Keyword(multi=True) - class ActionLabel(enum.Enum): - SEARCH = 'search' # counter:Search - VIEW = 'view' # counter:Investigation - DOWNLOAD = 'download' # counter:Request - WEB = 'web' # counter:Regular (aka "pageview") - API = 'api' # counter:TDM (aka "non-web api usage") - # TODO: count api usage, distinguish between web and non-web api requests - - # pageviews get additional info to support the "node analytics" view - # (see `api.metrics.views.NodeAnalyticsQuery`) - pageview_info = metrics.Object(PageviewInfo) - - class Meta: - dynamic = metrics.MetaField('strict') - source = metrics.MetaField(enabled=True) - - def save(self, *args, **kwargs): - raise RuntimeError('no saving to es6') diff --git a/poetry.lock b/poetry.lock index 6a2bc3e2d44..43d4f3ce08c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "amqp" @@ -1088,22 +1088,17 @@ name = "django-elasticsearch-metrics" version = "2026.0.6" description = "Django app for storing time-series metrics in Elasticsearch." optional = false -python-versions = ">=3.10,<4" +python-versions = "<4,>=3.10" groups = ["main"] -files = [] -develop = false +files = [ + {file = "django_elasticsearch_metrics-2026.0.6-py3-none-any.whl", hash = "sha256:69f798422c07d742dc9c517dee42a01add4485364ee7d4b1ff5d4f8f1029af7e"}, + {file = "django_elasticsearch_metrics-2026.0.6.tar.gz", hash = "sha256:8b186ef7c124c00aca73c3dab8865870f1979db8bb601b35e7e3f466e707d578"}, +] [package.extras] anydjango = ["django"] -elastic6 = ["elasticsearch6-dsl (>=6.3.0,<7.0.0)"] elastic8 = ["elasticsearch8 (>=8.0.0,<9.0.0)"] -[package.source] -type = "git" -url = "https://github.com/CenterForOpenScience/django-elasticsearch-metrics.git" -reference = "b5026b0878db738c7c62205819b72cdfbe1c2da6" -resolved_reference = "b5026b0878db738c7c62205819b72cdfbe1c2da6" - [[package]] name = "django-extensions" version = "3.2.3" @@ -1390,45 +1385,6 @@ files = [ [package.dependencies] urllib3 = ">=1.8,<2.0" -[[package]] -name = "elasticsearch6" -version = "6.8.2" -description = "Python client for Elasticsearch" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, <4" -groups = ["main"] -files = [ - {file = "elasticsearch6-6.8.2-py2.py3-none-any.whl", hash = "sha256:4edf2d61f854f642185d5af915b23c57e70d9f2b54f558b62ae55fa720583f5e"}, - {file = "elasticsearch6-6.8.2.tar.gz", hash = "sha256:7c215910b6bc18928d24d6c1d0b09b0684c824af609906d5e007a9a268109678"}, -] - -[package.dependencies] -urllib3 = ">=1.21.1" - -[package.extras] -develop = ["coverage", "mock", "nose", "nosexcover", "numpy", "pandas", "pyyaml", "requests (>=2.0.0,<3.0.0)", "sphinx (<1.7)", "sphinx-rtd-theme"] -requests = ["requests (>=2.4.0,<3.0.0)"] - -[[package]] -name = "elasticsearch6-dsl" -version = "6.4.0" -description = "Python client for Elasticsearch" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "elasticsearch6-dsl-6.4.0.tar.gz", hash = "sha256:4bbc60919b73484d028eca31f749f0eea80d8b0bfe0a9a33b54eb0afca1d9b5f"}, - {file = "elasticsearch6_dsl-6.4.0-py2.py3-none-any.whl", hash = "sha256:a5767ef65c50f7c8af7ba6c176bd8df2c1fb501c644bc196cbd675f15c0f2be1"}, -] - -[package.dependencies] -elasticsearch6 = ">=6.0.0,<7.0.0" -python-dateutil = "*" -six = "*" - -[package.extras] -develop = ["coverage (<5.0.0)", "mock", "pytest (>=3.0.0)", "pytest-cov", "pytz", "sphinx", "sphinx-rtd-theme"] - [[package]] name = "elasticsearch8" version = "8.19.3" @@ -4757,4 +4713,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "5599dfc677ced71d0e9097822fb1d1f6f0e22320eacca87299d1542b89e9f286" +content-hash = "8d4b8bd860af26dd39c8b2c2dca4187a08ac8ef39a88655e586273e833719c19" diff --git a/pyproject.toml b/pyproject.toml index 6d2b7cf701e..f73594c089f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,8 +38,6 @@ Markupsafe = "2.1.5" blinker = "1.7.0" furl = "2.1.3" elasticsearch2 = "2.5.1" -elasticsearch6= "6.8.2" -elasticsearch6-dsl = "6.4.0" elasticsearch8 = "8.19.3" elastic-transport = "8.17.1" google-api-python-client = "2.123.0" @@ -98,7 +96,7 @@ datacite = "1.1.3" rdflib = "7.0.0" colorlog = "6.8.2" # Metrics -django-elasticsearch-metrics = {git ="https://github.com/CenterForOpenScience/django-elasticsearch-metrics.git", rev = "b5026b0878db738c7c62205819b72cdfbe1c2da6"} +django-elasticsearch-metrics = "2026.0.6" # Impact Metrics CSV Export djangorestframework-csv = "3.0.2" gevent = "24.2.1" diff --git a/website/settings/defaults.py b/website/settings/defaults.py index 4b88b16bf1f..026187948a5 100644 --- a/website/settings/defaults.py +++ b/website/settings/defaults.py @@ -109,7 +109,6 @@ def parent_dir(path): SEARCH_ENGINE = 'elastic' # Can be 'elastic', or None ELASTIC_URI = '127.0.0.1:9200' -ELASTIC6_URI = os.environ.get('ELASTIC6_URI', '127.0.0.1:9201') ELASTIC8_URI = os.environ.get('ELASTIC8_URI') ELASTIC8_CERT_PATH = os.environ.get('ELASTIC8_CERT_PATH') ELASTIC8_ASSERT_HOSTNAME = os.environ.get('ELASTIC8_ASSERT_HOSTNAME') @@ -493,7 +492,6 @@ class CeleryConfig: } background_migration_modules = { - 'osf.management.commands.migrate_osfmetrics_fix_6to8', } try: @@ -610,7 +608,6 @@ class CeleryConfig: 'scripts.remove_after_use.merge_notification_subscription_provider_ct', 'scripts.disable_removed_beat_tasks', 'osf.management.commands.delete_withdrawn_or_failed_registration_files', - 'osf.management.commands.migrate_osfmetrics_fix_6to8', ) # Modules that need metrics and release requirements