From 531170d3e03b46e8ea0f773bc2caceaaf24eeb8a Mon Sep 17 00:00:00 2001 From: Graham Hukill Date: Wed, 11 Mar 2026 15:39:15 -0400 Subject: [PATCH 1/2] Update dependencies, pickup TDA v4 --- uv.lock | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/uv.lock b/uv.lock index c8402ef..395bd79 100644 --- a/uv.lock +++ b/uv.lock @@ -40,43 +40,43 @@ wheels = [ [[package]] name = "boto3" -version = "1.42.63" +version = "1.42.65" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7f/2a/33d5d4b16fd97dfd629421ebed2456392eae1553cc401d9f86010c18065e/boto3-1.42.63.tar.gz", hash = "sha256:cd008cfd0d7ea30f1c5e22daf0998c55b7c6c68cb68eea05110e33fe641173d5", size = 112778, upload-time = "2026-03-06T22:47:55.96Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/c9/8ff8a901cf62374f1289cf36391f855e1702c70f545c28d1b57608a84ff2/boto3-1.42.65.tar.gz", hash = "sha256:c740af6bdaebcc1a00f3827a5729050bf6fc820ee148bf7d06f28db11c80e2a1", size = 112805, upload-time = "2026-03-10T19:44:58.255Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/19/f1d8d2b24871d3d0ccb2cbd0b0cb64a3396d439384bd9643d2c25c641b84/boto3-1.42.63-py3-none-any.whl", hash = "sha256:d502a89a0acc701692ae020d15981f2a82e9eb3485acc651cfd0cf1a3afe79ee", size = 140554, upload-time = "2026-03-06T22:47:53.463Z" }, + { url = "https://files.pythonhosted.org/packages/46/bb/ace5921655df51e3c9b787b3f0bd6aa25548e5cf1dabae02e53fa88f2d98/boto3-1.42.65-py3-none-any.whl", hash = "sha256:cc7f2e0aec6c68ee5b10232cf3e01326acf6100bc785a770385b61a0474b31f4", size = 140556, upload-time = "2026-03-10T19:44:55.433Z" }, ] [[package]] name = "boto3-stubs" -version = "1.42.63" +version = "1.42.65" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore-stubs" }, { name = "types-s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/83/30/45d21d3df4598b6b37d8de9923603010e490dfc73a130c8ffa806750b1ff/boto3_stubs-1.42.63.tar.gz", hash = "sha256:b64726c86c0b3efbc6ccd4f0510fa5a8279caec46da36a91c037a4b395001e68", size = 101176, upload-time = "2026-03-06T22:50:00.514Z" } +sdist = { url = "https://files.pythonhosted.org/packages/73/a4/ff83514d9ce6750d497aba084f4e2c1e053d9a91770da32eaadd68fa4379/boto3_stubs-1.42.65.tar.gz", hash = "sha256:013bcb1f03bf8689d53ef24c2da5ce430a3a298813fdfc24ea81caca937952c1", size = 101190, upload-time = "2026-03-10T20:01:56.228Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/76/c2/933ecf99d0cc1ae1b463086d203b27a29ba05c1d1cdd8a3efb18bc1e8c98/boto3_stubs-1.42.63-py3-none-any.whl", hash = "sha256:132e9c57bfef5ea943745d6cdad16e62d4e82dca64f6ae9ddfba9789fcdc8378", size = 69913, upload-time = "2026-03-06T22:49:49.765Z" }, + { url = "https://files.pythonhosted.org/packages/7c/63/d215d55c0b6f2d6ab496ecd55397e8c153389382254434fd57abf4e3d95e/boto3_stubs-1.42.65-py3-none-any.whl", hash = "sha256:69b5a27619f1a9cfe0d895a1d7921f1a25c5101cdac52fa59463d79e0e25b472", size = 69917, upload-time = "2026-03-10T20:01:42.718Z" }, ] [[package]] name = "botocore" -version = "1.42.63" +version = "1.42.65" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/af/eb/a1c042f6638ada552399a9977335a6de2668a85bf80bece193c953531236/botocore-1.42.63.tar.gz", hash = "sha256:1fdfc33cff58d21e8622cf620ba2bba3cff324557932aaf935b5374e4610f059", size = 14965362, upload-time = "2026-03-06T22:47:44.158Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/81/2c832e2117d24da4fe800861e8ddd19bbaa308623b1198eb2c2cc6fcd3d4/botocore-1.42.65.tar.gz", hash = "sha256:7d52c148df07f70c375eeda58f99b439c7c7836c25df74cccfba3bb6e12444d2", size = 14970239, upload-time = "2026-03-10T19:44:43.686Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/60/17a2d3b94658bb999c6aee7bba6c76b271905debf0c8c8e6ac63ca8491bc/botocore-1.42.63-py3-none-any.whl", hash = "sha256:83f39d04f2b316bdfc59a3cac2d12238bde7126ac99d9a57d910dbd86d58c528", size = 14639889, upload-time = "2026-03-06T22:47:39.347Z" }, + { url = "https://files.pythonhosted.org/packages/8e/9e/2ca03a55408c0820d7f0a04ae52bc6dfc7e4fff1f007a90135a68e056c93/botocore-1.42.65-py3-none-any.whl", hash = "sha256:0283c332ce00cbd1b894e86b7bed89dd624a5ca3a4ee62ec4db3898d16652e98", size = 14644794, upload-time = "2026-03-10T19:44:37.442Z" }, ] [[package]] @@ -369,11 +369,11 @@ wheels = [ [[package]] name = "filelock" -version = "3.25.0" +version = "3.25.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/77/18/a1fd2231c679dcb9726204645721b12498aeac28e1ad0601038f94b42556/filelock-3.25.0.tar.gz", hash = "sha256:8f00faf3abf9dc730a1ffe9c354ae5c04e079ab7d3a683b7c32da5dd05f26af3", size = 40158, upload-time = "2026-03-01T15:08:45.916Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/8b/4c32ecde6bea6486a2a5d05340e695174351ff6b06cf651a74c005f9df00/filelock-3.25.1.tar.gz", hash = "sha256:b9a2e977f794ef94d77cdf7d27129ac648a61f585bff3ca24630c1629f701aa9", size = 40319, upload-time = "2026-03-09T19:38:47.309Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/0b/de6f54d4a8bedfe8645c41497f3c18d749f0bd3218170c667bf4b81d0cdd/filelock-3.25.0-py3-none-any.whl", hash = "sha256:5ccf8069f7948f494968fc0713c10e5c182a9c9d9eef3a636307a20c2490f047", size = 26427, upload-time = "2026-03-01T15:08:44.593Z" }, + { url = "https://files.pythonhosted.org/packages/a9/b8/2f664b56a3b4b32d28d3d106c71783073f712ba43ff6d34b9ea0ce36dc7b/filelock-3.25.1-py3-none-any.whl", hash = "sha256:18972df45473c4aa2c7921b609ee9ca4925910cc3a0fb226c96b92fc224ef7bf", size = 26720, upload-time = "2026-03-09T19:38:45.718Z" }, ] [[package]] @@ -1093,15 +1093,15 @@ wheels = [ [[package]] name = "python-discovery" -version = "1.1.1" +version = "1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, { name = "platformdirs" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ec/67/09765eacf4e44413c4f8943ba5a317fcb9c7b447c3b8b0b7fce7e3090b0b/python_discovery-1.1.1.tar.gz", hash = "sha256:584c08b141c5b7029f206b4e8b78b1a1764b22121e21519b89dec56936e95b0a", size = 56016, upload-time = "2026-03-07T00:00:56.354Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/7e/9f3b0dd3a074a6c3e1e79f35e465b1f2ee4b262d619de00cfce523cc9b24/python_discovery-1.1.3.tar.gz", hash = "sha256:7acca36e818cd88e9b2ba03e045ad7e93e1713e29c6bbfba5d90202310b7baa5", size = 56945, upload-time = "2026-03-10T15:08:15.038Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/75/0f/2bf7e3b5a4a65f623cb820feb5793e243fad58ae561015ee15a6152f67a2/python_discovery-1.1.1-py3-none-any.whl", hash = "sha256:69f11073fa2392251e405d4e847d60ffffd25fd762a0dc4d1a7d6b9c3f79f1a3", size = 30732, upload-time = "2026-03-07T00:00:55.143Z" }, + { url = "https://files.pythonhosted.org/packages/e7/80/73211fc5bfbfc562369b4aa61dc1e4bf07dc7b34df7b317e4539316b809c/python_discovery-1.1.3-py3-none-any.whl", hash = "sha256:90e795f0121bc84572e737c9aa9966311b9fde44ffb88a5953b3ec9b31c6945e", size = 31485, upload-time = "2026-03-10T15:08:13.06Z" }, ] [[package]] @@ -1322,8 +1322,8 @@ wheels = [ [[package]] name = "timdex-dataset-api" -version = "3.11.0" -source = { git = "https://github.com/MITLibraries/timdex-dataset-api.git#4cf98a4cb135596652130959644dd9c7b16f115b" } +version = "4.0.0" +source = { git = "https://github.com/MITLibraries/timdex-dataset-api.git#d58eccec67a2e1451d1c7f3c44846e07ccc311fb" } dependencies = [ { name = "attrs" }, { name = "boto3" }, @@ -1518,7 +1518,7 @@ wheels = [ [[package]] name = "virtualenv" -version = "21.1.0" +version = "21.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "distlib" }, @@ -1526,9 +1526,9 @@ dependencies = [ { name = "platformdirs" }, { name = "python-discovery" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2f/c9/18d4b36606d6091844daa3bd93cf7dc78e6f5da21d9f21d06c221104b684/virtualenv-21.1.0.tar.gz", hash = "sha256:1990a0188c8f16b6b9cf65c9183049007375b26aad415514d377ccacf1e4fb44", size = 5840471, upload-time = "2026-02-27T08:49:29.702Z" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/92/58199fe10049f9703c2666e809c4f686c54ef0a68b0f6afccf518c0b1eb9/virtualenv-21.2.0.tar.gz", hash = "sha256:1720dc3a62ef5b443092e3f499228599045d7fea4c79199770499df8becf9098", size = 5840618, upload-time = "2026-03-09T17:24:38.013Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/78/55/896b06bf93a49bec0f4ae2a6f1ed12bd05c8860744ac3a70eda041064e4d/virtualenv-21.1.0-py3-none-any.whl", hash = "sha256:164f5e14c5587d170cf98e60378eb91ea35bf037be313811905d3a24ea33cc07", size = 5825072, upload-time = "2026-02-27T08:49:27.516Z" }, + { url = "https://files.pythonhosted.org/packages/c6/59/7d02447a55b2e55755011a647479041bc92a82e143f96a8195cb33bd0a1c/virtualenv-21.2.0-py3-none-any.whl", hash = "sha256:1bd755b504931164a5a496d217c014d098426cddc79363ad66ac78125f9d908f", size = 5825084, upload-time = "2026-03-09T17:24:35.378Z" }, ] [[package]] From 4b0159b90d532cd28be17bb4168d7fc25016a9ee Mon Sep 17 00:00:00 2001 From: Graham Hukill Date: Thu, 12 Mar 2026 14:31:13 -0400 Subject: [PATCH 2/2] Filter opensearchpy DEBUG API request logging Why these changes are being introduced: The opensearchpy library has a built-in DEBUG logger that logs the full API request + response for each request. This is not terribly helpful for debugging purposes, as in code we could log both the requets and/or response as needed. A good example are bulk updates where ALL documents we send for update are logged. For a job like DSpace or Alma, the amount of logs are equal to the amount of data we are sending. That said, DEBUG logging for the opensearch library is quite helpful for other reasons, so it's not ideal to completely quiet that library's DEBUG logs. Ideall we would only target this specific, private logging method. How this addresses that need: * Pinpoints and filters out the opensearchpy logging method, _log_request_response() Side effects of this change: * Dramatically reduced logs in DEBUG mode, while still getting helpful opensearch DEBUG logs Relevant ticket(s): * https://mitlibraries.atlassian.net/browse/USE-373 --- tim/config.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tim/config.py b/tim/config.py index 0a3be9d..37d3063 100644 --- a/tim/config.py +++ b/tim/config.py @@ -5,7 +5,7 @@ import sentry_sdk OPENSEARCH_BULK_CONFIG_DEFAULTS = { - "OPENSEARCH_BULK_MAX_CHUNK_BYTES": 100 * 1024 * 1024, + "OPENSEARCH_BULK_MAX_CHUNK_BYTES": 20 * 1024 * 1024, "OPENSEARCH_BULK_MAX_RETRIES": 50, "OPENSEARCH_REQUEST_TIMEOUT": 120, } @@ -41,7 +41,7 @@ def configure_logger( """Configure application via passed application root logger. If verbose=True, 3rd party libraries can be quite chatty. For convenience, they can - be set to WARNING level by either passing a comma seperated list of logger names to + be set to WARNING level by either passing a comma separated list of logger names to 'warning_only_loggers' or by setting the env var WARNING_ONLY_LOGGERS. """ if verbose: @@ -50,6 +50,14 @@ def configure_logger( "%(asctime)s %(levelname)s %(name)s.%(funcName)s() " "line %(lineno)d: %(message)s" ) + + # Explicitly filter out opensearchpy's DEBUG logging of all API request payloads. + # This private method logs both the request + response payloads. If we need to + # know, we can both log data sent (request) or received (response). + logging.getLogger("opensearch").addFilter( + lambda record: record.funcName != "_log_request_response" + ) + else: root_logger.setLevel(logging.INFO) logging_format = "%(asctime)s %(levelname)s %(name)s.%(funcName)s(): %(message)s"