From 6f764624cb0db4045651f2cb74c008bac3eb89cd Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 10 Jul 2025 17:10:25 -0400 Subject: [PATCH 001/100] Add pycache to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1bc89af2e0..259d42c422 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .task/ build/ **/dist/ +**/__pycache__/ From 6fa551a02d22cd293a236e4154fb6b710e12b25e Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 10 Jul 2025 17:23:52 -0400 Subject: [PATCH 002/100] Preemptively update docs --- docs/src/dev-guide/building-package.md | 10 ++++++++++ integration-tests/.python-version | 1 + integration-tests/README.md | 0 integration-tests/pyproject.toml | 24 ++++++++++++++++++++++++ 4 files changed, 35 insertions(+) create mode 100644 integration-tests/.python-version create mode 100644 integration-tests/README.md create mode 100644 integration-tests/pyproject.toml diff --git a/docs/src/dev-guide/building-package.md b/docs/src/dev-guide/building-package.md index da3c28fa24..b1e8a07504 100644 --- a/docs/src/dev-guide/building-package.md +++ b/docs/src/dev-guide/building-package.md @@ -15,6 +15,7 @@ prebuilt version instead, check out the [releases](https://github.com/y-scope/cl * python3-venv (for the version of Python installed) * [Task] >= 3.38.0 and < 3.43.0 * We constrain the version due to unresolved [issues][clp-issue-872]. +* [uv] 0.7.19 or newer ## Setup @@ -61,6 +62,14 @@ where `` is `json` or `text`. The tar will be written to `build/clp----v.tar.gz`, with appropriate values for the fields in angle brackets. +## Test + +To test the package, run: + +```shell +task test-package +``` + ## Cleanup To clean up all build artifacts, run: @@ -71,3 +80,4 @@ task clean [clp-issue-872]: https://github.com/y-scope/clp/issues/872 [Task]: https://taskfile.dev/ +[uv]: https://docs.astral.sh/uv/ diff --git a/integration-tests/.python-version b/integration-tests/.python-version new file mode 100644 index 0000000000..c8cfe39591 --- /dev/null +++ b/integration-tests/.python-version @@ -0,0 +1 @@ +3.10 diff --git a/integration-tests/README.md b/integration-tests/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml new file mode 100644 index 0000000000..4c6d566b8b --- /dev/null +++ b/integration-tests/pyproject.toml @@ -0,0 +1,24 @@ +[project] +name = "integration-tests" +version = "0.1.0" +description = "Integration tests for the CLP project." +readme = "README.md" +authors = [ + { name = "YScope Inc.", email = "dev@yscope.com" } +] +requires-python = ">=3.10" +dependencies = [] + +[project.scripts] +integration-tests = "integration_tests:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pytest>=8.3.5", + "pytest-benchmark>=5.1.0", + "pytest-env>=1.1.5", +] From 01fba049c67bcdd89078e970d61e5516c970ea87 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 10 Jul 2025 17:52:26 -0400 Subject: [PATCH 003/100] Add project setup --- integration-tests/pyproject.toml | 1 - .../src/integration_tests/__init__.py | 2 + integration-tests/uv.lock | 182 ++++++++++++++++++ 3 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 integration-tests/src/integration_tests/__init__.py create mode 100644 integration-tests/uv.lock diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index 4c6d566b8b..d13bce5444 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -7,7 +7,6 @@ authors = [ { name = "YScope Inc.", email = "dev@yscope.com" } ] requires-python = ">=3.10" -dependencies = [] [project.scripts] integration-tests = "integration_tests:main" diff --git a/integration-tests/src/integration_tests/__init__.py b/integration-tests/src/integration_tests/__init__.py new file mode 100644 index 0000000000..17b793a3c7 --- /dev/null +++ b/integration-tests/src/integration_tests/__init__.py @@ -0,0 +1,2 @@ +def main() -> None: + print("Hello from integration-tests!") diff --git a/integration-tests/uv.lock b/integration-tests/uv.lock new file mode 100644 index 0000000000..0780d1f38d --- /dev/null +++ b/integration-tests/uv.lock @@ -0,0 +1,182 @@ +version = 1 +revision = 2 +requires-python = ">=3.10" + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, +] + +[[package]] +name = "integration-tests" +version = "0.1.0" +source = { editable = "." } + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-benchmark" }, + { name = "pytest-env" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.3.5" }, + { name = "pytest-benchmark", specifier = ">=5.1.0" }, + { name = "pytest-env", specifier = ">=1.1.5" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714, upload-time = "2025-06-18T05:48:06.109Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, +] + +[[package]] +name = "pytest-benchmark" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py-cpuinfo" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/d0/a8bd08d641b393db3be3819b03e2d9bb8760ca8479080a26a5f6e540e99c/pytest-benchmark-5.1.0.tar.gz", hash = "sha256:9ea661cdc292e8231f7cd4c10b0319e56a2118e2c09d9f50e1b3d150d2aca105", size = 337810, upload-time = "2024-10-30T11:51:48.521Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/d6/b41653199ea09d5969d4e385df9bbfd9a100f28ca7e824ce7c0a016e3053/pytest_benchmark-5.1.0-py3-none-any.whl", hash = "sha256:922de2dfa3033c227c96da942d1878191afa135a29485fb942e85dff1c592c89", size = 44259, upload-time = "2024-10-30T11:51:45.94Z" }, +] + +[[package]] +name = "pytest-env" +version = "1.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/31/27f28431a16b83cab7a636dce59cf397517807d247caa38ee67d65e71ef8/pytest_env-1.1.5.tar.gz", hash = "sha256:91209840aa0e43385073ac464a554ad2947cc2fd663a9debf88d03b01e0cc1cf", size = 8911, upload-time = "2024-09-17T22:39:18.566Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/b8/87cfb16045c9d4092cfcf526135d73b88101aac83bc1adcf82dfb5fd3833/pytest_env-1.1.5-py3-none-any.whl", hash = "sha256:ce90cf8772878515c24b31cd97c7fa1f4481cd68d588419fd45f10ecaee6bc30", size = 6141, upload-time = "2024-09-17T22:39:16.942Z" }, +] + +[[package]] +name = "tomli" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175, upload-time = "2024-11-27T22:38:36.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077, upload-time = "2024-11-27T22:37:54.956Z" }, + { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429, upload-time = "2024-11-27T22:37:56.698Z" }, + { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067, upload-time = "2024-11-27T22:37:57.63Z" }, + { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030, upload-time = "2024-11-27T22:37:59.344Z" }, + { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898, upload-time = "2024-11-27T22:38:00.429Z" }, + { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894, upload-time = "2024-11-27T22:38:02.094Z" }, + { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319, upload-time = "2024-11-27T22:38:03.206Z" }, + { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273, upload-time = "2024-11-27T22:38:04.217Z" }, + { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310, upload-time = "2024-11-27T22:38:05.908Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309, upload-time = "2024-11-27T22:38:06.812Z" }, + { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762, upload-time = "2024-11-27T22:38:07.731Z" }, + { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453, upload-time = "2024-11-27T22:38:09.384Z" }, + { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486, upload-time = "2024-11-27T22:38:10.329Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349, upload-time = "2024-11-27T22:38:11.443Z" }, + { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159, upload-time = "2024-11-27T22:38:13.099Z" }, + { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243, upload-time = "2024-11-27T22:38:14.766Z" }, + { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645, upload-time = "2024-11-27T22:38:15.843Z" }, + { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584, upload-time = "2024-11-27T22:38:17.645Z" }, + { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875, upload-time = "2024-11-27T22:38:19.159Z" }, + { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418, upload-time = "2024-11-27T22:38:20.064Z" }, + { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708, upload-time = "2024-11-27T22:38:21.659Z" }, + { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582, upload-time = "2024-11-27T22:38:22.693Z" }, + { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543, upload-time = "2024-11-27T22:38:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691, upload-time = "2024-11-27T22:38:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170, upload-time = "2024-11-27T22:38:27.921Z" }, + { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530, upload-time = "2024-11-27T22:38:29.591Z" }, + { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666, upload-time = "2024-11-27T22:38:30.639Z" }, + { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954, upload-time = "2024-11-27T22:38:31.702Z" }, + { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724, upload-time = "2024-11-27T22:38:32.837Z" }, + { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383, upload-time = "2024-11-27T22:38:34.455Z" }, + { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.14.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673, upload-time = "2025-07-04T13:28:34.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" }, +] From 4ac00054b67e5607d72c69dcead7fc91a650c784 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 10 Jul 2025 17:56:12 -0400 Subject: [PATCH 004/100] Add integration tests to python linting --- taskfiles/lint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 149a1ba5f8..5de90319db 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -776,6 +776,7 @@ tasks: - "components/job-orchestration/job_orchestration" - "tools/scripts" - "docs/conf" + - "integration-tests" cmd: |- . "{{.G_LINT_VENV_DIR}}/bin/activate" cd "{{.ITEM}}" From 6f7692452ad9a12efca80058158fe1691a36fd3f Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 10 Jul 2025 18:31:08 -0400 Subject: [PATCH 005/100] Add sample tests and task workflow to run them --- integration-tests/.pytest.ini | 12 ++++++++++++ .../tests/test_identity_transformation.py | 13 +++++++++++++ taskfile.yaml | 6 ++++++ 3 files changed, 31 insertions(+) create mode 100644 integration-tests/.pytest.ini create mode 100644 integration-tests/tests/test_identity_transformation.py diff --git a/integration-tests/.pytest.ini b/integration-tests/.pytest.ini new file mode 100644 index 0000000000..18b3d1e8a6 --- /dev/null +++ b/integration-tests/.pytest.ini @@ -0,0 +1,12 @@ +[pytest] +addopts = + --strict-config + --strict-markers + --capture=no + --verbose + --color=yes + --code-highlight=yes +markers = + binaries: mark tests that directly call the binaries in the package bin + clp: mark tests that use the CLP storage engine + clp_s: mark tests that use the CLP-S storage engine diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py new file mode 100644 index 0000000000..a5df55228f --- /dev/null +++ b/integration-tests/tests/test_identity_transformation.py @@ -0,0 +1,13 @@ +import pytest + +pytestmark = pytest.mark.binaries + + +@pytest.mark.clp +def test_always_pass_0() -> None: + assert True + + +@pytest.mark.clp_s +def test_always_pass_1() -> None: + assert True diff --git a/taskfile.yaml b/taskfile.yaml index e1bb43ac2b..7a0d7d25d9 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -15,6 +15,7 @@ vars: G_CORE_COMPONENT_DIR: "{{.G_COMPONENTS_DIR}}/core" G_CORE_COMPONENT_SUBMODULES_DIR: "{{.G_CORE_COMPONENT_DIR}}/submodules" G_WEBUI_SRC_DIR: "{{.G_COMPONENTS_DIR}}/webui" + G_INTEGRATION_TESTS_DIR: "{{.ROOT_DIR}}/integration-tests" # Build paths G_BUILD_DIR: "{{.ROOT_DIR}}/build" @@ -162,6 +163,11 @@ tasks: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + test-package: + dir: "{{.G_INTEGRATION_TESTS_DIR}}" + cmds: + - "uv run pytest" + clp-s-generate-parsers: vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" From 6c6701d5d25fc202a591292db835a87f00d24d02 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 10 Jul 2025 22:30:47 -0400 Subject: [PATCH 006/100] Big update --- integration-tests/tests/conftest.py | 7 ++++ .../fixtures/download_and_extract_dataset.py | 38 +++++++++++++++++++ .../tests/fixtures/fixture_types.py | 17 +++++++++ .../tests/fixtures/package_test_config.py | 23 +++++++++++ .../tests/fixtures/uncompressed_logs.py | 35 +++++++++++++++++ integration-tests/tests/fixtures/utils.py | 15 ++++++++ .../tests/test_identity_transformation.py | 27 ++++++++++++- taskfile.yaml | 5 ++- 8 files changed, 164 insertions(+), 3 deletions(-) create mode 100644 integration-tests/tests/conftest.py create mode 100644 integration-tests/tests/fixtures/download_and_extract_dataset.py create mode 100644 integration-tests/tests/fixtures/fixture_types.py create mode 100644 integration-tests/tests/fixtures/package_test_config.py create mode 100644 integration-tests/tests/fixtures/uncompressed_logs.py create mode 100644 integration-tests/tests/fixtures/utils.py diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py new file mode 100644 index 0000000000..1a042ff16f --- /dev/null +++ b/integration-tests/tests/conftest.py @@ -0,0 +1,7 @@ +from fixtures.download_and_extract_dataset import download_and_extract_dataset +from fixtures.package_test_config import config +from fixtures.uncompressed_logs import ( + hive_24hr, + postgresql, + spark_event_logs, +) diff --git a/integration-tests/tests/fixtures/download_and_extract_dataset.py b/integration-tests/tests/fixtures/download_and_extract_dataset.py new file mode 100644 index 0000000000..b5b2687398 --- /dev/null +++ b/integration-tests/tests/fixtures/download_and_extract_dataset.py @@ -0,0 +1,38 @@ +import shutil +from pathlib import Path + +import pytest + +from .fixture_types import DatasetParams +from .package_test_config import PackageTestConfig +from .utils import run_and_assert + + +@pytest.fixture(autouse=True) +def download_and_extract_dataset(request, config: PackageTestConfig) -> DatasetParams: + download_params = request.getfixturevalue(request.param) + dataset_name = download_params.dataset_name + if request.config.cache.get(dataset_name, False): + print(f"Uncompressed logs for dataset `{dataset_name}` is up-to-date.") + return download_params + + tar_path = str(config.uncompressed_logs_dir / f"{dataset_name}.tar.gz") + extract_path = str(config.uncompressed_logs_dir / dataset_name) + cmds = [ + "curl", + "--fail", + "--location", + "--output", + str(tar_path), + "--show-error", + download_params.tar_url, + ] + run_and_assert(cmds) + + try: + shutil.unpack_archive(tar_path, extract_path) + request.config.cache.set(dataset_name, True) + except: + assert False, f"Tar extraction failed for dataset `{dataset_name}`." + + return download_params diff --git a/integration-tests/tests/fixtures/fixture_types.py b/integration-tests/tests/fixtures/fixture_types.py new file mode 100644 index 0000000000..75bab0550e --- /dev/null +++ b/integration-tests/tests/fixtures/fixture_types.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class PackageTestConfig: + clp_bin_dir: Path + clp_package_dir: Path + clp_sbin_dir: Path + test_output_dir: Path + uncompressed_logs_dir: Path + + +@dataclass(frozen=True) +class DatasetParams: + dataset_name: str + tar_url: str diff --git a/integration-tests/tests/fixtures/package_test_config.py b/integration-tests/tests/fixtures/package_test_config.py new file mode 100644 index 0000000000..b455952ff0 --- /dev/null +++ b/integration-tests/tests/fixtures/package_test_config.py @@ -0,0 +1,23 @@ +from pathlib import Path + +import pytest + +from .fixture_types import PackageTestConfig +from .utils import get_env_var + + +@pytest.fixture(scope="session") +def config() -> PackageTestConfig: + clp_build_dir = Path(get_env_var("CLP_BUILD_DIR")) + clp_package_dir = Path(get_env_var("CLP_PACKAGE_BUILD_DIR")) + + config = PackageTestConfig( + clp_bin_dir=clp_package_dir / "bin", + clp_package_dir=clp_package_dir, + clp_sbin_dir=clp_package_dir / "sbin", + test_output_dir=clp_build_dir / "var" / "logs" / "pytest", + uncompressed_logs_dir=clp_build_dir / "var" / "data" / "pytest" / "downloads", + ) + config.test_output_dir.mkdir(parents=True, exist_ok=True) + config.uncompressed_logs_dir.mkdir(parents=True, exist_ok=True) + return config diff --git a/integration-tests/tests/fixtures/uncompressed_logs.py b/integration-tests/tests/fixtures/uncompressed_logs.py new file mode 100644 index 0000000000..d304843cbb --- /dev/null +++ b/integration-tests/tests/fixtures/uncompressed_logs.py @@ -0,0 +1,35 @@ +import pytest + +from .fixture_types import DatasetParams + + +@pytest.fixture(scope="session") +def hive_24hr() -> DatasetParams: + return DatasetParams( + dataset_name="hive-24hr", + tar_url="https://zenodo.org/records/7094921/files/hive-24hr.tar.gz?download=1", + ) + + +@pytest.fixture(scope="session") +def elasticsearch() -> DatasetParams: + return DatasetParams( + dataset_name="elasticsearch", + tar_url="https://zenodo.org/records/10516227/files/elasticsearch.tar.gz?download=1", + ) + + +@pytest.fixture(scope="session") +def spark_event_logs() -> DatasetParams: + return DatasetParams( + dataset_name="spark-event-logs", + tar_url="https://zenodo.org/records/10516346/files/spark-event-logs.tar.gz?download=1", + ) + + +@pytest.fixture(scope="session") +def postgresql() -> DatasetParams: + return DatasetParams( + dataset_name="postgresql", + tar_url="https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1", + ) diff --git a/integration-tests/tests/fixtures/utils.py b/integration-tests/tests/fixtures/utils.py new file mode 100644 index 0000000000..5e86d1e2f4 --- /dev/null +++ b/integration-tests/tests/fixtures/utils.py @@ -0,0 +1,15 @@ +import os +import subprocess +from typing import List + + +def get_env_var(var_name: str) -> str: + value = os.environ.get(var_name) + assert value is not None, f"Environment variable {var_name} is not set." + return value + + +def run_and_assert(cmd: List[str], **kwargs) -> subprocess.CompletedProcess: + proc = subprocess.run(cmd, **kwargs) + assert 0 == proc.returncode, f"Command failed: {' '.join(cmd)}" + return proc diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index a5df55228f..aaa2a71665 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -1,13 +1,36 @@ import pytest +from fixtures.fixture_types import DatasetParams pytestmark = pytest.mark.binaries +text_datasets = pytest.mark.parametrize( + "download_and_extract_dataset", + [ + # "hive_24hr", + ], + indirect=True, +) + +json_datasets = pytest.mark.parametrize( + "download_and_extract_dataset", + [ + # "spark_event_logs", + "postgresql", + ], + indirect=["download_and_extract_dataset"], +) + @pytest.mark.clp -def test_always_pass_0() -> None: +@text_datasets +def test_clp_identity_transform(download_and_extract_dataset, request) -> None: + # print(request.getfixturevalue(dataset)) assert True @pytest.mark.clp_s -def test_always_pass_1() -> None: +@json_datasets +def test_clp_s_identity_transform(download_and_extract_dataset: DatasetParams, request) -> None: + # print(request.getfixturevalue(download_and_extract_dataset)) + print(download_and_extract_dataset) assert True diff --git a/taskfile.yaml b/taskfile.yaml index 7a0d7d25d9..f2b7e75997 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -166,7 +166,10 @@ tasks: test-package: dir: "{{.G_INTEGRATION_TESTS_DIR}}" cmds: - - "uv run pytest" + - |- + CLP_BUILD_DIR="{{.G_BUILD_DIR}}" \ + CLP_PACKAGE_BUILD_DIR="{{.G_PACKAGE_BUILD_DIR}}" \ + uv run pytest clp-s-generate-parsers: vars: From c659eb4f046783915ce16c8a70a002bbc179aa50 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 13 Jul 2025 11:06:52 -0400 Subject: [PATCH 007/100] Add clp-s test code --- .../tests/test_identity_transformation.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index aaa2a71665..c4ce33a476 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -1,5 +1,6 @@ import pytest -from fixtures.fixture_types import DatasetParams +from fixtures.fixture_types import DatasetParams, PackageTestConfig +from fixtures.utils import run_and_assert pytestmark = pytest.mark.binaries @@ -30,7 +31,17 @@ def test_clp_identity_transform(download_and_extract_dataset, request) -> None: @pytest.mark.clp_s @json_datasets -def test_clp_s_identity_transform(download_and_extract_dataset: DatasetParams, request) -> None: - # print(request.getfixturevalue(download_and_extract_dataset)) - print(download_and_extract_dataset) - assert True +def test_clp_s_identity_transform( + request, config: PackageTestConfig, download_and_extract_dataset: DatasetParams +) -> None: + dataset_name = download_and_extract_dataset.dataset_name + download_dir = config.uncompressed_logs_dir / dataset_name + archives_dir = config.test_output_dir / f"{dataset_name}-archives" + extract_dir = config.test_output_dir / f"{dataset_name}-logs" + + compression_cmd = [config.clp_bin_dir / "clp-s", "c", archives_dir, download_dir] + print(compression_cmd) + run_and_assert(compression_cmd) + + decompression_cmd = [config.clp_bin_dir / "clp-s", "x", archives_dir, extract_dir] + run_and_assert(decompression_cmd) From dc62fd4e79aa68dda00b02efe27080d5cbe871c2 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 14 Jul 2025 03:37:55 -0400 Subject: [PATCH 008/100] Package restructure --- integration-tests/.pytest.ini | 3 + integration-tests/tests/conftest.py | 6 +- ...{package_test_config.py => base_config.py} | 16 ++--- .../tests/fixtures/dataset_logs.py | 70 +++++++++++++++++++ .../fixtures/download_and_extract_dataset.py | 38 ---------- .../tests/fixtures/uncompressed_logs.py | 35 ---------- .../tests/test_identity_transformation.py | 21 +++--- .../fixture_types.py => utils/config.py} | 6 +- .../tests/{fixtures => utils}/utils.py | 0 taskfile.yaml | 4 +- 10 files changed, 101 insertions(+), 98 deletions(-) rename integration-tests/tests/fixtures/{package_test_config.py => base_config.py} (53%) create mode 100644 integration-tests/tests/fixtures/dataset_logs.py delete mode 100644 integration-tests/tests/fixtures/download_and_extract_dataset.py delete mode 100644 integration-tests/tests/fixtures/uncompressed_logs.py rename integration-tests/tests/{fixtures/fixture_types.py => utils/config.py} (79%) rename integration-tests/tests/{fixtures => utils}/utils.py (100%) diff --git a/integration-tests/.pytest.ini b/integration-tests/.pytest.ini index 18b3d1e8a6..df982a1032 100644 --- a/integration-tests/.pytest.ini +++ b/integration-tests/.pytest.ini @@ -6,6 +6,9 @@ addopts = --verbose --color=yes --code-highlight=yes +env = + D:CLP_BUILD_DIR=../build + D:CLP_PACKAGE_DIR=../build/clp-package markers = binaries: mark tests that directly call the binaries in the package bin clp: mark tests that use the CLP storage engine diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index 1a042ff16f..be84853b3e 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -1,6 +1,6 @@ -from fixtures.download_and_extract_dataset import download_and_extract_dataset -from fixtures.package_test_config import config -from fixtures.uncompressed_logs import ( +from tests.fixtures.base_config import base_config +from tests.fixtures.dataset_logs import ( + download_and_extract_dataset, hive_24hr, postgresql, spark_event_logs, diff --git a/integration-tests/tests/fixtures/package_test_config.py b/integration-tests/tests/fixtures/base_config.py similarity index 53% rename from integration-tests/tests/fixtures/package_test_config.py rename to integration-tests/tests/fixtures/base_config.py index b455952ff0..58a7cf5d3f 100644 --- a/integration-tests/tests/fixtures/package_test_config.py +++ b/integration-tests/tests/fixtures/base_config.py @@ -2,22 +2,22 @@ import pytest -from .fixture_types import PackageTestConfig -from .utils import get_env_var +from tests.utils.config import BaseConfig +from tests.utils.utils import get_env_var @pytest.fixture(scope="session") -def config() -> PackageTestConfig: +def base_config() -> BaseConfig: clp_build_dir = Path(get_env_var("CLP_BUILD_DIR")) - clp_package_dir = Path(get_env_var("CLP_PACKAGE_BUILD_DIR")) + clp_package_dir = Path(get_env_var("CLP_PACKAGE_DIR")) - config = PackageTestConfig( + base_config = BaseConfig( clp_bin_dir=clp_package_dir / "bin", clp_package_dir=clp_package_dir, clp_sbin_dir=clp_package_dir / "sbin", test_output_dir=clp_build_dir / "var" / "logs" / "pytest", uncompressed_logs_dir=clp_build_dir / "var" / "data" / "pytest" / "downloads", ) - config.test_output_dir.mkdir(parents=True, exist_ok=True) - config.uncompressed_logs_dir.mkdir(parents=True, exist_ok=True) - return config + base_config.test_output_dir.mkdir(parents=True, exist_ok=True) + base_config.uncompressed_logs_dir.mkdir(parents=True, exist_ok=True) + return base_config diff --git a/integration-tests/tests/fixtures/dataset_logs.py b/integration-tests/tests/fixtures/dataset_logs.py new file mode 100644 index 0000000000..4ca4032e40 --- /dev/null +++ b/integration-tests/tests/fixtures/dataset_logs.py @@ -0,0 +1,70 @@ +import shutil +import pytest + +from tests.utils.config import ( + BaseConfig, + DatasetLogs, +) +from tests.utils.utils import run_and_assert + +@pytest.fixture(scope="session") +def hive_24hr() -> DatasetLogs: + return DatasetLogs( + name="hive-24hr", + tar_url="https://zenodo.org/records/7094921/files/hive-24hr.tar.gz?download=1", + ) + + +@pytest.fixture(scope="session") +def elasticsearch() -> DatasetLogs: + return DatasetLogs( + name="elasticsearch", + tar_url="https://zenodo.org/records/10516227/files/elasticsearch.tar.gz?download=1", + ) + + +@pytest.fixture(scope="session") +def spark_event_logs() -> DatasetLogs: + return DatasetLogs( + name="spark-event-logs", + tar_url="https://zenodo.org/records/10516346/files/spark-event-logs.tar.gz?download=1", + ) + + +@pytest.fixture(scope="session") +def postgresql() -> DatasetLogs: + return DatasetLogs( + name="postgresql", + tar_url="https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1", + ) + + +@pytest.fixture(autouse=True) +def download_and_extract_dataset(request, base_config: BaseConfig) -> DatasetLogs: + dataset_config = request.getfixturevalue(request.param) + dataset_name = dataset_config.name + if request.config.cache.get(dataset_name, False): + print(f"Uncompressed logs for dataset `{dataset_name}` is up-to-date.") + return dataset_config + + download_path = str(base_config.uncompressed_logs_dir / f"{dataset_name}.tar.gz") + extract_path = str(base_config.uncompressed_logs_dir / dataset_name) + # fmt: off + cmds = [ + "curl", + "--fail", + "--location", + "--output", str(download_path), + "--show-error", + dataset_config.tar_url, + ] + # fmt: on + run_and_assert(cmds) + + try: + shutil.unpack_archive(download_path, extract_path) + except: + assert False, f"Tar extraction failed for downloaded dataset `{dataset_name}`." + + request.config.cache.set(dataset_name, True) + return dataset_config diff --git a/integration-tests/tests/fixtures/download_and_extract_dataset.py b/integration-tests/tests/fixtures/download_and_extract_dataset.py deleted file mode 100644 index b5b2687398..0000000000 --- a/integration-tests/tests/fixtures/download_and_extract_dataset.py +++ /dev/null @@ -1,38 +0,0 @@ -import shutil -from pathlib import Path - -import pytest - -from .fixture_types import DatasetParams -from .package_test_config import PackageTestConfig -from .utils import run_and_assert - - -@pytest.fixture(autouse=True) -def download_and_extract_dataset(request, config: PackageTestConfig) -> DatasetParams: - download_params = request.getfixturevalue(request.param) - dataset_name = download_params.dataset_name - if request.config.cache.get(dataset_name, False): - print(f"Uncompressed logs for dataset `{dataset_name}` is up-to-date.") - return download_params - - tar_path = str(config.uncompressed_logs_dir / f"{dataset_name}.tar.gz") - extract_path = str(config.uncompressed_logs_dir / dataset_name) - cmds = [ - "curl", - "--fail", - "--location", - "--output", - str(tar_path), - "--show-error", - download_params.tar_url, - ] - run_and_assert(cmds) - - try: - shutil.unpack_archive(tar_path, extract_path) - request.config.cache.set(dataset_name, True) - except: - assert False, f"Tar extraction failed for dataset `{dataset_name}`." - - return download_params diff --git a/integration-tests/tests/fixtures/uncompressed_logs.py b/integration-tests/tests/fixtures/uncompressed_logs.py deleted file mode 100644 index d304843cbb..0000000000 --- a/integration-tests/tests/fixtures/uncompressed_logs.py +++ /dev/null @@ -1,35 +0,0 @@ -import pytest - -from .fixture_types import DatasetParams - - -@pytest.fixture(scope="session") -def hive_24hr() -> DatasetParams: - return DatasetParams( - dataset_name="hive-24hr", - tar_url="https://zenodo.org/records/7094921/files/hive-24hr.tar.gz?download=1", - ) - - -@pytest.fixture(scope="session") -def elasticsearch() -> DatasetParams: - return DatasetParams( - dataset_name="elasticsearch", - tar_url="https://zenodo.org/records/10516227/files/elasticsearch.tar.gz?download=1", - ) - - -@pytest.fixture(scope="session") -def spark_event_logs() -> DatasetParams: - return DatasetParams( - dataset_name="spark-event-logs", - tar_url="https://zenodo.org/records/10516346/files/spark-event-logs.tar.gz?download=1", - ) - - -@pytest.fixture(scope="session") -def postgresql() -> DatasetParams: - return DatasetParams( - dataset_name="postgresql", - tar_url="https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1", - ) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index c4ce33a476..b4abdf4591 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -1,6 +1,9 @@ import pytest -from fixtures.fixture_types import DatasetParams, PackageTestConfig -from fixtures.utils import run_and_assert +from tests.utils.config import ( + BaseConfig, + DatasetLogs, +) +from tests.utils.utils import run_and_assert pytestmark = pytest.mark.binaries @@ -32,16 +35,16 @@ def test_clp_identity_transform(download_and_extract_dataset, request) -> None: @pytest.mark.clp_s @json_datasets def test_clp_s_identity_transform( - request, config: PackageTestConfig, download_and_extract_dataset: DatasetParams + request, base_config: BaseConfig, download_and_extract_dataset: DatasetLogs ) -> None: - dataset_name = download_and_extract_dataset.dataset_name - download_dir = config.uncompressed_logs_dir / dataset_name - archives_dir = config.test_output_dir / f"{dataset_name}-archives" - extract_dir = config.test_output_dir / f"{dataset_name}-logs" + dataset_name = download_and_extract_dataset.name + download_dir = base_config.uncompressed_logs_dir / dataset_name + archives_dir = base_config.test_output_dir / f"{dataset_name}-archives" + extract_dir = base_config.test_output_dir / f"{dataset_name}-logs" - compression_cmd = [config.clp_bin_dir / "clp-s", "c", archives_dir, download_dir] + compression_cmd = [base_config.clp_bin_dir / "clp-s", "c", archives_dir, download_dir] print(compression_cmd) run_and_assert(compression_cmd) - decompression_cmd = [config.clp_bin_dir / "clp-s", "x", archives_dir, extract_dir] + decompression_cmd = [base_config.clp_bin_dir / "clp-s", "x", archives_dir, extract_dir] run_and_assert(decompression_cmd) diff --git a/integration-tests/tests/fixtures/fixture_types.py b/integration-tests/tests/utils/config.py similarity index 79% rename from integration-tests/tests/fixtures/fixture_types.py rename to integration-tests/tests/utils/config.py index 75bab0550e..3d06c12af2 100644 --- a/integration-tests/tests/fixtures/fixture_types.py +++ b/integration-tests/tests/utils/config.py @@ -3,7 +3,7 @@ @dataclass(frozen=True) -class PackageTestConfig: +class BaseConfig: clp_bin_dir: Path clp_package_dir: Path clp_sbin_dir: Path @@ -12,6 +12,6 @@ class PackageTestConfig: @dataclass(frozen=True) -class DatasetParams: - dataset_name: str +class DatasetLogs: + name: str tar_url: str diff --git a/integration-tests/tests/fixtures/utils.py b/integration-tests/tests/utils/utils.py similarity index 100% rename from integration-tests/tests/fixtures/utils.py rename to integration-tests/tests/utils/utils.py diff --git a/taskfile.yaml b/taskfile.yaml index f2b7e75997..ade68c295d 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -168,8 +168,8 @@ tasks: cmds: - |- CLP_BUILD_DIR="{{.G_BUILD_DIR}}" \ - CLP_PACKAGE_BUILD_DIR="{{.G_PACKAGE_BUILD_DIR}}" \ - uv run pytest + CLP_PACKAGE_DIR="{{.G_PACKAGE_BUILD_DIR}}" \ + uv run python -m pytest clp-s-generate-parsers: vars: From ded7d4c140c3ab6730c5536dfc6ac942ccddd6c4 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 14 Jul 2025 10:10:53 -0400 Subject: [PATCH 009/100] Complete clp-s testing (with bug) --- .../tests/fixtures/base_config.py | 5 +- .../tests/fixtures/dataset_logs.py | 3 +- .../tests/test_identity_transformation.py | 79 ++++++++++++++++--- integration-tests/tests/utils/utils.py | 10 +++ 4 files changed, 80 insertions(+), 17 deletions(-) diff --git a/integration-tests/tests/fixtures/base_config.py b/integration-tests/tests/fixtures/base_config.py index 58a7cf5d3f..f2434d5de5 100644 --- a/integration-tests/tests/fixtures/base_config.py +++ b/integration-tests/tests/fixtures/base_config.py @@ -1,15 +1,14 @@ from pathlib import Path import pytest - from tests.utils.config import BaseConfig from tests.utils.utils import get_env_var @pytest.fixture(scope="session") def base_config() -> BaseConfig: - clp_build_dir = Path(get_env_var("CLP_BUILD_DIR")) - clp_package_dir = Path(get_env_var("CLP_PACKAGE_DIR")) + clp_build_dir = Path(get_env_var("CLP_BUILD_DIR")).resolve() + clp_package_dir = Path(get_env_var("CLP_PACKAGE_DIR")).resolve() base_config = BaseConfig( clp_bin_dir=clp_package_dir / "bin", diff --git a/integration-tests/tests/fixtures/dataset_logs.py b/integration-tests/tests/fixtures/dataset_logs.py index 4ca4032e40..0709a2f49f 100644 --- a/integration-tests/tests/fixtures/dataset_logs.py +++ b/integration-tests/tests/fixtures/dataset_logs.py @@ -1,12 +1,13 @@ import shutil -import pytest +import pytest from tests.utils.config import ( BaseConfig, DatasetLogs, ) from tests.utils.utils import run_and_assert + @pytest.fixture(scope="session") def hive_24hr() -> DatasetLogs: return DatasetLogs( diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index b4abdf4591..f191656035 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -1,25 +1,31 @@ +import shutil +import tempfile + import pytest from tests.utils.config import ( BaseConfig, DatasetLogs, ) -from tests.utils.utils import run_and_assert +from tests.utils.utils import ( + diff_equal, + run_and_assert, +) pytestmark = pytest.mark.binaries text_datasets = pytest.mark.parametrize( "download_and_extract_dataset", [ - # "hive_24hr", + "hive_24hr", ], - indirect=True, + indirect=["download_and_extract_dataset"], ) json_datasets = pytest.mark.parametrize( "download_and_extract_dataset", [ - # "spark_event_logs", - "postgresql", + "spark_event_logs", + # "postgresql", ], indirect=["download_and_extract_dataset"], ) @@ -27,9 +33,31 @@ @pytest.mark.clp @text_datasets -def test_clp_identity_transform(download_and_extract_dataset, request) -> None: - # print(request.getfixturevalue(dataset)) - assert True +def test_clp_identity_transform( + request, base_config: BaseConfig, download_and_extract_dataset: DatasetLogs +) -> None: + binary_path_str = base_config.clp_bin_dir / "clp" + dataset_name = download_and_extract_dataset.name + download_dir = base_config.uncompressed_logs_dir / dataset_name + archives_dir = base_config.test_output_dir / f"{dataset_name}-archives" + shutil.rmtree(archives_dir, ignore_errors=True) + extract_dir = base_config.test_output_dir / f"{dataset_name}-logs" + shutil.rmtree(extract_dir, ignore_errors=True) + + # fmt: off + compression_cmd = [ + binary_path_str, + "c", + "--progress", + "--remove-path-prefix", str(download_dir), + str(archives_dir), + str(download_dir), + ] + # fmt: on + run_and_assert(compression_cmd) + run_and_assert([binary_path_str, "x", str(archives_dir), str(extract_dir)]) + + diff_equal(download_dir, extract_dir) @pytest.mark.clp_s @@ -37,14 +65,39 @@ def test_clp_identity_transform(download_and_extract_dataset, request) -> None: def test_clp_s_identity_transform( request, base_config: BaseConfig, download_and_extract_dataset: DatasetLogs ) -> None: + binary_path_str = base_config.clp_bin_dir / "clp-s" dataset_name = download_and_extract_dataset.name download_dir = base_config.uncompressed_logs_dir / dataset_name archives_dir = base_config.test_output_dir / f"{dataset_name}-archives" extract_dir = base_config.test_output_dir / f"{dataset_name}-logs" - compression_cmd = [base_config.clp_bin_dir / "clp-s", "c", archives_dir, download_dir] - print(compression_cmd) - run_and_assert(compression_cmd) + # shutil.rmtree(archives_dir, ignore_errors=True) + # shutil.rmtree(extract_dir, ignore_errors=True) + + # run_and_assert([binary_path_str, "c", str(archives_dir), str(download_dir)]) + # run_and_assert([binary_path_str, "x", str(archives_dir), str(extract_dir)]) + + # Recompress the decompressed single-file output and decompress it again to verify consistency. + # TODO: Remove this check once we can directly compare decompressed logs (which would preserve + # the directory structure) with the original downloaded logs. + # See also: https://docs.yscope.com/clp/main/user-guide/core-clp-s.html#current-limitations + single_file_archives_dir = base_config.test_output_dir / f"{dataset_name}-single-file-archives" + single_file_extract_dir = base_config.test_output_dir / f"{dataset_name}-single-file-logs" + + # shutil.rmtree(single_file_archives_dir, ignore_errors=True) + # shutil.rmtree(single_file_extract_dir, ignore_errors=True) + + # run_and_assert([binary_path_str, "c", single_file_archives_dir, extract_dir]) + # run_and_assert([binary_path_str, "x", single_file_archives_dir, single_file_extract_dir]) - decompression_cmd = [base_config.clp_bin_dir / "clp-s", "x", archives_dir, extract_dir] - run_and_assert(decompression_cmd) + # Key order within a single entry is not preserved. + with tempfile.NamedTemporaryFile( + suffix=".json", mode="w+", delete=True + ) as t1, tempfile.NamedTemporaryFile(suffix=".json", mode="w+", delete=True) as t2: + run_and_assert(["jq", "--sort-keys", ".", str(extract_dir / "original")], stdout=t1) + run_and_assert( + ["jq", "--sort-keys", ".", str(single_file_extract_dir / "original")], stdout=t2 + ) + t1.flush() + t2.flush() + diff_equal(t1.name, t2.name) diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index 5e86d1e2f4..4d4f43eb4a 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -1,8 +1,18 @@ import os import subprocess +from pathlib import Path from typing import List +def diff_equal(path1: Path, path2: Path) -> None: + cmd = ["diff", "--brief", "--recursive", str(path1), str(path2)] + proc = subprocess.run(cmd, stdout=subprocess.PIPE) + if 0 != proc.returncode: + if 1 == proc.returncode: + assert False, "Files/Directories don't match." + assert False, f"Command failed {' '.join(cmd)}" + + def get_env_var(var_name: str) -> str: value = os.environ.get(var_name) assert value is not None, f"Environment variable {var_name} is not set." From 6880a0f8ed7bc1f2e5a2f5bbd388c8ae20b40b58 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 14 Jul 2025 13:47:34 -0400 Subject: [PATCH 010/100] Make clp-s test workable with keys and rows sorting --- .../tests/test_identity_transformation.py | 73 ++++++++++++------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index f191656035..c6a29a2274 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -1,5 +1,7 @@ import shutil -import tempfile +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import IO import pytest from tests.utils.config import ( @@ -25,7 +27,7 @@ "download_and_extract_dataset", [ "spark_event_logs", - # "postgresql", + "postgresql", ], indirect=["download_and_extract_dataset"], ) @@ -36,12 +38,13 @@ def test_clp_identity_transform( request, base_config: BaseConfig, download_and_extract_dataset: DatasetLogs ) -> None: - binary_path_str = base_config.clp_bin_dir / "clp" + binary_path_str = str(base_config.clp_bin_dir / "clp") dataset_name = download_and_extract_dataset.name download_dir = base_config.uncompressed_logs_dir / dataset_name archives_dir = base_config.test_output_dir / f"{dataset_name}-archives" - shutil.rmtree(archives_dir, ignore_errors=True) extract_dir = base_config.test_output_dir / f"{dataset_name}-logs" + + shutil.rmtree(archives_dir, ignore_errors=True) shutil.rmtree(extract_dir, ignore_errors=True) # fmt: off @@ -59,45 +62,61 @@ def test_clp_identity_transform( diff_equal(download_dir, extract_dir) + shutil.rmtree(archives_dir, ignore_errors=True) + shutil.rmtree(extract_dir, ignore_errors=True) + @pytest.mark.clp_s @json_datasets def test_clp_s_identity_transform( request, base_config: BaseConfig, download_and_extract_dataset: DatasetLogs ) -> None: - binary_path_str = base_config.clp_bin_dir / "clp-s" + binary_path_str = str(base_config.clp_bin_dir / "clp-s") dataset_name = download_and_extract_dataset.name download_dir = base_config.uncompressed_logs_dir / dataset_name archives_dir = base_config.test_output_dir / f"{dataset_name}-archives" extract_dir = base_config.test_output_dir / f"{dataset_name}-logs" - # shutil.rmtree(archives_dir, ignore_errors=True) - # shutil.rmtree(extract_dir, ignore_errors=True) + shutil.rmtree(archives_dir, ignore_errors=True) + shutil.rmtree(extract_dir, ignore_errors=True) - # run_and_assert([binary_path_str, "c", str(archives_dir), str(download_dir)]) - # run_and_assert([binary_path_str, "x", str(archives_dir), str(extract_dir)]) + run_and_assert([binary_path_str, "c", str(archives_dir), str(download_dir)]) + run_and_assert([binary_path_str, "x", str(archives_dir), str(extract_dir)]) # Recompress the decompressed single-file output and decompress it again to verify consistency. # TODO: Remove this check once we can directly compare decompressed logs (which would preserve - # the directory structure) with the original downloaded logs. + # the directory structure and row/key order) with the original downloaded logs. # See also: https://docs.yscope.com/clp/main/user-guide/core-clp-s.html#current-limitations single_file_archives_dir = base_config.test_output_dir / f"{dataset_name}-single-file-archives" single_file_extract_dir = base_config.test_output_dir / f"{dataset_name}-single-file-logs" - # shutil.rmtree(single_file_archives_dir, ignore_errors=True) - # shutil.rmtree(single_file_extract_dir, ignore_errors=True) - - # run_and_assert([binary_path_str, "c", single_file_archives_dir, extract_dir]) - # run_and_assert([binary_path_str, "x", single_file_archives_dir, single_file_extract_dir]) - - # Key order within a single entry is not preserved. - with tempfile.NamedTemporaryFile( - suffix=".json", mode="w+", delete=True - ) as t1, tempfile.NamedTemporaryFile(suffix=".json", mode="w+", delete=True) as t2: - run_and_assert(["jq", "--sort-keys", ".", str(extract_dir / "original")], stdout=t1) - run_and_assert( - ["jq", "--sort-keys", ".", str(single_file_extract_dir / "original")], stdout=t2 - ) - t1.flush() - t2.flush() - diff_equal(t1.name, t2.name) + shutil.rmtree(single_file_archives_dir, ignore_errors=True) + shutil.rmtree(single_file_extract_dir, ignore_errors=True) + + run_and_assert([binary_path_str, "c", single_file_archives_dir, extract_dir]) + run_and_assert([binary_path_str, "x", single_file_archives_dir, single_file_extract_dir]) + + # Key and row orders are not preserved during `clp-s` operations, so sort before diffing. + with _sort_json_keys_and_rows(extract_dir / "original") as s1, _sort_json_keys_and_rows( + single_file_extract_dir / "original" + ) as s2: + diff_equal(s1.name, s2.name) + + shutil.rmtree(archives_dir, ignore_errors=True) + shutil.rmtree(extract_dir, ignore_errors=True) + shutil.rmtree(single_file_archives_dir, ignore_errors=True) + shutil.rmtree(single_file_extract_dir, ignore_errors=True) + + +def _sort_json_keys_and_rows(json_fp: Path) -> IO[bytes]: + with NamedTemporaryFile(mode="w+", delete=True) as keys_sorted, NamedTemporaryFile( + mode="w+", delete=True + ) as flattened: + keys_and_rows_sorted = NamedTemporaryFile(mode="w+", delete=True) + run_and_assert(["jq", "--sort-keys", ".", str(json_fp)], stdout=keys_sorted) + keys_sorted.flush() + run_and_assert(["jq", ".", keys_sorted.name], stdout=flattened) + flattened.flush() + run_and_assert(["sort", flattened.name], stdout=keys_and_rows_sorted) + keys_and_rows_sorted.flush() + return keys_and_rows_sorted From 9587f99c075e77ddb51a5a15d882ca354100e0ba Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 23 Jul 2025 20:14:15 -0400 Subject: [PATCH 011/100] Address some review comments --- docs/src/dev-guide/building-package.md | 2 +- integration-tests/README.md | 25 ++++++++++ .../src/integration_tests/__init__.py | 2 - integration-tests/tests/conftest.py | 2 +- .../tests/fixtures/base_config.py | 22 --------- .../tests/fixtures/dataset_logs.py | 47 ++++++++++--------- .../tests/fixtures/package_config.py | 22 +++++++++ .../tests/test_identity_transformation.py | 30 ++++++------ integration-tests/tests/utils/config.py | 2 +- taskfile.yaml | 10 +++- 10 files changed, 99 insertions(+), 65 deletions(-) delete mode 100644 integration-tests/tests/fixtures/base_config.py create mode 100644 integration-tests/tests/fixtures/package_config.py diff --git a/docs/src/dev-guide/building-package.md b/docs/src/dev-guide/building-package.md index b1e8a07504..3874fc9fb3 100644 --- a/docs/src/dev-guide/building-package.md +++ b/docs/src/dev-guide/building-package.md @@ -67,7 +67,7 @@ values for the fields in angle brackets. To test the package, run: ```shell -task test-package +task package-test ``` ## Cleanup diff --git a/integration-tests/README.md b/integration-tests/README.md index e69de29bb2..d45578eaa8 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -0,0 +1,25 @@ +# CLP Integration Tests + +This Python module provides end-to-end tests for the CLP package using the `pytest` framework. +It covers package lifecycle scripts and validates compression and search functionality for +performance and correctness on large [datasets]. + + +## Running tests +To build the package and run all integration tests: +``` +task package-test +``` + +With the package already built, you can leverage `pytest` markers to selectively run tests, e.g. +``` +# Inside the integration-tests top-level directory +uv run python -m pytest -m clp_s +``` +only runs `clp-s` related tests. + +To test CLP package at a custom location, override default paths with the environment variable +`CLP_PACKAGE_DIR`, and then run the `uv` command above. + + +[datasets]: https://docs.yscope.com/clp/main/user-guide/resources-datasets diff --git a/integration-tests/src/integration_tests/__init__.py b/integration-tests/src/integration_tests/__init__.py index 17b793a3c7..e69de29bb2 100644 --- a/integration-tests/src/integration_tests/__init__.py +++ b/integration-tests/src/integration_tests/__init__.py @@ -1,2 +0,0 @@ -def main() -> None: - print("Hello from integration-tests!") diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index be84853b3e..feefe06159 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -1,4 +1,4 @@ -from tests.fixtures.base_config import base_config +from tests.fixtures.package_config import package_config from tests.fixtures.dataset_logs import ( download_and_extract_dataset, hive_24hr, diff --git a/integration-tests/tests/fixtures/base_config.py b/integration-tests/tests/fixtures/base_config.py deleted file mode 100644 index f2434d5de5..0000000000 --- a/integration-tests/tests/fixtures/base_config.py +++ /dev/null @@ -1,22 +0,0 @@ -from pathlib import Path - -import pytest -from tests.utils.config import BaseConfig -from tests.utils.utils import get_env_var - - -@pytest.fixture(scope="session") -def base_config() -> BaseConfig: - clp_build_dir = Path(get_env_var("CLP_BUILD_DIR")).resolve() - clp_package_dir = Path(get_env_var("CLP_PACKAGE_DIR")).resolve() - - base_config = BaseConfig( - clp_bin_dir=clp_package_dir / "bin", - clp_package_dir=clp_package_dir, - clp_sbin_dir=clp_package_dir / "sbin", - test_output_dir=clp_build_dir / "var" / "logs" / "pytest", - uncompressed_logs_dir=clp_build_dir / "var" / "data" / "pytest" / "downloads", - ) - base_config.test_output_dir.mkdir(parents=True, exist_ok=True) - base_config.uncompressed_logs_dir.mkdir(parents=True, exist_ok=True) - return base_config diff --git a/integration-tests/tests/fixtures/dataset_logs.py b/integration-tests/tests/fixtures/dataset_logs.py index 0709a2f49f..9d5748cc1a 100644 --- a/integration-tests/tests/fixtures/dataset_logs.py +++ b/integration-tests/tests/fixtures/dataset_logs.py @@ -1,11 +1,12 @@ +import os import shutil +import subprocess import pytest from tests.utils.config import ( - BaseConfig, + PackageConfig, DatasetLogs, ) -from tests.utils.utils import run_and_assert @pytest.fixture(scope="session") @@ -40,32 +41,34 @@ def postgresql() -> DatasetLogs: ) -@pytest.fixture(autouse=True) -def download_and_extract_dataset(request, base_config: BaseConfig) -> DatasetLogs: +@pytest.fixture +def download_and_extract_dataset(request, package_config: PackageConfig) -> DatasetLogs: dataset_config = request.getfixturevalue(request.param) dataset_name = dataset_config.name if request.config.cache.get(dataset_name, False): - print(f"Uncompressed logs for dataset `{dataset_name}` is up-to-date.") + print(f"Uncompressed logs for dataset `{dataset_name}` is up-to-date. Skipping download.") return dataset_config - download_path = str(base_config.uncompressed_logs_dir / f"{dataset_name}.tar.gz") - extract_path = str(base_config.uncompressed_logs_dir / dataset_name) - # fmt: off - cmds = [ - "curl", - "--fail", - "--location", - "--output", str(download_path), - "--show-error", - dataset_config.tar_url, - ] - # fmt: on - run_and_assert(cmds) - try: - shutil.unpack_archive(download_path, extract_path) - except: - assert False, f"Tar extraction failed for downloaded dataset `{dataset_name}`." + download_path_str = str(package_config.uncompressed_logs_dir / f"{dataset_name}.tar.gz") + # fmt: off + curl_cmds = [ + "curl", + "--fail", + "--location", + "--output", download_path_str, + "--show-error", + dataset_config.tar_url, + ] + # fmt: on + subprocess.run(curl_cmds, check=True) + + extract_path_str = str(package_config.uncompressed_logs_dir / dataset_name) + shutil.unpack_archive(download_path_str, extract_path_str) + os.remove(download_path_str) + except Exception as e: + raise RuntimeError(f"Failed to download and extract dataset `{dataset_name}`: {e}") + print(f"Downloaded and extracted uncompressed logs for dataset `{dataset_name}`.") request.config.cache.set(dataset_name, True) return dataset_config diff --git a/integration-tests/tests/fixtures/package_config.py b/integration-tests/tests/fixtures/package_config.py new file mode 100644 index 0000000000..6ae29fe50b --- /dev/null +++ b/integration-tests/tests/fixtures/package_config.py @@ -0,0 +1,22 @@ +from pathlib import Path + +import pytest +from tests.utils.config import PackageConfig +from tests.utils.utils import get_env_var + + +@pytest.fixture(scope="session") +def package_config() -> PackageConfig: + clp_build_dir = Path(get_env_var("CLP_BUILD_DIR")).resolve() + clp_package_dir = Path(get_env_var("CLP_PACKAGE_DIR")).resolve() + + config = PackageConfig( + clp_bin_dir=clp_package_dir / "bin", + clp_package_dir=clp_package_dir, + clp_sbin_dir=clp_package_dir / "sbin", + test_output_dir=clp_build_dir / "integration-tests", + uncompressed_logs_dir=clp_build_dir / "integration-tests" / "downloads", + ) + config.test_output_dir.mkdir(parents=True, exist_ok=True) + config.uncompressed_logs_dir.mkdir(parents=True, exist_ok=True) + return config diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index c6a29a2274..0af4961c9b 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -5,7 +5,7 @@ import pytest from tests.utils.config import ( - BaseConfig, + PackageConfig, DatasetLogs, ) from tests.utils.utils import ( @@ -18,7 +18,7 @@ text_datasets = pytest.mark.parametrize( "download_and_extract_dataset", [ - "hive_24hr", + #"hive_24hr", ], indirect=["download_and_extract_dataset"], ) @@ -26,7 +26,7 @@ json_datasets = pytest.mark.parametrize( "download_and_extract_dataset", [ - "spark_event_logs", + #"spark_event_logs", "postgresql", ], indirect=["download_and_extract_dataset"], @@ -36,13 +36,13 @@ @pytest.mark.clp @text_datasets def test_clp_identity_transform( - request, base_config: BaseConfig, download_and_extract_dataset: DatasetLogs + request, package_config: PackageConfig, download_and_extract_dataset: DatasetLogs ) -> None: - binary_path_str = str(base_config.clp_bin_dir / "clp") + binary_path_str = str(package_config.clp_bin_dir / "clp") dataset_name = download_and_extract_dataset.name - download_dir = base_config.uncompressed_logs_dir / dataset_name - archives_dir = base_config.test_output_dir / f"{dataset_name}-archives" - extract_dir = base_config.test_output_dir / f"{dataset_name}-logs" + download_dir = package_config.uncompressed_logs_dir / dataset_name + archives_dir = package_config.test_output_dir / f"{dataset_name}-archives" + extract_dir = package_config.test_output_dir / f"{dataset_name}-logs" shutil.rmtree(archives_dir, ignore_errors=True) shutil.rmtree(extract_dir, ignore_errors=True) @@ -69,13 +69,13 @@ def test_clp_identity_transform( @pytest.mark.clp_s @json_datasets def test_clp_s_identity_transform( - request, base_config: BaseConfig, download_and_extract_dataset: DatasetLogs + request, package_config: PackageConfig, download_and_extract_dataset: DatasetLogs ) -> None: - binary_path_str = str(base_config.clp_bin_dir / "clp-s") + binary_path_str = str(package_config.clp_bin_dir / "clp-s") dataset_name = download_and_extract_dataset.name - download_dir = base_config.uncompressed_logs_dir / dataset_name - archives_dir = base_config.test_output_dir / f"{dataset_name}-archives" - extract_dir = base_config.test_output_dir / f"{dataset_name}-logs" + download_dir = package_config.uncompressed_logs_dir / dataset_name + archives_dir = package_config.test_output_dir / f"{dataset_name}-archives" + extract_dir = package_config.test_output_dir / f"{dataset_name}-logs" shutil.rmtree(archives_dir, ignore_errors=True) shutil.rmtree(extract_dir, ignore_errors=True) @@ -87,8 +87,8 @@ def test_clp_s_identity_transform( # TODO: Remove this check once we can directly compare decompressed logs (which would preserve # the directory structure and row/key order) with the original downloaded logs. # See also: https://docs.yscope.com/clp/main/user-guide/core-clp-s.html#current-limitations - single_file_archives_dir = base_config.test_output_dir / f"{dataset_name}-single-file-archives" - single_file_extract_dir = base_config.test_output_dir / f"{dataset_name}-single-file-logs" + single_file_archives_dir = package_config.test_output_dir / f"{dataset_name}-single-file-archives" + single_file_extract_dir = package_config.test_output_dir / f"{dataset_name}-single-file-logs" shutil.rmtree(single_file_archives_dir, ignore_errors=True) shutil.rmtree(single_file_extract_dir, ignore_errors=True) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 3d06c12af2..1966f5dead 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -3,7 +3,7 @@ @dataclass(frozen=True) -class BaseConfig: +class PackageConfig: clp_bin_dir: Path clp_package_dir: Path clp_sbin_dir: Path diff --git a/taskfile.yaml b/taskfile.yaml index 222674bf43..db906caca7 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -163,7 +163,9 @@ tasks: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] - test-package: + package-test: + deps: + - "package" dir: "{{.G_INTEGRATION_TESTS_DIR}}" cmds: - |- @@ -171,6 +173,12 @@ tasks: CLP_PACKAGE_DIR="{{.G_PACKAGE_BUILD_DIR}}" \ uv run python -m pytest + package-test-cache-clear: + dir: "{{.G_INTEGRATION_TESTS_DIR}}" + cmds: + - >- + uv run python -m pytest --cache-clear -q --collect-only + clp-s-generate-parsers: vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" From 6d73f311fb6579ac06b15352943c951a15c3334a Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 23 Jul 2025 21:31:59 -0400 Subject: [PATCH 012/100] Address more review comments --- integration-tests/.pytest.ini | 6 +-- integration-tests/tests/conftest.py | 2 +- .../tests/fixtures/dataset_logs.py | 3 +- .../tests/test_identity_transformation.py | 39 ++++++++++++------- integration-tests/tests/utils/utils.py | 13 ++++--- taskfile.yaml | 3 +- 6 files changed, 39 insertions(+), 27 deletions(-) diff --git a/integration-tests/.pytest.ini b/integration-tests/.pytest.ini index df982a1032..ddcc80392d 100644 --- a/integration-tests/.pytest.ini +++ b/integration-tests/.pytest.ini @@ -1,11 +1,11 @@ [pytest] addopts = + --capture=no + --code-highlight=yes + --color=yes --strict-config --strict-markers - --capture=no --verbose - --color=yes - --code-highlight=yes env = D:CLP_BUILD_DIR=../build D:CLP_PACKAGE_DIR=../build/clp-package diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index feefe06159..7b2322e7a3 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -1,7 +1,7 @@ -from tests.fixtures.package_config import package_config from tests.fixtures.dataset_logs import ( download_and_extract_dataset, hive_24hr, postgresql, spark_event_logs, ) +from tests.fixtures.package_config import package_config diff --git a/integration-tests/tests/fixtures/dataset_logs.py b/integration-tests/tests/fixtures/dataset_logs.py index 9d5748cc1a..6358573acf 100644 --- a/integration-tests/tests/fixtures/dataset_logs.py +++ b/integration-tests/tests/fixtures/dataset_logs.py @@ -4,8 +4,8 @@ import pytest from tests.utils.config import ( - PackageConfig, DatasetLogs, + PackageConfig, ) @@ -65,7 +65,6 @@ def download_and_extract_dataset(request, package_config: PackageConfig) -> Data extract_path_str = str(package_config.uncompressed_logs_dir / dataset_name) shutil.unpack_archive(download_path_str, extract_path_str) - os.remove(download_path_str) except Exception as e: raise RuntimeError(f"Failed to download and extract dataset `{dataset_name}`: {e}") diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 0af4961c9b..d1896ae077 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -1,15 +1,17 @@ import shutil +import subprocess +from contextlib import contextmanager from pathlib import Path from tempfile import NamedTemporaryFile from typing import IO import pytest from tests.utils.config import ( - PackageConfig, DatasetLogs, + PackageConfig, ) from tests.utils.utils import ( - diff_equal, + is_dir_tree_content_equal, run_and_assert, ) @@ -18,7 +20,7 @@ text_datasets = pytest.mark.parametrize( "download_and_extract_dataset", [ - #"hive_24hr", + # "hive_24hr", ], indirect=["download_and_extract_dataset"], ) @@ -26,7 +28,7 @@ json_datasets = pytest.mark.parametrize( "download_and_extract_dataset", [ - #"spark_event_logs", + # "spark_event_logs", "postgresql", ], indirect=["download_and_extract_dataset"], @@ -58,9 +60,13 @@ def test_clp_identity_transform( ] # fmt: on run_and_assert(compression_cmd) - run_and_assert([binary_path_str, "x", str(archives_dir), str(extract_dir)]) - diff_equal(download_dir, extract_dir) + extraction_cmd = [binary_path_str, "x", str(archives_dir), str(extract_dir)] + run_and_assert(extraction_cmd) + + assert is_dir_tree_content_equal( + download_dir, extract_dir + ), "Mismatch between clp compression input and decompression output." shutil.rmtree(archives_dir, ignore_errors=True) shutil.rmtree(extract_dir, ignore_errors=True) @@ -87,7 +93,9 @@ def test_clp_s_identity_transform( # TODO: Remove this check once we can directly compare decompressed logs (which would preserve # the directory structure and row/key order) with the original downloaded logs. # See also: https://docs.yscope.com/clp/main/user-guide/core-clp-s.html#current-limitations - single_file_archives_dir = package_config.test_output_dir / f"{dataset_name}-single-file-archives" + single_file_archives_dir = ( + package_config.test_output_dir / f"{dataset_name}-single-file-archives" + ) single_file_extract_dir = package_config.test_output_dir / f"{dataset_name}-single-file-logs" shutil.rmtree(single_file_archives_dir, ignore_errors=True) @@ -100,7 +108,9 @@ def test_clp_s_identity_transform( with _sort_json_keys_and_rows(extract_dir / "original") as s1, _sort_json_keys_and_rows( single_file_extract_dir / "original" ) as s2: - diff_equal(s1.name, s2.name) + assert is_dir_tree_content_equal( + s1.name, s2.name + ), "Mismatch between clp-s compression input and decompression output." shutil.rmtree(archives_dir, ignore_errors=True) shutil.rmtree(extract_dir, ignore_errors=True) @@ -108,15 +118,16 @@ def test_clp_s_identity_transform( shutil.rmtree(single_file_extract_dir, ignore_errors=True) -def _sort_json_keys_and_rows(json_fp: Path) -> IO[bytes]: +@contextmanager +def _sort_json_keys_and_rows(json_fp: Path) -> IO[str]: with NamedTemporaryFile(mode="w+", delete=True) as keys_sorted, NamedTemporaryFile( mode="w+", delete=True - ) as flattened: + ) as flattened, NamedTemporaryFile(mode="w+", delete=True) as keys_and_rows_sorted: keys_and_rows_sorted = NamedTemporaryFile(mode="w+", delete=True) - run_and_assert(["jq", "--sort-keys", ".", str(json_fp)], stdout=keys_sorted) + subprocess.run(["jq", "--sort-keys", ".", str(json_fp)], check=True, stdout=keys_sorted) keys_sorted.flush() - run_and_assert(["jq", ".", keys_sorted.name], stdout=flattened) + subprocess.run(["jq", ".", keys_sorted.name], check=True, stdout=flattened) flattened.flush() - run_and_assert(["sort", flattened.name], stdout=keys_and_rows_sorted) + subprocess.run(["sort", flattened.name], check=True, stdout=keys_and_rows_sorted) keys_and_rows_sorted.flush() - return keys_and_rows_sorted + yield keys_and_rows_sorted diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index 4d4f43eb4a..420bb8fd6d 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -4,13 +4,14 @@ from typing import List -def diff_equal(path1: Path, path2: Path) -> None: +def is_dir_tree_content_equal(path1: Path, path2: Path) -> bool: cmd = ["diff", "--brief", "--recursive", str(path1), str(path2)] proc = subprocess.run(cmd, stdout=subprocess.PIPE) - if 0 != proc.returncode: - if 1 == proc.returncode: - assert False, "Files/Directories don't match." - assert False, f"Command failed {' '.join(cmd)}" + if 0 == proc.returncode: + return True + if 1 == proc.returncode: + return False + raise RuntimeError(f"Command failed {' '.join(cmd)}: {proc.stderr.decode()}") def get_env_var(var_name: str) -> str: @@ -21,5 +22,5 @@ def get_env_var(var_name: str) -> str: def run_and_assert(cmd: List[str], **kwargs) -> subprocess.CompletedProcess: proc = subprocess.run(cmd, **kwargs) - assert 0 == proc.returncode, f"Command failed: {' '.join(cmd)}" + assert 0 == proc.returncode, f"Command failed: {' '.join(cmd)}: {proc.stderr.decode()}" return proc diff --git a/taskfile.yaml b/taskfile.yaml index db906caca7..70ccfc7240 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -177,7 +177,8 @@ tasks: dir: "{{.G_INTEGRATION_TESTS_DIR}}" cmds: - >- - uv run python -m pytest --cache-clear -q --collect-only + uv run python -m pytest --cache-clear --collect-only --override-ini addopts="" --quiet + > /dev/null clp-s-generate-parsers: vars: From 09fc85c33b07f993d4211c58e3e63c344bbdf51d Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 23 Jul 2025 22:17:20 -0400 Subject: [PATCH 013/100] turn download and extract fixture into a private helper function --- integration-tests/tests/conftest.py | 2 +- .../tests/fixtures/dataset_logs.py | 57 +++++++++++-------- .../tests/test_identity_transformation.py | 22 ++++--- 3 files changed, 49 insertions(+), 32 deletions(-) diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index 7b2322e7a3..b023584a51 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -1,5 +1,5 @@ from tests.fixtures.dataset_logs import ( - download_and_extract_dataset, + elasticsearch, hive_24hr, postgresql, spark_event_logs, diff --git a/integration-tests/tests/fixtures/dataset_logs.py b/integration-tests/tests/fixtures/dataset_logs.py index 6358573acf..3539aa53ef 100644 --- a/integration-tests/tests/fixtures/dataset_logs.py +++ b/integration-tests/tests/fixtures/dataset_logs.py @@ -10,44 +10,55 @@ @pytest.fixture(scope="session") -def hive_24hr() -> DatasetLogs: - return DatasetLogs( - name="hive-24hr", - tar_url="https://zenodo.org/records/7094921/files/hive-24hr.tar.gz?download=1", +def hive_24hr(request, package_config: PackageConfig) -> DatasetLogs: + return _download_and_extract_dataset( + request, + package_config, + "hive-24hr", + "https://zenodo.org/records/7094921/files/hive-24hr.tar.gz?download=1", ) @pytest.fixture(scope="session") -def elasticsearch() -> DatasetLogs: - return DatasetLogs( - name="elasticsearch", - tar_url="https://zenodo.org/records/10516227/files/elasticsearch.tar.gz?download=1", +def elasticsearch(request, package_config: PackageConfig) -> DatasetLogs: + return _download_and_extract_dataset( + request, + package_config, + "elasticsearch", + "https://zenodo.org/records/10516227/files/elasticsearch.tar.gz?download=1", ) @pytest.fixture(scope="session") -def spark_event_logs() -> DatasetLogs: - return DatasetLogs( - name="spark-event-logs", - tar_url="https://zenodo.org/records/10516346/files/spark-event-logs.tar.gz?download=1", +def spark_event_logs(request, package_config: PackageConfig) -> DatasetLogs: + return _download_and_extract_dataset( + request, + package_config, + "spark-event-logs", + "https://zenodo.org/records/10516346/files/spark-event-logs.tar.gz?download=1", ) @pytest.fixture(scope="session") -def postgresql() -> DatasetLogs: - return DatasetLogs( - name="postgresql", - tar_url="https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1", +def postgresql(request, package_config: PackageConfig) -> DatasetLogs: + return _download_and_extract_dataset( + request, + package_config, + "postgresql", + "https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1", ) -@pytest.fixture -def download_and_extract_dataset(request, package_config: PackageConfig) -> DatasetLogs: - dataset_config = request.getfixturevalue(request.param) - dataset_name = dataset_config.name +def _download_and_extract_dataset( + request, package_config: PackageConfig, dataset_name: str, dataset_tarball_url: str +) -> DatasetLogs: + dataset_logs = DatasetLogs( + name=dataset_name, + tar_url=dataset_tarball_url, + ) if request.config.cache.get(dataset_name, False): print(f"Uncompressed logs for dataset `{dataset_name}` is up-to-date. Skipping download.") - return dataset_config + return dataset_logs try: download_path_str = str(package_config.uncompressed_logs_dir / f"{dataset_name}.tar.gz") @@ -58,7 +69,7 @@ def download_and_extract_dataset(request, package_config: PackageConfig) -> Data "--location", "--output", download_path_str, "--show-error", - dataset_config.tar_url, + dataset_tarball_url, ] # fmt: on subprocess.run(curl_cmds, check=True) @@ -70,4 +81,4 @@ def download_and_extract_dataset(request, package_config: PackageConfig) -> Data print(f"Downloaded and extracted uncompressed logs for dataset `{dataset_name}`.") request.config.cache.set(dataset_name, True) - return dataset_config + return dataset_logs diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index d1896ae077..99ce36a012 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -18,30 +18,32 @@ pytestmark = pytest.mark.binaries text_datasets = pytest.mark.parametrize( - "download_and_extract_dataset", + "dataset_logs_fixture", [ # "hive_24hr", ], - indirect=["download_and_extract_dataset"], ) json_datasets = pytest.mark.parametrize( - "download_and_extract_dataset", + "dataset_logs_fixture", [ # "spark_event_logs", "postgresql", ], - indirect=["download_and_extract_dataset"], ) @pytest.mark.clp @text_datasets def test_clp_identity_transform( - request, package_config: PackageConfig, download_and_extract_dataset: DatasetLogs + request, + package_config: PackageConfig, + dataset_logs_fixture: str, ) -> None: + dataset_logs: DaatsetLogs = request.getfixturevalue(dataset_logs_fixture) + dataset_name = dataset_logs.name + binary_path_str = str(package_config.clp_bin_dir / "clp") - dataset_name = download_and_extract_dataset.name download_dir = package_config.uncompressed_logs_dir / dataset_name archives_dir = package_config.test_output_dir / f"{dataset_name}-archives" extract_dir = package_config.test_output_dir / f"{dataset_name}-logs" @@ -75,10 +77,14 @@ def test_clp_identity_transform( @pytest.mark.clp_s @json_datasets def test_clp_s_identity_transform( - request, package_config: PackageConfig, download_and_extract_dataset: DatasetLogs + request, + package_config: PackageConfig, + dataset_logs_fixture: str, ) -> None: + dataset_logs: DaatsetLogs = request.getfixturevalue(dataset_logs_fixture) + dataset_name = dataset_logs.name + binary_path_str = str(package_config.clp_bin_dir / "clp-s") - dataset_name = download_and_extract_dataset.name download_dir = package_config.uncompressed_logs_dir / dataset_name archives_dir = package_config.test_output_dir / f"{dataset_name}-archives" extract_dir = package_config.test_output_dir / f"{dataset_name}-logs" From 6216e68acd3087337bea5955d06b8b4f49672c59 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 23 Jul 2025 22:17:40 -0400 Subject: [PATCH 014/100] UNcomment larget datasets --- integration-tests/tests/test_identity_transformation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 99ce36a012..cb4f88de37 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -20,14 +20,14 @@ text_datasets = pytest.mark.parametrize( "dataset_logs_fixture", [ - # "hive_24hr", + "hive_24hr", ], ) json_datasets = pytest.mark.parametrize( "dataset_logs_fixture", [ - # "spark_event_logs", + "spark_event_logs", "postgresql", ], ) From 3a6753b7872558a07cc88e9be34a8559abe6ecc4 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 24 Jul 2025 16:24:52 +0800 Subject: [PATCH 015/100] Apply suggestions from code review Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- integration-tests/tests/test_identity_transformation.py | 2 +- integration-tests/tests/utils/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index cb4f88de37..b6c369eb10 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -40,7 +40,7 @@ def test_clp_identity_transform( package_config: PackageConfig, dataset_logs_fixture: str, ) -> None: - dataset_logs: DaatsetLogs = request.getfixturevalue(dataset_logs_fixture) + dataset_logs: DatasetLogs = request.getfixturevalue(dataset_logs_fixture) dataset_name = dataset_logs.name binary_path_str = str(package_config.clp_bin_dir / "clp") diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index 420bb8fd6d..0ee1d5154d 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -6,7 +6,7 @@ def is_dir_tree_content_equal(path1: Path, path2: Path) -> bool: cmd = ["diff", "--brief", "--recursive", str(path1), str(path2)] - proc = subprocess.run(cmd, stdout=subprocess.PIPE) + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if 0 == proc.returncode: return True if 1 == proc.returncode: @@ -22,5 +22,5 @@ def get_env_var(var_name: str) -> str: def run_and_assert(cmd: List[str], **kwargs) -> subprocess.CompletedProcess: proc = subprocess.run(cmd, **kwargs) - assert 0 == proc.returncode, f"Command failed: {' '.join(cmd)}: {proc.stderr.decode()}" + assert 0 == proc.returncode, f"Command failed: {' '.join(cmd)}" return proc From 734ce734503ec69871ebcd0b23ae553fe9c18735 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 28 Jul 2025 17:44:19 -0400 Subject: [PATCH 016/100] Move download dir and other attributes inside dataset_logs fixture --- .../tests/fixtures/dataset_logs.py | 50 +++++----- .../tests/test_identity_transformation.py | 95 ++++++++++--------- integration-tests/tests/utils/config.py | 54 ++++++++++- 3 files changed, 124 insertions(+), 75 deletions(-) diff --git a/integration-tests/tests/fixtures/dataset_logs.py b/integration-tests/tests/fixtures/dataset_logs.py index 3539aa53ef..32172e21f3 100644 --- a/integration-tests/tests/fixtures/dataset_logs.py +++ b/integration-tests/tests/fixtures/dataset_logs.py @@ -12,70 +12,68 @@ @pytest.fixture(scope="session") def hive_24hr(request, package_config: PackageConfig) -> DatasetLogs: return _download_and_extract_dataset( - request, - package_config, - "hive-24hr", - "https://zenodo.org/records/7094921/files/hive-24hr.tar.gz?download=1", + request=request, + package_config=package_config, + dataset_name="hive-24hr", + tarball_url="https://zenodo.org/records/7094921/files/hive-24hr.tar.gz?download=1", ) @pytest.fixture(scope="session") def elasticsearch(request, package_config: PackageConfig) -> DatasetLogs: return _download_and_extract_dataset( - request, - package_config, - "elasticsearch", - "https://zenodo.org/records/10516227/files/elasticsearch.tar.gz?download=1", + request=request, + package_config=package_config, + dataset_name="elasticsearch", + tarball_url="https://zenodo.org/records/10516227/files/elasticsearch.tar.gz?download=1", ) @pytest.fixture(scope="session") def spark_event_logs(request, package_config: PackageConfig) -> DatasetLogs: return _download_and_extract_dataset( - request, - package_config, - "spark-event-logs", - "https://zenodo.org/records/10516346/files/spark-event-logs.tar.gz?download=1", + request=request, + package_config=package_config, + dataset_name="spark-event-logs", + tarball_url="https://zenodo.org/records/10516346/files/spark-event-logs.tar.gz?download=1", ) @pytest.fixture(scope="session") def postgresql(request, package_config: PackageConfig) -> DatasetLogs: return _download_and_extract_dataset( - request, - package_config, - "postgresql", - "https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1", + request=request, + package_config=package_config, + dataset_name="postgresql", + tarball_url="https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1", ) def _download_and_extract_dataset( - request, package_config: PackageConfig, dataset_name: str, dataset_tarball_url: str + request, package_config: PackageConfig, dataset_name: str, tarball_url: str ) -> DatasetLogs: - dataset_logs = DatasetLogs( - name=dataset_name, - tar_url=dataset_tarball_url, + dataset_logs = DatasetLogs.create( + package_config=package_config, + dataset_name=dataset_name, + tarball_url=tarball_url, ) if request.config.cache.get(dataset_name, False): print(f"Uncompressed logs for dataset `{dataset_name}` is up-to-date. Skipping download.") return dataset_logs try: - download_path_str = str(package_config.uncompressed_logs_dir / f"{dataset_name}.tar.gz") # fmt: off curl_cmds = [ "curl", "--fail", "--location", - "--output", download_path_str, + "--output", str(dataset_logs.download_dir), "--show-error", - dataset_tarball_url, + tarball_url, ] # fmt: on subprocess.run(curl_cmds, check=True) - - extract_path_str = str(package_config.uncompressed_logs_dir / dataset_name) - shutil.unpack_archive(download_path_str, extract_path_str) + shutil.unpack_archive(str(dataset_logs.download_dir), str(dataset_logs.extraction_dir)) except Exception as e: raise RuntimeError(f"Failed to download and extract dataset `{dataset_name}`: {e}") diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index b6c369eb10..f337c9bad4 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -1,4 +1,3 @@ -import shutil import subprocess from contextlib import contextmanager from pathlib import Path @@ -20,14 +19,14 @@ text_datasets = pytest.mark.parametrize( "dataset_logs_fixture", [ - "hive_24hr", + # "hive_24hr", ], ) json_datasets = pytest.mark.parametrize( "dataset_logs_fixture", [ - "spark_event_logs", + # "spark_event_logs", "postgresql", ], ) @@ -42,36 +41,34 @@ def test_clp_identity_transform( ) -> None: dataset_logs: DatasetLogs = request.getfixturevalue(dataset_logs_fixture) dataset_name = dataset_logs.name + dataset_logs.clear_test_outputs() binary_path_str = str(package_config.clp_bin_dir / "clp") - download_dir = package_config.uncompressed_logs_dir / dataset_name - archives_dir = package_config.test_output_dir / f"{dataset_name}-archives" - extract_dir = package_config.test_output_dir / f"{dataset_name}-logs" - - shutil.rmtree(archives_dir, ignore_errors=True) - shutil.rmtree(extract_dir, ignore_errors=True) - # fmt: off compression_cmd = [ binary_path_str, "c", "--progress", - "--remove-path-prefix", str(download_dir), - str(archives_dir), - str(download_dir), + "--remove-path-prefix", str(dataset_logs.extraction_dir), + str(dataset_logs.compression_dir), + str(dataset_logs.extraction_dir), ] # fmt: on run_and_assert(compression_cmd) - extraction_cmd = [binary_path_str, "x", str(archives_dir), str(extract_dir)] + extraction_cmd = [ + binary_path_str, + "x", + str(dataset_logs.compression_dir), + str(dataset_logs.decompression_dir), + ] run_and_assert(extraction_cmd) assert is_dir_tree_content_equal( - download_dir, extract_dir + dataset_logs.extraction_dir, dataset_logs.decompression_dir ), "Mismatch between clp compression input and decompression output." - shutil.rmtree(archives_dir, ignore_errors=True) - shutil.rmtree(extract_dir, ignore_errors=True) + dataset_logs.clear_test_outputs() @pytest.mark.clp_s @@ -82,46 +79,51 @@ def test_clp_s_identity_transform( dataset_logs_fixture: str, ) -> None: dataset_logs: DaatsetLogs = request.getfixturevalue(dataset_logs_fixture) - dataset_name = dataset_logs.name - - binary_path_str = str(package_config.clp_bin_dir / "clp-s") - download_dir = package_config.uncompressed_logs_dir / dataset_name - archives_dir = package_config.test_output_dir / f"{dataset_name}-archives" - extract_dir = package_config.test_output_dir / f"{dataset_name}-logs" - - shutil.rmtree(archives_dir, ignore_errors=True) - shutil.rmtree(extract_dir, ignore_errors=True) - - run_and_assert([binary_path_str, "c", str(archives_dir), str(download_dir)]) - run_and_assert([binary_path_str, "x", str(archives_dir), str(extract_dir)]) + dataset_name = dataset_logs.dataset_name + _clp_s_compress_and_decompress(package_config, dataset_logs) - # Recompress the decompressed single-file output and decompress it again to verify consistency. + # Recompress the decompressed output that's consolidated into a single json file, and decompress + # it again to verify consistency. The compression input of the second iteration points to the + # decompression output of the first. # TODO: Remove this check once we can directly compare decompressed logs (which would preserve # the directory structure and row/key order) with the original downloaded logs. # See also: https://docs.yscope.com/clp/main/user-guide/core-clp-s.html#current-limitations - single_file_archives_dir = ( - package_config.test_output_dir / f"{dataset_name}-single-file-archives" + consolidated_json_logs = DatasetLogs.create( + package_config=package_config, + dataset_name=f"{dataset_name}-consolidated-json", + extraction_dir=dataset_logs.decompression_dir, ) - single_file_extract_dir = package_config.test_output_dir / f"{dataset_name}-single-file-logs" - - shutil.rmtree(single_file_archives_dir, ignore_errors=True) - shutil.rmtree(single_file_extract_dir, ignore_errors=True) - - run_and_assert([binary_path_str, "c", single_file_archives_dir, extract_dir]) - run_and_assert([binary_path_str, "x", single_file_archives_dir, single_file_extract_dir]) + _clp_s_compress_and_decompress(package_config, consolidated_json_logs) # Key and row orders are not preserved during `clp-s` operations, so sort before diffing. - with _sort_json_keys_and_rows(extract_dir / "original") as s1, _sort_json_keys_and_rows( - single_file_extract_dir / "original" - ) as s2: + with _sort_json_keys_and_rows( + dataset_logs.decompression_dir / "original" + ) as s1, _sort_json_keys_and_rows(consolidated_json_logs.decompression_dir / "original") as s2: assert is_dir_tree_content_equal( s1.name, s2.name ), "Mismatch between clp-s compression input and decompression output." - shutil.rmtree(archives_dir, ignore_errors=True) - shutil.rmtree(extract_dir, ignore_errors=True) - shutil.rmtree(single_file_archives_dir, ignore_errors=True) - shutil.rmtree(single_file_extract_dir, ignore_errors=True) + # Remove test outputs to save disk space + dataset_logs.clear_test_outputs() + consolidated_json_logs.clear_test_outputs() + + +def _clp_s_compress_and_decompress( + package_config: PackageConfig, dataset_logs: DatasetLogs +) -> None: + dataset_logs.clear_test_outputs() + binary_path_str = str(package_config.clp_bin_dir / "clp-s") + run_and_assert( + [binary_path_str, "c", str(dataset_logs.compression_dir), str(dataset_logs.extraction_dir)] + ) + run_and_assert( + [ + binary_path_str, + "x", + str(dataset_logs.compression_dir), + str(dataset_logs.decompression_dir), + ] + ) @contextmanager @@ -129,7 +131,6 @@ def _sort_json_keys_and_rows(json_fp: Path) -> IO[str]: with NamedTemporaryFile(mode="w+", delete=True) as keys_sorted, NamedTemporaryFile( mode="w+", delete=True ) as flattened, NamedTemporaryFile(mode="w+", delete=True) as keys_and_rows_sorted: - keys_and_rows_sorted = NamedTemporaryFile(mode="w+", delete=True) subprocess.run(["jq", "--sort-keys", ".", str(json_fp)], check=True, stdout=keys_sorted) keys_sorted.flush() subprocess.run(["jq", ".", keys_sorted.name], check=True, stdout=flattened) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 1966f5dead..a7c1bebf6c 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -1,3 +1,4 @@ +import shutil from dataclasses import dataclass from pathlib import Path @@ -13,5 +14,54 @@ class PackageConfig: @dataclass(frozen=True) class DatasetLogs: - name: str - tar_url: str + dataset_name: str + tarball_url: str + download_dir: Path + extraction_dir: Path + compression_dir: Path + decompression_dir: Path + + def clear_test_outputs(self) -> None: + """ + Remove compression and decompression directories for test init and cleanup. + """ + shutil.rmtree(self.compression_dir, ignore_errors=True) + shutil.rmtree(self.decompression_dir, ignore_errors=True) + + +@dataclass(frozen=True) +class DatasetLogs: + dataset_name: str + tarball_url: str + download_dir: Path + extraction_dir: Path + compression_dir: Path + decompression_dir: Path + + @classmethod + def create(cls, package_config, dataset_name: str, **kwargs): + dataset_name = dataset_name.strip() + if 0 == len(dataset_name): + raise ValueError("`dataset_name` cannot be empty.") + return cls( + dataset_name=dataset_name, + tarball_url=kwargs.get("tarball_url", ""), + download_dir=kwargs.get( + "download_dir", package_config.uncompressed_logs_dir / f"{dataset_name}.tar.gz" + ), + extraction_dir=kwargs.get( + "extraction_dir", package_config.uncompressed_logs_dir / dataset_name + ), + compression_dir=kwargs.get( + "compression_dir", package_config.test_output_dir / f"{dataset_name}-archives" + ), + decompression_dir=kwargs.get( + "decompression_dir", + package_config.test_output_dir / f"{dataset_name}-decompressed-logs", + ), + ) + + def clear_test_outputs(self) -> None: + """Remove compression and decompression directories for test init and cleanup.""" + shutil.rmtree(self.compression_dir, ignore_errors=True) + shutil.rmtree(self.decompression_dir, ignore_errors=True) From 777a0d31e62772636d932bdc7bf5a1fc2b6e369b Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 28 Jul 2025 18:27:34 -0400 Subject: [PATCH 017/100] Make json compare into a helper function --- .../tests/test_identity_transformation.py | 50 +++++-------------- integration-tests/tests/utils/utils.py | 19 ++++++- 2 files changed, 30 insertions(+), 39 deletions(-) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index f337c9bad4..9a6f1026f7 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -1,8 +1,4 @@ -import subprocess -from contextlib import contextmanager from pathlib import Path -from tempfile import NamedTemporaryFile -from typing import IO import pytest from tests.utils.config import ( @@ -11,6 +7,7 @@ ) from tests.utils.utils import ( is_dir_tree_content_equal, + is_json_file_structurally_equal, run_and_assert, ) @@ -26,7 +23,7 @@ json_datasets = pytest.mark.parametrize( "dataset_logs_fixture", [ - # "spark_event_logs", + "spark_event_logs", "postgresql", ], ) @@ -95,13 +92,10 @@ def test_clp_s_identity_transform( ) _clp_s_compress_and_decompress(package_config, consolidated_json_logs) - # Key and row orders are not preserved during `clp-s` operations, so sort before diffing. - with _sort_json_keys_and_rows( - dataset_logs.decompression_dir / "original" - ) as s1, _sort_json_keys_and_rows(consolidated_json_logs.decompression_dir / "original") as s2: - assert is_dir_tree_content_equal( - s1.name, s2.name - ), "Mismatch between clp-s compression input and decompression output." + assert is_json_file_structurally_equal( + dataset_logs.decompression_dir / "original", + consolidated_json_logs.decompression_dir / "original", + ), "Mismatch between clp-s compression input and decompression output." # Remove test outputs to save disk space dataset_logs.clear_test_outputs() @@ -112,29 +106,9 @@ def _clp_s_compress_and_decompress( package_config: PackageConfig, dataset_logs: DatasetLogs ) -> None: dataset_logs.clear_test_outputs() - binary_path_str = str(package_config.clp_bin_dir / "clp-s") - run_and_assert( - [binary_path_str, "c", str(dataset_logs.compression_dir), str(dataset_logs.extraction_dir)] - ) - run_and_assert( - [ - binary_path_str, - "x", - str(dataset_logs.compression_dir), - str(dataset_logs.decompression_dir), - ] - ) - - -@contextmanager -def _sort_json_keys_and_rows(json_fp: Path) -> IO[str]: - with NamedTemporaryFile(mode="w+", delete=True) as keys_sorted, NamedTemporaryFile( - mode="w+", delete=True - ) as flattened, NamedTemporaryFile(mode="w+", delete=True) as keys_and_rows_sorted: - subprocess.run(["jq", "--sort-keys", ".", str(json_fp)], check=True, stdout=keys_sorted) - keys_sorted.flush() - subprocess.run(["jq", ".", keys_sorted.name], check=True, stdout=flattened) - flattened.flush() - subprocess.run(["sort", flattened.name], check=True, stdout=keys_and_rows_sorted) - keys_and_rows_sorted.flush() - yield keys_and_rows_sorted + bin_path = str(package_config.clp_bin_dir / "clp-s") + ext_path = str(dataset_logs.extraction_dir) + comp_path = str(dataset_logs.compression_dir) + decomp_path = str(dataset_logs.decompression_dir) + run_and_assert([bin_path, "c", comp_path, ext_path]) + run_and_assert([bin_path, "x", comp_path, decomp_path]) diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index 0ee1d5154d..f1a0e3cc5d 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -1,7 +1,13 @@ import os import subprocess from pathlib import Path -from typing import List +from tempfile import NamedTemporaryFile +from typing import IO, List + + +def is_json_file_structurally_equal(json_fp1: Path, json_fp2: Path) -> bool: + with _sort_json_keys_and_rows(json_fp1) as s1, _sort_json_keys_and_rows(json_fp2) as s2: + return is_dir_tree_content_equal(s1.name, s2.name) def is_dir_tree_content_equal(path1: Path, path2: Path) -> bool: @@ -24,3 +30,14 @@ def run_and_assert(cmd: List[str], **kwargs) -> subprocess.CompletedProcess: proc = subprocess.run(cmd, **kwargs) assert 0 == proc.returncode, f"Command failed: {' '.join(cmd)}" return proc + + +def _sort_json_keys_and_rows(json_fp: Path) -> IO[str]: + sorted_fp = NamedTemporaryFile(mode="w+", delete=True) + jq_proc = subprocess.Popen( + ["jq", "--sort-keys", "--compact-output", ".", str(json_fp)], + stdout=subprocess.PIPE, + ) + subprocess.run(["sort"], stdin=jq_proc.stdout, stdout=sorted_fp, check=True) + sorted_fp.flush() + return sorted_fp From 97d881cdda47cfc16c4447fdcaaf6c72916b557b Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 28 Jul 2025 19:13:21 -0400 Subject: [PATCH 018/100] Add package basic validity check --- integration-tests/tests/fixtures/package_config.py | 6 ++++++ integration-tests/tests/test_identity_transformation.py | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/integration-tests/tests/fixtures/package_config.py b/integration-tests/tests/fixtures/package_config.py index 6ae29fe50b..eebd7012b7 100644 --- a/integration-tests/tests/fixtures/package_config.py +++ b/integration-tests/tests/fixtures/package_config.py @@ -10,6 +10,12 @@ def package_config() -> PackageConfig: clp_build_dir = Path(get_env_var("CLP_BUILD_DIR")).resolve() clp_package_dir = Path(get_env_var("CLP_PACKAGE_DIR")).resolve() + # Check for required directories + required_dirs = ["bin", "etc", "lib", "sbin"] + missing_dirs = [d for d in required_dirs if not (clp_package_dir / d).is_dir()] + if missing_dirs: + raise ValueError(f"CLP package at {clp_package_dir} is missing: {', '.join(missing_dirs)}") + config = PackageConfig( clp_bin_dir=clp_package_dir / "bin", clp_package_dir=clp_package_dir, diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 9a6f1026f7..0a1690b3af 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -16,13 +16,14 @@ text_datasets = pytest.mark.parametrize( "dataset_logs_fixture", [ - # "hive_24hr", + "hive_24hr", ], ) json_datasets = pytest.mark.parametrize( "dataset_logs_fixture", [ + "elasticsearch", "spark_event_logs", "postgresql", ], From 55a4b218b50bc64530f9cb2ee2ad72b6519e0732 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 28 Jul 2025 19:15:22 -0400 Subject: [PATCH 019/100] remove dup class def --- integration-tests/tests/utils/config.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index a7c1bebf6c..ffe9532340 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -12,23 +12,6 @@ class PackageConfig: uncompressed_logs_dir: Path -@dataclass(frozen=True) -class DatasetLogs: - dataset_name: str - tarball_url: str - download_dir: Path - extraction_dir: Path - compression_dir: Path - decompression_dir: Path - - def clear_test_outputs(self) -> None: - """ - Remove compression and decompression directories for test init and cleanup. - """ - shutil.rmtree(self.compression_dir, ignore_errors=True) - shutil.rmtree(self.decompression_dir, ignore_errors=True) - - @dataclass(frozen=True) class DatasetLogs: dataset_name: str From 3acbf0286df46e62959722d23e1a355719a9978a Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 1 Aug 2025 02:05:22 +0000 Subject: [PATCH 020/100] Address review comments --- integration-tests/pyproject.toml | 2 +- .../tests/fixtures/dataset_logs.py | 4 +- .../tests/fixtures/package_config.py | 4 +- .../tests/test_identity_transformation.py | 70 +++++++++++-------- integration-tests/tests/utils/config.py | 36 +++++++--- integration-tests/tests/utils/utils.py | 6 +- 6 files changed, 75 insertions(+), 47 deletions(-) diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index d13bce5444..1ce9988d46 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "integration-tests" version = "0.1.0" -description = "Integration tests for the CLP project." +description = "Integration tests for the CLP package." readme = "README.md" authors = [ { name = "YScope Inc.", email = "dev@yscope.com" } diff --git a/integration-tests/tests/fixtures/dataset_logs.py b/integration-tests/tests/fixtures/dataset_logs.py index 32172e21f3..17af073116 100644 --- a/integration-tests/tests/fixtures/dataset_logs.py +++ b/integration-tests/tests/fixtures/dataset_logs.py @@ -67,13 +67,13 @@ def _download_and_extract_dataset( "curl", "--fail", "--location", - "--output", str(dataset_logs.download_dir), + "--output", str(dataset_logs.tarball_path), "--show-error", tarball_url, ] # fmt: on subprocess.run(curl_cmds, check=True) - shutil.unpack_archive(str(dataset_logs.download_dir), str(dataset_logs.extraction_dir)) + shutil.unpack_archive(str(dataset_logs.tarball_path), str(dataset_logs.extraction_dir)) except Exception as e: raise RuntimeError(f"Failed to download and extract dataset `{dataset_name}`: {e}") diff --git a/integration-tests/tests/fixtures/package_config.py b/integration-tests/tests/fixtures/package_config.py index eebd7012b7..c40685ea93 100644 --- a/integration-tests/tests/fixtures/package_config.py +++ b/integration-tests/tests/fixtures/package_config.py @@ -20,9 +20,9 @@ def package_config() -> PackageConfig: clp_bin_dir=clp_package_dir / "bin", clp_package_dir=clp_package_dir, clp_sbin_dir=clp_package_dir / "sbin", - test_output_dir=clp_build_dir / "integration-tests", + test_root_dir=clp_build_dir / "integration-tests", uncompressed_logs_dir=clp_build_dir / "integration-tests" / "downloads", ) - config.test_output_dir.mkdir(parents=True, exist_ok=True) + config.test_root_dir.mkdir(parents=True, exist_ok=True) config.uncompressed_logs_dir.mkdir(parents=True, exist_ok=True) return config diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 0a1690b3af..54795c6765 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -2,6 +2,7 @@ import pytest from tests.utils.config import ( + CompressionTestPaths, DatasetLogs, PackageConfig, ) @@ -16,15 +17,15 @@ text_datasets = pytest.mark.parametrize( "dataset_logs_fixture", [ - "hive_24hr", + # "hive_24hr", ], ) json_datasets = pytest.mark.parametrize( "dataset_logs_fixture", [ - "elasticsearch", - "spark_event_logs", + # "elasticsearch", + # "spark_event_logs", "postgresql", ], ) @@ -38,8 +39,12 @@ def test_clp_identity_transform( dataset_logs_fixture: str, ) -> None: dataset_logs: DatasetLogs = request.getfixturevalue(dataset_logs_fixture) - dataset_name = dataset_logs.name - dataset_logs.clear_test_outputs() + test_paths = CompressionTestPaths.create( + package_config=package_config, + test_name=f"clp-{dataset_logs.name}", + logs_source_dir=dataset_logs.extraction_dir, + ) + test_paths.clear_test_outputs() binary_path_str = str(package_config.clp_bin_dir / "clp") # fmt: off @@ -47,26 +52,26 @@ def test_clp_identity_transform( binary_path_str, "c", "--progress", - "--remove-path-prefix", str(dataset_logs.extraction_dir), - str(dataset_logs.compression_dir), - str(dataset_logs.extraction_dir), + "--remove-path-prefix", str(test_paths.logs_source_dir), + str(test_paths.compression_dir), + str(test_paths.extraction_dir), ] # fmt: on run_and_assert(compression_cmd) - extraction_cmd = [ + decompression_cmd = [ binary_path_str, "x", - str(dataset_logs.compression_dir), - str(dataset_logs.decompression_dir), + str(test_paths.compression_dir), + str(test_paths.decompression_dir), ] - run_and_assert(extraction_cmd) + run_and_assert(decompression_cmd) assert is_dir_tree_content_equal( - dataset_logs.extraction_dir, dataset_logs.decompression_dir + test_paths.extraction_dir, test_paths.decompression_dir ), "Mismatch between clp compression input and decompression output." - dataset_logs.clear_test_outputs() + test_paths.clear_test_outputs() @pytest.mark.clp_s @@ -78,7 +83,13 @@ def test_clp_s_identity_transform( ) -> None: dataset_logs: DaatsetLogs = request.getfixturevalue(dataset_logs_fixture) dataset_name = dataset_logs.dataset_name - _clp_s_compress_and_decompress(package_config, dataset_logs) + + test_paths = CompressionTestPaths.create( + package_config=package_config, + test_name=f"clp-s-{dataset_name}", + logs_source_dir=dataset_logs.extraction_dir, + ) + _clp_s_compress_and_decompress(package_config, test_paths) # Recompress the decompressed output that's consolidated into a single json file, and decompress # it again to verify consistency. The compression input of the second iteration points to the @@ -86,30 +97,29 @@ def test_clp_s_identity_transform( # TODO: Remove this check once we can directly compare decompressed logs (which would preserve # the directory structure and row/key order) with the original downloaded logs. # See also: https://docs.yscope.com/clp/main/user-guide/core-clp-s.html#current-limitations - consolidated_json_logs = DatasetLogs.create( + consolidated_json_test_paths = CompressionTestPaths.create( package_config=package_config, - dataset_name=f"{dataset_name}-consolidated-json", - extraction_dir=dataset_logs.decompression_dir, + test_name=f"clp-s-{dataset_name}-consolidated-json", + logs_source_dir=test_paths.decompression_dir, ) - _clp_s_compress_and_decompress(package_config, consolidated_json_logs) + _clp_s_compress_and_decompress(package_config, consolidated_json_test_paths) assert is_json_file_structurally_equal( - dataset_logs.decompression_dir / "original", - consolidated_json_logs.decompression_dir / "original", + consolidated_json_test_paths.logs_source_dir / "original", + consolidated_json_test_paths.decompression_dir / "original", ), "Mismatch between clp-s compression input and decompression output." - # Remove test outputs to save disk space - dataset_logs.clear_test_outputs() - consolidated_json_logs.clear_test_outputs() + test_paths.clear_test_outputs() + consolidated_json_test_paths.clear_test_outputs() def _clp_s_compress_and_decompress( - package_config: PackageConfig, dataset_logs: DatasetLogs + package_config: PackageConfig, test_paths: CompressionTestPaths ) -> None: - dataset_logs.clear_test_outputs() + test_paths.clear_test_outputs() bin_path = str(package_config.clp_bin_dir / "clp-s") - ext_path = str(dataset_logs.extraction_dir) - comp_path = str(dataset_logs.compression_dir) - decomp_path = str(dataset_logs.decompression_dir) - run_and_assert([bin_path, "c", comp_path, ext_path]) + src_path = str(test_paths.logs_source_dir) + comp_path = str(test_paths.compression_dir) + decomp_path = str(test_paths.decompression_dir) + run_and_assert([bin_path, "c", comp_path, src_path]) run_and_assert([bin_path, "x", comp_path, decomp_path]) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index ffe9532340..3b37cc5a35 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -8,7 +8,7 @@ class PackageConfig: clp_bin_dir: Path clp_package_dir: Path clp_sbin_dir: Path - test_output_dir: Path + test_root_dir: Path uncompressed_logs_dir: Path @@ -16,31 +16,47 @@ class PackageConfig: class DatasetLogs: dataset_name: str tarball_url: str - download_dir: Path + tarball_path: Path extraction_dir: Path - compression_dir: Path - decompression_dir: Path @classmethod - def create(cls, package_config, dataset_name: str, **kwargs): + def create(cls, package_config: PackageConfig, dataset_name: str, tarball_url: str, **kwargs): dataset_name = dataset_name.strip() if 0 == len(dataset_name): raise ValueError("`dataset_name` cannot be empty.") return cls( dataset_name=dataset_name, - tarball_url=kwargs.get("tarball_url", ""), - download_dir=kwargs.get( - "download_dir", package_config.uncompressed_logs_dir / f"{dataset_name}.tar.gz" + tarball_url=tarball_url, + tarball_path=kwargs.get( + "tarball_path", package_config.uncompressed_logs_dir / f"{dataset_name}.tar.gz" ), extraction_dir=kwargs.get( "extraction_dir", package_config.uncompressed_logs_dir / dataset_name ), + ) + + +@dataclass(frozen=True) +class CompressionTestPaths: + test_name: str + logs_source_dir: Path + compression_dir: Path + decompression_dir: Path + + @classmethod + def create(cls, package_config: PackageConfig, test_name: str, logs_source_dir: Path, **kwargs): + test_name = test_name.strip() + if 0 == len(test_name): + raise ValueError("`test_name` cannot be empty.") + return cls( + test_name=test_name, + logs_source_dir=logs_source_dir, compression_dir=kwargs.get( - "compression_dir", package_config.test_output_dir / f"{dataset_name}-archives" + "compression_dir", package_config.test_root_dir / f"{test_name}-archives" ), decompression_dir=kwargs.get( "decompression_dir", - package_config.test_output_dir / f"{dataset_name}-decompressed-logs", + package_config.test_root_dir / f"{test_name}-decompressed-logs", ), ) diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index f1a0e3cc5d..a8b2b4f72f 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -6,8 +6,10 @@ def is_json_file_structurally_equal(json_fp1: Path, json_fp2: Path) -> bool: - with _sort_json_keys_and_rows(json_fp1) as s1, _sort_json_keys_and_rows(json_fp2) as s2: - return is_dir_tree_content_equal(s1.name, s2.name) + with _sort_json_keys_and_rows(json_fp1) as temp_file_1, _sort_json_keys_and_rows( + json_fp2 + ) as temp_file_2: + return is_dir_tree_content_equal(temp_file_1.name, temp_file_2.name) def is_dir_tree_content_equal(path1: Path, path2: Path) -> bool: From be64383f1efff63a7fe14b12f352b7ad62380717 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 31 Jul 2025 22:11:50 -0400 Subject: [PATCH 021/100] remove unnecessary class param customizations --- integration-tests/tests/utils/config.py | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 3b37cc5a35..f39f70834e 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -20,19 +20,15 @@ class DatasetLogs: extraction_dir: Path @classmethod - def create(cls, package_config: PackageConfig, dataset_name: str, tarball_url: str, **kwargs): + def create(cls, package_config: PackageConfig, dataset_name: str, tarball_url: str): dataset_name = dataset_name.strip() if 0 == len(dataset_name): raise ValueError("`dataset_name` cannot be empty.") return cls( dataset_name=dataset_name, tarball_url=tarball_url, - tarball_path=kwargs.get( - "tarball_path", package_config.uncompressed_logs_dir / f"{dataset_name}.tar.gz" - ), - extraction_dir=kwargs.get( - "extraction_dir", package_config.uncompressed_logs_dir / dataset_name - ), + tarball_path=package_config.uncompressed_logs_dir / f"{dataset_name}.tar.gz", + extraction_dir=package_config.uncompressed_logs_dir / dataset_name, ) @@ -44,23 +40,14 @@ class CompressionTestPaths: decompression_dir: Path @classmethod - def create(cls, package_config: PackageConfig, test_name: str, logs_source_dir: Path, **kwargs): - test_name = test_name.strip() - if 0 == len(test_name): - raise ValueError("`test_name` cannot be empty.") + def create(cls, package_config: PackageConfig, test_name: str, logs_source_dir: Path): return cls( test_name=test_name, logs_source_dir=logs_source_dir, - compression_dir=kwargs.get( - "compression_dir", package_config.test_root_dir / f"{test_name}-archives" - ), - decompression_dir=kwargs.get( - "decompression_dir", - package_config.test_root_dir / f"{test_name}-decompressed-logs", - ), + compression_dir=package_config.test_root_dir / f"{test_name}-archives", + decompression_dir=package_config.test_root_dir / f"{test_name}-decompressed-logs", ) def clear_test_outputs(self) -> None: - """Remove compression and decompression directories for test init and cleanup.""" shutil.rmtree(self.compression_dir, ignore_errors=True) shutil.rmtree(self.decompression_dir, ignore_errors=True) From 6f835784065fd7eafd62716c4d2cdfb7f1e02182 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 31 Jul 2025 22:12:25 -0400 Subject: [PATCH 022/100] Add back missing dataset tests --- integration-tests/tests/test_identity_transformation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 54795c6765..0f19a53d81 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -17,15 +17,15 @@ text_datasets = pytest.mark.parametrize( "dataset_logs_fixture", [ - # "hive_24hr", + "hive_24hr", ], ) json_datasets = pytest.mark.parametrize( "dataset_logs_fixture", [ - # "elasticsearch", - # "spark_event_logs", + "elasticsearch", + "spark_event_logs", "postgresql", ], ) From cb0d282e89a84e43fb042ae5d61fc9191ee07e86 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 31 Jul 2025 22:33:05 -0400 Subject: [PATCH 023/100] Update dev utils --- tools/yscope-dev-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index 76aecbd6d0..b965f19f3e 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit 76aecbd6d0db6e2078005431946b5c2e716c4291 +Subproject commit b965f19f3e11c548f32bf7ab8e28ff38906be018 From 57cf51bae5f662fe20f945071617b13864597367 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 4 Aug 2025 13:48:02 -0400 Subject: [PATCH 024/100] Rename fixtures and fields according to offline discussions --- integration-tests/tests/conftest.py | 4 +- .../tests/fixtures/package_config.py | 4 +- .../{dataset_logs.py => test_logs.py} | 44 +++++++++---------- .../tests/test_identity_transformation.py | 26 +++++------ integration-tests/tests/utils/config.py | 20 ++++----- 5 files changed, 49 insertions(+), 49 deletions(-) rename integration-tests/tests/fixtures/{dataset_logs.py => test_logs.py} (62%) diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index b023584a51..aa23a8d1ba 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -1,7 +1,7 @@ -from tests.fixtures.dataset_logs import ( +from tests.fixtures.package_config import package_config +from tests.fixtures.test_logs import ( elasticsearch, hive_24hr, postgresql, spark_event_logs, ) -from tests.fixtures.package_config import package_config diff --git a/integration-tests/tests/fixtures/package_config.py b/integration-tests/tests/fixtures/package_config.py index c40685ea93..d44d2fb6c4 100644 --- a/integration-tests/tests/fixtures/package_config.py +++ b/integration-tests/tests/fixtures/package_config.py @@ -20,9 +20,9 @@ def package_config() -> PackageConfig: clp_bin_dir=clp_package_dir / "bin", clp_package_dir=clp_package_dir, clp_sbin_dir=clp_package_dir / "sbin", + logs_source_dir=clp_build_dir / "integration-tests" / "downloads", test_root_dir=clp_build_dir / "integration-tests", - uncompressed_logs_dir=clp_build_dir / "integration-tests" / "downloads", ) config.test_root_dir.mkdir(parents=True, exist_ok=True) - config.uncompressed_logs_dir.mkdir(parents=True, exist_ok=True) + config.logs_source_dir.mkdir(parents=True, exist_ok=True) return config diff --git a/integration-tests/tests/fixtures/dataset_logs.py b/integration-tests/tests/fixtures/test_logs.py similarity index 62% rename from integration-tests/tests/fixtures/dataset_logs.py rename to integration-tests/tests/fixtures/test_logs.py index 17af073116..97d0a554c0 100644 --- a/integration-tests/tests/fixtures/dataset_logs.py +++ b/integration-tests/tests/fixtures/test_logs.py @@ -4,62 +4,62 @@ import pytest from tests.utils.config import ( - DatasetLogs, PackageConfig, + TestLogs, ) @pytest.fixture(scope="session") -def hive_24hr(request, package_config: PackageConfig) -> DatasetLogs: +def hive_24hr(request, package_config: PackageConfig) -> TestLogs: return _download_and_extract_dataset( request=request, package_config=package_config, - dataset_name="hive-24hr", + name="hive-24hr", tarball_url="https://zenodo.org/records/7094921/files/hive-24hr.tar.gz?download=1", ) @pytest.fixture(scope="session") -def elasticsearch(request, package_config: PackageConfig) -> DatasetLogs: +def elasticsearch(request, package_config: PackageConfig) -> TestLogs: return _download_and_extract_dataset( request=request, package_config=package_config, - dataset_name="elasticsearch", + name="elasticsearch", tarball_url="https://zenodo.org/records/10516227/files/elasticsearch.tar.gz?download=1", ) @pytest.fixture(scope="session") -def spark_event_logs(request, package_config: PackageConfig) -> DatasetLogs: +def spark_event_logs(request, package_config: PackageConfig) -> TestLogs: return _download_and_extract_dataset( request=request, package_config=package_config, - dataset_name="spark-event-logs", + name="spark-event-logs", tarball_url="https://zenodo.org/records/10516346/files/spark-event-logs.tar.gz?download=1", ) @pytest.fixture(scope="session") -def postgresql(request, package_config: PackageConfig) -> DatasetLogs: +def postgresql(request, package_config: PackageConfig) -> TestLogs: return _download_and_extract_dataset( request=request, package_config=package_config, - dataset_name="postgresql", + name="postgresql", tarball_url="https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1", ) def _download_and_extract_dataset( - request, package_config: PackageConfig, dataset_name: str, tarball_url: str -) -> DatasetLogs: - dataset_logs = DatasetLogs.create( + request, package_config: PackageConfig, name: str, tarball_url: str +) -> TestLogs: + test_logs = TestLogs.create( package_config=package_config, - dataset_name=dataset_name, + name=name, tarball_url=tarball_url, ) - if request.config.cache.get(dataset_name, False): - print(f"Uncompressed logs for dataset `{dataset_name}` is up-to-date. Skipping download.") - return dataset_logs + if request.config.cache.get(name, False): + print(f"Test logs `{name}` is up-to-date. Skipping download.") + return test_logs try: # fmt: off @@ -67,16 +67,16 @@ def _download_and_extract_dataset( "curl", "--fail", "--location", - "--output", str(dataset_logs.tarball_path), + "--output", str(test_logs.tarball_path), "--show-error", tarball_url, ] # fmt: on subprocess.run(curl_cmds, check=True) - shutil.unpack_archive(str(dataset_logs.tarball_path), str(dataset_logs.extraction_dir)) + shutil.unpack_archive(str(test_logs.tarball_path), str(test_logs.extraction_dir)) except Exception as e: - raise RuntimeError(f"Failed to download and extract dataset `{dataset_name}`: {e}") + raise RuntimeError(f"Failed to download and extract dataset `{name}`: {e}") - print(f"Downloaded and extracted uncompressed logs for dataset `{dataset_name}`.") - request.config.cache.set(dataset_name, True) - return dataset_logs + print(f"Downloaded and extracted uncompressed logs for dataset `{name}`.") + request.config.cache.set(name, True) + return test_logs diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 0f19a53d81..738ad4b253 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -3,8 +3,8 @@ import pytest from tests.utils.config import ( CompressionTestPaths, - DatasetLogs, PackageConfig, + TestLogs, ) from tests.utils.utils import ( is_dir_tree_content_equal, @@ -15,14 +15,14 @@ pytestmark = pytest.mark.binaries text_datasets = pytest.mark.parametrize( - "dataset_logs_fixture", + "test_logs_fixture", [ "hive_24hr", ], ) json_datasets = pytest.mark.parametrize( - "dataset_logs_fixture", + "test_logs_fixture", [ "elasticsearch", "spark_event_logs", @@ -36,13 +36,13 @@ def test_clp_identity_transform( request, package_config: PackageConfig, - dataset_logs_fixture: str, + test_logs_fixture: str, ) -> None: - dataset_logs: DatasetLogs = request.getfixturevalue(dataset_logs_fixture) + test_logs: TestLogs = request.getfixturevalue(test_logs_fixture) test_paths = CompressionTestPaths.create( package_config=package_config, - test_name=f"clp-{dataset_logs.name}", - logs_source_dir=dataset_logs.extraction_dir, + test_name=f"clp-{test_logs.name}", + logs_source_dir=test_logs.extraction_dir, ) test_paths.clear_test_outputs() @@ -79,15 +79,15 @@ def test_clp_identity_transform( def test_clp_s_identity_transform( request, package_config: PackageConfig, - dataset_logs_fixture: str, + test_logs_fixture: str, ) -> None: - dataset_logs: DaatsetLogs = request.getfixturevalue(dataset_logs_fixture) - dataset_name = dataset_logs.dataset_name + test_logs: TestLogs = request.getfixturevalue(test_logs_fixture) + name = test_logs.name test_paths = CompressionTestPaths.create( package_config=package_config, - test_name=f"clp-s-{dataset_name}", - logs_source_dir=dataset_logs.extraction_dir, + test_name=f"clp-s-{name}", + logs_source_dir=test_logs.extraction_dir, ) _clp_s_compress_and_decompress(package_config, test_paths) @@ -99,7 +99,7 @@ def test_clp_s_identity_transform( # See also: https://docs.yscope.com/clp/main/user-guide/core-clp-s.html#current-limitations consolidated_json_test_paths = CompressionTestPaths.create( package_config=package_config, - test_name=f"clp-s-{dataset_name}-consolidated-json", + test_name=f"clp-s-{name}-consolidated-json", logs_source_dir=test_paths.decompression_dir, ) _clp_s_compress_and_decompress(package_config, consolidated_json_test_paths) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index f39f70834e..827b438a6e 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -8,27 +8,27 @@ class PackageConfig: clp_bin_dir: Path clp_package_dir: Path clp_sbin_dir: Path + logs_source_dir: Path test_root_dir: Path - uncompressed_logs_dir: Path @dataclass(frozen=True) -class DatasetLogs: - dataset_name: str +class TestLogs: + name: str tarball_url: str tarball_path: Path extraction_dir: Path @classmethod - def create(cls, package_config: PackageConfig, dataset_name: str, tarball_url: str): - dataset_name = dataset_name.strip() - if 0 == len(dataset_name): - raise ValueError("`dataset_name` cannot be empty.") + def create(cls, package_config: PackageConfig, name: str, tarball_url: str): + name = name.strip() + if 0 == len(name): + raise ValueError("`name` cannot be empty.") return cls( - dataset_name=dataset_name, + name=name, tarball_url=tarball_url, - tarball_path=package_config.uncompressed_logs_dir / f"{dataset_name}.tar.gz", - extraction_dir=package_config.uncompressed_logs_dir / dataset_name, + tarball_path=package_config.logs_source_dir / f"{name}.tar.gz", + extraction_dir=package_config.logs_source_dir / name, ) From bdbe7d2dcc1ffceb9e2d21df2db8870e74655356 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 5 Aug 2025 00:37:52 -0400 Subject: [PATCH 025/100] Address most review comments --- .../tests/fixtures/package_config.py | 6 ++- integration-tests/tests/fixtures/test_logs.py | 4 +- .../tests/test_identity_transformation.py | 45 ++++++++++--------- integration-tests/tests/utils/config.py | 12 ++--- 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/integration-tests/tests/fixtures/package_config.py b/integration-tests/tests/fixtures/package_config.py index d44d2fb6c4..ca785f5c6a 100644 --- a/integration-tests/tests/fixtures/package_config.py +++ b/integration-tests/tests/fixtures/package_config.py @@ -13,8 +13,10 @@ def package_config() -> PackageConfig: # Check for required directories required_dirs = ["bin", "etc", "lib", "sbin"] missing_dirs = [d for d in required_dirs if not (clp_package_dir / d).is_dir()] - if missing_dirs: - raise ValueError(f"CLP package at {clp_package_dir} is missing: {', '.join(missing_dirs)}") + if len(missing_dirs) > 0: + raise ValueError( + f"CLP package at {clp_package_dir} is incomplete. Missing: {', '.join(missing_dirs)}" + ) config = PackageConfig( clp_bin_dir=clp_package_dir / "bin", diff --git a/integration-tests/tests/fixtures/test_logs.py b/integration-tests/tests/fixtures/test_logs.py index 97d0a554c0..9abb294a75 100644 --- a/integration-tests/tests/fixtures/test_logs.py +++ b/integration-tests/tests/fixtures/test_logs.py @@ -53,9 +53,9 @@ def _download_and_extract_dataset( request, package_config: PackageConfig, name: str, tarball_url: str ) -> TestLogs: test_logs = TestLogs.create( - package_config=package_config, name=name, tarball_url=tarball_url, + package_config=package_config, ) if request.config.cache.get(name, False): print(f"Test logs `{name}` is up-to-date. Skipping download.") @@ -73,7 +73,7 @@ def _download_and_extract_dataset( ] # fmt: on subprocess.run(curl_cmds, check=True) - shutil.unpack_archive(str(test_logs.tarball_path), str(test_logs.extraction_dir)) + shutil.unpack_archive(test_logs.tarball_path, test_logs.extraction_dir) except Exception as e: raise RuntimeError(f"Failed to download and extract dataset `{name}`: {e}") diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 738ad4b253..3cb27b4d7f 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -2,7 +2,7 @@ import pytest from tests.utils.config import ( - CompressionTestPaths, + CompressionTestConfig, PackageConfig, TestLogs, ) @@ -39,10 +39,10 @@ def test_clp_identity_transform( test_logs_fixture: str, ) -> None: test_logs: TestLogs = request.getfixturevalue(test_logs_fixture) - test_paths = CompressionTestPaths.create( - package_config=package_config, + test_paths = CompressionTestConfig.create( test_name=f"clp-{test_logs.name}", logs_source_dir=test_logs.extraction_dir, + package_config=package_config, ) test_paths.clear_test_outputs() @@ -54,7 +54,7 @@ def test_clp_identity_transform( "--progress", "--remove-path-prefix", str(test_paths.logs_source_dir), str(test_paths.compression_dir), - str(test_paths.extraction_dir), + str(test_paths.logs_source_dir), ] # fmt: on run_and_assert(compression_cmd) @@ -67,9 +67,11 @@ def test_clp_identity_transform( ] run_and_assert(decompression_cmd) + input_path = test_paths.logs_source_dir + output_path = test_paths.decompression_dir assert is_dir_tree_content_equal( - test_paths.extraction_dir, test_paths.decompression_dir - ), "Mismatch between clp compression input and decompression output." + input_path, output_path, + ), "Mismatch between clp input {input_path} and output {output_path}." test_paths.clear_test_outputs() @@ -82,12 +84,12 @@ def test_clp_s_identity_transform( test_logs_fixture: str, ) -> None: test_logs: TestLogs = request.getfixturevalue(test_logs_fixture) - name = test_logs.name + test_logs_name = test_logs.name - test_paths = CompressionTestPaths.create( - package_config=package_config, - test_name=f"clp-s-{name}", + test_paths = CompressionTestConfig.create( + test_name=f"clp-s-{test_logs_name}", logs_source_dir=test_logs.extraction_dir, + package_config=package_config, ) _clp_s_compress_and_decompress(package_config, test_paths) @@ -97,29 +99,30 @@ def test_clp_s_identity_transform( # TODO: Remove this check once we can directly compare decompressed logs (which would preserve # the directory structure and row/key order) with the original downloaded logs. # See also: https://docs.yscope.com/clp/main/user-guide/core-clp-s.html#current-limitations - consolidated_json_test_paths = CompressionTestPaths.create( - package_config=package_config, - test_name=f"clp-s-{name}-consolidated-json", + consolidated_json_test_paths = CompressionTestConfig.create( + test_name=f"clp-s-{test_logs_name}-consolidated-json", logs_source_dir=test_paths.decompression_dir, + package_config=package_config, ) _clp_s_compress_and_decompress(package_config, consolidated_json_test_paths) + input_path = consolidated_json_test_paths.logs_source_dir / "original" + output_path = consolidated_json_test_paths.decompression_dir / "original" assert is_json_file_structurally_equal( - consolidated_json_test_paths.logs_source_dir / "original", - consolidated_json_test_paths.decompression_dir / "original", - ), "Mismatch between clp-s compression input and decompression output." + input_path, output_path + ), f"Mismatch between clp-s input {input_path} and output {output_path}." test_paths.clear_test_outputs() consolidated_json_test_paths.clear_test_outputs() def _clp_s_compress_and_decompress( - package_config: PackageConfig, test_paths: CompressionTestPaths + package_config: PackageConfig, test_paths: CompressionTestConfig ) -> None: test_paths.clear_test_outputs() bin_path = str(package_config.clp_bin_dir / "clp-s") src_path = str(test_paths.logs_source_dir) - comp_path = str(test_paths.compression_dir) - decomp_path = str(test_paths.decompression_dir) - run_and_assert([bin_path, "c", comp_path, src_path]) - run_and_assert([bin_path, "x", comp_path, decomp_path]) + compression_path = str(test_paths.compression_dir) + decompression_path = str(test_paths.decompression_dir) + run_and_assert([bin_path, "c", compression_path, src_path]) + run_and_assert([bin_path, "x", compression_path, decompression_path]) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 827b438a6e..7dcc0f023e 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -20,7 +20,7 @@ class TestLogs: extraction_dir: Path @classmethod - def create(cls, package_config: PackageConfig, name: str, tarball_url: str): + def create(cls, name: str, tarball_url: str, package_config: PackageConfig): name = name.strip() if 0 == len(name): raise ValueError("`name` cannot be empty.") @@ -33,14 +33,14 @@ def create(cls, package_config: PackageConfig, name: str, tarball_url: str): @dataclass(frozen=True) -class CompressionTestPaths: +class CompressionTestConfig: test_name: str logs_source_dir: Path compression_dir: Path decompression_dir: Path @classmethod - def create(cls, package_config: PackageConfig, test_name: str, logs_source_dir: Path): + def create(cls, test_name: str, logs_source_dir: Path, package_config: PackageConfig): return cls( test_name=test_name, logs_source_dir=logs_source_dir, @@ -49,5 +49,7 @@ def create(cls, package_config: PackageConfig, test_name: str, logs_source_dir: ) def clear_test_outputs(self) -> None: - shutil.rmtree(self.compression_dir, ignore_errors=True) - shutil.rmtree(self.decompression_dir, ignore_errors=True) + if self.compression_dir.exists(): + shutil.rmtree(self.compression_dir) + if self.decompression_dir.exists(): + shutil.rmtree(self.decompression_dir) From 9c92e93d1e06b1d07f7cf6ecc204a6a41fa1e74a Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 5 Aug 2025 11:40:25 -0400 Subject: [PATCH 026/100] Furthur renaming from package_config to test_config --- integration-tests/tests/conftest.py | 2 +- .../{package_config.py => test_config.py} | 8 ++--- integration-tests/tests/fixtures/test_logs.py | 22 ++++++------- .../tests/test_identity_transformation.py | 31 ++++++++++--------- integration-tests/tests/utils/config.py | 26 ++++++++++------ 5 files changed, 48 insertions(+), 41 deletions(-) rename integration-tests/tests/fixtures/{package_config.py => test_config.py} (81%) diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index aa23a8d1ba..8330f6359d 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -1,4 +1,4 @@ -from tests.fixtures.package_config import package_config +from tests.fixtures.test_config import test_config from tests.fixtures.test_logs import ( elasticsearch, hive_24hr, diff --git a/integration-tests/tests/fixtures/package_config.py b/integration-tests/tests/fixtures/test_config.py similarity index 81% rename from integration-tests/tests/fixtures/package_config.py rename to integration-tests/tests/fixtures/test_config.py index ca785f5c6a..348d9ee847 100644 --- a/integration-tests/tests/fixtures/package_config.py +++ b/integration-tests/tests/fixtures/test_config.py @@ -1,12 +1,12 @@ from pathlib import Path import pytest -from tests.utils.config import PackageConfig +from tests.utils.config import TestConfig from tests.utils.utils import get_env_var @pytest.fixture(scope="session") -def package_config() -> PackageConfig: +def test_config() -> TestConfig: clp_build_dir = Path(get_env_var("CLP_BUILD_DIR")).resolve() clp_package_dir = Path(get_env_var("CLP_PACKAGE_DIR")).resolve() @@ -18,10 +18,8 @@ def package_config() -> PackageConfig: f"CLP package at {clp_package_dir} is incomplete. Missing: {', '.join(missing_dirs)}" ) - config = PackageConfig( - clp_bin_dir=clp_package_dir / "bin", + config = TestConfig( clp_package_dir=clp_package_dir, - clp_sbin_dir=clp_package_dir / "sbin", logs_source_dir=clp_build_dir / "integration-tests" / "downloads", test_root_dir=clp_build_dir / "integration-tests", ) diff --git a/integration-tests/tests/fixtures/test_logs.py b/integration-tests/tests/fixtures/test_logs.py index 9abb294a75..1cfc1ac47e 100644 --- a/integration-tests/tests/fixtures/test_logs.py +++ b/integration-tests/tests/fixtures/test_logs.py @@ -4,58 +4,58 @@ import pytest from tests.utils.config import ( - PackageConfig, + TestConfig, TestLogs, ) @pytest.fixture(scope="session") -def hive_24hr(request, package_config: PackageConfig) -> TestLogs: +def hive_24hr(request, test_config: TestConfig) -> TestLogs: return _download_and_extract_dataset( request=request, - package_config=package_config, + test_config=test_config, name="hive-24hr", tarball_url="https://zenodo.org/records/7094921/files/hive-24hr.tar.gz?download=1", ) @pytest.fixture(scope="session") -def elasticsearch(request, package_config: PackageConfig) -> TestLogs: +def elasticsearch(request, test_config: TestConfig) -> TestLogs: return _download_and_extract_dataset( request=request, - package_config=package_config, + test_config=test_config, name="elasticsearch", tarball_url="https://zenodo.org/records/10516227/files/elasticsearch.tar.gz?download=1", ) @pytest.fixture(scope="session") -def spark_event_logs(request, package_config: PackageConfig) -> TestLogs: +def spark_event_logs(request, test_config: TestConfig) -> TestLogs: return _download_and_extract_dataset( request=request, - package_config=package_config, + test_config=test_config, name="spark-event-logs", tarball_url="https://zenodo.org/records/10516346/files/spark-event-logs.tar.gz?download=1", ) @pytest.fixture(scope="session") -def postgresql(request, package_config: PackageConfig) -> TestLogs: +def postgresql(request, test_config: TestConfig) -> TestLogs: return _download_and_extract_dataset( request=request, - package_config=package_config, + test_config=test_config, name="postgresql", tarball_url="https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1", ) def _download_and_extract_dataset( - request, package_config: PackageConfig, name: str, tarball_url: str + request, test_config: TestConfig, name: str, tarball_url: str ) -> TestLogs: test_logs = TestLogs.create( name=name, tarball_url=tarball_url, - package_config=package_config, + test_config=test_config, ) if request.config.cache.get(name, False): print(f"Test logs `{name}` is up-to-date. Skipping download.") diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 3cb27b4d7f..e8a46d9163 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -3,7 +3,7 @@ import pytest from tests.utils.config import ( CompressionTestConfig, - PackageConfig, + TestConfig, TestLogs, ) from tests.utils.utils import ( @@ -17,15 +17,15 @@ text_datasets = pytest.mark.parametrize( "test_logs_fixture", [ - "hive_24hr", + # "hive_24hr", ], ) json_datasets = pytest.mark.parametrize( "test_logs_fixture", [ - "elasticsearch", - "spark_event_logs", + # "elasticsearch", + # "spark_event_logs", "postgresql", ], ) @@ -35,18 +35,18 @@ @text_datasets def test_clp_identity_transform( request, - package_config: PackageConfig, + test_config: TestConfig, test_logs_fixture: str, ) -> None: test_logs: TestLogs = request.getfixturevalue(test_logs_fixture) test_paths = CompressionTestConfig.create( test_name=f"clp-{test_logs.name}", logs_source_dir=test_logs.extraction_dir, - package_config=package_config, + test_config=test_config, ) test_paths.clear_test_outputs() - binary_path_str = str(package_config.clp_bin_dir / "clp") + binary_path_str = str(test_config.get_clp_binary_path()) # fmt: off compression_cmd = [ binary_path_str, @@ -70,7 +70,8 @@ def test_clp_identity_transform( input_path = test_paths.logs_source_dir output_path = test_paths.decompression_dir assert is_dir_tree_content_equal( - input_path, output_path, + input_path, + output_path, ), "Mismatch between clp input {input_path} and output {output_path}." test_paths.clear_test_outputs() @@ -80,7 +81,7 @@ def test_clp_identity_transform( @json_datasets def test_clp_s_identity_transform( request, - package_config: PackageConfig, + test_config: TestConfig, test_logs_fixture: str, ) -> None: test_logs: TestLogs = request.getfixturevalue(test_logs_fixture) @@ -89,9 +90,9 @@ def test_clp_s_identity_transform( test_paths = CompressionTestConfig.create( test_name=f"clp-s-{test_logs_name}", logs_source_dir=test_logs.extraction_dir, - package_config=package_config, + test_config=test_config, ) - _clp_s_compress_and_decompress(package_config, test_paths) + _clp_s_compress_and_decompress(test_config, test_paths) # Recompress the decompressed output that's consolidated into a single json file, and decompress # it again to verify consistency. The compression input of the second iteration points to the @@ -102,9 +103,9 @@ def test_clp_s_identity_transform( consolidated_json_test_paths = CompressionTestConfig.create( test_name=f"clp-s-{test_logs_name}-consolidated-json", logs_source_dir=test_paths.decompression_dir, - package_config=package_config, + test_config=test_config, ) - _clp_s_compress_and_decompress(package_config, consolidated_json_test_paths) + _clp_s_compress_and_decompress(test_config, consolidated_json_test_paths) input_path = consolidated_json_test_paths.logs_source_dir / "original" output_path = consolidated_json_test_paths.decompression_dir / "original" @@ -117,10 +118,10 @@ def test_clp_s_identity_transform( def _clp_s_compress_and_decompress( - package_config: PackageConfig, test_paths: CompressionTestConfig + test_config: TestConfig, test_paths: CompressionTestConfig ) -> None: test_paths.clear_test_outputs() - bin_path = str(package_config.clp_bin_dir / "clp-s") + bin_path = str(test_config.get_clp_s_binary_path()) src_path = str(test_paths.logs_source_dir) compression_path = str(test_paths.compression_dir) decompression_path = str(test_paths.decompression_dir) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 7dcc0f023e..37508adaba 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -4,31 +4,39 @@ @dataclass(frozen=True) -class PackageConfig: - clp_bin_dir: Path +class TestConfig: + __test__ = False + clp_package_dir: Path - clp_sbin_dir: Path logs_source_dir: Path test_root_dir: Path + def get_clp_binary_path(self) -> Path: + return self.clp_package_dir / "bin" / "clp" + + def get_clp_s_binary_path(self) -> Path: + return self.clp_package_dir / "bin" / "clp-s" + @dataclass(frozen=True) class TestLogs: + __test__ = False + name: str tarball_url: str tarball_path: Path extraction_dir: Path @classmethod - def create(cls, name: str, tarball_url: str, package_config: PackageConfig): + def create(cls, name: str, tarball_url: str, test_config: TestConfig): name = name.strip() if 0 == len(name): raise ValueError("`name` cannot be empty.") return cls( name=name, tarball_url=tarball_url, - tarball_path=package_config.logs_source_dir / f"{name}.tar.gz", - extraction_dir=package_config.logs_source_dir / name, + tarball_path=test_config.logs_source_dir / f"{name}.tar.gz", + extraction_dir=test_config.logs_source_dir / name, ) @@ -40,12 +48,12 @@ class CompressionTestConfig: decompression_dir: Path @classmethod - def create(cls, test_name: str, logs_source_dir: Path, package_config: PackageConfig): + def create(cls, test_name: str, logs_source_dir: Path, test_config: TestConfig): return cls( test_name=test_name, logs_source_dir=logs_source_dir, - compression_dir=package_config.test_root_dir / f"{test_name}-archives", - decompression_dir=package_config.test_root_dir / f"{test_name}-decompressed-logs", + compression_dir=test_config.test_root_dir / f"{test_name}-archives", + decompression_dir=test_config.test_root_dir / f"{test_name}-decompressed-logs", ) def clear_test_outputs(self) -> None: From 0857b495068c864033f3efcefe81674140f5d878 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 5 Aug 2025 13:27:03 -0400 Subject: [PATCH 027/100] Use __post_init__ to improve dataclass design --- .../tests/fixtures/test_config.py | 21 +----- integration-tests/tests/fixtures/test_logs.py | 2 +- .../tests/test_identity_transformation.py | 12 ++-- integration-tests/tests/utils/config.py | 65 ++++++++++++------- 4 files changed, 52 insertions(+), 48 deletions(-) diff --git a/integration-tests/tests/fixtures/test_config.py b/integration-tests/tests/fixtures/test_config.py index 348d9ee847..e9e45ad09f 100644 --- a/integration-tests/tests/fixtures/test_config.py +++ b/integration-tests/tests/fixtures/test_config.py @@ -7,22 +7,7 @@ @pytest.fixture(scope="session") def test_config() -> TestConfig: - clp_build_dir = Path(get_env_var("CLP_BUILD_DIR")).resolve() - clp_package_dir = Path(get_env_var("CLP_PACKAGE_DIR")).resolve() - - # Check for required directories - required_dirs = ["bin", "etc", "lib", "sbin"] - missing_dirs = [d for d in required_dirs if not (clp_package_dir / d).is_dir()] - if len(missing_dirs) > 0: - raise ValueError( - f"CLP package at {clp_package_dir} is incomplete. Missing: {', '.join(missing_dirs)}" - ) - - config = TestConfig( - clp_package_dir=clp_package_dir, - logs_source_dir=clp_build_dir / "integration-tests" / "downloads", - test_root_dir=clp_build_dir / "integration-tests", + return TestConfig( + clp_package_dir=Path(get_env_var("CLP_PACKAGE_DIR")).resolve(), + test_root_dir=Path(get_env_var("CLP_BUILD_DIR")).resolve() / "integration-tests", ) - config.test_root_dir.mkdir(parents=True, exist_ok=True) - config.logs_source_dir.mkdir(parents=True, exist_ok=True) - return config diff --git a/integration-tests/tests/fixtures/test_logs.py b/integration-tests/tests/fixtures/test_logs.py index 1cfc1ac47e..088e4fb72a 100644 --- a/integration-tests/tests/fixtures/test_logs.py +++ b/integration-tests/tests/fixtures/test_logs.py @@ -52,7 +52,7 @@ def postgresql(request, test_config: TestConfig) -> TestLogs: def _download_and_extract_dataset( request, test_config: TestConfig, name: str, tarball_url: str ) -> TestLogs: - test_logs = TestLogs.create( + test_logs = TestLogs( name=name, tarball_url=tarball_url, test_config=test_config, diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index e8a46d9163..cea65f15c1 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -17,15 +17,15 @@ text_datasets = pytest.mark.parametrize( "test_logs_fixture", [ - # "hive_24hr", + "hive_24hr", ], ) json_datasets = pytest.mark.parametrize( "test_logs_fixture", [ - # "elasticsearch", - # "spark_event_logs", + "elasticsearch", + "spark_event_logs", "postgresql", ], ) @@ -39,7 +39,7 @@ def test_clp_identity_transform( test_logs_fixture: str, ) -> None: test_logs: TestLogs = request.getfixturevalue(test_logs_fixture) - test_paths = CompressionTestConfig.create( + test_paths = CompressionTestConfig( test_name=f"clp-{test_logs.name}", logs_source_dir=test_logs.extraction_dir, test_config=test_config, @@ -87,7 +87,7 @@ def test_clp_s_identity_transform( test_logs: TestLogs = request.getfixturevalue(test_logs_fixture) test_logs_name = test_logs.name - test_paths = CompressionTestConfig.create( + test_paths = CompressionTestConfig( test_name=f"clp-s-{test_logs_name}", logs_source_dir=test_logs.extraction_dir, test_config=test_config, @@ -100,7 +100,7 @@ def test_clp_s_identity_transform( # TODO: Remove this check once we can directly compare decompressed logs (which would preserve # the directory structure and row/key order) with the original downloaded logs. # See also: https://docs.yscope.com/clp/main/user-guide/core-clp-s.html#current-limitations - consolidated_json_test_paths = CompressionTestConfig.create( + consolidated_json_test_paths = CompressionTestConfig( test_name=f"clp-s-{test_logs_name}-consolidated-json", logs_source_dir=test_paths.decompression_dir, test_config=test_config, diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 37508adaba..41c3c67dd6 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -1,5 +1,5 @@ import shutil -from dataclasses import dataclass +from dataclasses import dataclass, field, InitVar from pathlib import Path @@ -8,8 +8,24 @@ class TestConfig: __test__ = False clp_package_dir: Path - logs_source_dir: Path test_root_dir: Path + logs_source_dir: Path = field(default=Path("/")) + + def __post_init__(self): + if Path("/") == self.logs_source_dir: + object.__setattr__(self, "logs_source_dir", self.test_root_dir / "downloads") + + # Check for required directories + required_dirs = ["bin", "etc", "lib", "sbin"] + missing_dirs = [d for d in required_dirs if not (self.clp_package_dir / d).is_dir()] + if len(missing_dirs) > 0: + raise ValueError( + f"CLP package at {self.clp_package_dir} is incomplete. " + f"Missing: {', '.join(missing_dirs)}" + ) + + self.test_root_dir.mkdir(parents=True, exist_ok=True) + self.logs_source_dir.mkdir(parents=True, exist_ok=True) def get_clp_binary_path(self) -> Path: return self.clp_package_dir / "bin" / "clp" @@ -24,36 +40,39 @@ class TestLogs: name: str tarball_url: str - tarball_path: Path - extraction_dir: Path + test_config: InitVar[TestConfig] + tarball_path: Path = field(init=False, repr=True) + extraction_dir: Path = field(init=False, repr=True) - @classmethod - def create(cls, name: str, tarball_url: str, test_config: TestConfig): - name = name.strip() + def __post_init__(self, test_config: TestConfig): + name = self.name.strip() if 0 == len(name): raise ValueError("`name` cannot be empty.") - return cls( - name=name, - tarball_url=tarball_url, - tarball_path=test_config.logs_source_dir / f"{name}.tar.gz", - extraction_dir=test_config.logs_source_dir / name, - ) + + object.__setattr__(self, "name", name) + object.__setattr__(self, "tarball_path", test_config.logs_source_dir / f"{name}.tar.gz") + object.__setattr__(self, "extraction_dir", test_config.logs_source_dir / name) @dataclass(frozen=True) class CompressionTestConfig: test_name: str logs_source_dir: Path - compression_dir: Path - decompression_dir: Path - - @classmethod - def create(cls, test_name: str, logs_source_dir: Path, test_config: TestConfig): - return cls( - test_name=test_name, - logs_source_dir=logs_source_dir, - compression_dir=test_config.test_root_dir / f"{test_name}-archives", - decompression_dir=test_config.test_root_dir / f"{test_name}-decompressed-logs", + test_config: InitVar[TestConfig] + compression_dir: Path = field(init=False, repr=True) + decompression_dir: Path = field(init=False, repr=True) + + def __post_init__(self, test_config: TestConfig): + test_name = self.test_name.strip() + if 0 == len(test_name): + raise ValueError("`test_name` cannot be empty.") + + object.__setattr__(self, "test_name", test_name) + object.__setattr__( + self, "compression_dir", test_config.test_root_dir / f"{test_name}-archives" + ) + object.__setattr__( + self, "decompression_dir", test_config.test_root_dir / f"{test_name}-decompressed-logs" ) def clear_test_outputs(self) -> None: From e747bfc3f321348a78bfc1704a7c155afddac244 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 6 Aug 2025 16:37:54 +0800 Subject: [PATCH 028/100] Update integration-tests/tests/test_identity_transformation.py Co-authored-by: haiqi96 <14502009+haiqi96@users.noreply.github.com> --- integration-tests/tests/test_identity_transformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index cea65f15c1..53b802917b 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -72,7 +72,7 @@ def test_clp_identity_transform( assert is_dir_tree_content_equal( input_path, output_path, - ), "Mismatch between clp input {input_path} and output {output_path}." + ), f"Mismatch between clp input {input_path} and output {output_path}." test_paths.clear_test_outputs() From 1b10e016b91b5e62231e584dd5a17b90fa1ff757 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 7 Aug 2025 21:20:12 -0400 Subject: [PATCH 029/100] Rename to avoid classes starting with Test --- integration-tests/tests/conftest.py | 4 +- ...t_config.py => integration_test_config.py} | 6 +-- ...{test_logs.py => integration_test_logs.py} | 40 ++++++++++-------- .../tests/test_identity_transformation.py | 42 +++++++++---------- integration-tests/tests/utils/config.py | 28 ++++++------- 5 files changed, 62 insertions(+), 58 deletions(-) rename integration-tests/tests/fixtures/{test_config.py => integration_test_config.py} (66%) rename integration-tests/tests/fixtures/{test_logs.py => integration_test_logs.py} (59%) diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index 8330f6359d..a73d07f72e 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -1,5 +1,5 @@ -from tests.fixtures.test_config import test_config -from tests.fixtures.test_logs import ( +from tests.fixtures.integration_test_config import integration_test_config +from tests.fixtures.integration_test_logs import ( elasticsearch, hive_24hr, postgresql, diff --git a/integration-tests/tests/fixtures/test_config.py b/integration-tests/tests/fixtures/integration_test_config.py similarity index 66% rename from integration-tests/tests/fixtures/test_config.py rename to integration-tests/tests/fixtures/integration_test_config.py index e9e45ad09f..fbc851efee 100644 --- a/integration-tests/tests/fixtures/test_config.py +++ b/integration-tests/tests/fixtures/integration_test_config.py @@ -1,13 +1,13 @@ from pathlib import Path import pytest -from tests.utils.config import TestConfig +from tests.utils.config import IntegrationTestConfig from tests.utils.utils import get_env_var @pytest.fixture(scope="session") -def test_config() -> TestConfig: - return TestConfig( +def integration_test_config() -> IntegrationTestConfig: + return IntegrationTestConfig( clp_package_dir=Path(get_env_var("CLP_PACKAGE_DIR")).resolve(), test_root_dir=Path(get_env_var("CLP_BUILD_DIR")).resolve() / "integration-tests", ) diff --git a/integration-tests/tests/fixtures/test_logs.py b/integration-tests/tests/fixtures/integration_test_logs.py similarity index 59% rename from integration-tests/tests/fixtures/test_logs.py rename to integration-tests/tests/fixtures/integration_test_logs.py index 088e4fb72a..b772fc13d5 100644 --- a/integration-tests/tests/fixtures/test_logs.py +++ b/integration-tests/tests/fixtures/integration_test_logs.py @@ -4,62 +4,64 @@ import pytest from tests.utils.config import ( - TestConfig, - TestLogs, + IntegrationTestConfig, + IntegrationTestLogs, ) @pytest.fixture(scope="session") -def hive_24hr(request, test_config: TestConfig) -> TestLogs: +def hive_24hr(request, integration_test_config: IntegrationTestConfig) -> IntegrationTestLogs: return _download_and_extract_dataset( request=request, - test_config=test_config, + integration_test_config=integration_test_config, name="hive-24hr", tarball_url="https://zenodo.org/records/7094921/files/hive-24hr.tar.gz?download=1", ) @pytest.fixture(scope="session") -def elasticsearch(request, test_config: TestConfig) -> TestLogs: +def elasticsearch(request, integration_test_config: IntegrationTestConfig) -> IntegrationTestLogs: return _download_and_extract_dataset( request=request, - test_config=test_config, + integration_test_config=integration_test_config, name="elasticsearch", tarball_url="https://zenodo.org/records/10516227/files/elasticsearch.tar.gz?download=1", ) @pytest.fixture(scope="session") -def spark_event_logs(request, test_config: TestConfig) -> TestLogs: +def spark_event_logs( + request, integration_test_config: IntegrationTestConfig +) -> IntegrationTestLogs: return _download_and_extract_dataset( request=request, - test_config=test_config, + integration_test_config=integration_test_config, name="spark-event-logs", tarball_url="https://zenodo.org/records/10516346/files/spark-event-logs.tar.gz?download=1", ) @pytest.fixture(scope="session") -def postgresql(request, test_config: TestConfig) -> TestLogs: +def postgresql(request, integration_test_config: IntegrationTestConfig) -> IntegrationTestLogs: return _download_and_extract_dataset( request=request, - test_config=test_config, + integration_test_config=integration_test_config, name="postgresql", tarball_url="https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1", ) def _download_and_extract_dataset( - request, test_config: TestConfig, name: str, tarball_url: str -) -> TestLogs: - test_logs = TestLogs( + request, integration_test_config: IntegrationTestConfig, name: str, tarball_url: str +) -> IntegrationTestLogs: + integration_test_logs = IntegrationTestLogs( name=name, tarball_url=tarball_url, - test_config=test_config, + integration_test_config=integration_test_config, ) if request.config.cache.get(name, False): print(f"Test logs `{name}` is up-to-date. Skipping download.") - return test_logs + return integration_test_logs try: # fmt: off @@ -67,16 +69,18 @@ def _download_and_extract_dataset( "curl", "--fail", "--location", - "--output", str(test_logs.tarball_path), + "--output", str(integration_test_logs.tarball_path), "--show-error", tarball_url, ] # fmt: on subprocess.run(curl_cmds, check=True) - shutil.unpack_archive(test_logs.tarball_path, test_logs.extraction_dir) + shutil.unpack_archive( + integration_test_logs.tarball_path, integration_test_logs.extraction_dir + ) except Exception as e: raise RuntimeError(f"Failed to download and extract dataset `{name}`: {e}") print(f"Downloaded and extracted uncompressed logs for dataset `{name}`.") request.config.cache.set(name, True) - return test_logs + return integration_test_logs diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 53b802917b..4dc90c3d04 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -3,8 +3,8 @@ import pytest from tests.utils.config import ( CompressionTestConfig, - TestConfig, - TestLogs, + IntegrationTestConfig, + IntegrationTestLogs, ) from tests.utils.utils import ( is_dir_tree_content_equal, @@ -17,15 +17,15 @@ text_datasets = pytest.mark.parametrize( "test_logs_fixture", [ - "hive_24hr", + # "hive_24hr", ], ) json_datasets = pytest.mark.parametrize( "test_logs_fixture", [ - "elasticsearch", - "spark_event_logs", + # "elasticsearch", + # "spark_event_logs", "postgresql", ], ) @@ -35,18 +35,18 @@ @text_datasets def test_clp_identity_transform( request, - test_config: TestConfig, + integration_test_config: IntegrationTestConfig, test_logs_fixture: str, ) -> None: - test_logs: TestLogs = request.getfixturevalue(test_logs_fixture) + integration_test_logs: IntegrationTestLogs = request.getfixturevalue(test_logs_fixture) test_paths = CompressionTestConfig( - test_name=f"clp-{test_logs.name}", - logs_source_dir=test_logs.extraction_dir, - test_config=test_config, + test_name=f"clp-{integration_test_logs.name}", + logs_source_dir=integration_test_logs.extraction_dir, + integration_test_config=integration_test_config, ) test_paths.clear_test_outputs() - binary_path_str = str(test_config.get_clp_binary_path()) + binary_path_str = str(integration_test_config.get_clp_binary_path()) # fmt: off compression_cmd = [ binary_path_str, @@ -81,18 +81,18 @@ def test_clp_identity_transform( @json_datasets def test_clp_s_identity_transform( request, - test_config: TestConfig, + integration_test_config: IntegrationTestConfig, test_logs_fixture: str, ) -> None: - test_logs: TestLogs = request.getfixturevalue(test_logs_fixture) - test_logs_name = test_logs.name + integration_test_logs: IntegrationTestLogs = request.getfixturevalue(test_logs_fixture) + test_logs_name = integration_test_logs.name test_paths = CompressionTestConfig( test_name=f"clp-s-{test_logs_name}", - logs_source_dir=test_logs.extraction_dir, - test_config=test_config, + logs_source_dir=integration_test_logs.extraction_dir, + integration_test_config=integration_test_config, ) - _clp_s_compress_and_decompress(test_config, test_paths) + _clp_s_compress_and_decompress(integration_test_config, test_paths) # Recompress the decompressed output that's consolidated into a single json file, and decompress # it again to verify consistency. The compression input of the second iteration points to the @@ -103,9 +103,9 @@ def test_clp_s_identity_transform( consolidated_json_test_paths = CompressionTestConfig( test_name=f"clp-s-{test_logs_name}-consolidated-json", logs_source_dir=test_paths.decompression_dir, - test_config=test_config, + integration_test_config=integration_test_config, ) - _clp_s_compress_and_decompress(test_config, consolidated_json_test_paths) + _clp_s_compress_and_decompress(integration_test_config, consolidated_json_test_paths) input_path = consolidated_json_test_paths.logs_source_dir / "original" output_path = consolidated_json_test_paths.decompression_dir / "original" @@ -118,10 +118,10 @@ def test_clp_s_identity_transform( def _clp_s_compress_and_decompress( - test_config: TestConfig, test_paths: CompressionTestConfig + integration_test_config: IntegrationTestConfig, test_paths: CompressionTestConfig ) -> None: test_paths.clear_test_outputs() - bin_path = str(test_config.get_clp_s_binary_path()) + bin_path = str(integration_test_config.get_clp_s_binary_path()) src_path = str(test_paths.logs_source_dir) compression_path = str(test_paths.compression_dir) decompression_path = str(test_paths.decompression_dir) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 41c3c67dd6..55caa626c7 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -4,9 +4,7 @@ @dataclass(frozen=True) -class TestConfig: - __test__ = False - +class IntegrationTestConfig: clp_package_dir: Path test_root_dir: Path logs_source_dir: Path = field(default=Path("/")) @@ -35,44 +33,46 @@ def get_clp_s_binary_path(self) -> Path: @dataclass(frozen=True) -class TestLogs: - __test__ = False - +class IntegrationTestLogs: name: str tarball_url: str - test_config: InitVar[TestConfig] + integration_test_config: InitVar[IntegrationTestConfig] tarball_path: Path = field(init=False, repr=True) extraction_dir: Path = field(init=False, repr=True) - def __post_init__(self, test_config: TestConfig): + def __post_init__(self, integration_test_config: IntegrationTestConfig): name = self.name.strip() if 0 == len(name): raise ValueError("`name` cannot be empty.") object.__setattr__(self, "name", name) - object.__setattr__(self, "tarball_path", test_config.logs_source_dir / f"{name}.tar.gz") - object.__setattr__(self, "extraction_dir", test_config.logs_source_dir / name) + object.__setattr__( + self, "tarball_path", integration_test_config.logs_source_dir / f"{name}.tar.gz" + ) + object.__setattr__(self, "extraction_dir", integration_test_config.logs_source_dir / name) @dataclass(frozen=True) class CompressionTestConfig: test_name: str logs_source_dir: Path - test_config: InitVar[TestConfig] + integration_test_config: InitVar[IntegrationTestConfig] compression_dir: Path = field(init=False, repr=True) decompression_dir: Path = field(init=False, repr=True) - def __post_init__(self, test_config: TestConfig): + def __post_init__(self, integration_test_config: IntegrationTestConfig): test_name = self.test_name.strip() if 0 == len(test_name): raise ValueError("`test_name` cannot be empty.") object.__setattr__(self, "test_name", test_name) object.__setattr__( - self, "compression_dir", test_config.test_root_dir / f"{test_name}-archives" + self, "compression_dir", integration_test_config.test_root_dir / f"{test_name}-archives" ) object.__setattr__( - self, "decompression_dir", test_config.test_root_dir / f"{test_name}-decompressed-logs" + self, + "decompression_dir", + integration_test_config.test_root_dir / f"{test_name}-decompressed-logs", ) def clear_test_outputs(self) -> None: From 61909bd413520b9f8521955fcc8e05c852597fee Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 7 Aug 2025 21:29:03 -0400 Subject: [PATCH 030/100] Address review comment --- integration-tests/tests/utils/config.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 55caa626c7..c1791a15a2 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -1,17 +1,18 @@ import shutil from dataclasses import dataclass, field, InitVar from pathlib import Path +from typing import Optional @dataclass(frozen=True) class IntegrationTestConfig: clp_package_dir: Path test_root_dir: Path - logs_source_dir: Path = field(default=Path("/")) + logs_download_dir: Optional[Path] = None def __post_init__(self): - if Path("/") == self.logs_source_dir: - object.__setattr__(self, "logs_source_dir", self.test_root_dir / "downloads") + if self.logs_download_dir is None: + object.__setattr__(self, "logs_download_dir", self.test_root_dir / "downloads") # Check for required directories required_dirs = ["bin", "etc", "lib", "sbin"] @@ -23,7 +24,7 @@ def __post_init__(self): ) self.test_root_dir.mkdir(parents=True, exist_ok=True) - self.logs_source_dir.mkdir(parents=True, exist_ok=True) + self.logs_download_dir.mkdir(parents=True, exist_ok=True) def get_clp_binary_path(self) -> Path: return self.clp_package_dir / "bin" / "clp" @@ -47,9 +48,9 @@ def __post_init__(self, integration_test_config: IntegrationTestConfig): object.__setattr__(self, "name", name) object.__setattr__( - self, "tarball_path", integration_test_config.logs_source_dir / f"{name}.tar.gz" + self, "tarball_path", integration_test_config.logs_download_dir / f"{name}.tar.gz" ) - object.__setattr__(self, "extraction_dir", integration_test_config.logs_source_dir / name) + object.__setattr__(self, "extraction_dir", integration_test_config.logs_download_dir / name) @dataclass(frozen=True) From 54b26a92f83b593336fd15b34c4692dd2e40e2b2 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 7 Aug 2025 21:29:12 -0400 Subject: [PATCH 031/100] Uncomment tests --- integration-tests/tests/test_identity_transformation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 4dc90c3d04..525b6f80bd 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -17,15 +17,15 @@ text_datasets = pytest.mark.parametrize( "test_logs_fixture", [ - # "hive_24hr", + "hive_24hr", ], ) json_datasets = pytest.mark.parametrize( "test_logs_fixture", [ - # "elasticsearch", - # "spark_event_logs", + "elasticsearch", + "spark_event_logs", "postgresql", ], ) From 14b014128dc8c06c0aecf17494e417ec7f70c9ba Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 8 Aug 2025 15:16:36 -0400 Subject: [PATCH 032/100] Split out package config --- .../tests/fixtures/integration_test_config.py | 5 ++-- .../tests/test_identity_transformation.py | 4 ++-- integration-tests/tests/utils/config.py | 24 ++++++++++++------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/integration-tests/tests/fixtures/integration_test_config.py b/integration-tests/tests/fixtures/integration_test_config.py index fbc851efee..55606b4f6b 100644 --- a/integration-tests/tests/fixtures/integration_test_config.py +++ b/integration-tests/tests/fixtures/integration_test_config.py @@ -1,13 +1,14 @@ from pathlib import Path import pytest -from tests.utils.config import IntegrationTestConfig +from tests.utils.config import IntegrationTestConfig, PackageConfig from tests.utils.utils import get_env_var @pytest.fixture(scope="session") def integration_test_config() -> IntegrationTestConfig: + package_config = PackageConfig(clp_package_dir=Path(get_env_var("CLP_PACKAGE_DIR")).resolve()) return IntegrationTestConfig( - clp_package_dir=Path(get_env_var("CLP_PACKAGE_DIR")).resolve(), + package_config=package_config, test_root_dir=Path(get_env_var("CLP_BUILD_DIR")).resolve() / "integration-tests", ) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 525b6f80bd..b41771ce51 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -46,7 +46,7 @@ def test_clp_identity_transform( ) test_paths.clear_test_outputs() - binary_path_str = str(integration_test_config.get_clp_binary_path()) + binary_path_str = str(integration_test_config.package_config.get_clp_binary_path()) # fmt: off compression_cmd = [ binary_path_str, @@ -121,7 +121,7 @@ def _clp_s_compress_and_decompress( integration_test_config: IntegrationTestConfig, test_paths: CompressionTestConfig ) -> None: test_paths.clear_test_outputs() - bin_path = str(integration_test_config.get_clp_s_binary_path()) + bin_path = str(integration_test_config.package_config.get_clp_s_binary_path()) src_path = str(test_paths.logs_source_dir) compression_path = str(test_paths.compression_dir) decompression_path = str(test_paths.decompression_dir) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index c1791a15a2..cbe7a978c2 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -5,15 +5,10 @@ @dataclass(frozen=True) -class IntegrationTestConfig: +class PackageConfig: clp_package_dir: Path - test_root_dir: Path - logs_download_dir: Optional[Path] = None def __post_init__(self): - if self.logs_download_dir is None: - object.__setattr__(self, "logs_download_dir", self.test_root_dir / "downloads") - # Check for required directories required_dirs = ["bin", "etc", "lib", "sbin"] missing_dirs = [d for d in required_dirs if not (self.clp_package_dir / d).is_dir()] @@ -23,9 +18,6 @@ def __post_init__(self): f"Missing: {', '.join(missing_dirs)}" ) - self.test_root_dir.mkdir(parents=True, exist_ok=True) - self.logs_download_dir.mkdir(parents=True, exist_ok=True) - def get_clp_binary_path(self) -> Path: return self.clp_package_dir / "bin" / "clp" @@ -33,6 +25,20 @@ def get_clp_s_binary_path(self) -> Path: return self.clp_package_dir / "bin" / "clp-s" +@dataclass(frozen=True) +class IntegrationTestConfig: + package_config: PackageConfig + test_root_dir: Path + logs_download_dir: Optional[Path] = None + + def __post_init__(self): + if self.logs_download_dir is None: + object.__setattr__(self, "logs_download_dir", self.test_root_dir / "downloads") + + self.test_root_dir.mkdir(parents=True, exist_ok=True) + self.logs_download_dir.mkdir(parents=True, exist_ok=True) + + @dataclass(frozen=True) class IntegrationTestLogs: name: str From 709d9c80810966df9e0564cf8dc7b3b5e88adcec Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sat, 9 Aug 2025 11:18:28 +0800 Subject: [PATCH 033/100] Apply suggestions from code review Co-authored-by: haiqi96 <14502009+haiqi96@users.noreply.github.com> --- integration-tests/tests/utils/config.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index cbe7a978c2..aecf9ef60b 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -9,6 +9,16 @@ class PackageConfig: clp_package_dir: Path def __post_init__(self): + # Check if package directory exists and is valid + if not self.clp_package_dir.exists(): + raise ValueError( + f"CLP package directory does not exist: {self.clp_package_dir}" + ) + if not self.clp_package_dir.is_dir(): + raise ValueError( + f"CLP package path is not a directory: {self.clp_package_dir}" + ) + # Check for required directories required_dirs = ["bin", "etc", "lib", "sbin"] missing_dirs = [d for d in required_dirs if not (self.clp_package_dir / d).is_dir()] From 1a3392746f1ba495c745aa29fc4136aad32ea823 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 8 Aug 2025 23:19:26 -0400 Subject: [PATCH 034/100] Use singular term for name --- integration-tests/tests/test_identity_transformation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index b41771ce51..b1bf7b1fa1 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -85,10 +85,10 @@ def test_clp_s_identity_transform( test_logs_fixture: str, ) -> None: integration_test_logs: IntegrationTestLogs = request.getfixturevalue(test_logs_fixture) - test_logs_name = integration_test_logs.name + test_log_name = integration_test_logs.name test_paths = CompressionTestConfig( - test_name=f"clp-s-{test_logs_name}", + test_name=f"clp-s-{test_log_name}", logs_source_dir=integration_test_logs.extraction_dir, integration_test_config=integration_test_config, ) @@ -101,7 +101,7 @@ def test_clp_s_identity_transform( # the directory structure and row/key order) with the original downloaded logs. # See also: https://docs.yscope.com/clp/main/user-guide/core-clp-s.html#current-limitations consolidated_json_test_paths = CompressionTestConfig( - test_name=f"clp-s-{test_logs_name}-consolidated-json", + test_name=f"clp-s-{test_log_name}-consolidated-json", logs_source_dir=test_paths.decompression_dir, integration_test_config=integration_test_config, ) From f0c5a8f56bd10e85176e9636187b5b0e6eff2f9a Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sat, 9 Aug 2025 03:55:39 +0000 Subject: [PATCH 035/100] abbreviate var --- integration-tests/tests/utils/config.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index aecf9ef60b..666e2c8d49 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -9,22 +9,20 @@ class PackageConfig: clp_package_dir: Path def __post_init__(self): + clp_package_dir = self.clp_package_dir + # Check if package directory exists and is valid - if not self.clp_package_dir.exists(): - raise ValueError( - f"CLP package directory does not exist: {self.clp_package_dir}" - ) - if not self.clp_package_dir.is_dir(): - raise ValueError( - f"CLP package path is not a directory: {self.clp_package_dir}" - ) - + if not clp_package_dir.exists(): + raise ValueError(f"CLP package directory does not exist: {clp_package_dir}") + if not clp_package_dir.is_dir(): + raise ValueError(f"CLP package path is not a directory: {clp_package_dir}") + # Check for required directories required_dirs = ["bin", "etc", "lib", "sbin"] - missing_dirs = [d for d in required_dirs if not (self.clp_package_dir / d).is_dir()] + missing_dirs = [d for d in required_dirs if not (clp_package_dir / d).is_dir()] if len(missing_dirs) > 0: raise ValueError( - f"CLP package at {self.clp_package_dir} is incomplete. " + f"CLP package at {clp_package_dir} is incomplete. " f"Missing: {', '.join(missing_dirs)}" ) From 9ca7500716e03b08ec659ab541c0514f1b101432 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sat, 9 Aug 2025 00:00:11 -0400 Subject: [PATCH 036/100] Revert test_log_name singular noun change --- integration-tests/tests/test_identity_transformation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index b1bf7b1fa1..b41771ce51 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -85,10 +85,10 @@ def test_clp_s_identity_transform( test_logs_fixture: str, ) -> None: integration_test_logs: IntegrationTestLogs = request.getfixturevalue(test_logs_fixture) - test_log_name = integration_test_logs.name + test_logs_name = integration_test_logs.name test_paths = CompressionTestConfig( - test_name=f"clp-s-{test_log_name}", + test_name=f"clp-s-{test_logs_name}", logs_source_dir=integration_test_logs.extraction_dir, integration_test_config=integration_test_config, ) @@ -101,7 +101,7 @@ def test_clp_s_identity_transform( # the directory structure and row/key order) with the original downloaded logs. # See also: https://docs.yscope.com/clp/main/user-guide/core-clp-s.html#current-limitations consolidated_json_test_paths = CompressionTestConfig( - test_name=f"clp-s-{test_log_name}-consolidated-json", + test_name=f"clp-s-{test_logs_name}-consolidated-json", logs_source_dir=test_paths.decompression_dir, integration_test_config=integration_test_config, ) From 28119eadbe45c8ed490e6d858115f6c476f3d909 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 10 Aug 2025 17:56:10 +0800 Subject: [PATCH 037/100] Apply suggestions from code review --- .../tests/test_identity_transformation.py | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index b41771ce51..52feba702d 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -34,7 +34,7 @@ @pytest.mark.clp @text_datasets def test_clp_identity_transform( - request, + request: pytest.FixtureRequest, integration_test_config: IntegrationTestConfig, test_logs_fixture: str, ) -> None: @@ -46,25 +46,23 @@ def test_clp_identity_transform( ) test_paths.clear_test_outputs() - binary_path_str = str(integration_test_config.package_config.get_clp_binary_path()) + bin_path = str(integration_test_config.package_config.get_clp_binary_path()) + src_path = str(test_paths.logs_source_dir) + compression_path = str(test_paths.compression_dir) + decompression_path = str(test_paths.decompression_dir) # fmt: off compression_cmd = [ - binary_path_str, + bin_path, "c", "--progress", - "--remove-path-prefix", str(test_paths.logs_source_dir), - str(test_paths.compression_dir), - str(test_paths.logs_source_dir), + "--remove-path-prefix", src_path, + compression_path, + src_path, ] # fmt: on run_and_assert(compression_cmd) - decompression_cmd = [ - binary_path_str, - "x", - str(test_paths.compression_dir), - str(test_paths.decompression_dir), - ] + decompression_cmd = [bin_path, "x", compression_path, decompression_path] run_and_assert(decompression_cmd) input_path = test_paths.logs_source_dir @@ -80,7 +78,7 @@ def test_clp_identity_transform( @pytest.mark.clp_s @json_datasets def test_clp_s_identity_transform( - request, + request: pytest.FixtureRequest, integration_test_config: IntegrationTestConfig, test_logs_fixture: str, ) -> None: @@ -107,8 +105,9 @@ def test_clp_s_identity_transform( ) _clp_s_compress_and_decompress(integration_test_config, consolidated_json_test_paths) - input_path = consolidated_json_test_paths.logs_source_dir / "original" - output_path = consolidated_json_test_paths.decompression_dir / "original" + _consolidated_json_file_name = "original" + input_path = consolidated_json_test_paths.logs_source_dir / _consolidated_json_file_name + output_path = consolidated_json_test_paths.decompression_dir / _consolidated_json_file_name assert is_json_file_structurally_equal( input_path, output_path ), f"Mismatch between clp-s input {input_path} and output {output_path}." From 0cedb12e0236cc6b5099d8ffe1497c13a90ee3f8 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 10 Aug 2025 18:30:20 +0800 Subject: [PATCH 038/100] Lint fix --- integration-tests/tests/test_identity_transformation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 52feba702d..b2eae0f3fc 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -106,8 +106,8 @@ def test_clp_s_identity_transform( _clp_s_compress_and_decompress(integration_test_config, consolidated_json_test_paths) _consolidated_json_file_name = "original" - input_path = consolidated_json_test_paths.logs_source_dir / _consolidated_json_file_name - output_path = consolidated_json_test_paths.decompression_dir / _consolidated_json_file_name + input_path = consolidated_json_test_paths.logs_source_dir / _consolidated_json_file_name + output_path = consolidated_json_test_paths.decompression_dir / _consolidated_json_file_name assert is_json_file_structurally_equal( input_path, output_path ), f"Mismatch between clp-s input {input_path} and output {output_path}." From 0378bede76549c7ef8313cdd546917374fd16078 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sat, 16 Aug 2025 04:54:27 -0400 Subject: [PATCH 039/100] Address most review comments --- docs/src/dev-guide/building-package.md | 10 +---- docs/src/dev-guide/components-core/index.md | 6 +++ integration-tests/.python-version | 2 +- integration-tests/README.md | 40 +++++++++++++------ integration-tests/pyproject.toml | 5 +-- .../tests/fixtures/integration_test_logs.py | 22 ---------- .../tests/test_identity_transformation.py | 2 - 7 files changed, 38 insertions(+), 49 deletions(-) diff --git a/docs/src/dev-guide/building-package.md b/docs/src/dev-guide/building-package.md index 838687034a..a84b6be539 100644 --- a/docs/src/dev-guide/building-package.md +++ b/docs/src/dev-guide/building-package.md @@ -15,7 +15,7 @@ prebuilt version instead, check out the [releases](https://github.com/y-scope/cl * python3-venv (for the version of Python installed) * [Task] >= 3.40.0 and < 3.43.0 * We constrain the version due to unresolved [issues][clp-issue-872]. -* [uv] 0.7.19 or newer +* [uv] >= 0.8 ## Setup @@ -62,14 +62,6 @@ where `` is `json` or `text`. The tar will be written to `build/clp----v.tar.gz`, with appropriate values for the fields in angle brackets. -## Test - -To test the package, run: - -```shell -task package-test -``` - ## Cleanup To clean up all build artifacts, run: diff --git a/docs/src/dev-guide/components-core/index.md b/docs/src/dev-guide/components-core/index.md index 20db3ed9a8..12c6525059 100644 --- a/docs/src/dev-guide/components-core/index.md +++ b/docs/src/dev-guide/components-core/index.md @@ -105,6 +105,12 @@ the relevant paths on your machine. make -j ``` +## Test + +```shell +task tests:integration:core +``` + :::{toctree} :hidden: diff --git a/integration-tests/.python-version b/integration-tests/.python-version index c8cfe39591..bd28b9c5c2 100644 --- a/integration-tests/.python-version +++ b/integration-tests/.python-version @@ -1 +1 @@ -3.10 +3.9 diff --git a/integration-tests/README.md b/integration-tests/README.md index d45578eaa8..79cca0e575 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -1,25 +1,41 @@ -# CLP Integration Tests - -This Python module provides end-to-end tests for the CLP package using the `pytest` framework. -It covers package lifecycle scripts and validates compression and search functionality for -performance and correctness on large [datasets]. +# CLP integration tests +This Python project provides end-to-end tests for CLP using the `pytest` framework. ## Running tests -To build the package and run all integration tests: + +To run all integration tests: + ``` -task package-test +task tests:integration ``` -With the package already built, you can leverage `pytest` markers to selectively run tests, e.g. +To test the core CLP binaries: + ``` -# Inside the integration-tests top-level directory +task tests:integration:core +``` + +## Testing customizations + +### Using pytest markers + +One can leverage `pytest` markers to selectively run tests, e.g. + +``` +# From the integration-tests directory uv run python -m pytest -m clp_s ``` + only runs `clp-s` related tests. -To test CLP package at a custom location, override default paths with the environment variable -`CLP_PACKAGE_DIR`, and then run the `uv` command above. +Before running tests, ensure all package script dependencies and binaries are built. + +### Specifying custom test directories + +You can override the default test paths by setting the following environment variables: +- **`CLP_PACKAGE_DIR`** — Path to a custom CLP package directory (where package scripts live). +- **`CLP_CORE_BINS_DIR`** — Path to CLP core binaries, for testing only the core executables. -[datasets]: https://docs.yscope.com/clp/main/user-guide/resources-datasets +These variables let you point the test suite at non-default build or install locations. diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index 1ce9988d46..cb40817662 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -1,12 +1,12 @@ [project] name = "integration-tests" version = "0.1.0" -description = "Integration tests for the CLP package." +description = "Integration tests for CLP." readme = "README.md" authors = [ { name = "YScope Inc.", email = "dev@yscope.com" } ] -requires-python = ">=3.10" +requires-python = ">=3.9" [project.scripts] integration-tests = "integration_tests:main" @@ -18,6 +18,5 @@ build-backend = "hatchling.build" [dependency-groups] dev = [ "pytest>=8.3.5", - "pytest-benchmark>=5.1.0", "pytest-env>=1.1.5", ] diff --git a/integration-tests/tests/fixtures/integration_test_logs.py b/integration-tests/tests/fixtures/integration_test_logs.py index b772fc13d5..2220898432 100644 --- a/integration-tests/tests/fixtures/integration_test_logs.py +++ b/integration-tests/tests/fixtures/integration_test_logs.py @@ -19,28 +19,6 @@ def hive_24hr(request, integration_test_config: IntegrationTestConfig) -> Integr ) -@pytest.fixture(scope="session") -def elasticsearch(request, integration_test_config: IntegrationTestConfig) -> IntegrationTestLogs: - return _download_and_extract_dataset( - request=request, - integration_test_config=integration_test_config, - name="elasticsearch", - tarball_url="https://zenodo.org/records/10516227/files/elasticsearch.tar.gz?download=1", - ) - - -@pytest.fixture(scope="session") -def spark_event_logs( - request, integration_test_config: IntegrationTestConfig -) -> IntegrationTestLogs: - return _download_and_extract_dataset( - request=request, - integration_test_config=integration_test_config, - name="spark-event-logs", - tarball_url="https://zenodo.org/records/10516346/files/spark-event-logs.tar.gz?download=1", - ) - - @pytest.fixture(scope="session") def postgresql(request, integration_test_config: IntegrationTestConfig) -> IntegrationTestLogs: return _download_and_extract_dataset( diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index b2eae0f3fc..603a8e14a9 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -24,8 +24,6 @@ json_datasets = pytest.mark.parametrize( "test_logs_fixture", [ - "elasticsearch", - "spark_event_logs", "postgresql", ], ) From 64dbb7222cb50cdaa6984e1e27dd94aa2b879f96 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sat, 16 Aug 2025 05:33:11 -0400 Subject: [PATCH 040/100] Test using CLP core bins instead of package --- integration-tests/.pytest.ini | 3 +- integration-tests/tests/conftest.py | 2 -- .../tests/fixtures/integration_test_config.py | 8 ++++- .../tests/test_identity_transformation.py | 6 ++-- integration-tests/tests/utils/config.py | 36 +++++++++++++++---- integration-tests/uv.lock | 26 +------------- 6 files changed, 43 insertions(+), 38 deletions(-) diff --git a/integration-tests/.pytest.ini b/integration-tests/.pytest.ini index ddcc80392d..f2dd45df35 100644 --- a/integration-tests/.pytest.ini +++ b/integration-tests/.pytest.ini @@ -8,8 +8,9 @@ addopts = --verbose env = D:CLP_BUILD_DIR=../build + D:CLP_CORE_BINS_DIR=../build/core D:CLP_PACKAGE_DIR=../build/clp-package markers = - binaries: mark tests that directly call the binaries in the package bin clp: mark tests that use the CLP storage engine clp_s: mark tests that use the CLP-S storage engine + core: mark tests that test the CLP core binaries diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index a73d07f72e..8a746499f8 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -1,7 +1,5 @@ from tests.fixtures.integration_test_config import integration_test_config from tests.fixtures.integration_test_logs import ( - elasticsearch, hive_24hr, postgresql, - spark_event_logs, ) diff --git a/integration-tests/tests/fixtures/integration_test_config.py b/integration-tests/tests/fixtures/integration_test_config.py index 55606b4f6b..0226a116d1 100644 --- a/integration-tests/tests/fixtures/integration_test_config.py +++ b/integration-tests/tests/fixtures/integration_test_config.py @@ -1,14 +1,20 @@ from pathlib import Path import pytest -from tests.utils.config import IntegrationTestConfig, PackageConfig +from tests.utils.config import ( + CoreConfig, + IntegrationTestConfig, + PackageConfig, +) from tests.utils.utils import get_env_var @pytest.fixture(scope="session") def integration_test_config() -> IntegrationTestConfig: + core_config = CoreConfig(clp_core_bins_dir=Path(get_env_var("CLP_CORE_BINS_DIR")).resolve()) package_config = PackageConfig(clp_package_dir=Path(get_env_var("CLP_PACKAGE_DIR")).resolve()) return IntegrationTestConfig( + core_config=core_config, package_config=package_config, test_root_dir=Path(get_env_var("CLP_BUILD_DIR")).resolve() / "integration-tests", ) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 603a8e14a9..1040d8d43e 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -12,7 +12,7 @@ run_and_assert, ) -pytestmark = pytest.mark.binaries +pytestmark = pytest.mark.core text_datasets = pytest.mark.parametrize( "test_logs_fixture", @@ -44,7 +44,7 @@ def test_clp_identity_transform( ) test_paths.clear_test_outputs() - bin_path = str(integration_test_config.package_config.get_clp_binary_path()) + bin_path = str(integration_test_config.core_config.get_clp_binary_path()) src_path = str(test_paths.logs_source_dir) compression_path = str(test_paths.compression_dir) decompression_path = str(test_paths.decompression_dir) @@ -118,7 +118,7 @@ def _clp_s_compress_and_decompress( integration_test_config: IntegrationTestConfig, test_paths: CompressionTestConfig ) -> None: test_paths.clear_test_outputs() - bin_path = str(integration_test_config.package_config.get_clp_s_binary_path()) + bin_path = str(integration_test_config.core_config.get_clp_s_binary_path()) src_path = str(test_paths.logs_source_dir) compression_path = str(test_paths.compression_dir) decompression_path = str(test_paths.decompression_dir) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 666e2c8d49..7ea1b762b8 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -4,6 +4,35 @@ from typing import Optional +@dataclass(frozen=True) +class CoreConfig: + clp_core_bins_dir: Path + + def __post_init__(self): + clp_core_bins_dir = self.clp_core_bins_dir + + # Check if CLP core directory exists and is valid + if not clp_core_bins_dir.exists(): + raise ValueError(f"CLP core bins directory does not exist: {clp_core_bins_dir}") + if not clp_core_bins_dir.is_dir(): + raise ValueError(f"CLP core bins path is not a directory: {clp_core_bins_dir}") + + # Check for required directories + required_binaries = ["clg", "clo", "clp", "clp-s", "indexer", "reducer-server"] + missing_binaries = [b for b in required_binaries if not (clp_core_bins_dir / b).is_file()] + if len(missing_binaries) > 0: + raise ValueError( + f"CLP core bins at {clp_core_bins_dir} is incomplete. " + f"Missing binaries: {', '.join(missing_dirs)}" + ) + + def get_clp_binary_path(self) -> Path: + return self.clp_core_bins_dir / "clp" + + def get_clp_s_binary_path(self) -> Path: + return self.clp_core_bins_dir / "clp-s" + + @dataclass(frozen=True) class PackageConfig: clp_package_dir: Path @@ -26,15 +55,10 @@ def __post_init__(self): f"Missing: {', '.join(missing_dirs)}" ) - def get_clp_binary_path(self) -> Path: - return self.clp_package_dir / "bin" / "clp" - - def get_clp_s_binary_path(self) -> Path: - return self.clp_package_dir / "bin" / "clp-s" - @dataclass(frozen=True) class IntegrationTestConfig: + core_config: CoreConfig package_config: PackageConfig test_root_dir: Path logs_download_dir: Optional[Path] = None diff --git a/integration-tests/uv.lock b/integration-tests/uv.lock index 0780d1f38d..0589aef612 100644 --- a/integration-tests/uv.lock +++ b/integration-tests/uv.lock @@ -1,6 +1,6 @@ version = 1 revision = 2 -requires-python = ">=3.10" +requires-python = ">=3.9" [[package]] name = "colorama" @@ -40,7 +40,6 @@ source = { editable = "." } [package.dev-dependencies] dev = [ { name = "pytest" }, - { name = "pytest-benchmark" }, { name = "pytest-env" }, ] @@ -49,7 +48,6 @@ dev = [ [package.metadata.requires-dev] dev = [ { name = "pytest", specifier = ">=8.3.5" }, - { name = "pytest-benchmark", specifier = ">=5.1.0" }, { name = "pytest-env", specifier = ">=1.1.5" }, ] @@ -71,15 +69,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] -[[package]] -name = "py-cpuinfo" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, -] - [[package]] name = "pygments" version = "2.19.2" @@ -107,19 +96,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, ] -[[package]] -name = "pytest-benchmark" -version = "5.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "py-cpuinfo" }, - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/39/d0/a8bd08d641b393db3be3819b03e2d9bb8760ca8479080a26a5f6e540e99c/pytest-benchmark-5.1.0.tar.gz", hash = "sha256:9ea661cdc292e8231f7cd4c10b0319e56a2118e2c09d9f50e1b3d150d2aca105", size = 337810, upload-time = "2024-10-30T11:51:48.521Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/d6/b41653199ea09d5969d4e385df9bbfd9a100f28ca7e824ce7c0a016e3053/pytest_benchmark-5.1.0-py3-none-any.whl", hash = "sha256:922de2dfa3033c227c96da942d1878191afa135a29485fb942e85dff1c592c89", size = 44259, upload-time = "2024-10-30T11:51:45.94Z" }, -] - [[package]] name = "pytest-env" version = "1.1.5" From 5ae418c2a3516a7b388cee28fe8f72c7bf8bc51b Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sat, 16 Aug 2025 06:19:22 -0400 Subject: [PATCH 041/100] Rename tasks --- taskfile.yaml | 19 +------------------ taskfiles/tests/integration.yaml | 27 +++++++++++++++++++++++++++ taskfiles/tests/main.yaml | 4 ++++ 3 files changed, 32 insertions(+), 18 deletions(-) create mode 100644 taskfiles/tests/integration.yaml create mode 100644 taskfiles/tests/main.yaml diff --git a/taskfile.yaml b/taskfile.yaml index 70ccfc7240..b9a17d1e75 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -7,6 +7,7 @@ includes: deps: "taskfiles/deps/main.yaml" docs: "taskfiles/docs.yaml" lint: "taskfiles/lint.yaml" + tests: "taskfiles/tests/main.yaml" utils: "tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" vars: @@ -15,7 +16,6 @@ vars: G_CORE_COMPONENT_DIR: "{{.G_COMPONENTS_DIR}}/core" G_CORE_COMPONENT_SUBMODULES_DIR: "{{.G_CORE_COMPONENT_DIR}}/submodules" G_WEBUI_SRC_DIR: "{{.G_COMPONENTS_DIR}}/webui" - G_INTEGRATION_TESTS_DIR: "{{.ROOT_DIR}}/integration-tests" # Build paths G_BUILD_DIR: "{{.ROOT_DIR}}/build" @@ -163,23 +163,6 @@ tasks: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] - package-test: - deps: - - "package" - dir: "{{.G_INTEGRATION_TESTS_DIR}}" - cmds: - - |- - CLP_BUILD_DIR="{{.G_BUILD_DIR}}" \ - CLP_PACKAGE_DIR="{{.G_PACKAGE_BUILD_DIR}}" \ - uv run python -m pytest - - package-test-cache-clear: - dir: "{{.G_INTEGRATION_TESTS_DIR}}" - cmds: - - >- - uv run python -m pytest --cache-clear --collect-only --override-ini addopts="" --quiet - > /dev/null - clp-s-generate-parsers: vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" diff --git a/taskfiles/tests/integration.yaml b/taskfiles/tests/integration.yaml new file mode 100644 index 0000000000..4fe83379cd --- /dev/null +++ b/taskfiles/tests/integration.yaml @@ -0,0 +1,27 @@ +version: "3" + +vars: + G_INTEGRATION_TESTS_DIR: "{{.ROOT_DIR}}/integration-tests" + +tasks: + default: + deps: + - "core" + + cache-clear: + dir: "{{.G_INTEGRATION_TESTS_DIR}}" + cmds: + - >- + uv run python -m pytest --cache-clear --collect-only --override-ini addopts="" --quiet + > /dev/null + + core: + deps: + - "::core" + dir: "{{.G_INTEGRATION_TESTS_DIR}}" + cmds: + - |- + CLP_BUILD_DIR="{{.G_BUILD_DIR}}" \ + CLP_CORE_BINS_DIR="{{.G_CORE_COMPONENT_BUILD_DIR}}" \ + CLP_PACKAGE_DIR="{{.G_PACKAGE_BUILD_DIR}}" \ + uv run python -m pytest -m "core" diff --git a/taskfiles/tests/main.yaml b/taskfiles/tests/main.yaml new file mode 100644 index 0000000000..9f561314ea --- /dev/null +++ b/taskfiles/tests/main.yaml @@ -0,0 +1,4 @@ +version: "3" + +includes: + integration: "integration.yaml" From 812450c48e174e6fbaa37ecc88fa081e6aa73529 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sat, 16 Aug 2025 06:21:28 -0400 Subject: [PATCH 042/100] Add uv requirement to core building --- docs/src/dev-guide/components-core/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/src/dev-guide/components-core/index.md b/docs/src/dev-guide/components-core/index.md index 12c6525059..2c481b35f4 100644 --- a/docs/src/dev-guide/components-core/index.md +++ b/docs/src/dev-guide/components-core/index.md @@ -11,6 +11,7 @@ CLP core is the low-level component that performs compression, decompression, an * std::source_location * [Task] >= 3.40.0 and < 3.43.0 * We constrain the version due to unresolved [issues][clp-issue-872]. +* [uv] >= 0.8 To build, we require some source dependencies, packages from package managers, and libraries built from source. @@ -123,3 +124,4 @@ regex-utils [clp-issue-872]: https://github.com/y-scope/clp/issues/872 [feature-req]: https://github.com/y-scope/clp/issues/new?assignees=&labels=enhancement&template=feature-request.yml [Task]: https://taskfile.dev/ +[uv]: https://docs.astral.sh/uv/ From 011d0737dd071d803ef64773f7261cd6bba8389e Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sat, 16 Aug 2025 06:23:02 -0400 Subject: [PATCH 043/100] Add README shell script lang hint --- integration-tests/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-tests/README.md b/integration-tests/README.md index 79cca0e575..330b2c1c39 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -6,13 +6,13 @@ This Python project provides end-to-end tests for CLP using the `pytest` framewo To run all integration tests: -``` +```shell task tests:integration ``` To test the core CLP binaries: -``` +```shell task tests:integration:core ``` @@ -22,7 +22,7 @@ task tests:integration:core One can leverage `pytest` markers to selectively run tests, e.g. -``` +```shell # From the integration-tests directory uv run python -m pytest -m clp_s ``` From fce64899855513b3be598060dcb9e80b6d29e2a4 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sat, 16 Aug 2025 18:27:15 +0800 Subject: [PATCH 044/100] Apply suggestions from code review Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- integration-tests/tests/fixtures/integration_test_logs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integration-tests/tests/fixtures/integration_test_logs.py b/integration-tests/tests/fixtures/integration_test_logs.py index 2220898432..17c449dae7 100644 --- a/integration-tests/tests/fixtures/integration_test_logs.py +++ b/integration-tests/tests/fixtures/integration_test_logs.py @@ -53,11 +53,13 @@ def _download_and_extract_dataset( ] # fmt: on subprocess.run(curl_cmds, check=True) + if integration_test_logs.extraction_dir.exists(): + shutil.rmtree(integration_test_logs.extraction_dir) shutil.unpack_archive( integration_test_logs.tarball_path, integration_test_logs.extraction_dir ) except Exception as e: - raise RuntimeError(f"Failed to download and extract dataset `{name}`: {e}") + raise RuntimeError(f"Failed to download and extract dataset `{name}`: {e}") from e print(f"Downloaded and extracted uncompressed logs for dataset `{name}`.") request.config.cache.set(name, True) From 9b40d166c199a32a356fb1982cbb01a35562dd61 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sat, 16 Aug 2025 14:22:03 -0400 Subject: [PATCH 045/100] Address coderabbit ai copmments --- integration-tests/tests/conftest.py | 9 ++++----- integration-tests/tests/utils/config.py | 2 +- integration-tests/tests/utils/utils.py | 3 ++- taskfiles/tests/integration.yaml | 10 ++++++---- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index 8a746499f8..e4517b45b4 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -1,5 +1,4 @@ -from tests.fixtures.integration_test_config import integration_test_config -from tests.fixtures.integration_test_logs import ( - hive_24hr, - postgresql, -) +pytest_plugins = [ + "tests.fixtures.integration_test_config", + "tests.fixtures.integration_test_logs", +] diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 7ea1b762b8..516405fb8b 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -23,7 +23,7 @@ def __post_init__(self): if len(missing_binaries) > 0: raise ValueError( f"CLP core bins at {clp_core_bins_dir} is incomplete. " - f"Missing binaries: {', '.join(missing_dirs)}" + f"Missing binaries: {', '.join(missing_binaries)}" ) def get_clp_binary_path(self) -> Path: diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index a8b2b4f72f..2e658cc671 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -24,7 +24,8 @@ def is_dir_tree_content_equal(path1: Path, path2: Path) -> bool: def get_env_var(var_name: str) -> str: value = os.environ.get(var_name) - assert value is not None, f"Environment variable {var_name} is not set." + if value is None: + raise RuntimeError(f"Environment variable {var_name} is not set.") return value diff --git a/taskfiles/tests/integration.yaml b/taskfiles/tests/integration.yaml index 4fe83379cd..46df657de0 100644 --- a/taskfiles/tests/integration.yaml +++ b/taskfiles/tests/integration.yaml @@ -3,6 +3,11 @@ version: "3" vars: G_INTEGRATION_TESTS_DIR: "{{.ROOT_DIR}}/integration-tests" +env: + CLP_BUILD_DIR: "{{.G_BUILD_DIR}}" + CLP_CORE_BINS_DIR: "{{.G_CORE_COMPONENT_BUILD_DIR}}" + CLP_PACKAGE_DIR: "{{.G_PACKAGE_BUILD_DIR}}" + tasks: default: deps: @@ -20,8 +25,5 @@ tasks: - "::core" dir: "{{.G_INTEGRATION_TESTS_DIR}}" cmds: - - |- - CLP_BUILD_DIR="{{.G_BUILD_DIR}}" \ - CLP_CORE_BINS_DIR="{{.G_CORE_COMPONENT_BUILD_DIR}}" \ - CLP_PACKAGE_DIR="{{.G_PACKAGE_BUILD_DIR}}" \ + - >- uv run python -m pytest -m "core" From 53a6768a6cb66b5cce045d7f8a60c77c313213a0 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sat, 16 Aug 2025 15:11:36 -0400 Subject: [PATCH 046/100] Add helper for validating directory exists --- integration-tests/tests/utils/config.py | 40 ++++++++++--------------- integration-tests/tests/utils/utils.py | 7 +++++ 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 516405fb8b..f45876b66a 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -3,6 +3,8 @@ from pathlib import Path from typing import Optional +from tests.utils.utils import validate_dir_exists + @dataclass(frozen=True) class CoreConfig: @@ -10,14 +12,9 @@ class CoreConfig: def __post_init__(self): clp_core_bins_dir = self.clp_core_bins_dir + validate_dir_exists(clp_core_bins_dir) - # Check if CLP core directory exists and is valid - if not clp_core_bins_dir.exists(): - raise ValueError(f"CLP core bins directory does not exist: {clp_core_bins_dir}") - if not clp_core_bins_dir.is_dir(): - raise ValueError(f"CLP core bins path is not a directory: {clp_core_bins_dir}") - - # Check for required directories + # Check for required CLP core binaries required_binaries = ["clg", "clo", "clp", "clp-s", "indexer", "reducer-server"] missing_binaries = [b for b in required_binaries if not (clp_core_bins_dir / b).is_file()] if len(missing_binaries) > 0: @@ -39,14 +36,9 @@ class PackageConfig: def __post_init__(self): clp_package_dir = self.clp_package_dir + validate_dir_exists(clp_package_dir) - # Check if package directory exists and is valid - if not clp_package_dir.exists(): - raise ValueError(f"CLP package directory does not exist: {clp_package_dir}") - if not clp_package_dir.is_dir(): - raise ValueError(f"CLP package path is not a directory: {clp_package_dir}") - - # Check for required directories + # Check for required package script directories required_dirs = ["bin", "etc", "lib", "sbin"] missing_dirs = [d for d in required_dirs if not (clp_package_dir / d).is_dir()] if len(missing_dirs) > 0: @@ -84,11 +76,12 @@ def __post_init__(self, integration_test_config: IntegrationTestConfig): if 0 == len(name): raise ValueError("`name` cannot be empty.") + logs_download_dir = integration_test_config.logs_download_dir + validate_dir_exists(logs_download_dir) + object.__setattr__(self, "name", name) - object.__setattr__( - self, "tarball_path", integration_test_config.logs_download_dir / f"{name}.tar.gz" - ) - object.__setattr__(self, "extraction_dir", integration_test_config.logs_download_dir / name) + object.__setattr__(self, "tarball_path", logs_download_dir / f"{name}.tar.gz") + object.__setattr__(self, "extraction_dir", logs_download_dir / name) @dataclass(frozen=True) @@ -104,14 +97,13 @@ def __post_init__(self, integration_test_config: IntegrationTestConfig): if 0 == len(test_name): raise ValueError("`test_name` cannot be empty.") + test_root_dir = integration_test_config.test_root_dir + validate_dir_exists(test_root_dir) + object.__setattr__(self, "test_name", test_name) + object.__setattr__(self, "compression_dir", test_root_dir / f"{test_name}-archives") object.__setattr__( - self, "compression_dir", integration_test_config.test_root_dir / f"{test_name}-archives" - ) - object.__setattr__( - self, - "decompression_dir", - integration_test_config.test_root_dir / f"{test_name}-decompressed-logs", + self, "decompression_dir", test_root_dir / f"{test_name}-decompressed-logs" ) def clear_test_outputs(self) -> None: diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index 2e658cc671..bf434d1606 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -35,6 +35,13 @@ def run_and_assert(cmd: List[str], **kwargs) -> subprocess.CompletedProcess: return proc +def validate_dir_exists(dir_path: Path) -> None: + if not dir_path.exists(): + raise ValueError(f"Directory does not exist: {dir_path}") + if not dir_path.is_dir(): + raise ValueError(f"Path is not a directory: {dir_path}") + + def _sort_json_keys_and_rows(json_fp: Path) -> IO[str]: sorted_fp = NamedTemporaryFile(mode="w+", delete=True) jq_proc = subprocess.Popen( From 29c2dedfa15d49ce6960affc711c23dc71ef98d4 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 05:34:57 -0400 Subject: [PATCH 047/100] Add docstring for dataclasses --- integration-tests/tests/utils/config.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index f45876b66a..2f5c26b581 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -8,6 +8,8 @@ @dataclass(frozen=True) class CoreConfig: + """The configuration for the clp core binaries subject to testing.""" + #: clp_core_bins_dir: Path def __post_init__(self): @@ -32,6 +34,8 @@ def get_clp_s_binary_path(self) -> Path: @dataclass(frozen=True) class PackageConfig: + """The configuration for the clp package subject to testing.""" + #: clp_package_dir: Path def __post_init__(self): @@ -50,9 +54,14 @@ def __post_init__(self): @dataclass(frozen=True) class IntegrationTestConfig: + """The general configuration for integration tests.""" + #: core_config: CoreConfig + #: package_config: PackageConfig + #: Root directory for integration tests output. test_root_dir: Path + #: Directory to store the downloaded logs. logs_download_dir: Optional[Path] = None def __post_init__(self): @@ -65,10 +74,15 @@ def __post_init__(self): @dataclass(frozen=True) class IntegrationTestLogs: + """Metadata for the downloaded logs used for integration tests.""" + #: name: str + #: tarball_url: str integration_test_config: InitVar[IntegrationTestConfig] + #: tarball_path: Path = field(init=False, repr=True) + #: extraction_dir: Path = field(init=False, repr=True) def __post_init__(self, integration_test_config: IntegrationTestConfig): @@ -86,10 +100,15 @@ def __post_init__(self, integration_test_config: IntegrationTestConfig): @dataclass(frozen=True) class CompressionTestConfig: + """Compression test configuration providing per-test metadata for artifacts and directories.""" + #: test_name: str + #: Directory containing the original (uncompressed) log files used by this test. logs_source_dir: Path integration_test_config: InitVar[IntegrationTestConfig] + #: Path to store compressed archives generated by the test. compression_dir: Path = field(init=False, repr=True) + #: Path to store decompressed logs generated by the test. decompression_dir: Path = field(init=False, repr=True) def __post_init__(self, integration_test_config: IntegrationTestConfig): @@ -107,6 +126,7 @@ def __post_init__(self, integration_test_config: IntegrationTestConfig): ) def clear_test_outputs(self) -> None: + """Remove any existing output directories created by this compression test.""" if self.compression_dir.exists(): shutil.rmtree(self.compression_dir) if self.decompression_dir.exists(): From 85c37f2ac0ad4ba4258457444bd2986f015770eb Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 06:59:33 -0400 Subject: [PATCH 048/100] Add docstrings for test utils and improve functions --- integration-tests/tests/utils/utils.py | 53 ++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index bf434d1606..aa884ad51f 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -6,13 +6,24 @@ def is_json_file_structurally_equal(json_fp1: Path, json_fp2: Path) -> bool: + """ + :param json_fp1: + :param json_fp2: + :return: Whether two JSON files are structurally equal after sorting has been applied. + """ with _sort_json_keys_and_rows(json_fp1) as temp_file_1, _sort_json_keys_and_rows( json_fp2 ) as temp_file_2: - return is_dir_tree_content_equal(temp_file_1.name, temp_file_2.name) + return is_dir_tree_content_equal(Path(temp_file_1.name), Path(temp_file_2.name)) def is_dir_tree_content_equal(path1: Path, path2: Path) -> bool: + """ + :param path1: + :param path2: + :return: Whether two files/directories hold the exactly same content. + :raise: RuntimeError if the diff command fails due to execution errors. + """ cmd = ["diff", "--brief", "--recursive", str(path1), str(path2)] proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if 0 == proc.returncode: @@ -23,19 +34,36 @@ def is_dir_tree_content_equal(path1: Path, path2: Path) -> bool: def get_env_var(var_name: str) -> str: + """ + :param var_name: + :return: The string value of the specified environment variable. + :raise: ValueError if the environment variable is not set + """ value = os.environ.get(var_name) if value is None: - raise RuntimeError(f"Environment variable {var_name} is not set.") + raise ValueError(f"Environment variable {var_name} is not set.") return value def run_and_assert(cmd: List[str], **kwargs) -> subprocess.CompletedProcess: + """ + Runs a command with subprocess and asserts that it succeeds with pytest. + + :param cmd: Command and arguments to execute. + :param kwargs: Additional keyword arguments passed through to the subprocess. + :return: The completed process object, for inspection or further handling. + :raise: AssertionError if the command exits with a non-zero return code. + """ proc = subprocess.run(cmd, **kwargs) assert 0 == proc.returncode, f"Command failed: {' '.join(cmd)}" return proc def validate_dir_exists(dir_path: Path) -> None: + """ + :param dir_path: + :raise: ValueError if the path does not exist or is not a directory. + """ if not dir_path.exists(): raise ValueError(f"Directory does not exist: {dir_path}") if not dir_path.is_dir(): @@ -43,11 +71,30 @@ def validate_dir_exists(dir_path: Path) -> None: def _sort_json_keys_and_rows(json_fp: Path) -> IO[str]: + """ + Normalize a JSON file to a stable, deterministically ordered form for comparison. + + :param json_fp: + :return: A named temoprary file (delete on close) that contains the sorted JSON content. + :raise: RuntimeError if either jq or sort fails due to execution errors. + """ sorted_fp = NamedTemporaryFile(mode="w+", delete=True) jq_proc = subprocess.Popen( ["jq", "--sort-keys", "--compact-output", ".", str(json_fp)], stdout=subprocess.PIPE, ) - subprocess.run(["sort"], stdin=jq_proc.stdout, stdout=sorted_fp, check=True) + try: + sort_proc = subprocess.run(["sort"], stdin=jq_proc.stdout, stdout=sorted_fp, check=True) + sort_rc = sort_proc.returncode + if sort_rc != 0: + raise RuntimeError(f"sort failed with exist code {sort_rc} for {json_fp}") + finally: + if jq_proc.stdout is not None: + jq_proc.stdout.close() + jq_rc = jq_proc.wait() + if jq_rc != 0: + raise RuntimeError(f"jq failed with exit code {jq_rc} for {json_fp}") + sorted_fp.flush() + sorted_fp.seek(0) return sorted_fp From 37ffcacb15000bd9273c268381631b90dc02aba7 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 07:00:03 -0400 Subject: [PATCH 049/100] Lint fix --- integration-tests/tests/utils/config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 2f5c26b581..ae227a437d 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -9,6 +9,7 @@ @dataclass(frozen=True) class CoreConfig: """The configuration for the clp core binaries subject to testing.""" + #: clp_core_bins_dir: Path @@ -35,6 +36,7 @@ def get_clp_s_binary_path(self) -> Path: @dataclass(frozen=True) class PackageConfig: """The configuration for the clp package subject to testing.""" + #: clp_package_dir: Path @@ -55,6 +57,7 @@ def __post_init__(self): @dataclass(frozen=True) class IntegrationTestConfig: """The general configuration for integration tests.""" + #: core_config: CoreConfig #: @@ -75,6 +78,7 @@ def __post_init__(self): @dataclass(frozen=True) class IntegrationTestLogs: """Metadata for the downloaded logs used for integration tests.""" + #: name: str #: @@ -101,6 +105,7 @@ def __post_init__(self, integration_test_config: IntegrationTestConfig): @dataclass(frozen=True) class CompressionTestConfig: """Compression test configuration providing per-test metadata for artifacts and directories.""" + #: test_name: str #: Directory containing the original (uncompressed) log files used by this test. From 35f1690f8815176f72dbe3f8bc153a9e6d62d913 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 11:17:54 +0000 Subject: [PATCH 050/100] Add mypy and ruff linters --- integration-tests/pyproject.toml | 40 +++++++++++++ integration-tests/uv.lock | 99 ++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index cb40817662..c7ea37ea8f 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -17,6 +17,46 @@ build-backend = "hatchling.build" [dependency-groups] dev = [ + "mypy>=1.16.0", + "ruff>=0.11.12", "pytest>=8.3.5", "pytest-env>=1.1.5", ] + +[tool.mypy] +strict = true + +# Additional output +pretty = true +show_error_code_links = true +show_error_context = true +show_error_end = true + +[tool.ruff] +line-length = 100 + +[tool.ruff.lint] +select = ["ALL"] +ignore = [ + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful +] +isort.order-by-type = false + +[tool.ruff.format] +docstring-code-format = true +docstring-code-line-length = 100 diff --git a/integration-tests/uv.lock b/integration-tests/uv.lock index 0589aef612..c6cd245d81 100644 --- a/integration-tests/uv.lock +++ b/integration-tests/uv.lock @@ -39,16 +39,80 @@ source = { editable = "." } [package.dev-dependencies] dev = [ + { name = "mypy" }, { name = "pytest" }, { name = "pytest-env" }, + { name = "ruff" }, ] [package.metadata] [package.metadata.requires-dev] dev = [ + { name = "mypy", specifier = ">=1.16.0" }, { name = "pytest", specifier = ">=8.3.5" }, { name = "pytest-env", specifier = ">=1.1.5" }, + { name = "ruff", specifier = ">=0.11.12" }, +] + +[[package]] +name = "mypy" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8e/22/ea637422dedf0bf36f3ef238eab4e455e2a0dcc3082b5cc067615347ab8e/mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01", size = 3352570, upload-time = "2025-07-31T07:54:19.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/a9/3d7aa83955617cdf02f94e50aab5c830d205cfa4320cf124ff64acce3a8e/mypy-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3fbe6d5555bf608c47203baa3e72dbc6ec9965b3d7c318aa9a4ca76f465bd972", size = 11003299, upload-time = "2025-07-31T07:54:06.425Z" }, + { url = "https://files.pythonhosted.org/packages/83/e8/72e62ff837dd5caaac2b4a5c07ce769c8e808a00a65e5d8f94ea9c6f20ab/mypy-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80ef5c058b7bce08c83cac668158cb7edea692e458d21098c7d3bce35a5d43e7", size = 10125451, upload-time = "2025-07-31T07:53:52.974Z" }, + { url = "https://files.pythonhosted.org/packages/7d/10/f3f3543f6448db11881776f26a0ed079865926b0c841818ee22de2c6bbab/mypy-1.17.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a580f8a70c69e4a75587bd925d298434057fe2a428faaf927ffe6e4b9a98df", size = 11916211, upload-time = "2025-07-31T07:53:18.879Z" }, + { url = "https://files.pythonhosted.org/packages/06/bf/63e83ed551282d67bb3f7fea2cd5561b08d2bb6eb287c096539feb5ddbc5/mypy-1.17.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd86bb649299f09d987a2eebb4d52d10603224500792e1bee18303bbcc1ce390", size = 12652687, upload-time = "2025-07-31T07:53:30.544Z" }, + { url = "https://files.pythonhosted.org/packages/69/66/68f2eeef11facf597143e85b694a161868b3b006a5fbad50e09ea117ef24/mypy-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a76906f26bd8d51ea9504966a9c25419f2e668f012e0bdf3da4ea1526c534d94", size = 12896322, upload-time = "2025-07-31T07:53:50.74Z" }, + { url = "https://files.pythonhosted.org/packages/a3/87/8e3e9c2c8bd0d7e071a89c71be28ad088aaecbadf0454f46a540bda7bca6/mypy-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:e79311f2d904ccb59787477b7bd5d26f3347789c06fcd7656fa500875290264b", size = 9507962, upload-time = "2025-07-31T07:53:08.431Z" }, + { url = "https://files.pythonhosted.org/packages/46/cf/eadc80c4e0a70db1c08921dcc220357ba8ab2faecb4392e3cebeb10edbfa/mypy-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad37544be07c5d7fba814eb370e006df58fed8ad1ef33ed1649cb1889ba6ff58", size = 10921009, upload-time = "2025-07-31T07:53:23.037Z" }, + { url = "https://files.pythonhosted.org/packages/5d/c1/c869d8c067829ad30d9bdae051046561552516cfb3a14f7f0347b7d973ee/mypy-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:064e2ff508e5464b4bd807a7c1625bc5047c5022b85c70f030680e18f37273a5", size = 10047482, upload-time = "2025-07-31T07:53:26.151Z" }, + { url = "https://files.pythonhosted.org/packages/98/b9/803672bab3fe03cee2e14786ca056efda4bb511ea02dadcedde6176d06d0/mypy-1.17.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70401bbabd2fa1aa7c43bb358f54037baf0586f41e83b0ae67dd0534fc64edfd", size = 11832883, upload-time = "2025-07-31T07:53:47.948Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/fcdac695beca66800918c18697b48833a9a6701de288452b6715a98cfee1/mypy-1.17.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e92bdc656b7757c438660f775f872a669b8ff374edc4d18277d86b63edba6b8b", size = 12566215, upload-time = "2025-07-31T07:54:04.031Z" }, + { url = "https://files.pythonhosted.org/packages/7f/37/a932da3d3dace99ee8eb2043b6ab03b6768c36eb29a02f98f46c18c0da0e/mypy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c1fdf4abb29ed1cb091cf432979e162c208a5ac676ce35010373ff29247bcad5", size = 12751956, upload-time = "2025-07-31T07:53:36.263Z" }, + { url = "https://files.pythonhosted.org/packages/8c/cf/6438a429e0f2f5cab8bc83e53dbebfa666476f40ee322e13cac5e64b79e7/mypy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:ff2933428516ab63f961644bc49bc4cbe42bbffb2cd3b71cc7277c07d16b1a8b", size = 9507307, upload-time = "2025-07-31T07:53:59.734Z" }, + { url = "https://files.pythonhosted.org/packages/17/a2/7034d0d61af8098ec47902108553122baa0f438df8a713be860f7407c9e6/mypy-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:69e83ea6553a3ba79c08c6e15dbd9bfa912ec1e493bf75489ef93beb65209aeb", size = 11086295, upload-time = "2025-07-31T07:53:28.124Z" }, + { url = "https://files.pythonhosted.org/packages/14/1f/19e7e44b594d4b12f6ba8064dbe136505cec813549ca3e5191e40b1d3cc2/mypy-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b16708a66d38abb1e6b5702f5c2c87e133289da36f6a1d15f6a5221085c6403", size = 10112355, upload-time = "2025-07-31T07:53:21.121Z" }, + { url = "https://files.pythonhosted.org/packages/5b/69/baa33927e29e6b4c55d798a9d44db5d394072eef2bdc18c3e2048c9ed1e9/mypy-1.17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89e972c0035e9e05823907ad5398c5a73b9f47a002b22359b177d40bdaee7056", size = 11875285, upload-time = "2025-07-31T07:53:55.293Z" }, + { url = "https://files.pythonhosted.org/packages/90/13/f3a89c76b0a41e19490b01e7069713a30949d9a6c147289ee1521bcea245/mypy-1.17.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03b6d0ed2b188e35ee6d5c36b5580cffd6da23319991c49ab5556c023ccf1341", size = 12737895, upload-time = "2025-07-31T07:53:43.623Z" }, + { url = "https://files.pythonhosted.org/packages/23/a1/c4ee79ac484241301564072e6476c5a5be2590bc2e7bfd28220033d2ef8f/mypy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c837b896b37cd103570d776bda106eabb8737aa6dd4f248451aecf53030cdbeb", size = 12931025, upload-time = "2025-07-31T07:54:17.125Z" }, + { url = "https://files.pythonhosted.org/packages/89/b8/7409477be7919a0608900e6320b155c72caab4fef46427c5cc75f85edadd/mypy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:665afab0963a4b39dff7c1fa563cc8b11ecff7910206db4b2e64dd1ba25aed19", size = 9584664, upload-time = "2025-07-31T07:54:12.842Z" }, + { url = "https://files.pythonhosted.org/packages/5b/82/aec2fc9b9b149f372850291827537a508d6c4d3664b1750a324b91f71355/mypy-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93378d3203a5c0800c6b6d850ad2f19f7a3cdf1a3701d3416dbf128805c6a6a7", size = 11075338, upload-time = "2025-07-31T07:53:38.873Z" }, + { url = "https://files.pythonhosted.org/packages/07/ac/ee93fbde9d2242657128af8c86f5d917cd2887584cf948a8e3663d0cd737/mypy-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15d54056f7fe7a826d897789f53dd6377ec2ea8ba6f776dc83c2902b899fee81", size = 10113066, upload-time = "2025-07-31T07:54:14.707Z" }, + { url = "https://files.pythonhosted.org/packages/5a/68/946a1e0be93f17f7caa56c45844ec691ca153ee8b62f21eddda336a2d203/mypy-1.17.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:209a58fed9987eccc20f2ca94afe7257a8f46eb5df1fb69958650973230f91e6", size = 11875473, upload-time = "2025-07-31T07:53:14.504Z" }, + { url = "https://files.pythonhosted.org/packages/9f/0f/478b4dce1cb4f43cf0f0d00fba3030b21ca04a01b74d1cd272a528cf446f/mypy-1.17.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:099b9a5da47de9e2cb5165e581f158e854d9e19d2e96b6698c0d64de911dd849", size = 12744296, upload-time = "2025-07-31T07:53:03.896Z" }, + { url = "https://files.pythonhosted.org/packages/ca/70/afa5850176379d1b303f992a828de95fc14487429a7139a4e0bdd17a8279/mypy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa6ffadfbe6994d724c5a1bb6123a7d27dd68fc9c059561cd33b664a79578e14", size = 12914657, upload-time = "2025-07-31T07:54:08.576Z" }, + { url = "https://files.pythonhosted.org/packages/53/f9/4a83e1c856a3d9c8f6edaa4749a4864ee98486e9b9dbfbc93842891029c2/mypy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:9a2b7d9180aed171f033c9f2fc6c204c1245cf60b0cb61cf2e7acc24eea78e0a", size = 9593320, upload-time = "2025-07-31T07:53:01.341Z" }, + { url = "https://files.pythonhosted.org/packages/38/56/79c2fac86da57c7d8c48622a05873eaab40b905096c33597462713f5af90/mypy-1.17.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:15a83369400454c41ed3a118e0cc58bd8123921a602f385cb6d6ea5df050c733", size = 11040037, upload-time = "2025-07-31T07:54:10.942Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c3/adabe6ff53638e3cad19e3547268482408323b1e68bf082c9119000cd049/mypy-1.17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b918670f692fc9fba55c3298d8a3beae295c5cded0a55dccdc5bbead814acd", size = 10131550, upload-time = "2025-07-31T07:53:41.307Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c5/2e234c22c3bdeb23a7817af57a58865a39753bde52c74e2c661ee0cfc640/mypy-1.17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62761474061feef6f720149d7ba876122007ddc64adff5ba6f374fda35a018a0", size = 11872963, upload-time = "2025-07-31T07:53:16.878Z" }, + { url = "https://files.pythonhosted.org/packages/ab/26/c13c130f35ca8caa5f2ceab68a247775648fdcd6c9a18f158825f2bc2410/mypy-1.17.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c49562d3d908fd49ed0938e5423daed8d407774a479b595b143a3d7f87cdae6a", size = 12710189, upload-time = "2025-07-31T07:54:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/82/df/c7d79d09f6de8383fe800521d066d877e54d30b4fb94281c262be2df84ef/mypy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:397fba5d7616a5bc60b45c7ed204717eaddc38f826e3645402c426057ead9a91", size = 12900322, upload-time = "2025-07-31T07:53:10.551Z" }, + { url = "https://files.pythonhosted.org/packages/b8/98/3d5a48978b4f708c55ae832619addc66d677f6dc59f3ebad71bae8285ca6/mypy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:9d6b20b97d373f41617bd0708fd46aa656059af57f2ef72aa8c7d6a2b73b74ed", size = 9751879, upload-time = "2025-07-31T07:52:56.683Z" }, + { url = "https://files.pythonhosted.org/packages/29/cb/673e3d34e5d8de60b3a61f44f80150a738bff568cd6b7efb55742a605e98/mypy-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5d1092694f166a7e56c805caaf794e0585cabdbf1df36911c414e4e9abb62ae9", size = 10992466, upload-time = "2025-07-31T07:53:57.574Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d0/fe1895836eea3a33ab801561987a10569df92f2d3d4715abf2cfeaa29cb2/mypy-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:79d44f9bfb004941ebb0abe8eff6504223a9c1ac51ef967d1263c6572bbebc99", size = 10117638, upload-time = "2025-07-31T07:53:34.256Z" }, + { url = "https://files.pythonhosted.org/packages/97/f3/514aa5532303aafb95b9ca400a31054a2bd9489de166558c2baaeea9c522/mypy-1.17.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b01586eed696ec905e61bd2568f48740f7ac4a45b3a468e6423a03d3788a51a8", size = 11915673, upload-time = "2025-07-31T07:52:59.361Z" }, + { url = "https://files.pythonhosted.org/packages/ab/c3/c0805f0edec96fe8e2c048b03769a6291523d509be8ee7f56ae922fa3882/mypy-1.17.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43808d9476c36b927fbcd0b0255ce75efe1b68a080154a38ae68a7e62de8f0f8", size = 12649022, upload-time = "2025-07-31T07:53:45.92Z" }, + { url = "https://files.pythonhosted.org/packages/45/3e/d646b5a298ada21a8512fa7e5531f664535a495efa672601702398cea2b4/mypy-1.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:feb8cc32d319edd5859da2cc084493b3e2ce5e49a946377663cc90f6c15fb259", size = 12895536, upload-time = "2025-07-31T07:53:06.17Z" }, + { url = "https://files.pythonhosted.org/packages/14/55/e13d0dcd276975927d1f4e9e2ec4fd409e199f01bdc671717e673cc63a22/mypy-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d7598cf74c3e16539d4e2f0b8d8c318e00041553d83d4861f87c7a72e95ac24d", size = 9512564, upload-time = "2025-07-31T07:53:12.346Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f3/8fcd2af0f5b806f6cf463efaffd3c9548a28f84220493ecd38d127b6b66d/mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9", size = 2283411, upload-time = "2025-07-31T07:53:24.664Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] [[package]] @@ -60,6 +124,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -109,6 +182,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/b8/87cfb16045c9d4092cfcf526135d73b88101aac83bc1adcf82dfb5fd3833/pytest_env-1.1.5-py3-none-any.whl", hash = "sha256:ce90cf8772878515c24b31cd97c7fa1f4481cd68d588419fd45f10ecaee6bc30", size = 6141, upload-time = "2024-09-17T22:39:16.942Z" }, ] +[[package]] +name = "ruff" +version = "0.12.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/45/2e403fa7007816b5fbb324cb4f8ed3c7402a927a0a0cb2b6279879a8bfdc/ruff-0.12.9.tar.gz", hash = "sha256:fbd94b2e3c623f659962934e52c2bea6fc6da11f667a427a368adaf3af2c866a", size = 5254702, upload-time = "2025-08-14T16:08:55.2Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/20/53bf098537adb7b6a97d98fcdebf6e916fcd11b2e21d15f8c171507909cc/ruff-0.12.9-py3-none-linux_armv6l.whl", hash = "sha256:fcebc6c79fcae3f220d05585229463621f5dbf24d79fdc4936d9302e177cfa3e", size = 11759705, upload-time = "2025-08-14T16:08:12.968Z" }, + { url = "https://files.pythonhosted.org/packages/20/4d/c764ee423002aac1ec66b9d541285dd29d2c0640a8086c87de59ebbe80d5/ruff-0.12.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:aed9d15f8c5755c0e74467731a007fcad41f19bcce41cd75f768bbd687f8535f", size = 12527042, upload-time = "2025-08-14T16:08:16.54Z" }, + { url = "https://files.pythonhosted.org/packages/8b/45/cfcdf6d3eb5fc78a5b419e7e616d6ccba0013dc5b180522920af2897e1be/ruff-0.12.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:5b15ea354c6ff0d7423814ba6d44be2807644d0c05e9ed60caca87e963e93f70", size = 11724457, upload-time = "2025-08-14T16:08:18.686Z" }, + { url = "https://files.pythonhosted.org/packages/72/e6/44615c754b55662200c48bebb02196dbb14111b6e266ab071b7e7297b4ec/ruff-0.12.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d596c2d0393c2502eaabfef723bd74ca35348a8dac4267d18a94910087807c53", size = 11949446, upload-time = "2025-08-14T16:08:21.059Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d1/9b7d46625d617c7df520d40d5ac6cdcdf20cbccb88fad4b5ecd476a6bb8d/ruff-0.12.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1b15599931a1a7a03c388b9c5df1bfa62be7ede6eb7ef753b272381f39c3d0ff", size = 11566350, upload-time = "2025-08-14T16:08:23.433Z" }, + { url = "https://files.pythonhosted.org/packages/59/20/b73132f66f2856bc29d2d263c6ca457f8476b0bbbe064dac3ac3337a270f/ruff-0.12.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d02faa2977fb6f3f32ddb7828e212b7dd499c59eb896ae6c03ea5c303575756", size = 13270430, upload-time = "2025-08-14T16:08:25.837Z" }, + { url = "https://files.pythonhosted.org/packages/a2/21/eaf3806f0a3d4c6be0a69d435646fba775b65f3f2097d54898b0fd4bb12e/ruff-0.12.9-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:17d5b6b0b3a25259b69ebcba87908496e6830e03acfb929ef9fd4c58675fa2ea", size = 14264717, upload-time = "2025-08-14T16:08:27.907Z" }, + { url = "https://files.pythonhosted.org/packages/d2/82/1d0c53bd37dcb582b2c521d352fbf4876b1e28bc0d8894344198f6c9950d/ruff-0.12.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:72db7521860e246adbb43f6ef464dd2a532ef2ef1f5dd0d470455b8d9f1773e0", size = 13684331, upload-time = "2025-08-14T16:08:30.352Z" }, + { url = "https://files.pythonhosted.org/packages/3b/2f/1c5cf6d8f656306d42a686f1e207f71d7cebdcbe7b2aa18e4e8a0cb74da3/ruff-0.12.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a03242c1522b4e0885af63320ad754d53983c9599157ee33e77d748363c561ce", size = 12739151, upload-time = "2025-08-14T16:08:32.55Z" }, + { url = "https://files.pythonhosted.org/packages/47/09/25033198bff89b24d734e6479e39b1968e4c992e82262d61cdccaf11afb9/ruff-0.12.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fc83e4e9751e6c13b5046d7162f205d0a7bac5840183c5beebf824b08a27340", size = 12954992, upload-time = "2025-08-14T16:08:34.816Z" }, + { url = "https://files.pythonhosted.org/packages/52/8e/d0dbf2f9dca66c2d7131feefc386523404014968cd6d22f057763935ab32/ruff-0.12.9-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:881465ed56ba4dd26a691954650de6ad389a2d1fdb130fe51ff18a25639fe4bb", size = 12899569, upload-time = "2025-08-14T16:08:36.852Z" }, + { url = "https://files.pythonhosted.org/packages/a0/bd/b614d7c08515b1428ed4d3f1d4e3d687deffb2479703b90237682586fa66/ruff-0.12.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:43f07a3ccfc62cdb4d3a3348bf0588358a66da756aa113e071b8ca8c3b9826af", size = 11751983, upload-time = "2025-08-14T16:08:39.314Z" }, + { url = "https://files.pythonhosted.org/packages/58/d6/383e9f818a2441b1a0ed898d7875f11273f10882f997388b2b51cb2ae8b5/ruff-0.12.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:07adb221c54b6bba24387911e5734357f042e5669fa5718920ee728aba3cbadc", size = 11538635, upload-time = "2025-08-14T16:08:41.297Z" }, + { url = "https://files.pythonhosted.org/packages/20/9c/56f869d314edaa9fc1f491706d1d8a47747b9d714130368fbd69ce9024e9/ruff-0.12.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f5cd34fabfdea3933ab85d72359f118035882a01bff15bd1d2b15261d85d5f66", size = 12534346, upload-time = "2025-08-14T16:08:43.39Z" }, + { url = "https://files.pythonhosted.org/packages/bd/4b/d8b95c6795a6c93b439bc913ee7a94fda42bb30a79285d47b80074003ee7/ruff-0.12.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:f6be1d2ca0686c54564da8e7ee9e25f93bdd6868263805f8c0b8fc6a449db6d7", size = 13017021, upload-time = "2025-08-14T16:08:45.889Z" }, + { url = "https://files.pythonhosted.org/packages/c7/c1/5f9a839a697ce1acd7af44836f7c2181cdae5accd17a5cb85fcbd694075e/ruff-0.12.9-py3-none-win32.whl", hash = "sha256:cc7a37bd2509974379d0115cc5608a1a4a6c4bff1b452ea69db83c8855d53f93", size = 11734785, upload-time = "2025-08-14T16:08:48.062Z" }, + { url = "https://files.pythonhosted.org/packages/fa/66/cdddc2d1d9a9f677520b7cfc490d234336f523d4b429c1298de359a3be08/ruff-0.12.9-py3-none-win_amd64.whl", hash = "sha256:6fb15b1977309741d7d098c8a3cb7a30bc112760a00fb6efb7abc85f00ba5908", size = 12840654, upload-time = "2025-08-14T16:08:50.158Z" }, + { url = "https://files.pythonhosted.org/packages/ac/fd/669816bc6b5b93b9586f3c1d87cd6bc05028470b3ecfebb5938252c47a35/ruff-0.12.9-py3-none-win_arm64.whl", hash = "sha256:63c8c819739d86b96d500cce885956a1a48ab056bbcbc61b747ad494b2485089", size = 11949623, upload-time = "2025-08-14T16:08:52.233Z" }, +] + [[package]] name = "tomli" version = "2.2.1" From 95b6886ed26ad835610184cbc3cbf336b9a933c8 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 11:19:29 +0000 Subject: [PATCH 051/100] ruff lint --- integration-tests/tests/test_identity_transformation.py | 6 +++--- integration-tests/tests/utils/utils.py | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 1040d8d43e..f6f5e411a8 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -106,9 +106,9 @@ def test_clp_s_identity_transform( _consolidated_json_file_name = "original" input_path = consolidated_json_test_paths.logs_source_dir / _consolidated_json_file_name output_path = consolidated_json_test_paths.decompression_dir / _consolidated_json_file_name - assert is_json_file_structurally_equal( - input_path, output_path - ), f"Mismatch between clp-s input {input_path} and output {output_path}." + assert is_json_file_structurally_equal(input_path, output_path), ( + f"Mismatch between clp-s input {input_path} and output {output_path}." + ) test_paths.clear_test_outputs() consolidated_json_test_paths.clear_test_outputs() diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index aa884ad51f..1d5f20b556 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -11,9 +11,10 @@ def is_json_file_structurally_equal(json_fp1: Path, json_fp2: Path) -> bool: :param json_fp2: :return: Whether two JSON files are structurally equal after sorting has been applied. """ - with _sort_json_keys_and_rows(json_fp1) as temp_file_1, _sort_json_keys_and_rows( - json_fp2 - ) as temp_file_2: + with ( + _sort_json_keys_and_rows(json_fp1) as temp_file_1, + _sort_json_keys_and_rows(json_fp2) as temp_file_2, + ): return is_dir_tree_content_equal(Path(temp_file_1.name), Path(temp_file_2.name)) From 0a68983ca5b7b39890aafa7033e01f39365df958 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 11:26:06 +0000 Subject: [PATCH 052/100] Add missing __init__ files --- integration-tests/tests/__init__.py | 0 integration-tests/tests/fixtures/__init__.py | 0 integration-tests/tests/utils/__init__.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 integration-tests/tests/__init__.py create mode 100644 integration-tests/tests/fixtures/__init__.py create mode 100644 integration-tests/tests/utils/__init__.py diff --git a/integration-tests/tests/__init__.py b/integration-tests/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/integration-tests/tests/fixtures/__init__.py b/integration-tests/tests/fixtures/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/integration-tests/tests/utils/__init__.py b/integration-tests/tests/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From e4f48912cc2d1368c2cd271a1cb7e3966fa01dff Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 12:41:11 +0000 Subject: [PATCH 053/100] Pass mypy test --- .../tests/fixtures/integration_test_logs.py | 15 ++++++++++++--- integration-tests/tests/utils/config.py | 17 ++++++++++------- integration-tests/tests/utils/utils.py | 4 ++-- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/integration-tests/tests/fixtures/integration_test_logs.py b/integration-tests/tests/fixtures/integration_test_logs.py index 17c449dae7..38c47d5ee5 100644 --- a/integration-tests/tests/fixtures/integration_test_logs.py +++ b/integration-tests/tests/fixtures/integration_test_logs.py @@ -10,7 +10,10 @@ @pytest.fixture(scope="session") -def hive_24hr(request, integration_test_config: IntegrationTestConfig) -> IntegrationTestLogs: +def hive_24hr( + request: pytest.FixtureRequest, + integration_test_config: IntegrationTestConfig, +) -> IntegrationTestLogs: return _download_and_extract_dataset( request=request, integration_test_config=integration_test_config, @@ -20,7 +23,10 @@ def hive_24hr(request, integration_test_config: IntegrationTestConfig) -> Integr @pytest.fixture(scope="session") -def postgresql(request, integration_test_config: IntegrationTestConfig) -> IntegrationTestLogs: +def postgresql( + request: pytest.FixtureRequest, + integration_test_config: IntegrationTestConfig, +) -> IntegrationTestLogs: return _download_and_extract_dataset( request=request, integration_test_config=integration_test_config, @@ -30,7 +36,10 @@ def postgresql(request, integration_test_config: IntegrationTestConfig) -> Integ def _download_and_extract_dataset( - request, integration_test_config: IntegrationTestConfig, name: str, tarball_url: str + request: pytest.FixtureRequest, + integration_test_config: IntegrationTestConfig, + name: str, + tarball_url: str, ) -> IntegrationTestLogs: integration_test_logs = IntegrationTestLogs( name=name, diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index ae227a437d..b2dbec84ed 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -13,7 +13,7 @@ class CoreConfig: #: clp_core_bins_dir: Path - def __post_init__(self): + def __post_init__(self) -> None: clp_core_bins_dir = self.clp_core_bins_dir validate_dir_exists(clp_core_bins_dir) @@ -40,7 +40,7 @@ class PackageConfig: #: clp_package_dir: Path - def __post_init__(self): + def __post_init__(self) -> None: clp_package_dir = self.clp_package_dir validate_dir_exists(clp_package_dir) @@ -64,11 +64,14 @@ class IntegrationTestConfig: package_config: PackageConfig #: Root directory for integration tests output. test_root_dir: Path + logs_download_dir_init: InitVar[Optional[Path]] = None #: Directory to store the downloaded logs. - logs_download_dir: Optional[Path] = None + logs_download_dir: Path = field(init=False, repr=True) - def __post_init__(self): - if self.logs_download_dir is None: + def __post_init__(self, logs_download_dir_init: Optional[Path]) -> None: + if logs_download_dir_init is not None: + object.__setattr__(self, "logs_download_dir", logs_download_dir_init) + else: object.__setattr__(self, "logs_download_dir", self.test_root_dir / "downloads") self.test_root_dir.mkdir(parents=True, exist_ok=True) @@ -89,7 +92,7 @@ class IntegrationTestLogs: #: extraction_dir: Path = field(init=False, repr=True) - def __post_init__(self, integration_test_config: IntegrationTestConfig): + def __post_init__(self, integration_test_config: IntegrationTestConfig) -> None: name = self.name.strip() if 0 == len(name): raise ValueError("`name` cannot be empty.") @@ -116,7 +119,7 @@ class CompressionTestConfig: #: Path to store decompressed logs generated by the test. decompression_dir: Path = field(init=False, repr=True) - def __post_init__(self, integration_test_config: IntegrationTestConfig): + def __post_init__(self, integration_test_config: IntegrationTestConfig) -> None: test_name = self.test_name.strip() if 0 == len(test_name): raise ValueError("`test_name` cannot be empty.") diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index 1d5f20b556..e32ccd352f 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -2,7 +2,7 @@ import subprocess from pathlib import Path from tempfile import NamedTemporaryFile -from typing import IO, List +from typing import Any, IO, List def is_json_file_structurally_equal(json_fp1: Path, json_fp2: Path) -> bool: @@ -46,7 +46,7 @@ def get_env_var(var_name: str) -> str: return value -def run_and_assert(cmd: List[str], **kwargs) -> subprocess.CompletedProcess: +def run_and_assert(cmd: List[str], **kwargs: Any) -> subprocess.CompletedProcess[Any]: """ Runs a command with subprocess and asserts that it succeeds with pytest. From ca4301431d8d49760ec7a8cfa360bc1e1ae8369b Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 12:46:24 +0000 Subject: [PATCH 054/100] Disable warning about assert --- integration-tests/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index c7ea37ea8f..bcac3408c0 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -49,6 +49,7 @@ ignore = [ "FIX002", # Allow todo statements "PERF401", # Allow for loops when creating lists "PERF403", # Allow for loops when creating dicts + "S101", # Allow usage of pytest `assert` "S311", # Allow usage of `random` package "SIM102", # Allow collapsible if statements for readability "TD002", # Author unnecessary for todo statement From ebbc541b528b009ae91536d41e82007b17cbdf84 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 15:19:42 +0000 Subject: [PATCH 055/100] Add mypy taskflow and fix all ruff complaints --- integration-tests/.pytest.ini | 5 ++ integration-tests/pyproject.toml | 9 +++- .../tests/fixtures/integration_test_config.py | 2 + .../tests/fixtures/integration_test_logs.py | 14 +++-- .../tests/test_identity_transformation.py | 23 ++++++-- integration-tests/tests/utils/config.py | 33 +++++++----- integration-tests/tests/utils/utils.py | 52 ++++++++++++------- lint-requirements.txt | 2 +- taskfile.yaml | 1 + taskfiles/lint.yaml | 1 + taskfiles/tests/integration.yaml | 3 -- 11 files changed, 100 insertions(+), 45 deletions(-) diff --git a/integration-tests/.pytest.ini b/integration-tests/.pytest.ini index f2dd45df35..ab8499dc06 100644 --- a/integration-tests/.pytest.ini +++ b/integration-tests/.pytest.ini @@ -10,7 +10,12 @@ env = D:CLP_BUILD_DIR=../build D:CLP_CORE_BINS_DIR=../build/core D:CLP_PACKAGE_DIR=../build/clp-package +log_cli = True +log_cli_date_format = %Y-%m-%d %H:%M:%S,%f +log_cli_format = %(name)s %(asctime)s [%(levelname)s] %(message)s +log_cli_level = INFO markers = clp: mark tests that use the CLP storage engine clp_s: mark tests that use the CLP-S storage engine core: mark tests that test the CLP core binaries + shit: shit diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index bcac3408c0..0db8ed285f 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -38,7 +38,12 @@ line-length = 100 [tool.ruff.lint] select = ["ALL"] ignore = [ + "ANN401", # Allow using `Any` type for function signatures "COM812", # Redundant and conflicts with ruff format + "TC003", # Ignore performance overhead of imports only used for type checking + "D100", # Allow missing docstrings in modules + "D104", # Allow missing docstrings in public packages + "D105", # Allow missing docstrings in magic methods beceause they are usually self-explanatory "D203", # No blank line before docstrings (D211) "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists "D212", # Enforce docstring summary line on the next line after quotes (D213) @@ -47,10 +52,12 @@ ignore = [ "D415", # First line of docstrings may not end in a period, question mark, or exclamation point "FBT", # Allow bool positional parameters since other value positions are allowed "FIX002", # Allow todo statements + "G004", # Allow logging statement to use f-string "PERF401", # Allow for loops when creating lists "PERF403", # Allow for loops when creating dicts - "S101", # Allow usage of pytest `assert` + "S101", # Allow usage of pytest `assert` "S311", # Allow usage of `random` package + "S603", # Automatically trust inputs of subprocess execution "SIM102", # Allow collapsible if statements for readability "TD002", # Author unnecessary for todo statement "TD003", # Issue link unnecessary for todo statement diff --git a/integration-tests/tests/fixtures/integration_test_config.py b/integration-tests/tests/fixtures/integration_test_config.py index 0226a116d1..3e5f6cc345 100644 --- a/integration-tests/tests/fixtures/integration_test_config.py +++ b/integration-tests/tests/fixtures/integration_test_config.py @@ -1,6 +1,7 @@ from pathlib import Path import pytest + from tests.utils.config import ( CoreConfig, IntegrationTestConfig, @@ -11,6 +12,7 @@ @pytest.fixture(scope="session") def integration_test_config() -> IntegrationTestConfig: + """Fixture that provides an IntegrationTestConfig shared across tests.""" core_config = CoreConfig(clp_core_bins_dir=Path(get_env_var("CLP_CORE_BINS_DIR")).resolve()) package_config = PackageConfig(clp_package_dir=Path(get_env_var("CLP_PACKAGE_DIR")).resolve()) return IntegrationTestConfig( diff --git a/integration-tests/tests/fixtures/integration_test_logs.py b/integration-tests/tests/fixtures/integration_test_logs.py index 38c47d5ee5..17771edb37 100644 --- a/integration-tests/tests/fixtures/integration_test_logs.py +++ b/integration-tests/tests/fixtures/integration_test_logs.py @@ -1,19 +1,23 @@ -import os +import logging import shutil import subprocess import pytest + from tests.utils.config import ( IntegrationTestConfig, IntegrationTestLogs, ) +logger = logging.getLogger(__name__) + @pytest.fixture(scope="session") def hive_24hr( request: pytest.FixtureRequest, integration_test_config: IntegrationTestConfig, ) -> IntegrationTestLogs: + """Fixture that provides `hive_24hr` test logs shared across tests.""" return _download_and_extract_dataset( request=request, integration_test_config=integration_test_config, @@ -27,6 +31,7 @@ def postgresql( request: pytest.FixtureRequest, integration_test_config: IntegrationTestConfig, ) -> IntegrationTestLogs: + """Fixture that provides `postgresql` test logs shared across tests.""" return _download_and_extract_dataset( request=request, integration_test_config=integration_test_config, @@ -47,7 +52,7 @@ def _download_and_extract_dataset( integration_test_config=integration_test_config, ) if request.config.cache.get(name, False): - print(f"Test logs `{name}` is up-to-date. Skipping download.") + logger.info(f"Test logs `{name}` is up-to-date. Skipping download.") return integration_test_logs try: @@ -68,8 +73,9 @@ def _download_and_extract_dataset( integration_test_logs.tarball_path, integration_test_logs.extraction_dir ) except Exception as e: - raise RuntimeError(f"Failed to download and extract dataset `{name}`: {e}") from e + err_msg = f"Failed to download and extract dataset `{name}`." + raise RuntimeError(err_msg) from e - print(f"Downloaded and extracted uncompressed logs for dataset `{name}`.") + logger.info(f"Downloaded and extracted uncompressed logs for dataset `{name}`.") request.config.cache.set(name, True) return integration_test_logs diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index f6f5e411a8..e92047e50c 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -1,6 +1,5 @@ -from pathlib import Path - import pytest + from tests.utils.config import ( CompressionTestConfig, IntegrationTestConfig, @@ -36,6 +35,13 @@ def test_clp_identity_transform( integration_test_config: IntegrationTestConfig, test_logs_fixture: str, ) -> None: + """ + Validate that compression and decompression by the core binary `clp` are lossless. + + :param request: + :param integration_test_config: + :param test_logs_fixture: + """ integration_test_logs: IntegrationTestLogs = request.getfixturevalue(test_logs_fixture) test_paths = CompressionTestConfig( test_name=f"clp-{integration_test_logs.name}", @@ -80,6 +86,13 @@ def test_clp_s_identity_transform( integration_test_config: IntegrationTestConfig, test_logs_fixture: str, ) -> None: + """ + Validate that compression and decompression by the core binary `clp-s` are lossless. + + :param request: + :param integration_test_config: + :param test_logs_fixture: + """ integration_test_logs: IntegrationTestLogs = request.getfixturevalue(test_logs_fixture) test_logs_name = integration_test_logs.name @@ -106,9 +119,9 @@ def test_clp_s_identity_transform( _consolidated_json_file_name = "original" input_path = consolidated_json_test_paths.logs_source_dir / _consolidated_json_file_name output_path = consolidated_json_test_paths.decompression_dir / _consolidated_json_file_name - assert is_json_file_structurally_equal(input_path, output_path), ( - f"Mismatch between clp-s input {input_path} and output {output_path}." - ) + assert is_json_file_structurally_equal( + input_path, output_path + ), f"Mismatch between clp-s input {input_path} and output {output_path}." test_paths.clear_test_outputs() consolidated_json_test_paths.clear_test_outputs() diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index b2dbec84ed..7005b18650 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -1,7 +1,8 @@ +from __future__ import annotations + import shutil from dataclasses import dataclass, field, InitVar from pathlib import Path -from typing import Optional from tests.utils.utils import validate_dir_exists @@ -21,15 +22,18 @@ def __post_init__(self) -> None: required_binaries = ["clg", "clo", "clp", "clp-s", "indexer", "reducer-server"] missing_binaries = [b for b in required_binaries if not (clp_core_bins_dir / b).is_file()] if len(missing_binaries) > 0: - raise ValueError( - f"CLP core bins at {clp_core_bins_dir} is incomplete. " - f"Missing binaries: {', '.join(missing_binaries)}" + err_msg = ( + f"CLP core bins at {clp_core_bins_dir} is incomplete." + f" Missing binaries: {', '.join(missing_binaries)}" ) + raise ValueError(err_msg) def get_clp_binary_path(self) -> Path: + """:return: The absolute path to the core binary `clp`.""" return self.clp_core_bins_dir / "clp" def get_clp_s_binary_path(self) -> Path: + """:return: The absolute path to the core binary `clp-s`.""" return self.clp_core_bins_dir / "clp-s" @@ -48,10 +52,11 @@ def __post_init__(self) -> None: required_dirs = ["bin", "etc", "lib", "sbin"] missing_dirs = [d for d in required_dirs if not (clp_package_dir / d).is_dir()] if len(missing_dirs) > 0: - raise ValueError( - f"CLP package at {clp_package_dir} is incomplete. " - f"Missing: {', '.join(missing_dirs)}" + err_msg = ( + f"CLP package at {clp_package_dir} is incomplete." + f" Missing directories: {', '.join(missing_dirs)}" ) + raise ValueError(err_msg) @dataclass(frozen=True) @@ -64,11 +69,11 @@ class IntegrationTestConfig: package_config: PackageConfig #: Root directory for integration tests output. test_root_dir: Path - logs_download_dir_init: InitVar[Optional[Path]] = None + logs_download_dir_init: InitVar[Path | None] = None #: Directory to store the downloaded logs. logs_download_dir: Path = field(init=False, repr=True) - def __post_init__(self, logs_download_dir_init: Optional[Path]) -> None: + def __post_init__(self, logs_download_dir_init: Path | None) -> None: if logs_download_dir_init is not None: object.__setattr__(self, "logs_download_dir", logs_download_dir_init) else: @@ -94,8 +99,9 @@ class IntegrationTestLogs: def __post_init__(self, integration_test_config: IntegrationTestConfig) -> None: name = self.name.strip() - if 0 == len(name): - raise ValueError("`name` cannot be empty.") + if len(name) == 0: + err_msg = "`name` cannot be empty." + raise ValueError(err_msg) logs_download_dir = integration_test_config.logs_download_dir validate_dir_exists(logs_download_dir) @@ -121,8 +127,9 @@ class CompressionTestConfig: def __post_init__(self, integration_test_config: IntegrationTestConfig) -> None: test_name = self.test_name.strip() - if 0 == len(test_name): - raise ValueError("`test_name` cannot be empty.") + if len(test_name) == 0: + err_msg = "`test_name` cannot be empty." + raise ValueError(err_msg) test_root_dir = integration_test_config.test_root_dir validate_dir_exists(test_root_dir) diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index e32ccd352f..d8dcf6987d 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -1,8 +1,11 @@ import os +import shutil import subprocess from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Any, IO, List +from typing import Any, IO + +import pytest def is_json_file_structurally_equal(json_fp1: Path, json_fp2: Path) -> bool: @@ -26,12 +29,13 @@ def is_dir_tree_content_equal(path1: Path, path2: Path) -> bool: :raise: RuntimeError if the diff command fails due to execution errors. """ cmd = ["diff", "--brief", "--recursive", str(path1), str(path2)] - proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if 0 == proc.returncode: + proc = subprocess.run(cmd, check=False, capture_output=True) + if proc.returncode == 0: return True - if 1 == proc.returncode: + if proc.returncode == 1: return False - raise RuntimeError(f"Command failed {' '.join(cmd)}: {proc.stderr.decode()}") + err_msg = f"Command failed {' '.join(cmd)}: {proc.stderr.decode()}" + raise RuntimeError(err_msg) def get_env_var(var_name: str) -> str: @@ -42,11 +46,12 @@ def get_env_var(var_name: str) -> str: """ value = os.environ.get(var_name) if value is None: - raise ValueError(f"Environment variable {var_name} is not set.") + err_msg = f"Environment variable {var_name} is not set." + raise ValueError(err_msg) return value -def run_and_assert(cmd: List[str], **kwargs: Any) -> subprocess.CompletedProcess[Any]: +def run_and_assert(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess[Any]: """ Runs a command with subprocess and asserts that it succeeds with pytest. @@ -55,8 +60,10 @@ def run_and_assert(cmd: List[str], **kwargs: Any) -> subprocess.CompletedProcess :return: The completed process object, for inspection or further handling. :raise: AssertionError if the command exits with a non-zero return code. """ - proc = subprocess.run(cmd, **kwargs) - assert 0 == proc.returncode, f"Command failed: {' '.join(cmd)}" + try: + proc = subprocess.run(cmd, check=True, **kwargs) + except subprocess.CalledProcessError as e: + pytest.fail(f"Command failed: {' '.join(cmd)}: {e}") return proc @@ -66,9 +73,10 @@ def validate_dir_exists(dir_path: Path) -> None: :raise: ValueError if the path does not exist or is not a directory. """ if not dir_path.exists(): - raise ValueError(f"Directory does not exist: {dir_path}") - if not dir_path.is_dir(): - raise ValueError(f"Path is not a directory: {dir_path}") + err_msg = f"Directory does not exist: {dir_path}" + elif not dir_path.is_dir(): + err_msg = f"Path is not a directory: {dir_path}" + raise ValueError(err_msg) def _sort_json_keys_and_rows(json_fp: Path) -> IO[str]: @@ -77,24 +85,32 @@ def _sort_json_keys_and_rows(json_fp: Path) -> IO[str]: :param json_fp: :return: A named temoprary file (delete on close) that contains the sorted JSON content. - :raise: RuntimeError if either jq or sort fails due to execution errors. + :raise: RuntimeError if either jq or sort is missing or fails due to execution errors. """ - sorted_fp = NamedTemporaryFile(mode="w+", delete=True) + jq_bin = shutil.which("jq") + sort_bin = shutil.which("sort") + if jq_bin is None or sort_bin is None: + err_msg = "jq/sort executable not found" + raise RuntimeError(err_msg) + + sorted_fp = NamedTemporaryFile(mode="w+", delete=True) # noqa: SIM115 jq_proc = subprocess.Popen( - ["jq", "--sort-keys", "--compact-output", ".", str(json_fp)], + [jq_bin, "--sort-keys", "--compact-output", ".", str(json_fp)], stdout=subprocess.PIPE, ) try: - sort_proc = subprocess.run(["sort"], stdin=jq_proc.stdout, stdout=sorted_fp, check=True) + sort_proc = subprocess.run([sort_bin], stdin=jq_proc.stdout, stdout=sorted_fp, check=True) sort_rc = sort_proc.returncode if sort_rc != 0: - raise RuntimeError(f"sort failed with exist code {sort_rc} for {json_fp}") + err_msg = f"sort failed with exist code {sort_rc} for {json_fp}" + raise RuntimeError(err_msg) finally: if jq_proc.stdout is not None: jq_proc.stdout.close() jq_rc = jq_proc.wait() if jq_rc != 0: - raise RuntimeError(f"jq failed with exit code {jq_rc} for {json_fp}") + err_msg = f"jq failed with exit code {jq_rc} for {json_fp}" + raise RuntimeError(err_msg) sorted_fp.flush() sorted_fp.seek(0) diff --git a/lint-requirements.txt b/lint-requirements.txt index 1916faa61c..b4555b3d46 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -2,5 +2,5 @@ black>=24.4.2 clang-format>=20.1 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 -ruff>=0.4.4 +ruff>=0.11.12 yamllint>=1.35.1 diff --git a/taskfile.yaml b/taskfile.yaml index a0e15461ff..650f019d9c 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -15,6 +15,7 @@ vars: G_COMPONENTS_DIR: "{{.ROOT_DIR}}/components" G_CORE_COMPONENT_DIR: "{{.G_COMPONENTS_DIR}}/core" G_CORE_COMPONENT_SUBMODULES_DIR: "{{.G_CORE_COMPONENT_DIR}}/submodules" + G_INTEGRATION_TESTS_DIR: "{{.ROOT_DIR}}/integration-tests" G_WEBUI_SRC_DIR: "{{.G_COMPONENTS_DIR}}/webui" # Build paths diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 0034781776..97dc2c5788 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -787,6 +787,7 @@ tasks: . "{{.G_LINT_VENV_DIR}}/bin/activate" cd "{{.ITEM}}" black --color --line-length 100 {{.BLACK_FLAGS}} . + uv run --directory {{.G_INTEGRATION_TESTS_DIR}} mypy . ruff check {{.RUFF_FLAGS}} . venv: diff --git a/taskfiles/tests/integration.yaml b/taskfiles/tests/integration.yaml index 46df657de0..193eba81ae 100644 --- a/taskfiles/tests/integration.yaml +++ b/taskfiles/tests/integration.yaml @@ -1,8 +1,5 @@ version: "3" -vars: - G_INTEGRATION_TESTS_DIR: "{{.ROOT_DIR}}/integration-tests" - env: CLP_BUILD_DIR: "{{.G_BUILD_DIR}}" CLP_CORE_BINS_DIR: "{{.G_CORE_COMPONENT_BUILD_DIR}}" From baa6e9da420e5ca76cab0fc6ec5f59c28f7ceb72 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 23:23:05 +0800 Subject: [PATCH 056/100] Update integration-tests/.pytest.ini --- integration-tests/.pytest.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-tests/.pytest.ini b/integration-tests/.pytest.ini index ab8499dc06..6235100f27 100644 --- a/integration-tests/.pytest.ini +++ b/integration-tests/.pytest.ini @@ -18,4 +18,3 @@ markers = clp: mark tests that use the CLP storage engine clp_s: mark tests that use the CLP-S storage engine core: mark tests that test the CLP core binaries - shit: shit From d22c5344271881b151d35a31b2ff240ae082e49b Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 11:27:44 -0400 Subject: [PATCH 057/100] Address coderabbit comment --- integration-tests/tests/utils/utils.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index d8dcf6987d..45b9698d6e 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -84,7 +84,7 @@ def _sort_json_keys_and_rows(json_fp: Path) -> IO[str]: Normalize a JSON file to a stable, deterministically ordered form for comparison. :param json_fp: - :return: A named temoprary file (delete on close) that contains the sorted JSON content. + :return: A named temporary file (delete on close) that contains the sorted JSON content. :raise: RuntimeError if either jq or sort is missing or fails due to execution errors. """ jq_bin = shutil.which("jq") @@ -100,10 +100,6 @@ def _sort_json_keys_and_rows(json_fp: Path) -> IO[str]: ) try: sort_proc = subprocess.run([sort_bin], stdin=jq_proc.stdout, stdout=sorted_fp, check=True) - sort_rc = sort_proc.returncode - if sort_rc != 0: - err_msg = f"sort failed with exist code {sort_rc} for {json_fp}" - raise RuntimeError(err_msg) finally: if jq_proc.stdout is not None: jq_proc.stdout.close() From 98330695c37c888c9582a74741490f190ee4c86c Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 15:48:49 +0000 Subject: [PATCH 058/100] Lint fix --- integration-tests/tests/utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index 45b9698d6e..272ee5cd8c 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -99,7 +99,7 @@ def _sort_json_keys_and_rows(json_fp: Path) -> IO[str]: stdout=subprocess.PIPE, ) try: - sort_proc = subprocess.run([sort_bin], stdin=jq_proc.stdout, stdout=sorted_fp, check=True) + subprocess.run([sort_bin], stdin=jq_proc.stdout, stdout=sorted_fp, check=True) finally: if jq_proc.stdout is not None: jq_proc.stdout.close() From 7c818d46b354eebd0dcadacad33eeb54e4fd4e47 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 15:54:52 +0000 Subject: [PATCH 059/100] logic fix --- integration-tests/tests/utils/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index 272ee5cd8c..80b43cf069 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -74,9 +74,10 @@ def validate_dir_exists(dir_path: Path) -> None: """ if not dir_path.exists(): err_msg = f"Directory does not exist: {dir_path}" - elif not dir_path.is_dir(): + raise ValueError(err_msg) + if not dir_path.is_dir(): err_msg = f"Path is not a directory: {dir_path}" - raise ValueError(err_msg) + raise ValueError(err_msg) def _sort_json_keys_and_rows(json_fp: Path) -> IO[str]: From 37c0e23a7cd5e8dd977e156ace02b7cf17f1a8be Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 15:57:51 +0000 Subject: [PATCH 060/100] Improve docstrings --- integration-tests/tests/test_identity_transformation.py | 6 ++++-- integration-tests/tests/utils/utils.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index e92047e50c..df9259e5d2 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -36,7 +36,8 @@ def test_clp_identity_transform( test_logs_fixture: str, ) -> None: """ - Validate that compression and decompression by the core binary `clp` are lossless. + Validate that compression and decompression by the core binary `clp` run successfully and are + lossless. :param request: :param integration_test_config: @@ -87,7 +88,8 @@ def test_clp_s_identity_transform( test_logs_fixture: str, ) -> None: """ - Validate that compression and decompression by the core binary `clp-s` are lossless. + Validate that compression and decompression by the core binary `clp-s` run successfully and are + lossless. :param request: :param integration_test_config: diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index 80b43cf069..01d2fc49c4 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -58,7 +58,7 @@ def run_and_assert(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess :param cmd: Command and arguments to execute. :param kwargs: Additional keyword arguments passed through to the subprocess. :return: The completed process object, for inspection or further handling. - :raise: AssertionError if the command exits with a non-zero return code. + :raise: pytest.fail if the command exits with a non-zero return code. """ try: proc = subprocess.run(cmd, check=True, **kwargs) From c608df6d394b57888c9a08c609f30b6ffe6312d7 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 16:08:38 +0000 Subject: [PATCH 061/100] Add linting section to README --- integration-tests/README.md | 40 +++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/integration-tests/README.md b/integration-tests/README.md index 330b2c1c39..d9a36983a5 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -2,6 +2,15 @@ This Python project provides end-to-end tests for CLP using the `pytest` framework. +# Contributing +Follow the steps below to develop and contribute to the project. + +## Requirements + +* Python >= 3.9 +* [Task] >= 3.40.0 +* [uv] >= 0.8 + ## Running tests To run all integration tests: @@ -39,3 +48,34 @@ You can override the default test paths by setting the following environment var - **`CLP_CORE_BINS_DIR`** — Path to CLP core binaries, for testing only the core executables. These variables let you point the test suite at non-default build or install locations. + +## Linting + +Before submitting a pull request, ensure you’ve run the linting commands below and have fixed all +violations and suppressed all warnings. + +The following commands should be run from the integration-tests directory. + +To run ruff format checks: +```shell +uv run ruff format --check . +``` + +To run ruff format fixes: +```shell +uv run ruff format . +``` + +To run mypy static checks: +```shell +uv run mypy . +``` + +To run ruff static checks: +```shell +uv run ruff --check . +``` + + +[Task]: https://taskfile.dev +[uv]: https://docs.astral.sh/uv From 1cae84e68bbd675d14251ba012dbc453194f0563 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 18 Aug 2025 00:11:56 +0800 Subject: [PATCH 062/100] Apply suggestions from code review Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- integration-tests/pyproject.toml | 2 +- integration-tests/tests/utils/config.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index 0db8ed285f..59b9c31daf 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -43,7 +43,7 @@ ignore = [ "TC003", # Ignore performance overhead of imports only used for type checking "D100", # Allow missing docstrings in modules "D104", # Allow missing docstrings in public packages - "D105", # Allow missing docstrings in magic methods beceause they are usually self-explanatory + "D105", # Allow missing docstrings in magic methods because they are usually self-explanatory "D203", # No blank line before docstrings (D211) "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists "D212", # Enforce docstring summary line on the next line after quotes (D213) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 7005b18650..8c1173984d 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -99,10 +99,9 @@ class IntegrationTestLogs: def __post_init__(self, integration_test_config: IntegrationTestConfig) -> None: name = self.name.strip() - if len(name) == 0: + if 0 == len(name): err_msg = "`name` cannot be empty." raise ValueError(err_msg) - logs_download_dir = integration_test_config.logs_download_dir validate_dir_exists(logs_download_dir) @@ -127,10 +126,9 @@ class CompressionTestConfig: def __post_init__(self, integration_test_config: IntegrationTestConfig) -> None: test_name = self.test_name.strip() - if len(test_name) == 0: + if 0 == len(test_name): err_msg = "`test_name` cannot be empty." raise ValueError(err_msg) - test_root_dir = integration_test_config.test_root_dir validate_dir_exists(test_root_dir) From 1aa50e61b71947517ecb11025e331520121d21ae Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 16:16:10 +0000 Subject: [PATCH 063/100] Add yoda-condition check skips --- integration-tests/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index 59b9c31daf..175d18fd1a 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -59,6 +59,7 @@ ignore = [ "S311", # Allow usage of `random` package "S603", # Automatically trust inputs of subprocess execution "SIM102", # Allow collapsible if statements for readability + "SIM300", # Skip Yoda-condition format fixes "TD002", # Author unnecessary for todo statement "TD003", # Issue link unnecessary for todo statement "UP015", # Explicit open modes are helpful From d49cedd1832dc393ed78d5475efe16f18076628d Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 18 Aug 2025 00:19:17 +0800 Subject: [PATCH 064/100] Update integration-tests/README.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- integration-tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/README.md b/integration-tests/README.md index d9a36983a5..9f6216a6a3 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -76,6 +76,6 @@ To run ruff static checks: uv run ruff --check . ``` - +[Task]: https://taskfile.dev [Task]: https://taskfile.dev [uv]: https://docs.astral.sh/uv From 5ea32897b2bd5a97337588cef5801cd456c0962d Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 16:20:36 +0000 Subject: [PATCH 065/100] Space out README code section --- integration-tests/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration-tests/README.md b/integration-tests/README.md index 9f6216a6a3..76e51192db 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -57,21 +57,25 @@ violations and suppressed all warnings. The following commands should be run from the integration-tests directory. To run ruff format checks: + ```shell uv run ruff format --check . ``` To run ruff format fixes: + ```shell uv run ruff format . ``` To run mypy static checks: + ```shell uv run mypy . ``` To run ruff static checks: + ```shell uv run ruff --check . ``` From 8f81696e99c4f583088fde4ccce858089d917d07 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 18 Aug 2025 00:52:45 +0800 Subject: [PATCH 066/100] Update integration-tests/pyproject.toml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- integration-tests/pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index 175d18fd1a..f2a647de42 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -64,7 +64,8 @@ ignore = [ "TD003", # Issue link unnecessary for todo statement "UP015", # Explicit open modes are helpful ] -isort.order-by-type = false +[tool.ruff.lint.isort] +order-by-type = false [tool.ruff.format] docstring-code-format = true From 1d63ccc33bfcb7fc29e483e6b89ddd1a02464033 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 13:11:46 -0400 Subject: [PATCH 067/100] Make use of python class property --- integration-tests/tests/test_identity_transformation.py | 4 ++-- integration-tests/tests/utils/config.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index df9259e5d2..6312360d44 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -51,7 +51,7 @@ def test_clp_identity_transform( ) test_paths.clear_test_outputs() - bin_path = str(integration_test_config.core_config.get_clp_binary_path()) + bin_path = str(integration_test_config.core_config.clp_binary_path) src_path = str(test_paths.logs_source_dir) compression_path = str(test_paths.compression_dir) decompression_path = str(test_paths.decompression_dir) @@ -133,7 +133,7 @@ def _clp_s_compress_and_decompress( integration_test_config: IntegrationTestConfig, test_paths: CompressionTestConfig ) -> None: test_paths.clear_test_outputs() - bin_path = str(integration_test_config.core_config.get_clp_s_binary_path()) + bin_path = str(integration_test_config.core_config.clp_s_binary_path) src_path = str(test_paths.logs_source_dir) compression_path = str(test_paths.compression_dir) decompression_path = str(test_paths.decompression_dir) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 8c1173984d..29c0ce8c77 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -28,11 +28,13 @@ def __post_init__(self) -> None: ) raise ValueError(err_msg) - def get_clp_binary_path(self) -> Path: + @property + def clp_binary_path(self) -> Path: """:return: The absolute path to the core binary `clp`.""" return self.clp_core_bins_dir / "clp" - def get_clp_s_binary_path(self) -> Path: + @property + def clp_s_binary_path(self) -> Path: """:return: The absolute path to the core binary `clp-s`.""" return self.clp_core_bins_dir / "clp-s" From 41393411eb5df147277ee8db4b6af5e4ac54dbe6 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 18 Aug 2025 03:14:59 +0800 Subject: [PATCH 068/100] Improve taskfile --- taskfiles/lint.yaml | 5 ++++- taskfiles/tests/integration.yaml | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 97dc2c5788..6e070f2efb 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -787,8 +787,11 @@ tasks: . "{{.G_LINT_VENV_DIR}}/bin/activate" cd "{{.ITEM}}" black --color --line-length 100 {{.BLACK_FLAGS}} . - uv run --directory {{.G_INTEGRATION_TESTS_DIR}} mypy . ruff check {{.RUFF_FLAGS}} . + - for: + - "{{.G_INTEGRATION_TESTS_DIR}}" + cmd: |- + uv run --directory {{.ITEM}} mypy . venv: internal: true diff --git a/taskfiles/tests/integration.yaml b/taskfiles/tests/integration.yaml index 193eba81ae..f83c087884 100644 --- a/taskfiles/tests/integration.yaml +++ b/taskfiles/tests/integration.yaml @@ -8,7 +8,7 @@ env: tasks: default: deps: - - "core" + - task: "core" cache-clear: dir: "{{.G_INTEGRATION_TESTS_DIR}}" @@ -19,7 +19,7 @@ tasks: core: deps: - - "::core" + - task: "::core" dir: "{{.G_INTEGRATION_TESTS_DIR}}" cmds: - >- From d5b7f760a9e395f21666813d796e9abc49553085 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 15:23:58 -0400 Subject: [PATCH 069/100] Fix tab spaces --- taskfiles/lint.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 6e070f2efb..1d6d705a33 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -788,10 +788,10 @@ tasks: cd "{{.ITEM}}" black --color --line-length 100 {{.BLACK_FLAGS}} . ruff check {{.RUFF_FLAGS}} . - - for: - - "{{.G_INTEGRATION_TESTS_DIR}}" - cmd: |- - uv run --directory {{.ITEM}} mypy . + - for: + - "{{.G_INTEGRATION_TESTS_DIR}}" + cmd: |- + uv run --directory {{.ITEM}} mypy . venv: internal: true From 9f30065f9f666b52ba09c3fc4c2dfab1803a1a2a Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 15:24:37 -0400 Subject: [PATCH 070/100] Create taskfile python linting for projects using uv --- integration-tests/pyproject.toml | 1 + integration-tests/uv.lock | 84 ++++++++++++++++++++++++++++++++ lint-requirements.txt | 2 +- taskfiles/lint.yaml | 10 ++-- 4 files changed, 93 insertions(+), 4 deletions(-) diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index f2a647de42..a6d22c97ca 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -17,6 +17,7 @@ build-backend = "hatchling.build" [dependency-groups] dev = [ + "black>=24.4.2", "mypy>=1.16.0", "ruff>=0.11.12", "pytest>=8.3.5", diff --git a/integration-tests/uv.lock b/integration-tests/uv.lock index c6cd245d81..580fb96b6c 100644 --- a/integration-tests/uv.lock +++ b/integration-tests/uv.lock @@ -1,6 +1,79 @@ version = 1 revision = 2 requires-python = ">=3.9" +resolution-markers = [ + "python_full_version >= '3.10'", + "python_full_version < '3.10'", +] + +[[package]] +name = "black" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "click", version = "8.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "mypy-extensions" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/94/49/26a7b0f3f35da4b5a65f081943b7bcd22d7002f5f0fb8098ec1ff21cb6ef/black-25.1.0.tar.gz", hash = "sha256:33496d5cd1222ad73391352b4ae8da15253c5de89b93a80b3e2c8d9a19ec2666", size = 649449, upload-time = "2025-01-29T04:15:40.373Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/3b/4ba3f93ac8d90410423fdd31d7541ada9bcee1df32fb90d26de41ed40e1d/black-25.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:759e7ec1e050a15f89b770cefbf91ebee8917aac5c20483bc2d80a6c3a04df32", size = 1629419, upload-time = "2025-01-29T05:37:06.642Z" }, + { url = "https://files.pythonhosted.org/packages/b4/02/0bde0485146a8a5e694daed47561785e8b77a0466ccc1f3e485d5ef2925e/black-25.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e519ecf93120f34243e6b0054db49c00a35f84f195d5bce7e9f5cfc578fc2da", size = 1461080, upload-time = "2025-01-29T05:37:09.321Z" }, + { url = "https://files.pythonhosted.org/packages/52/0e/abdf75183c830eaca7589144ff96d49bce73d7ec6ad12ef62185cc0f79a2/black-25.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:055e59b198df7ac0b7efca5ad7ff2516bca343276c466be72eb04a3bcc1f82d7", size = 1766886, upload-time = "2025-01-29T04:18:24.432Z" }, + { url = "https://files.pythonhosted.org/packages/dc/a6/97d8bb65b1d8a41f8a6736222ba0a334db7b7b77b8023ab4568288f23973/black-25.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:db8ea9917d6f8fc62abd90d944920d95e73c83a5ee3383493e35d271aca872e9", size = 1419404, upload-time = "2025-01-29T04:19:04.296Z" }, + { url = "https://files.pythonhosted.org/packages/7e/4f/87f596aca05c3ce5b94b8663dbfe242a12843caaa82dd3f85f1ffdc3f177/black-25.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a39337598244de4bae26475f77dda852ea00a93bd4c728e09eacd827ec929df0", size = 1614372, upload-time = "2025-01-29T05:37:11.71Z" }, + { url = "https://files.pythonhosted.org/packages/e7/d0/2c34c36190b741c59c901e56ab7f6e54dad8df05a6272a9747ecef7c6036/black-25.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96c1c7cd856bba8e20094e36e0f948718dc688dba4a9d78c3adde52b9e6c2299", size = 1442865, upload-time = "2025-01-29T05:37:14.309Z" }, + { url = "https://files.pythonhosted.org/packages/21/d4/7518c72262468430ead45cf22bd86c883a6448b9eb43672765d69a8f1248/black-25.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bce2e264d59c91e52d8000d507eb20a9aca4a778731a08cfff7e5ac4a4bb7096", size = 1749699, upload-time = "2025-01-29T04:18:17.688Z" }, + { url = "https://files.pythonhosted.org/packages/58/db/4f5beb989b547f79096e035c4981ceb36ac2b552d0ac5f2620e941501c99/black-25.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:172b1dbff09f86ce6f4eb8edf9dede08b1fce58ba194c87d7a4f1a5aa2f5b3c2", size = 1428028, upload-time = "2025-01-29T04:18:51.711Z" }, + { url = "https://files.pythonhosted.org/packages/83/71/3fe4741df7adf015ad8dfa082dd36c94ca86bb21f25608eb247b4afb15b2/black-25.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4b60580e829091e6f9238c848ea6750efed72140b91b048770b64e74fe04908b", size = 1650988, upload-time = "2025-01-29T05:37:16.707Z" }, + { url = "https://files.pythonhosted.org/packages/13/f3/89aac8a83d73937ccd39bbe8fc6ac8860c11cfa0af5b1c96d081facac844/black-25.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e2978f6df243b155ef5fa7e558a43037c3079093ed5d10fd84c43900f2d8ecc", size = 1453985, upload-time = "2025-01-29T05:37:18.273Z" }, + { url = "https://files.pythonhosted.org/packages/6f/22/b99efca33f1f3a1d2552c714b1e1b5ae92efac6c43e790ad539a163d1754/black-25.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b48735872ec535027d979e8dcb20bf4f70b5ac75a8ea99f127c106a7d7aba9f", size = 1783816, upload-time = "2025-01-29T04:18:33.823Z" }, + { url = "https://files.pythonhosted.org/packages/18/7e/a27c3ad3822b6f2e0e00d63d58ff6299a99a5b3aee69fa77cd4b0076b261/black-25.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:ea0213189960bda9cf99be5b8c8ce66bb054af5e9e861249cd23471bd7b0b3ba", size = 1440860, upload-time = "2025-01-29T04:19:12.944Z" }, + { url = "https://files.pythonhosted.org/packages/98/87/0edf98916640efa5d0696e1abb0a8357b52e69e82322628f25bf14d263d1/black-25.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f0b18a02996a836cc9c9c78e5babec10930862827b1b724ddfe98ccf2f2fe4f", size = 1650673, upload-time = "2025-01-29T05:37:20.574Z" }, + { url = "https://files.pythonhosted.org/packages/52/e5/f7bf17207cf87fa6e9b676576749c6b6ed0d70f179a3d812c997870291c3/black-25.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:afebb7098bfbc70037a053b91ae8437c3857482d3a690fefc03e9ff7aa9a5fd3", size = 1453190, upload-time = "2025-01-29T05:37:22.106Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ee/adda3d46d4a9120772fae6de454c8495603c37c4c3b9c60f25b1ab6401fe/black-25.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:030b9759066a4ee5e5aca28c3c77f9c64789cdd4de8ac1df642c40b708be6171", size = 1782926, upload-time = "2025-01-29T04:18:58.564Z" }, + { url = "https://files.pythonhosted.org/packages/cc/64/94eb5f45dcb997d2082f097a3944cfc7fe87e071907f677e80788a2d7b7a/black-25.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:a22f402b410566e2d1c950708c77ebf5ebd5d0d88a6a2e87c86d9fb48afa0d18", size = 1442613, upload-time = "2025-01-29T04:19:27.63Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b6/ae7507470a4830dbbfe875c701e84a4a5fb9183d1497834871a715716a92/black-25.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a1ee0a0c330f7b5130ce0caed9936a904793576ef4d2b98c40835d6a65afa6a0", size = 1628593, upload-time = "2025-01-29T05:37:23.672Z" }, + { url = "https://files.pythonhosted.org/packages/24/c1/ae36fa59a59f9363017ed397750a0cd79a470490860bc7713967d89cdd31/black-25.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3df5f1bf91d36002b0a75389ca8663510cf0531cca8aa5c1ef695b46d98655f", size = 1460000, upload-time = "2025-01-29T05:37:25.829Z" }, + { url = "https://files.pythonhosted.org/packages/ac/b6/98f832e7a6c49aa3a464760c67c7856363aa644f2f3c74cf7d624168607e/black-25.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9e6827d563a2c820772b32ce8a42828dc6790f095f441beef18f96aa6f8294e", size = 1765963, upload-time = "2025-01-29T04:18:38.116Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e9/2cb0a017eb7024f70e0d2e9bdb8c5a5b078c5740c7f8816065d06f04c557/black-25.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:bacabb307dca5ebaf9c118d2d2f6903da0d62c9faa82bd21a33eecc319559355", size = 1419419, upload-time = "2025-01-29T04:18:30.191Z" }, + { url = "https://files.pythonhosted.org/packages/09/71/54e999902aed72baf26bca0d50781b01838251a462612966e9fc4891eadd/black-25.1.0-py3-none-any.whl", hash = "sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717", size = 207646, upload-time = "2025-01-29T04:15:38.082Z" }, +] + +[[package]] +name = "click" +version = "8.1.8" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188, upload-time = "2024-12-21T18:38:41.666Z" }, +] + +[[package]] +name = "click" +version = "8.2.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.10'", +] +dependencies = [ + { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/6c/8ca2efa64cf75a977a0d7fac081354553ebe483345c734fb6b6515d96bbc/click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202", size = 286342, upload-time = "2025-05-20T23:19:49.832Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" }, +] [[package]] name = "colorama" @@ -39,6 +112,7 @@ source = { editable = "." } [package.dev-dependencies] dev = [ + { name = "black" }, { name = "mypy" }, { name = "pytest" }, { name = "pytest-env" }, @@ -49,6 +123,7 @@ dev = [ [package.metadata.requires-dev] dev = [ + { name = "black", specifier = ">=24.4.2" }, { name = "mypy", specifier = ">=1.16.0" }, { name = "pytest", specifier = ">=8.3.5" }, { name = "pytest-env", specifier = ">=1.1.5" }, @@ -133,6 +208,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, ] +[[package]] +name = "platformdirs" +version = "4.3.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/8b/3c73abc9c759ecd3f1f7ceff6685840859e8070c4d947c93fae71f6a0bf2/platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc", size = 21362, upload-time = "2025-05-07T22:47:42.121Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/39/979e8e21520d4e47a0bbe349e2713c0aac6f3d853d0e5b34d76206c439aa/platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4", size = 18567, upload-time = "2025-05-07T22:47:40.376Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" diff --git a/lint-requirements.txt b/lint-requirements.txt index b4555b3d46..1916faa61c 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -2,5 +2,5 @@ black>=24.4.2 clang-format>=20.1 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 -ruff>=0.11.12 +ruff>=0.4.4 yamllint>=1.35.1 diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 1d6d705a33..6b6b1d27a2 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -770,6 +770,9 @@ tasks: py: internal: true + vars: + UV_PYTHON_PROJECTS: + - "{{.G_INTEGRATION_TESTS_DIR}}" requires: vars: ["BLACK_FLAGS", "RUFF_FLAGS"] deps: ["venv"] @@ -782,16 +785,17 @@ tasks: - "tools/deployment" - "tools/scripts" - "docs/conf" - - "integration-tests" cmd: |- . "{{.G_LINT_VENV_DIR}}/bin/activate" cd "{{.ITEM}}" black --color --line-length 100 {{.BLACK_FLAGS}} . ruff check {{.RUFF_FLAGS}} . - for: - - "{{.G_INTEGRATION_TESTS_DIR}}" + var: UV_PYTHON_PROJECTS cmd: |- - uv run --directory {{.ITEM}} mypy . + uv run --directory "{{.ITEM}}" black --color --line-length 100 {{.BLACK_FLAGS}} . + uv run --directory "{{.ITEM}}" mypy . + uv run --directory "{{.ITEM}}" ruff check {{.RUFF_FLAGS}} . venv: internal: true From 664873438355154799c832eecd8feec887f298dc Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 17 Aug 2025 19:56:31 +0000 Subject: [PATCH 071/100] satisfy yaml linter --- taskfiles/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 6b6b1d27a2..8345017ec7 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -791,7 +791,7 @@ tasks: black --color --line-length 100 {{.BLACK_FLAGS}} . ruff check {{.RUFF_FLAGS}} . - for: - var: UV_PYTHON_PROJECTS + var: "UV_PYTHON_PROJECTS" cmd: |- uv run --directory "{{.ITEM}}" black --color --line-length 100 {{.BLACK_FLAGS}} . uv run --directory "{{.ITEM}}" mypy . From 6da26a1340f06115455bc0f0ac577fe1ded576ea Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 18 Aug 2025 07:00:27 -0400 Subject: [PATCH 072/100] Refactor docs. --- docs/src/dev-guide/index.md | 16 ++++ docs/src/dev-guide/testing/index.md | 19 +++++ .../dev-guide/testing/integration-tests.md | 49 +++++++++++ docs/src/dev-guide/testing/unit-tests.md | 5 ++ integration-tests/README.md | 84 +------------------ 5 files changed, 91 insertions(+), 82 deletions(-) create mode 100644 docs/src/dev-guide/testing/index.md create mode 100644 docs/src/dev-guide/testing/integration-tests.md create mode 100644 docs/src/dev-guide/testing/unit-tests.md diff --git a/docs/src/dev-guide/index.md b/docs/src/dev-guide/index.md index ae892e5707..fbcd9b9dc1 100644 --- a/docs/src/dev-guide/index.md +++ b/docs/src/dev-guide/index.md @@ -15,6 +15,13 @@ Building Docs about building CLP. ::: +:::{grid-item-card} +:link: testing +Testing +^^^ +Docs about testing CLP. +::: + :::{grid-item-card} :link: contributing-getting-started Contributing @@ -50,6 +57,15 @@ Any design docs describing parts of this project. building-package ::: +:::{toctree} +:caption: Testing +:hidden: + +testing/index +testing/unit-tests +testing/integration-tests +::: + :::{toctree} :caption: Contributing :hidden: diff --git a/docs/src/dev-guide/testing/index.md b/docs/src/dev-guide/testing/index.md new file mode 100644 index 0000000000..0fd9e03f82 --- /dev/null +++ b/docs/src/dev-guide/testing/index.md @@ -0,0 +1,19 @@ +# Overview + +::::{grid} 1 1 1 1 +:gutter: 2 + +:::{grid-item-card} +:link: unit-tests +Unit tests +^^^ +Docs about running unit tests for each component. +::: + +:::{grid-item-card} +:link: integration-tests +Integration tests +^^^ +Docs about running CLP's integration tests. +::: +:::: diff --git a/docs/src/dev-guide/testing/integration-tests.md b/docs/src/dev-guide/testing/integration-tests.md new file mode 100644 index 0000000000..ddb5139e16 --- /dev/null +++ b/docs/src/dev-guide/testing/integration-tests.md @@ -0,0 +1,49 @@ +# Integration tests + +The `integration-tests` directory contains a Python project that provides end-to-end tests for +CLP via the `pytest` framework. + +## Running tests + +To run all integration tests: + +```shell +task tests:integration +``` + +To test the core CLP binaries: + +```shell +task tests:integration:core +``` + +### Using `pytest` markers + +To run more specific sets of tests, you can use `pytest` directly with `pytest` markers. + +:::{note} +Before running tests using `pytest`, ensure that the CLP package and/or core binaries have been +built. +::: + +Ensure all commands below are run from inside the `integration-tests` directory. + +To list all available markers: + +```shell +uv run python -m pytest --markers +``` + +To run tests related to a specific marker (e.g., `clp_s`): +run: + +```shell +uv run python -m pytest -m clp_s +``` + +### Specifying custom CLP binary paths + +You can override the default binary paths by setting the following environment variables: + +* **`CLP_PACKAGE_DIR`**: Directory of the CLP package to test. +* **`CLP_CORE_BINS_DIR`**: Directory containing the CLP core binaries to test. diff --git a/docs/src/dev-guide/testing/unit-tests.md b/docs/src/dev-guide/testing/unit-tests.md new file mode 100644 index 0000000000..8a2bb742e1 --- /dev/null +++ b/docs/src/dev-guide/testing/unit-tests.md @@ -0,0 +1,5 @@ +# Unit tests + +:::{warning} +🚧 This section is under construction. +::: diff --git a/integration-tests/README.md b/integration-tests/README.md index 76e51192db..411373afed 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -1,85 +1,5 @@ # CLP integration tests -This Python project provides end-to-end tests for CLP using the `pytest` framework. +This Python project provides end-to-end tests for CLP via the `pytest` framework. -# Contributing -Follow the steps below to develop and contribute to the project. - -## Requirements - -* Python >= 3.9 -* [Task] >= 3.40.0 -* [uv] >= 0.8 - -## Running tests - -To run all integration tests: - -```shell -task tests:integration -``` - -To test the core CLP binaries: - -```shell -task tests:integration:core -``` - -## Testing customizations - -### Using pytest markers - -One can leverage `pytest` markers to selectively run tests, e.g. - -```shell -# From the integration-tests directory -uv run python -m pytest -m clp_s -``` - -only runs `clp-s` related tests. - -Before running tests, ensure all package script dependencies and binaries are built. - -### Specifying custom test directories - -You can override the default test paths by setting the following environment variables: - -- **`CLP_PACKAGE_DIR`** — Path to a custom CLP package directory (where package scripts live). -- **`CLP_CORE_BINS_DIR`** — Path to CLP core binaries, for testing only the core executables. - -These variables let you point the test suite at non-default build or install locations. - -## Linting - -Before submitting a pull request, ensure you’ve run the linting commands below and have fixed all -violations and suppressed all warnings. - -The following commands should be run from the integration-tests directory. - -To run ruff format checks: - -```shell -uv run ruff format --check . -``` - -To run ruff format fixes: - -```shell -uv run ruff format . -``` - -To run mypy static checks: - -```shell -uv run mypy . -``` - -To run ruff static checks: - -```shell -uv run ruff --check . -``` - -[Task]: https://taskfile.dev -[Task]: https://taskfile.dev -[uv]: https://docs.astral.sh/uv +For more information, see https://docs.yscope.com/clp/main/dev-guide/testing/integration-tests. From 789f4d1cd55bb84f6635913b5f69bc81f81aee0f Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 20 Aug 2025 08:26:02 +0800 Subject: [PATCH 073/100] Apply suggestions from code review Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- docs/src/dev-guide/testing/integration-tests.md | 4 ---- integration-tests/pyproject.toml | 5 ++--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/src/dev-guide/testing/integration-tests.md b/docs/src/dev-guide/testing/integration-tests.md index ddb5139e16..41ee819dab 100644 --- a/docs/src/dev-guide/testing/integration-tests.md +++ b/docs/src/dev-guide/testing/integration-tests.md @@ -35,11 +35,7 @@ uv run python -m pytest --markers ``` To run tests related to a specific marker (e.g., `clp_s`): -run: -```shell -uv run python -m pytest -m clp_s -``` ### Specifying custom CLP binary paths diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index a6d22c97ca..af9830fa55 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -20,7 +20,7 @@ dev = [ "black>=24.4.2", "mypy>=1.16.0", "ruff>=0.11.12", - "pytest>=8.3.5", + "pytest>=8.4.1", "pytest-env>=1.1.5", ] @@ -65,8 +65,7 @@ ignore = [ "TD003", # Issue link unnecessary for todo statement "UP015", # Explicit open modes are helpful ] -[tool.ruff.lint.isort] -order-by-type = false +isort.order-by-type = false [tool.ruff.format] docstring-code-format = true From d7619cb468f7e4bb032eb2a3094e11080cb581eb Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 20 Aug 2025 04:24:56 -0400 Subject: [PATCH 074/100] Address review concern --- integration-tests/pyproject.toml | 7 +- .../src/integration_tests/__init__.py | 1 + integration-tests/tests/__init__.py | 1 + integration-tests/tests/conftest.py | 2 + integration-tests/tests/fixtures/__init__.py | 1 + .../tests/fixtures/integration_test_config.py | 2 + .../tests/fixtures/integration_test_logs.py | 6 +- .../tests/test_identity_transformation.py | 7 +- integration-tests/tests/utils/__init__.py | 1 + integration-tests/tests/utils/assert_utils.py | 28 +++++++ integration-tests/tests/utils/config.py | 10 +++ integration-tests/tests/utils/utils.py | 22 +---- integration-tests/uv.lock | 82 +------------------ taskfiles/lint.yaml | 1 - 14 files changed, 61 insertions(+), 110 deletions(-) create mode 100644 integration-tests/tests/utils/assert_utils.py diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index af9830fa55..906705a0f2 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -17,7 +17,6 @@ build-backend = "hatchling.build" [dependency-groups] dev = [ - "black>=24.4.2", "mypy>=1.16.0", "ruff>=0.11.12", "pytest>=8.4.1", @@ -41,10 +40,6 @@ select = ["ALL"] ignore = [ "ANN401", # Allow using `Any` type for function signatures "COM812", # Redundant and conflicts with ruff format - "TC003", # Ignore performance overhead of imports only used for type checking - "D100", # Allow missing docstrings in modules - "D104", # Allow missing docstrings in public packages - "D105", # Allow missing docstrings in magic methods because they are usually self-explanatory "D203", # No blank line before docstrings (D211) "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists "D212", # Enforce docstring summary line on the next line after quotes (D213) @@ -53,7 +48,6 @@ ignore = [ "D415", # First line of docstrings may not end in a period, question mark, or exclamation point "FBT", # Allow bool positional parameters since other value positions are allowed "FIX002", # Allow todo statements - "G004", # Allow logging statement to use f-string "PERF401", # Allow for loops when creating lists "PERF403", # Allow for loops when creating dicts "S101", # Allow usage of pytest `assert` @@ -61,6 +55,7 @@ ignore = [ "S603", # Automatically trust inputs of subprocess execution "SIM102", # Allow collapsible if statements for readability "SIM300", # Skip Yoda-condition format fixes + "TC003", # Ignore performance overhead of imports only used for type checking "TD002", # Author unnecessary for todo statement "TD003", # Issue link unnecessary for todo statement "UP015", # Explicit open modes are helpful diff --git a/integration-tests/src/integration_tests/__init__.py b/integration-tests/src/integration_tests/__init__.py index e69de29bb2..dc20358fd2 100644 --- a/integration-tests/src/integration_tests/__init__.py +++ b/integration-tests/src/integration_tests/__init__.py @@ -0,0 +1 @@ +"""Integration test package for CLP.""" diff --git a/integration-tests/tests/__init__.py b/integration-tests/tests/__init__.py index e69de29bb2..5d36df00ef 100644 --- a/integration-tests/tests/__init__.py +++ b/integration-tests/tests/__init__.py @@ -0,0 +1 @@ +"""Top-level package for CLP integration tests.""" diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index e4517b45b4..2cb709a249 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -1,3 +1,5 @@ +"""Make the fixtures defined in `tests/fixtures/` globally available without imports.""" + pytest_plugins = [ "tests.fixtures.integration_test_config", "tests.fixtures.integration_test_logs", diff --git a/integration-tests/tests/fixtures/__init__.py b/integration-tests/tests/fixtures/__init__.py index e69de29bb2..a74b1a0c32 100644 --- a/integration-tests/tests/fixtures/__init__.py +++ b/integration-tests/tests/fixtures/__init__.py @@ -0,0 +1 @@ +"""Fixtures for CLP integration tests.""" diff --git a/integration-tests/tests/fixtures/integration_test_config.py b/integration-tests/tests/fixtures/integration_test_config.py index 3e5f6cc345..e6ae92cae6 100644 --- a/integration-tests/tests/fixtures/integration_test_config.py +++ b/integration-tests/tests/fixtures/integration_test_config.py @@ -1,3 +1,5 @@ +"""Define the integration test configuration fixture.""" + from pathlib import Path import pytest diff --git a/integration-tests/tests/fixtures/integration_test_logs.py b/integration-tests/tests/fixtures/integration_test_logs.py index 17771edb37..6df52ecb05 100644 --- a/integration-tests/tests/fixtures/integration_test_logs.py +++ b/integration-tests/tests/fixtures/integration_test_logs.py @@ -1,3 +1,5 @@ +"""Define test logs fixtures.""" + import logging import shutil import subprocess @@ -52,7 +54,7 @@ def _download_and_extract_dataset( integration_test_config=integration_test_config, ) if request.config.cache.get(name, False): - logger.info(f"Test logs `{name}` is up-to-date. Skipping download.") + logger.info("Test logs `%s` is up-to-date. Skipping download.", name) return integration_test_logs try: @@ -76,6 +78,6 @@ def _download_and_extract_dataset( err_msg = f"Failed to download and extract dataset `{name}`." raise RuntimeError(err_msg) from e - logger.info(f"Downloaded and extracted uncompressed logs for dataset `{name}`.") + logger.info("Downloaded and extracted uncompressed logs for dataset `%s`.", name) request.config.cache.set(name, True) return integration_test_logs diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 6312360d44..8b1a1758ef 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -1,5 +1,11 @@ +""" +Integration tests verifying that CLP core compression binaries perform lossless round-trip +compression and decompression. +""" + import pytest +from tests.utils.assert_utils import run_and_assert from tests.utils.config import ( CompressionTestConfig, IntegrationTestConfig, @@ -8,7 +14,6 @@ from tests.utils.utils import ( is_dir_tree_content_equal, is_json_file_structurally_equal, - run_and_assert, ) pytestmark = pytest.mark.core diff --git a/integration-tests/tests/utils/__init__.py b/integration-tests/tests/utils/__init__.py index e69de29bb2..15f5b9e1db 100644 --- a/integration-tests/tests/utils/__init__.py +++ b/integration-tests/tests/utils/__init__.py @@ -0,0 +1 @@ +"""Utility functions and configuration classes for CLP integration tests.""" diff --git a/integration-tests/tests/utils/assert_utils.py b/integration-tests/tests/utils/assert_utils.py new file mode 100644 index 0000000000..c3dc7ee8ea --- /dev/null +++ b/integration-tests/tests/utils/assert_utils.py @@ -0,0 +1,28 @@ +""" +Provide utility functions used across `integration-tests` that involve pytest assertions and +failures. +""" + +import subprocess +from typing import Any + +import pytest + + +def run_and_assert(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess[Any]: + """ + Runs a command with subprocess and asserts that it succeeds with pytest. + + :param cmd: Command and arguments to execute. + :param kwargs: Additional keyword arguments passed through to the subprocess. + :return: The completed process object, for inspection or further handling. + :raise: pytest.fail if the command exits with a non-zero return code. + """ + try: + proc = subprocess.run(cmd, check=True, **kwargs) + except subprocess.CalledProcessError as e: + pytest.fail(f"Command failed: {' '.join(cmd)}: {e}") + return proc + + + diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 29c0ce8c77..211cb3ff28 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -1,3 +1,5 @@ +"""Define all python classes used in `integration-tests`.""" + from __future__ import annotations import shutil @@ -15,6 +17,10 @@ class CoreConfig: clp_core_bins_dir: Path def __post_init__(self) -> None: + """ + Validate that the CLP core binaries directory exists and contains all required + executables. + """ clp_core_bins_dir = self.clp_core_bins_dir validate_dir_exists(clp_core_bins_dir) @@ -47,6 +53,7 @@ class PackageConfig: clp_package_dir: Path def __post_init__(self) -> None: + """Validate that the CLP package directory exists and contains all required directories.""" clp_package_dir = self.clp_package_dir validate_dir_exists(clp_package_dir) @@ -76,6 +83,7 @@ class IntegrationTestConfig: logs_download_dir: Path = field(init=False, repr=True) def __post_init__(self, logs_download_dir_init: Path | None) -> None: + """Initialize and create required directories for integration tests.""" if logs_download_dir_init is not None: object.__setattr__(self, "logs_download_dir", logs_download_dir_init) else: @@ -100,6 +108,7 @@ class IntegrationTestLogs: extraction_dir: Path = field(init=False, repr=True) def __post_init__(self, integration_test_config: IntegrationTestConfig) -> None: + """Initialize and set tarball and extraction paths for integration test logs.""" name = self.name.strip() if 0 == len(name): err_msg = "`name` cannot be empty." @@ -127,6 +136,7 @@ class CompressionTestConfig: decompression_dir: Path = field(init=False, repr=True) def __post_init__(self, integration_test_config: IntegrationTestConfig) -> None: + """Initialize and set required directory paths for compression tests.""" test_name = self.test_name.strip() if 0 == len(test_name): err_msg = "`test_name` cannot be empty." diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index 01d2fc49c4..460b7c0d7c 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -1,11 +1,11 @@ +"""Provide general utility functions used across `integration-tests`.""" + import os import shutil import subprocess from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Any, IO - -import pytest +from typing import IO def is_json_file_structurally_equal(json_fp1: Path, json_fp2: Path) -> bool: @@ -51,22 +51,6 @@ def get_env_var(var_name: str) -> str: return value -def run_and_assert(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess[Any]: - """ - Runs a command with subprocess and asserts that it succeeds with pytest. - - :param cmd: Command and arguments to execute. - :param kwargs: Additional keyword arguments passed through to the subprocess. - :return: The completed process object, for inspection or further handling. - :raise: pytest.fail if the command exits with a non-zero return code. - """ - try: - proc = subprocess.run(cmd, check=True, **kwargs) - except subprocess.CalledProcessError as e: - pytest.fail(f"Command failed: {' '.join(cmd)}: {e}") - return proc - - def validate_dir_exists(dir_path: Path) -> None: """ :param dir_path: diff --git a/integration-tests/uv.lock b/integration-tests/uv.lock index 580fb96b6c..67070d7ec5 100644 --- a/integration-tests/uv.lock +++ b/integration-tests/uv.lock @@ -6,75 +6,6 @@ resolution-markers = [ "python_full_version < '3.10'", ] -[[package]] -name = "black" -version = "25.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "click", version = "8.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "mypy-extensions" }, - { name = "packaging" }, - { name = "pathspec" }, - { name = "platformdirs" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/94/49/26a7b0f3f35da4b5a65f081943b7bcd22d7002f5f0fb8098ec1ff21cb6ef/black-25.1.0.tar.gz", hash = "sha256:33496d5cd1222ad73391352b4ae8da15253c5de89b93a80b3e2c8d9a19ec2666", size = 649449, upload-time = "2025-01-29T04:15:40.373Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/3b/4ba3f93ac8d90410423fdd31d7541ada9bcee1df32fb90d26de41ed40e1d/black-25.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:759e7ec1e050a15f89b770cefbf91ebee8917aac5c20483bc2d80a6c3a04df32", size = 1629419, upload-time = "2025-01-29T05:37:06.642Z" }, - { url = "https://files.pythonhosted.org/packages/b4/02/0bde0485146a8a5e694daed47561785e8b77a0466ccc1f3e485d5ef2925e/black-25.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e519ecf93120f34243e6b0054db49c00a35f84f195d5bce7e9f5cfc578fc2da", size = 1461080, upload-time = "2025-01-29T05:37:09.321Z" }, - { url = "https://files.pythonhosted.org/packages/52/0e/abdf75183c830eaca7589144ff96d49bce73d7ec6ad12ef62185cc0f79a2/black-25.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:055e59b198df7ac0b7efca5ad7ff2516bca343276c466be72eb04a3bcc1f82d7", size = 1766886, upload-time = "2025-01-29T04:18:24.432Z" }, - { url = "https://files.pythonhosted.org/packages/dc/a6/97d8bb65b1d8a41f8a6736222ba0a334db7b7b77b8023ab4568288f23973/black-25.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:db8ea9917d6f8fc62abd90d944920d95e73c83a5ee3383493e35d271aca872e9", size = 1419404, upload-time = "2025-01-29T04:19:04.296Z" }, - { url = "https://files.pythonhosted.org/packages/7e/4f/87f596aca05c3ce5b94b8663dbfe242a12843caaa82dd3f85f1ffdc3f177/black-25.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a39337598244de4bae26475f77dda852ea00a93bd4c728e09eacd827ec929df0", size = 1614372, upload-time = "2025-01-29T05:37:11.71Z" }, - { url = "https://files.pythonhosted.org/packages/e7/d0/2c34c36190b741c59c901e56ab7f6e54dad8df05a6272a9747ecef7c6036/black-25.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96c1c7cd856bba8e20094e36e0f948718dc688dba4a9d78c3adde52b9e6c2299", size = 1442865, upload-time = "2025-01-29T05:37:14.309Z" }, - { url = "https://files.pythonhosted.org/packages/21/d4/7518c72262468430ead45cf22bd86c883a6448b9eb43672765d69a8f1248/black-25.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bce2e264d59c91e52d8000d507eb20a9aca4a778731a08cfff7e5ac4a4bb7096", size = 1749699, upload-time = "2025-01-29T04:18:17.688Z" }, - { url = "https://files.pythonhosted.org/packages/58/db/4f5beb989b547f79096e035c4981ceb36ac2b552d0ac5f2620e941501c99/black-25.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:172b1dbff09f86ce6f4eb8edf9dede08b1fce58ba194c87d7a4f1a5aa2f5b3c2", size = 1428028, upload-time = "2025-01-29T04:18:51.711Z" }, - { url = "https://files.pythonhosted.org/packages/83/71/3fe4741df7adf015ad8dfa082dd36c94ca86bb21f25608eb247b4afb15b2/black-25.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4b60580e829091e6f9238c848ea6750efed72140b91b048770b64e74fe04908b", size = 1650988, upload-time = "2025-01-29T05:37:16.707Z" }, - { url = "https://files.pythonhosted.org/packages/13/f3/89aac8a83d73937ccd39bbe8fc6ac8860c11cfa0af5b1c96d081facac844/black-25.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e2978f6df243b155ef5fa7e558a43037c3079093ed5d10fd84c43900f2d8ecc", size = 1453985, upload-time = "2025-01-29T05:37:18.273Z" }, - { url = "https://files.pythonhosted.org/packages/6f/22/b99efca33f1f3a1d2552c714b1e1b5ae92efac6c43e790ad539a163d1754/black-25.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b48735872ec535027d979e8dcb20bf4f70b5ac75a8ea99f127c106a7d7aba9f", size = 1783816, upload-time = "2025-01-29T04:18:33.823Z" }, - { url = "https://files.pythonhosted.org/packages/18/7e/a27c3ad3822b6f2e0e00d63d58ff6299a99a5b3aee69fa77cd4b0076b261/black-25.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:ea0213189960bda9cf99be5b8c8ce66bb054af5e9e861249cd23471bd7b0b3ba", size = 1440860, upload-time = "2025-01-29T04:19:12.944Z" }, - { url = "https://files.pythonhosted.org/packages/98/87/0edf98916640efa5d0696e1abb0a8357b52e69e82322628f25bf14d263d1/black-25.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f0b18a02996a836cc9c9c78e5babec10930862827b1b724ddfe98ccf2f2fe4f", size = 1650673, upload-time = "2025-01-29T05:37:20.574Z" }, - { url = "https://files.pythonhosted.org/packages/52/e5/f7bf17207cf87fa6e9b676576749c6b6ed0d70f179a3d812c997870291c3/black-25.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:afebb7098bfbc70037a053b91ae8437c3857482d3a690fefc03e9ff7aa9a5fd3", size = 1453190, upload-time = "2025-01-29T05:37:22.106Z" }, - { url = "https://files.pythonhosted.org/packages/e3/ee/adda3d46d4a9120772fae6de454c8495603c37c4c3b9c60f25b1ab6401fe/black-25.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:030b9759066a4ee5e5aca28c3c77f9c64789cdd4de8ac1df642c40b708be6171", size = 1782926, upload-time = "2025-01-29T04:18:58.564Z" }, - { url = "https://files.pythonhosted.org/packages/cc/64/94eb5f45dcb997d2082f097a3944cfc7fe87e071907f677e80788a2d7b7a/black-25.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:a22f402b410566e2d1c950708c77ebf5ebd5d0d88a6a2e87c86d9fb48afa0d18", size = 1442613, upload-time = "2025-01-29T04:19:27.63Z" }, - { url = "https://files.pythonhosted.org/packages/d3/b6/ae7507470a4830dbbfe875c701e84a4a5fb9183d1497834871a715716a92/black-25.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a1ee0a0c330f7b5130ce0caed9936a904793576ef4d2b98c40835d6a65afa6a0", size = 1628593, upload-time = "2025-01-29T05:37:23.672Z" }, - { url = "https://files.pythonhosted.org/packages/24/c1/ae36fa59a59f9363017ed397750a0cd79a470490860bc7713967d89cdd31/black-25.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3df5f1bf91d36002b0a75389ca8663510cf0531cca8aa5c1ef695b46d98655f", size = 1460000, upload-time = "2025-01-29T05:37:25.829Z" }, - { url = "https://files.pythonhosted.org/packages/ac/b6/98f832e7a6c49aa3a464760c67c7856363aa644f2f3c74cf7d624168607e/black-25.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9e6827d563a2c820772b32ce8a42828dc6790f095f441beef18f96aa6f8294e", size = 1765963, upload-time = "2025-01-29T04:18:38.116Z" }, - { url = "https://files.pythonhosted.org/packages/ce/e9/2cb0a017eb7024f70e0d2e9bdb8c5a5b078c5740c7f8816065d06f04c557/black-25.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:bacabb307dca5ebaf9c118d2d2f6903da0d62c9faa82bd21a33eecc319559355", size = 1419419, upload-time = "2025-01-29T04:18:30.191Z" }, - { url = "https://files.pythonhosted.org/packages/09/71/54e999902aed72baf26bca0d50781b01838251a462612966e9fc4891eadd/black-25.1.0-py3-none-any.whl", hash = "sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717", size = 207646, upload-time = "2025-01-29T04:15:38.082Z" }, -] - -[[package]] -name = "click" -version = "8.1.8" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188, upload-time = "2024-12-21T18:38:41.666Z" }, -] - -[[package]] -name = "click" -version = "8.2.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.10'", -] -dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/60/6c/8ca2efa64cf75a977a0d7fac081354553ebe483345c734fb6b6515d96bbc/click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202", size = 286342, upload-time = "2025-05-20T23:19:49.832Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -112,7 +43,6 @@ source = { editable = "." } [package.dev-dependencies] dev = [ - { name = "black" }, { name = "mypy" }, { name = "pytest" }, { name = "pytest-env" }, @@ -123,9 +53,8 @@ dev = [ [package.metadata.requires-dev] dev = [ - { name = "black", specifier = ">=24.4.2" }, { name = "mypy", specifier = ">=1.16.0" }, - { name = "pytest", specifier = ">=8.3.5" }, + { name = "pytest", specifier = ">=8.4.1" }, { name = "pytest-env", specifier = ">=1.1.5" }, { name = "ruff", specifier = ">=0.11.12" }, ] @@ -208,15 +137,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, ] -[[package]] -name = "platformdirs" -version = "4.3.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fe/8b/3c73abc9c759ecd3f1f7ceff6685840859e8070c4d947c93fae71f6a0bf2/platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc", size = 21362, upload-time = "2025-05-07T22:47:42.121Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/39/979e8e21520d4e47a0bbe349e2713c0aac6f3d853d0e5b34d76206c439aa/platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4", size = 18567, upload-time = "2025-05-07T22:47:40.376Z" }, -] - [[package]] name = "pluggy" version = "1.6.0" diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 8345017ec7..57ff143ebd 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -793,7 +793,6 @@ tasks: - for: var: "UV_PYTHON_PROJECTS" cmd: |- - uv run --directory "{{.ITEM}}" black --color --line-length 100 {{.BLACK_FLAGS}} . uv run --directory "{{.ITEM}}" mypy . uv run --directory "{{.ITEM}}" ruff check {{.RUFF_FLAGS}} . From c8bd52f372cb1da3bc7959a111e305edb419a3c0 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 20 Aug 2025 17:03:43 +0800 Subject: [PATCH 075/100] Apply suggestions from code review Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .../tests/fixtures/integration_test_config.py | 10 +++++++--- .../tests/fixtures/integration_test_logs.py | 2 +- integration-tests/tests/utils/config.py | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/integration-tests/tests/fixtures/integration_test_config.py b/integration-tests/tests/fixtures/integration_test_config.py index e6ae92cae6..821e597788 100644 --- a/integration-tests/tests/fixtures/integration_test_config.py +++ b/integration-tests/tests/fixtures/integration_test_config.py @@ -15,10 +15,14 @@ @pytest.fixture(scope="session") def integration_test_config() -> IntegrationTestConfig: """Fixture that provides an IntegrationTestConfig shared across tests.""" - core_config = CoreConfig(clp_core_bins_dir=Path(get_env_var("CLP_CORE_BINS_DIR")).resolve()) - package_config = PackageConfig(clp_package_dir=Path(get_env_var("CLP_PACKAGE_DIR")).resolve()) + core_config = CoreConfig( + clp_core_bins_dir=Path(get_env_var("CLP_CORE_BINS_DIR")).expanduser().resolve() + ) + package_config = PackageConfig( + clp_package_dir=Path(get_env_var("CLP_PACKAGE_DIR")).expanduser().resolve() + ) return IntegrationTestConfig( core_config=core_config, package_config=package_config, - test_root_dir=Path(get_env_var("CLP_BUILD_DIR")).resolve() / "integration-tests", + test_root_dir=Path(get_env_var("CLP_BUILD_DIR")).expanduser().resolve() / "integration-tests", ) diff --git a/integration-tests/tests/fixtures/integration_test_logs.py b/integration-tests/tests/fixtures/integration_test_logs.py index 6df52ecb05..ae726edcab 100644 --- a/integration-tests/tests/fixtures/integration_test_logs.py +++ b/integration-tests/tests/fixtures/integration_test_logs.py @@ -54,7 +54,7 @@ def _download_and_extract_dataset( integration_test_config=integration_test_config, ) if request.config.cache.get(name, False): - logger.info("Test logs `%s` is up-to-date. Skipping download.", name) + logger.info("Test logs `%s` are up-to-date. Skipping download.", name) return integration_test_logs try: diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 211cb3ff28..8f0127f8c3 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -153,6 +153,6 @@ def __post_init__(self, integration_test_config: IntegrationTestConfig) -> None: def clear_test_outputs(self) -> None: """Remove any existing output directories created by this compression test.""" if self.compression_dir.exists(): - shutil.rmtree(self.compression_dir) + shutil.rmtree(self.compression_dir, ignore_errors=True) if self.decompression_dir.exists(): - shutil.rmtree(self.decompression_dir) + shutil.rmtree(self.decompression_dir, ignore_errors=True) From 8400e01bc91f215d0b406d0e5d6ac4239fa37780 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 20 Aug 2025 10:27:15 +0000 Subject: [PATCH 076/100] Address review comments --- .../tests/fixtures/integration_test_config.py | 3 +- .../tests/fixtures/integration_test_logs.py | 5 +- .../tests/test_identity_transformation.py | 6 +- integration-tests/tests/utils/assert_utils.py | 3 - integration-tests/tests/utils/config.py | 12 ++-- integration-tests/tests/utils/utils.py | 60 +++++++++++++------ taskfiles/lint.yaml | 13 ++-- 7 files changed, 64 insertions(+), 38 deletions(-) diff --git a/integration-tests/tests/fixtures/integration_test_config.py b/integration-tests/tests/fixtures/integration_test_config.py index 821e597788..f097021b3d 100644 --- a/integration-tests/tests/fixtures/integration_test_config.py +++ b/integration-tests/tests/fixtures/integration_test_config.py @@ -21,8 +21,9 @@ def integration_test_config() -> IntegrationTestConfig: package_config = PackageConfig( clp_package_dir=Path(get_env_var("CLP_PACKAGE_DIR")).expanduser().resolve() ) + test_root_dir = Path(get_env_var("CLP_BUILD_DIR")).expanduser().resolve() / "integration-tests" return IntegrationTestConfig( core_config=core_config, package_config=package_config, - test_root_dir=Path(get_env_var("CLP_BUILD_DIR")).expanduser().resolve() / "integration-tests", + test_root_dir=test_root_dir, ) diff --git a/integration-tests/tests/fixtures/integration_test_logs.py b/integration-tests/tests/fixtures/integration_test_logs.py index ae726edcab..09d632ec40 100644 --- a/integration-tests/tests/fixtures/integration_test_logs.py +++ b/integration-tests/tests/fixtures/integration_test_logs.py @@ -10,6 +10,7 @@ IntegrationTestConfig, IntegrationTestLogs, ) +from tests.utils.utils import unlink logger = logging.getLogger(__name__) @@ -69,8 +70,8 @@ def _download_and_extract_dataset( ] # fmt: on subprocess.run(curl_cmds, check=True) - if integration_test_logs.extraction_dir.exists(): - shutil.rmtree(integration_test_logs.extraction_dir) + + unlink(integration_test_logs.extraction_dir) shutil.unpack_archive( integration_test_logs.tarball_path, integration_test_logs.extraction_dir ) diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index 8b1a1758ef..a394698c3b 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -126,9 +126,9 @@ def test_clp_s_identity_transform( _consolidated_json_file_name = "original" input_path = consolidated_json_test_paths.logs_source_dir / _consolidated_json_file_name output_path = consolidated_json_test_paths.decompression_dir / _consolidated_json_file_name - assert is_json_file_structurally_equal( - input_path, output_path - ), f"Mismatch between clp-s input {input_path} and output {output_path}." + assert is_json_file_structurally_equal(input_path, output_path), ( + f"Mismatch between clp-s input {input_path} and output {output_path}." + ) test_paths.clear_test_outputs() consolidated_json_test_paths.clear_test_outputs() diff --git a/integration-tests/tests/utils/assert_utils.py b/integration-tests/tests/utils/assert_utils.py index c3dc7ee8ea..65573e8878 100644 --- a/integration-tests/tests/utils/assert_utils.py +++ b/integration-tests/tests/utils/assert_utils.py @@ -23,6 +23,3 @@ def run_and_assert(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess except subprocess.CalledProcessError as e: pytest.fail(f"Command failed: {' '.join(cmd)}: {e}") return proc - - - diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 8f0127f8c3..ae0bfc6f54 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -2,11 +2,13 @@ from __future__ import annotations -import shutil from dataclasses import dataclass, field, InitVar from pathlib import Path -from tests.utils.utils import validate_dir_exists +from tests.utils.utils import ( + unlink, + validate_dir_exists, +) @dataclass(frozen=True) @@ -152,7 +154,5 @@ def __post_init__(self, integration_test_config: IntegrationTestConfig) -> None: def clear_test_outputs(self) -> None: """Remove any existing output directories created by this compression test.""" - if self.compression_dir.exists(): - shutil.rmtree(self.compression_dir, ignore_errors=True) - if self.decompression_dir.exists(): - shutil.rmtree(self.decompression_dir, ignore_errors=True) + unlink(self.compression_dir) + unlink(self.decompression_dir) diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index 460b7c0d7c..5635f310ff 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -8,17 +8,17 @@ from typing import IO -def is_json_file_structurally_equal(json_fp1: Path, json_fp2: Path) -> bool: +def get_env_var(var_name: str) -> str: """ - :param json_fp1: - :param json_fp2: - :return: Whether two JSON files are structurally equal after sorting has been applied. + :param var_name: + :return: The string value of the specified environment variable. + :raise: ValueError if the environment variable is not set """ - with ( - _sort_json_keys_and_rows(json_fp1) as temp_file_1, - _sort_json_keys_and_rows(json_fp2) as temp_file_2, - ): - return is_dir_tree_content_equal(Path(temp_file_1.name), Path(temp_file_2.name)) + value = os.environ.get(var_name) + if value is None: + err_msg = f"Environment variable {var_name} is not set." + raise ValueError(err_msg) + return value def is_dir_tree_content_equal(path1: Path, path2: Path) -> bool: @@ -38,17 +38,41 @@ def is_dir_tree_content_equal(path1: Path, path2: Path) -> bool: raise RuntimeError(err_msg) -def get_env_var(var_name: str) -> str: +def is_json_file_structurally_equal(json_fp1: Path, json_fp2: Path) -> bool: """ - :param var_name: - :return: The string value of the specified environment variable. - :raise: ValueError if the environment variable is not set + :param json_fp1: + :param json_fp2: + :return: Whether two JSON files are structurally equal after sorting has been applied. """ - value = os.environ.get(var_name) - if value is None: - err_msg = f"Environment variable {var_name} is not set." - raise ValueError(err_msg) - return value + with ( + _sort_json_keys_and_rows(json_fp1) as temp_file_1, + _sort_json_keys_and_rows(json_fp2) as temp_file_2, + ): + return is_dir_tree_content_equal(Path(temp_file_1.name), Path(temp_file_2.name)) + + +def unlink(rm_path: Path, force: bool = True) -> None: + """ + Remove a file or directory at `path`. + + :param rm_path: + :param force: Whether to force remove with sudo priviledges in case the normal operation fails. + Defaults to True. + """ + try: + shutil.rmtree(rm_path) + except FileNotFoundError: + pass + except PermissionError: + if not force: + raise + + sudo_rm_cmds = ["sudo", "rm", "-rf", str(rm_path)] + try: + subprocess.run(sudo_rm_cmds, check=True) + except subprocess.CalledProcessError as e: + err_msg = f"Failed to remove {rm_path} due to lack of superuser privileges (sudo)." + raise OSError(err_msg) from e def validate_dir_exists(dir_path: Path) -> None: diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 57ff143ebd..9e323c38d5 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -78,14 +78,16 @@ tasks: - task: "py" vars: BLACK_FLAGS: "--check --diff" - RUFF_FLAGS: "" + RUFF_CHECK_FLAGS: "" + RUFF_FORMAT_FLAGS: "--check --diff" fix-py: cmds: - task: "py" vars: BLACK_FLAGS: "" - RUFF_FLAGS: "--fix" + RUFF_CHECK_FLAGS: "--exit-non-zero-on-fix --fix" + RUFF_FORMAT_FLAGS: "--exit-non-zero-on-fix" yaml: aliases: @@ -774,7 +776,7 @@ tasks: UV_PYTHON_PROJECTS: - "{{.G_INTEGRATION_TESTS_DIR}}" requires: - vars: ["BLACK_FLAGS", "RUFF_FLAGS"] + vars: ["BLACK_FLAGS", "RUFF_CHECK_FLAGS", "RUFF_FORMAT_FLAGS"] deps: ["venv"] cmds: - for: @@ -789,12 +791,13 @@ tasks: . "{{.G_LINT_VENV_DIR}}/bin/activate" cd "{{.ITEM}}" black --color --line-length 100 {{.BLACK_FLAGS}} . - ruff check {{.RUFF_FLAGS}} . + ruff check {{.RUFF_CHECK_FLAGS}} . - for: var: "UV_PYTHON_PROJECTS" cmd: |- + uv run --directory "{{.ITEM}}" ruff format {{.RUFF_FORMAT_FLAGS}} . + uv run --directory "{{.ITEM}}" ruff check {{.RUFF_CHECK_FLAGS}} . uv run --directory "{{.ITEM}}" mypy . - uv run --directory "{{.ITEM}}" ruff check {{.RUFF_FLAGS}} . venv: internal: true From 95d19ba4cdb36ae5c20fad4156d4ac82a24f3de0 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 21 Aug 2025 06:05:57 +0800 Subject: [PATCH 077/100] Update integration-tests/tests/utils/config.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- integration-tests/tests/utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index ae0bfc6f54..3ac617d4d2 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -31,7 +31,7 @@ def __post_init__(self) -> None: missing_binaries = [b for b in required_binaries if not (clp_core_bins_dir / b).is_file()] if len(missing_binaries) > 0: err_msg = ( - f"CLP core bins at {clp_core_bins_dir} is incomplete." + f"CLP core binaries at {clp_core_bins_dir} are incomplete." f" Missing binaries: {', '.join(missing_binaries)}" ) raise ValueError(err_msg) From 9f022f81dbe26b502bf069886d1087afc7ec660f Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 20 Aug 2025 19:58:22 -0400 Subject: [PATCH 078/100] Make integration task depend on the whole package --- taskfiles/tests/integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/tests/integration.yaml b/taskfiles/tests/integration.yaml index f83c087884..58a3a1fafa 100644 --- a/taskfiles/tests/integration.yaml +++ b/taskfiles/tests/integration.yaml @@ -19,7 +19,7 @@ tasks: core: deps: - - task: "::core" + - task: "::package" dir: "{{.G_INTEGRATION_TESTS_DIR}}" cmds: - >- From cbd0e8e7c8c15d43d84d24e8bd67f0d6ccb96f09 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Sep 2025 22:54:17 +0800 Subject: [PATCH 079/100] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- docs/src/dev-guide/testing/integration-tests.md | 3 +-- integration-tests/tests/utils/assert_utils.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/src/dev-guide/testing/integration-tests.md b/docs/src/dev-guide/testing/integration-tests.md index 41ee819dab..cee893dcc4 100644 --- a/docs/src/dev-guide/testing/integration-tests.md +++ b/docs/src/dev-guide/testing/integration-tests.md @@ -36,10 +36,9 @@ uv run python -m pytest --markers To run tests related to a specific marker (e.g., `clp_s`): - ### Specifying custom CLP binary paths You can override the default binary paths by setting the following environment variables: -* **`CLP_PACKAGE_DIR`**: Directory of the CLP package to test. * **`CLP_CORE_BINS_DIR`**: Directory containing the CLP core binaries to test. +* **`CLP_PACKAGE_DIR`**: Directory of the CLP package to test. diff --git a/integration-tests/tests/utils/assert_utils.py b/integration-tests/tests/utils/assert_utils.py index 65573e8878..f86a66530b 100644 --- a/integration-tests/tests/utils/assert_utils.py +++ b/integration-tests/tests/utils/assert_utils.py @@ -1,6 +1,5 @@ """ -Provide utility functions used across `integration-tests` that involve pytest assertions and -failures. +Utilities that raise pytest assertions on failure. """ import subprocess From b53c04a8e8e5803ba75df55f776ac98153546739 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Sep 2025 10:59:28 -0400 Subject: [PATCH 080/100] Rename assert_utils to asserting_utils --- .../tests/test_identity_transformation.py | 2 +- integration-tests/tests/utils/assert_utils.py | 24 ------------------- 2 files changed, 1 insertion(+), 25 deletions(-) delete mode 100644 integration-tests/tests/utils/assert_utils.py diff --git a/integration-tests/tests/test_identity_transformation.py b/integration-tests/tests/test_identity_transformation.py index a394698c3b..953ffd913f 100644 --- a/integration-tests/tests/test_identity_transformation.py +++ b/integration-tests/tests/test_identity_transformation.py @@ -5,7 +5,7 @@ import pytest -from tests.utils.assert_utils import run_and_assert +from tests.utils.asserting_utils import run_and_assert from tests.utils.config import ( CompressionTestConfig, IntegrationTestConfig, diff --git a/integration-tests/tests/utils/assert_utils.py b/integration-tests/tests/utils/assert_utils.py deleted file mode 100644 index f86a66530b..0000000000 --- a/integration-tests/tests/utils/assert_utils.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Utilities that raise pytest assertions on failure. -""" - -import subprocess -from typing import Any - -import pytest - - -def run_and_assert(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess[Any]: - """ - Runs a command with subprocess and asserts that it succeeds with pytest. - - :param cmd: Command and arguments to execute. - :param kwargs: Additional keyword arguments passed through to the subprocess. - :return: The completed process object, for inspection or further handling. - :raise: pytest.fail if the command exits with a non-zero return code. - """ - try: - proc = subprocess.run(cmd, check=True, **kwargs) - except subprocess.CalledProcessError as e: - pytest.fail(f"Command failed: {' '.join(cmd)}: {e}") - return proc From 73dd3d1f9527196682a81bb29a12482200b13589 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Sep 2025 11:00:56 -0400 Subject: [PATCH 081/100] Change validate to validates in docstring start --- integration-tests/tests/utils/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 3ac617d4d2..b087a106f2 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -20,7 +20,7 @@ class CoreConfig: def __post_init__(self) -> None: """ - Validate that the CLP core binaries directory exists and contains all required + Validates that the CLP core binaries directory exists and contains all required executables. """ clp_core_bins_dir = self.clp_core_bins_dir @@ -55,7 +55,7 @@ class PackageConfig: clp_package_dir: Path def __post_init__(self) -> None: - """Validate that the CLP package directory exists and contains all required directories.""" + """Validates that the CLP package directory exists and contains all required directories.""" clp_package_dir = self.clp_package_dir validate_dir_exists(clp_package_dir) From 9d280d72511bbaa12f0c26c41e61d5aeeb81bfe0 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Sep 2025 11:02:45 -0400 Subject: [PATCH 082/100] abbreviate validate_dir_exists --- integration-tests/tests/utils/utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index 5635f310ff..1dca8ba162 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -80,11 +80,8 @@ def validate_dir_exists(dir_path: Path) -> None: :param dir_path: :raise: ValueError if the path does not exist or is not a directory. """ - if not dir_path.exists(): - err_msg = f"Directory does not exist: {dir_path}" - raise ValueError(err_msg) if not dir_path.is_dir(): - err_msg = f"Path is not a directory: {dir_path}" + err_msg = f"Path does not exist or is not a directory: {dir_path}" raise ValueError(err_msg) From 0de00bb97210b7da54d47c287f7d3959fe1329cc Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Sep 2025 11:08:38 -0400 Subject: [PATCH 083/100] localize integration tests taskfile vars --- taskfile.yaml | 1 - taskfiles/tests/integration.yaml | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/taskfile.yaml b/taskfile.yaml index f05d33e3cb..0a063d5b0b 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -15,7 +15,6 @@ vars: G_COMPONENTS_DIR: "{{.ROOT_DIR}}/components" G_CORE_COMPONENT_DIR: "{{.G_COMPONENTS_DIR}}/core" G_CORE_COMPONENT_SUBMODULES_DIR: "{{.G_CORE_COMPONENT_DIR}}/submodules" - G_INTEGRATION_TESTS_DIR: "{{.ROOT_DIR}}/integration-tests" G_WEBUI_SRC_DIR: "{{.G_COMPONENTS_DIR}}/webui" # Build paths diff --git a/taskfiles/tests/integration.yaml b/taskfiles/tests/integration.yaml index 58a3a1fafa..68d318cfac 100644 --- a/taskfiles/tests/integration.yaml +++ b/taskfiles/tests/integration.yaml @@ -1,9 +1,10 @@ version: "3" -env: +vars: CLP_BUILD_DIR: "{{.G_BUILD_DIR}}" CLP_CORE_BINS_DIR: "{{.G_CORE_COMPONENT_BUILD_DIR}}" CLP_PACKAGE_DIR: "{{.G_PACKAGE_BUILD_DIR}}" + G_INTEGRATION_TESTS_DIR: "{{.ROOT_DIR}}/integration-tests" tasks: default: From 2f2c97f160e64b81dc066aa65a076df1aaebb879 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Sep 2025 11:08:56 -0400 Subject: [PATCH 084/100] Add back missing asserting_utils.py --- .../tests/utils/asserting_utils.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 integration-tests/tests/utils/asserting_utils.py diff --git a/integration-tests/tests/utils/asserting_utils.py b/integration-tests/tests/utils/asserting_utils.py new file mode 100644 index 0000000000..f86a66530b --- /dev/null +++ b/integration-tests/tests/utils/asserting_utils.py @@ -0,0 +1,24 @@ +""" +Utilities that raise pytest assertions on failure. +""" + +import subprocess +from typing import Any + +import pytest + + +def run_and_assert(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess[Any]: + """ + Runs a command with subprocess and asserts that it succeeds with pytest. + + :param cmd: Command and arguments to execute. + :param kwargs: Additional keyword arguments passed through to the subprocess. + :return: The completed process object, for inspection or further handling. + :raise: pytest.fail if the command exits with a non-zero return code. + """ + try: + proc = subprocess.run(cmd, check=True, **kwargs) + except subprocess.CalledProcessError as e: + pytest.fail(f"Command failed: {' '.join(cmd)}: {e}") + return proc From bafa27212ab9092f6d7b3b4ba2fe7b59cdbfb1c7 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Sep 2025 23:24:26 +0800 Subject: [PATCH 085/100] Update docs/src/dev-docs/index.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- docs/src/dev-docs/index.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/src/dev-docs/index.md b/docs/src/dev-docs/index.md index f8ed54102c..6d4c381cd7 100644 --- a/docs/src/dev-docs/index.md +++ b/docs/src/dev-docs/index.md @@ -61,11 +61,10 @@ building-package :caption: Testing :hidden: -testing/index -testing/unit-tests -testing/integration-tests +../dev-guide/testing/index +../dev-guide/testing/unit-tests +../dev-guide/testing/integration-tests ::: - :::{toctree} :caption: Contributing :hidden: From 98b228d1b1fc456c508c28ea54d2b5c431fd1502 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Sep 2025 11:24:49 -0400 Subject: [PATCH 086/100] lint fix --- integration-tests/tests/utils/asserting_utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integration-tests/tests/utils/asserting_utils.py b/integration-tests/tests/utils/asserting_utils.py index f86a66530b..fe39ec4576 100644 --- a/integration-tests/tests/utils/asserting_utils.py +++ b/integration-tests/tests/utils/asserting_utils.py @@ -1,6 +1,4 @@ -""" -Utilities that raise pytest assertions on failure. -""" +"""Utilities that raise pytest assertions on failure.""" import subprocess from typing import Any From 51ec20c0c04abdd6b62c1e47f3eab82b0cd1caff Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 3 Sep 2025 03:04:04 -0400 Subject: [PATCH 087/100] Remove unrelated changes --- docs/src/dev-docs/building-package.md | 3 ++- docs/src/dev-docs/components-core/index.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/src/dev-docs/building-package.md b/docs/src/dev-docs/building-package.md index e20300aac6..37c1b88cf4 100644 --- a/docs/src/dev-docs/building-package.md +++ b/docs/src/dev-docs/building-package.md @@ -13,7 +13,7 @@ prebuilt version instead, check out the [releases](https://github.com/y-scope/cl * Python 3.9 or newer * python3-dev * python3-venv (for the version of Python installed) -* [Task] == 3.44.0 +* [Task] 3.44.0 * [uv] >= 0.8 ## Setup @@ -69,5 +69,6 @@ To clean up all build artifacts, run: task clean ``` +[clp-issue-872]: https://github.com/y-scope/clp/issues/872 [Task]: https://taskfile.dev/ [uv]: https://docs.astral.sh/uv/ diff --git a/docs/src/dev-docs/components-core/index.md b/docs/src/dev-docs/components-core/index.md index d7736a7bfc..34fb1f7457 100644 --- a/docs/src/dev-docs/components-core/index.md +++ b/docs/src/dev-docs/components-core/index.md @@ -9,7 +9,7 @@ CLP core is the low-level component that performs compression, decompression, an * A recent compiler that fully supports C++20 features such as * std::span * std::source_location -* [Task] == 3.44.0 +* [Task] 3.44.0 * [uv] >= 0.8 To build, we require some source dependencies, packages from package managers, and libraries built @@ -125,6 +125,7 @@ ubuntu-jammy-deps-install regex-utils ::: +[clp-issue-872]: https://github.com/y-scope/clp/issues/872 [feature-req]: https://github.com/y-scope/clp/issues/new?assignees=&labels=enhancement&template=feature-request.yml [Task]: https://taskfile.dev/ [uv]: https://docs.astral.sh/uv/ From fa81d0084a3dc54df30d55acca5a6e779e022cfe Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 3 Sep 2025 15:07:49 +0800 Subject: [PATCH 088/100] Update docs/src/dev-docs/index.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- docs/src/dev-docs/index.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/src/dev-docs/index.md b/docs/src/dev-docs/index.md index 6d4c381cd7..b3472c684d 100644 --- a/docs/src/dev-docs/index.md +++ b/docs/src/dev-docs/index.md @@ -16,12 +16,10 @@ Docs about building CLP. ::: :::{grid-item-card} -:link: testing +:link: ../dev-guide/testing/index Testing -^^^ Docs about testing CLP. ::: - :::{grid-item-card} :link: contributing-getting-started Contributing From 7ca37dae511500f23c2d4855be8f930c920a479b Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Thu, 4 Sep 2025 05:17:27 -0400 Subject: [PATCH 089/100] Fix docs. --- docs/src/dev-docs/index.md | 11 +++++++---- docs/src/{dev-guide => dev-docs}/testing/index.md | 0 .../testing/integration-tests.md | 4 ++++ .../src/{dev-guide => dev-docs}/testing/unit-tests.md | 0 integration-tests/README.md | 2 +- 5 files changed, 12 insertions(+), 5 deletions(-) rename docs/src/{dev-guide => dev-docs}/testing/index.md (100%) rename docs/src/{dev-guide => dev-docs}/testing/integration-tests.md (95%) rename docs/src/{dev-guide => dev-docs}/testing/unit-tests.md (100%) diff --git a/docs/src/dev-docs/index.md b/docs/src/dev-docs/index.md index b3472c684d..a4ef9e60fe 100644 --- a/docs/src/dev-docs/index.md +++ b/docs/src/dev-docs/index.md @@ -16,10 +16,12 @@ Docs about building CLP. ::: :::{grid-item-card} -:link: ../dev-guide/testing/index +:link: testing/index Testing +^^^ Docs about testing CLP. ::: + :::{grid-item-card} :link: contributing-getting-started Contributing @@ -59,10 +61,11 @@ building-package :caption: Testing :hidden: -../dev-guide/testing/index -../dev-guide/testing/unit-tests -../dev-guide/testing/integration-tests +testing/index +testing/unit-tests +testing/integration-tests ::: + :::{toctree} :caption: Contributing :hidden: diff --git a/docs/src/dev-guide/testing/index.md b/docs/src/dev-docs/testing/index.md similarity index 100% rename from docs/src/dev-guide/testing/index.md rename to docs/src/dev-docs/testing/index.md diff --git a/docs/src/dev-guide/testing/integration-tests.md b/docs/src/dev-docs/testing/integration-tests.md similarity index 95% rename from docs/src/dev-guide/testing/integration-tests.md rename to docs/src/dev-docs/testing/integration-tests.md index cee893dcc4..145aa41bbe 100644 --- a/docs/src/dev-guide/testing/integration-tests.md +++ b/docs/src/dev-docs/testing/integration-tests.md @@ -36,6 +36,10 @@ uv run python -m pytest --markers To run tests related to a specific marker (e.g., `clp_s`): +```shell +uv run python -m pytest -m clp_s +``` + ### Specifying custom CLP binary paths You can override the default binary paths by setting the following environment variables: diff --git a/docs/src/dev-guide/testing/unit-tests.md b/docs/src/dev-docs/testing/unit-tests.md similarity index 100% rename from docs/src/dev-guide/testing/unit-tests.md rename to docs/src/dev-docs/testing/unit-tests.md diff --git a/integration-tests/README.md b/integration-tests/README.md index 411373afed..39933eff01 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -2,4 +2,4 @@ This Python project provides end-to-end tests for CLP via the `pytest` framework. -For more information, see https://docs.yscope.com/clp/main/dev-guide/testing/integration-tests. +For more information, see https://docs.yscope.com/clp/main/dev-docs/testing/integration-tests. From 936519369aafc181747714c9e6c591ad64667122 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 8 Sep 2025 06:11:09 -0400 Subject: [PATCH 090/100] Apply Rabbit's suggestion. --- integration-tests/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integration-tests/README.md b/integration-tests/README.md index 39933eff01..b020afa046 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -2,4 +2,6 @@ This Python project provides end-to-end tests for CLP via the `pytest` framework. -For more information, see https://docs.yscope.com/clp/main/dev-docs/testing/integration-tests. +For more information, see the [integration test docs][integration-test-docs]. + +[integration-test-docs]: https://docs.yscope.com/clp/main/dev-docs/testing/integration-tests. From b641bd96b80f0e660b4ef5a234b85443487fa5d9 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 8 Sep 2025 06:29:15 -0400 Subject: [PATCH 091/100] Alphabetize .gitignore. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2a0f64f2bd..cfb9aff9af 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ +__pycache__/ .clang-format .clang-tidy .lint-venv/ .task/ -__pycache__/ build/ dist/ From c2073cdf66bbae4e27b70e33ab79dd46dd3f8978 Mon Sep 17 00:00:00 2001 From: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 8 Sep 2025 06:44:59 -0400 Subject: [PATCH 092/100] Update integration-tests/README.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- integration-tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/README.md b/integration-tests/README.md index b020afa046..479704d8f5 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -4,4 +4,4 @@ This Python project provides end-to-end tests for CLP via the `pytest` framework For more information, see the [integration test docs][integration-test-docs]. -[integration-test-docs]: https://docs.yscope.com/clp/main/dev-docs/testing/integration-tests. +[integration-test-docs]: https://docs.yscope.com/clp/main/dev-docs/testing/integration-tests From fb274d13fd73c410c17bd11ed787718371fbf8cc Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 10 Sep 2025 12:32:16 -0400 Subject: [PATCH 093/100] Address review comments --- taskfiles/lint.yaml | 6 +++--- taskfiles/tests/integration.yaml | 15 +++++++++------ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index c5133a3a38..e5b3dd5952 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -79,15 +79,15 @@ tasks: vars: BLACK_FLAGS: "--check --diff" RUFF_CHECK_FLAGS: "" - RUFF_FORMAT_FLAGS: "--check --diff" + RUFF_FORMAT_FLAGS: "--diff" fix-py: cmds: - task: "py" vars: BLACK_FLAGS: "" - RUFF_CHECK_FLAGS: "--exit-non-zero-on-fix --fix" - RUFF_FORMAT_FLAGS: "--exit-non-zero-on-fix" + RUFF_CHECK_FLAGS: "--fix" + RUFF_FORMAT_FLAGS: "" yaml: aliases: diff --git a/taskfiles/tests/integration.yaml b/taskfiles/tests/integration.yaml index 68d318cfac..0d2c5f8da1 100644 --- a/taskfiles/tests/integration.yaml +++ b/taskfiles/tests/integration.yaml @@ -4,7 +4,7 @@ vars: CLP_BUILD_DIR: "{{.G_BUILD_DIR}}" CLP_CORE_BINS_DIR: "{{.G_CORE_COMPONENT_BUILD_DIR}}" CLP_PACKAGE_DIR: "{{.G_PACKAGE_BUILD_DIR}}" - G_INTEGRATION_TESTS_DIR: "{{.ROOT_DIR}}/integration-tests" + INTEGRATION_TESTS_DIR: "{{.ROOT_DIR}}/integration-tests" tasks: default: @@ -12,7 +12,7 @@ tasks: - task: "core" cache-clear: - dir: "{{.G_INTEGRATION_TESTS_DIR}}" + dir: "{{.INTEGRATION_TESTS_DIR}}" cmds: - >- uv run python -m pytest --cache-clear --collect-only --override-ini addopts="" --quiet @@ -20,8 +20,11 @@ tasks: core: deps: + # TODO: https://github.com/y-scope/clp/issues/1281 - task: "::package" - dir: "{{.G_INTEGRATION_TESTS_DIR}}" - cmds: - - >- - uv run python -m pytest -m "core" + dir: "{{.INTEGRATION_TESTS_DIR}}" + env: + CLP_BUILD_DIR: "{{.CLP_BUILD_DIR}}" + CLP_CORE_BINS_DIR: "{{.CLP_CORE_BINS_DIR}}" + CLP_PACKAGE_DIR: "{{.CLP_PACKAGE_DIR}}" + cmd: "uv run python -m pytest -m core" From 4840360202d8b63a6ac5242ec1a3e7d36430e57e Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 10 Sep 2025 13:15:30 -0400 Subject: [PATCH 094/100] typo fix --- taskfiles/tests/integration.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/taskfiles/tests/integration.yaml b/taskfiles/tests/integration.yaml index 0d2c5f8da1..b1442487c6 100644 --- a/taskfiles/tests/integration.yaml +++ b/taskfiles/tests/integration.yaml @@ -4,7 +4,7 @@ vars: CLP_BUILD_DIR: "{{.G_BUILD_DIR}}" CLP_CORE_BINS_DIR: "{{.G_CORE_COMPONENT_BUILD_DIR}}" CLP_PACKAGE_DIR: "{{.G_PACKAGE_BUILD_DIR}}" - INTEGRATION_TESTS_DIR: "{{.ROOT_DIR}}/integration-tests" + G_INTEGRATION_TESTS_DIR: "{{.ROOT_DIR}}/integration-tests" tasks: default: @@ -12,7 +12,7 @@ tasks: - task: "core" cache-clear: - dir: "{{.INTEGRATION_TESTS_DIR}}" + dir: "{{.G_INTEGRATION_TESTS_DIR}}" cmds: - >- uv run python -m pytest --cache-clear --collect-only --override-ini addopts="" --quiet @@ -22,7 +22,7 @@ tasks: deps: # TODO: https://github.com/y-scope/clp/issues/1281 - task: "::package" - dir: "{{.INTEGRATION_TESTS_DIR}}" + dir: "{{.G_INTEGRATION_TESTS_DIR}}" env: CLP_BUILD_DIR: "{{.CLP_BUILD_DIR}}" CLP_CORE_BINS_DIR: "{{.CLP_CORE_BINS_DIR}}" From 47ccb1ca8bad1f94f74135f0156b47095a1a6940 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 11 Sep 2025 15:16:55 -0400 Subject: [PATCH 095/100] Move python linting checks specific for unit tests into their own category --- integration-tests/pyproject.toml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index 906705a0f2..06b5752775 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -50,18 +50,22 @@ ignore = [ "FIX002", # Allow todo statements "PERF401", # Allow for loops when creating lists "PERF403", # Allow for loops when creating dicts - "S101", # Allow usage of pytest `assert` "S311", # Allow usage of `random` package "S603", # Automatically trust inputs of subprocess execution "SIM102", # Allow collapsible if statements for readability "SIM300", # Skip Yoda-condition format fixes - "TC003", # Ignore performance overhead of imports only used for type checking "TD002", # Author unnecessary for todo statement "TD003", # Issue link unnecessary for todo statement "UP015", # Explicit open modes are helpful ] isort.order-by-type = false +[tool.ruff.lint.per-file-ignores] +"tests/**" = [ + "S101", # Allow usage of pytest `assert` + "TC003", # Ignore performance overhead of imports only used for type checking +] + [tool.ruff.format] docstring-code-format = true docstring-code-line-length = 100 From 37f59bba9f63eb9a8ae61a107f05ca635a21ece1 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 15 Sep 2025 00:03:21 +0800 Subject: [PATCH 096/100] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- taskfiles/tests/integration.yaml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/taskfiles/tests/integration.yaml b/taskfiles/tests/integration.yaml index b1442487c6..823c6853b0 100644 --- a/taskfiles/tests/integration.yaml +++ b/taskfiles/tests/integration.yaml @@ -1,9 +1,6 @@ version: "3" vars: - CLP_BUILD_DIR: "{{.G_BUILD_DIR}}" - CLP_CORE_BINS_DIR: "{{.G_CORE_COMPONENT_BUILD_DIR}}" - CLP_PACKAGE_DIR: "{{.G_PACKAGE_BUILD_DIR}}" G_INTEGRATION_TESTS_DIR: "{{.ROOT_DIR}}/integration-tests" tasks: @@ -24,7 +21,7 @@ tasks: - task: "::package" dir: "{{.G_INTEGRATION_TESTS_DIR}}" env: - CLP_BUILD_DIR: "{{.CLP_BUILD_DIR}}" - CLP_CORE_BINS_DIR: "{{.CLP_CORE_BINS_DIR}}" - CLP_PACKAGE_DIR: "{{.CLP_PACKAGE_DIR}}" + CLP_BUILD_DIR: "{{.G_BUILD_DIR}}" + CLP_CORE_BINS_DIR: "{{.G_CORE_COMPONENT_BUILD_DIR}}" + CLP_PACKAGE_DIR: "{{.G_PACKAGE_BUILD_DIR}}" cmd: "uv run python -m pytest -m core" From 8c98e4d619a67e1256b9ad108e113fdd603aaf8e Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 14 Sep 2025 12:49:39 -0400 Subject: [PATCH 097/100] Address review comment --- integration-tests/tests/fixtures/integration_test_logs.py | 1 + taskfile.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/integration-tests/tests/fixtures/integration_test_logs.py b/integration-tests/tests/fixtures/integration_test_logs.py index 09d632ec40..8be7ed472a 100644 --- a/integration-tests/tests/fixtures/integration_test_logs.py +++ b/integration-tests/tests/fixtures/integration_test_logs.py @@ -75,6 +75,7 @@ def _download_and_extract_dataset( shutil.unpack_archive( integration_test_logs.tarball_path, integration_test_logs.extraction_dir ) + subprocess.run(["chmod","-R", "gu+w", integration_test_logs.extraction_dir], check=True) except Exception as e: err_msg = f"Failed to download and extract dataset `{name}`." raise RuntimeError(err_msg) from e diff --git a/taskfile.yaml b/taskfile.yaml index e4980d3855..c01dda1d7b 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -64,6 +64,7 @@ tasks: vars: COMPONENT: "job-orchestration" - task: "clean-webui" + - task: "tests:integration:cache-clear" clean-core: cmds: @@ -606,6 +607,7 @@ tasks: vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + clean-python-component: internal: true label: "clean-{{.COMPONENT}}" From 60501f13698c3bf81bb0aed628dde3f9e3e91f69 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 14 Sep 2025 13:00:47 -0400 Subject: [PATCH 098/100] lint fix --- integration-tests/tests/fixtures/integration_test_logs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/tests/fixtures/integration_test_logs.py b/integration-tests/tests/fixtures/integration_test_logs.py index 8be7ed472a..d79549c686 100644 --- a/integration-tests/tests/fixtures/integration_test_logs.py +++ b/integration-tests/tests/fixtures/integration_test_logs.py @@ -75,7 +75,7 @@ def _download_and_extract_dataset( shutil.unpack_archive( integration_test_logs.tarball_path, integration_test_logs.extraction_dir ) - subprocess.run(["chmod","-R", "gu+w", integration_test_logs.extraction_dir], check=True) + subprocess.run(["chmod", "-R", "gu+w", integration_test_logs.extraction_dir], check=True) except Exception as e: err_msg = f"Failed to download and extract dataset `{name}`." raise RuntimeError(err_msg) from e From 31e69e35e5dc0263abce52a6e8927c5c1bbcc3d5 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 14 Sep 2025 15:51:53 -0400 Subject: [PATCH 099/100] use shutil to find chmod binary --- integration-tests/tests/fixtures/integration_test_logs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/integration-tests/tests/fixtures/integration_test_logs.py b/integration-tests/tests/fixtures/integration_test_logs.py index d79549c686..db17278295 100644 --- a/integration-tests/tests/fixtures/integration_test_logs.py +++ b/integration-tests/tests/fixtures/integration_test_logs.py @@ -75,11 +75,17 @@ def _download_and_extract_dataset( shutil.unpack_archive( integration_test_logs.tarball_path, integration_test_logs.extraction_dir ) - subprocess.run(["chmod", "-R", "gu+w", integration_test_logs.extraction_dir], check=True) except Exception as e: err_msg = f"Failed to download and extract dataset `{name}`." raise RuntimeError(err_msg) from e + # Allow the extracted content to be deletable or overwritable + chmod_bin = shutil.which("chmod") + if chmod_bin is None: + err_msg = "chmod executable not found" + raise RuntimeError(err_msg) + subprocess.run([chmod_bin, "-R", "gu+w", integration_test_logs.extraction_dir], check=True) + logger.info("Downloaded and extracted uncompressed logs for dataset `%s`.", name) request.config.cache.set(name, True) return integration_test_logs From f424f21843abd033bf3efe7f7702ff945d028731 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 15 Sep 2025 07:56:56 -0400 Subject: [PATCH 100/100] use shutil to find the curl executable --- integration-tests/tests/fixtures/integration_test_logs.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/integration-tests/tests/fixtures/integration_test_logs.py b/integration-tests/tests/fixtures/integration_test_logs.py index db17278295..21c288e4c5 100644 --- a/integration-tests/tests/fixtures/integration_test_logs.py +++ b/integration-tests/tests/fixtures/integration_test_logs.py @@ -58,10 +58,15 @@ def _download_and_extract_dataset( logger.info("Test logs `%s` are up-to-date. Skipping download.", name) return integration_test_logs + curl_bin = shutil.which("curl") + if curl_bin is None: + err_msg = "curl executable not found" + raise RuntimeError(err_msg) + try: # fmt: off curl_cmds = [ - "curl", + curl_bin, "--fail", "--location", "--output", str(integration_test_logs.tarball_path),