Skip to content

Commit a2f5938

Browse files
Pigbibicodex
Pigbibi
and
codex
committed
Fix internal dependency matrix check for pyproject.toml consumers
Co-Authored-By: Codex <noreply@openai.com>
1 parent b6f7fcd commit a2f5938

3 files changed

Lines changed: 92 additions & 24 deletions

File tree

‎internal_dependency_matrix.json‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -34,14 +34,14 @@
3434
"path": "requirements.txt",
3535
"package": "us-equity-strategies",
3636
"source_repo": "UsEquityStrategies",
37-
"ref": "6568c315ce3be6f7ae5b799374cf7fb44232c170"
37+
"ref": "46887bc3f5454d5b59623b1f5efb7c65912c6b8b"
3838
},
3939
{
4040
"consumer_repo": "CharlesSchwabPlatform",
4141
"path": "requirements.txt",
4242
"package": "quant-platform-kit",
4343
"source_repo": "QuantPlatformKit",
44-
"ref": "b0eacd2fe4884f7f2447b704a232e9a121f396c4"
44+
"ref": "0063af3b4a974650ea58a7d3f26dd1b94f65d3e8"
4545
},
4646
{
4747
"consumer_repo": "CnEquitySnapshotPipelines",
@@ -55,21 +55,21 @@
5555
"path": "pyproject.toml",
5656
"package": "quant-platform-kit",
5757
"source_repo": "QuantPlatformKit",
58-
"ref": "b0eacd2fe4884f7f2447b704a232e9a121f396c4"
58+
"ref": "0063af3b4a974650ea58a7d3f26dd1b94f65d3e8"
5959
},
6060
{
6161
"consumer_repo": "FirstradePlatform",
6262
"path": "requirements.txt",
6363
"package": "us-equity-strategies",
6464
"source_repo": "UsEquityStrategies",
65-
"ref": "6568c315ce3be6f7ae5b799374cf7fb44232c170"
65+
"ref": "46887bc3f5454d5b59623b1f5efb7c65912c6b8b"
6666
},
6767
{
6868
"consumer_repo": "FirstradePlatform",
6969
"path": "requirements.txt",
7070
"package": "quant-platform-kit",
7171
"source_repo": "QuantPlatformKit",
72-
"ref": "b0eacd2fe4884f7f2447b704a232e9a121f396c4"
72+
"ref": "0063af3b4a974650ea58a7d3f26dd1b94f65d3e8"
7373
},
7474
{
7575
"consumer_repo": "HkEquitySnapshotPipelines",
@@ -83,42 +83,42 @@
8383
"path": "requirements.txt",
8484
"package": "hk-equity-strategies",
8585
"source_repo": "HkEquityStrategies",
86-
"ref": "e9e3058c1eaf3f43b25d50df5eb14442816e568e"
86+
"ref": "61993bf261aeccf64b5a75428b9405f4e1d1d682"
8787
},
8888
{
8989
"consumer_repo": "InteractiveBrokersPlatform",
9090
"path": "requirements.txt",
9191
"package": "us-equity-strategies",
9292
"source_repo": "UsEquityStrategies",
93-
"ref": "6568c315ce3be6f7ae5b799374cf7fb44232c170"
93+
"ref": "46887bc3f5454d5b59623b1f5efb7c65912c6b8b"
9494
},
9595
{
9696
"consumer_repo": "InteractiveBrokersPlatform",
9797
"path": "requirements.txt",
9898
"package": "quant-platform-kit",
9999
"source_repo": "QuantPlatformKit",
100-
"ref": "b0eacd2fe4884f7f2447b704a232e9a121f396c4"
100+
"ref": "0063af3b4a974650ea58a7d3f26dd1b94f65d3e8"
101101
},
102102
{
103103
"consumer_repo": "LongBridgePlatform",
104104
"path": "requirements.txt",
105105
"package": "hk-equity-strategies",
106106
"source_repo": "HkEquityStrategies",
107-
"ref": "e9e3058c1eaf3f43b25d50df5eb14442816e568e"
107+
"ref": "61993bf261aeccf64b5a75428b9405f4e1d1d682"
108108
},
109109
{
110110
"consumer_repo": "LongBridgePlatform",
111111
"path": "requirements.txt",
112112
"package": "quant-platform-kit",
113113
"source_repo": "QuantPlatformKit",
114-
"ref": "b0eacd2fe4884f7f2447b704a232e9a121f396c4"
114+
"ref": "0063af3b4a974650ea58a7d3f26dd1b94f65d3e8"
115115
},
116116
{
117117
"consumer_repo": "LongBridgePlatform",
118118
"path": "requirements.txt",
119119
"package": "us-equity-strategies",
120120
"source_repo": "UsEquityStrategies",
121-
"ref": "6568c315ce3be6f7ae5b799374cf7fb44232c170"
121+
"ref": "46887bc3f5454d5b59623b1f5efb7c65912c6b8b"
122122
},
123123
{
124124
"consumer_repo": "QmtPlatform",

‎python/scripts/check_internal_dependency_matrix.py‎

Lines changed: 56 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
r"(?P<source_repo>[A-Za-z0-9_.-]+)\.git@(?P<ref>[A-Za-z0-9_.-]+)"
2121
)
2222
TRACKED_DEPENDENCY_PATHS = ("requirements.txt", "requirements-lock.txt", "pyproject.toml")
23+
LEGACY_DEPENDENCY_PATHS = ("requirements.txt", "requirements-lock.txt")
24+
PYPROJECT_FALLBACK_PATH = "pyproject.toml"
2325

2426

2527
def _sort_dependency_pins(pins: list[DependencyPin]) -> list[DependencyPin]:
@@ -126,6 +128,40 @@ def parse_dependency_pins(consumer_repo: str, path: str, text: str) -> list[Depe
126128
]
127129

128130

131+
def _parse_repo_pins(projects_root: Path, consumer_repo: str, relative_path: str) -> list[DependencyPin]:
132+
path = projects_root / consumer_repo / relative_path
133+
if not path.is_file():
134+
return []
135+
return parse_dependency_pins(consumer_repo, relative_path, path.read_text(encoding="utf-8"))
136+
137+
138+
def _fallback_path_for_legacy_requirements(projects_root: Path, consumer_repo: str, expected_path: str) -> str | None:
139+
if expected_path not in LEGACY_DEPENDENCY_PATHS:
140+
return None
141+
pyproject_path = projects_root / consumer_repo / PYPROJECT_FALLBACK_PATH
142+
if pyproject_path.is_file():
143+
return PYPROJECT_FALLBACK_PATH
144+
return None
145+
146+
147+
def _compare_dependency_pins(
148+
*, expected_pins: list[DependencyPin], actual_pins: list[DependencyPin], issues: list[str]
149+
) -> None:
150+
actual_by_key = {pin.key: pin for pin in actual_pins}
151+
expected_by_key = {pin.key: pin for pin in expected_pins}
152+
153+
for key, expected in sorted(expected_by_key.items()):
154+
actual = actual_by_key.get(key)
155+
if actual is None:
156+
issues.append(f"missing {expected.label()} expected @{expected.ref}")
157+
elif actual.ref != expected.ref:
158+
issues.append(f"ref mismatch {expected.label()}: expected @{expected.ref}, found @{actual.ref}")
159+
160+
for key, actual in sorted(actual_by_key.items()):
161+
if key not in expected_by_key:
162+
issues.append(f"untracked internal dependency {actual.label()} @{actual.ref}")
163+
164+
129165
def check_matrix(*, matrix_pins: list[DependencyPin], projects_root: Path) -> MatrixReport:
130166
expected_by_file: dict[tuple[str, str], list[DependencyPin]] = {}
131167
for pin in matrix_pins:
@@ -137,23 +173,30 @@ def check_matrix(*, matrix_pins: list[DependencyPin], projects_root: Path) -> Ma
137173
for (consumer_repo, relative_path), expected_pins in sorted(expected_by_file.items()):
138174
path = projects_root / consumer_repo / relative_path
139175
if not path.exists():
176+
fallback_path = _fallback_path_for_legacy_requirements(projects_root, consumer_repo, relative_path)
177+
if fallback_path is not None:
178+
fallback_pins = _parse_repo_pins(projects_root, consumer_repo, fallback_path)
179+
if fallback_pins:
180+
_compare_dependency_pins(
181+
expected_pins=[
182+
DependencyPin(
183+
consumer_repo=pin.consumer_repo,
184+
path=fallback_path,
185+
package=pin.package,
186+
source_repo=pin.source_repo,
187+
ref=pin.ref,
188+
)
189+
for pin in expected_pins
190+
],
191+
actual_pins=fallback_pins,
192+
issues=issues,
193+
)
194+
continue
140195
missing_files.append(f"{consumer_repo}/{relative_path}")
141196
continue
142197
checked_files += 1
143198
actual_pins = parse_dependency_pins(consumer_repo, relative_path, path.read_text(encoding="utf-8"))
144-
actual_by_key = {pin.key: pin for pin in actual_pins}
145-
expected_by_key = {pin.key: pin for pin in expected_pins}
146-
147-
for key, expected in sorted(expected_by_key.items()):
148-
actual = actual_by_key.get(key)
149-
if actual is None:
150-
issues.append(f"missing {expected.label()} expected @{expected.ref}")
151-
elif actual.ref != expected.ref:
152-
issues.append(f"ref mismatch {expected.label()}: expected @{expected.ref}, found @{actual.ref}")
153-
154-
for key, actual in sorted(actual_by_key.items()):
155-
if key not in expected_by_key:
156-
issues.append(f"untracked internal dependency {actual.label()} @{actual.ref}")
199+
_compare_dependency_pins(expected_pins=expected_pins, actual_pins=actual_pins, issues=issues)
157200

158201
return MatrixReport(checked_files=checked_files, missing_files=missing_files, issues=issues)
159202

‎python/tests/test_internal_dependency_matrix.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,31 @@ def test_require_consumer_files_treats_missing_paths_as_issues(self):
101101
self.assertEqual(report.missing_files, ["ExamplePlatform/requirements.txt"])
102102
self.assertEqual(report.issues, [])
103103

104+
def test_check_matrix_accepts_pyproject_for_migrated_legacy_requirements(self):
105+
projects_root = self._make_projects_root(
106+
{
107+
"ExamplePlatform/pyproject.toml": (
108+
"quant-platform-kit @ git+https://github.com/QuantStrategyLab/"
109+
"QuantPlatformKit.git@v0.7.35"
110+
)
111+
}
112+
)
113+
expected = [
114+
check_internal_dependency_matrix.DependencyPin(
115+
consumer_repo="ExamplePlatform",
116+
path="requirements.txt",
117+
package="quant-platform-kit",
118+
source_repo="QuantPlatformKit",
119+
ref="v0.7.35",
120+
)
121+
]
122+
123+
report = check_internal_dependency_matrix.check_matrix(matrix_pins=expected, projects_root=projects_root)
124+
125+
self.assertEqual(report.checked_files, 0)
126+
self.assertEqual(report.missing_files, [])
127+
self.assertEqual(report.issues, [])
128+
104129
def test_collect_dependency_pins_from_projects(self):
105130
projects_root = self._make_projects_root(
106131
{

0 commit comments

Comments
 (0)