Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .github/workflows/validate.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,14 @@ permissions:
contents: read

jobs:
actionlint:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v6
- name: Actionlint
uses: docker://rhysd/actionlint:1.7.12

python:
runs-on: ubuntu-latest
timeout-minutes: 15
Expand Down
7 changes: 7 additions & 0 deletions docs/internal_dependency_pin_policy.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,13 @@ QuantStrategyLab shares Python packages across platforms, strategies, and pipeli
python3 scripts/check_internal_dependency_matrix.py --projects-root .. --strict
```

When drift is found repeatedly, you can regenerate the matrix from local consumer files in a single step:

```bash
python3 scripts/check_internal_dependency_matrix.py --projects-root .. --generate --json > /tmp/internal_dependency_matrix.json
python3 scripts/check_internal_dependency_matrix.py --projects-root .. --sync
```

The checker compares matrix entries against consumer `requirements.txt`, `requirements-lock.txt`, and `pyproject.toml` files in sibling repositories. With `--strict`, ref mismatches fail CI even when sibling repos are not checked out locally.

## Pin formats
Expand Down
7 changes: 7 additions & 0 deletions docs/internal_dependency_pin_policy.zh-CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,13 @@ QuantStrategyLab 通过 git URL pin 在平台、策略与 pipeline 之间共享
python3 scripts/check_internal_dependency_matrix.py --projects-root .. --strict
```

若本地 consumer 文件与 matrix 漂移,可先从本地消费方依赖文件重建并同步:

```bash
python3 scripts/check_internal_dependency_matrix.py --projects-root .. --generate --json > /tmp/internal_dependency_matrix.json
python3 scripts/check_internal_dependency_matrix.py --projects-root .. --sync
```

该脚本会将 matrix 条目与各 consumer 仓库中的 `requirements.txt`、`requirements-lock.txt`、`pyproject.toml` 对比。启用 `--strict` 时,即使本地未 checkout sibling 仓库,ref 不一致也会导致 CI 失败。

## Pin 格式
Expand Down
3 changes: 3 additions & 0 deletions python/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# quant-runtime-settings

Python helper package for QuantStrategyLab runtime target settings and validation scripts.
1 change: 1 addition & 0 deletions python/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
name = "quant-runtime-settings"
version = "0.1.0"
description = "Declarative runtime target settings for QuantStrategyLab deployments"
readme = "README.md"
requires-python = ">=3.11"

[tool.ruff]
Expand Down
55 changes: 55 additions & 0 deletions python/scripts/check_internal_dependency_matrix.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,14 @@
r"git\+https://github\.com/QuantStrategyLab/"
r"(?P<source_repo>[A-Za-z0-9_.-]+)\.git@(?P<ref>[A-Za-z0-9_.-]+)"
)
TRACKED_DEPENDENCY_PATHS = ("requirements.txt", "requirements-lock.txt", "pyproject.toml")


def _sort_dependency_pins(pins: list[DependencyPin]) -> list[DependencyPin]:
return sorted(
set(pins),
key=lambda pin: (pin.consumer_repo, pin.path, pin.package, pin.source_repo, pin.ref),
)


@dataclass(frozen=True)
Expand Down Expand Up @@ -48,6 +56,33 @@ def ok(self) -> bool:
return not self.issues


def matrix_payload(pins: list[DependencyPin]) -> dict[str, Any]:
return {
"schema_version": 1,
"dependencies": [
{
"consumer_repo": pin.consumer_repo,
"path": pin.path,
"package": pin.package,
"source_repo": pin.source_repo,
"ref": pin.ref,
}
for pin in _sort_dependency_pins(pins)
],
}


def collect_dependency_pins_from_projects(projects_root: Path) -> list[DependencyPin]:
pins: list[DependencyPin] = []
for project_dir in sorted(p for p in projects_root.iterdir() if p.is_dir() and not p.name.startswith(".")):
for relative_path in TRACKED_DEPENDENCY_PATHS:
path = project_dir / relative_path
if not path.is_file():
continue
pins.extend(parse_dependency_pins(project_dir.name, relative_path, path.read_text(encoding="utf-8")))
return _sort_dependency_pins(pins)


def load_matrix(path: Path) -> list[DependencyPin]:
payload = json.loads(path.read_text(encoding="utf-8"))
if payload.get("schema_version") != 1:
Expand Down Expand Up @@ -127,6 +162,16 @@ def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Report QuantStrategyLab internal dependency pin drift.")
parser.add_argument("--matrix", type=Path, default=DEFAULT_MATRIX_PATH)
parser.add_argument("--projects-root", type=Path, default=DEFAULT_PROJECTS_ROOT)
parser.add_argument(
"--generate",
action="store_true",
help="Generate internal dependency matrix payload from local consumer dependency files.",
)
parser.add_argument(
"--sync",
action="store_true",
help="Overwrite --matrix with generated internal dependency payload.",
)
parser.add_argument("--json", action="store_true", help="Print machine-readable report.")
parser.add_argument("--strict", action="store_true", help="Exit non-zero when drift is detected.")
parser.add_argument(
Expand All @@ -139,6 +184,16 @@ def build_parser() -> argparse.ArgumentParser:

def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
if args.generate or args.sync:
generated_payload = matrix_payload(collect_dependency_pins_from_projects(projects_root=args.projects_root))
rendered_payload = json.dumps(generated_payload, ensure_ascii=False, indent=2)
print(rendered_payload)
if args.sync:
args.matrix.write_text(rendered_payload + "\n", encoding="utf-8")

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Guard --sync against empty consumer scans

When --sync is run from a standalone QuantRuntimeSettings checkout or before sibling consumer repos are cloned, collect_dependency_pins_from_projects returns no pins and this line overwrites internal_dependency_matrix.json with dependencies: []. I checked the follow-up strict path: with an empty matrix, --strict --require-consumer-files checks zero files and exits 0, so a mistaken sync can remove all matrix coverage without CI catching it. Please fail or require the expected consumers before writing.

Useful? React with 👍 / 👎.

if not args.json:
print(f"synced matrix -> {args.matrix}")
return 0

report = check_matrix(matrix_pins=load_matrix(args.matrix), projects_root=args.projects_root)
issues = list(report.issues)
if args.require_consumer_files and report.missing_files:
Expand Down
61 changes: 61 additions & 0 deletions python/tests/test_internal_dependency_matrix.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import annotations

import importlib.util
import json
import sys
import unittest
from pathlib import Path
Expand Down Expand Up @@ -100,6 +101,66 @@ def test_require_consumer_files_treats_missing_paths_as_issues(self):
self.assertEqual(report.missing_files, ["ExamplePlatform/requirements.txt"])
self.assertEqual(report.issues, [])

def test_collect_dependency_pins_from_projects(self):
projects_root = self._make_projects_root(
{
"ExampleA/pyproject.toml": "quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@a11",
"ExampleB/requirements.txt": "us-equity-strategies @ git+https://github.com/QuantStrategyLab/UsEquityStrategies.git@b22",
"ExampleB/requirements-lock.txt": "crypto-strategies @ git+https://github.com/QuantStrategyLab/CryptoStrategies.git@c33",
}
)

pins = check_internal_dependency_matrix.collect_dependency_pins_from_projects(projects_root)
rows = [(pin.consumer_repo, pin.path, pin.package, pin.source_repo, pin.ref) for pin in pins]

self.assertEqual(rows, [
("ExampleA", "pyproject.toml", "quant-platform-kit", "QuantPlatformKit", "a11"),
("ExampleB", "requirements-lock.txt", "crypto-strategies", "CryptoStrategies", "c33"),
("ExampleB", "requirements.txt", "us-equity-strategies", "UsEquityStrategies", "b22"),
])

def test_sync_rewrites_matrix_with_stable_order(self):
projects_root = self._make_projects_root(
{
"ExampleB/requirements.txt": "us-equity-strategies @ git+https://github.com/QuantStrategyLab/UsEquityStrategies.git@b22",
"ExampleA/pyproject.toml": "quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@a11",
}
)
matrix_path = projects_root / "internal_dependency_matrix.json"
matrix_path.write_text(
(
"{\n"
' "schema_version": 1,\n'
' "dependencies": [\n'
' {\n'
' "consumer_repo": "ExampleB",\n'
' "path": "requirements.txt",\n'
' "package": "us-equity-strategies",\n'
' "source_repo": "UsEquityStrategies",\n'
' "ref": "b22"\n'
" }\n"
" ]\n"
"}\n"
),
encoding="utf-8",
)

projects_payload = check_internal_dependency_matrix.matrix_payload(
check_internal_dependency_matrix.collect_dependency_pins_from_projects(projects_root)
)
exit_code = check_internal_dependency_matrix.main(
["--sync", "--projects-root", str(projects_root), "--matrix", str(matrix_path)]
)
synced_payload = json.loads(matrix_path.read_text(encoding="utf-8"))

self.assertEqual(exit_code, 0)
self.assertEqual(synced_payload["schema_version"], 1)
self.assertEqual(synced_payload["dependencies"], projects_payload["dependencies"])
self.assertLess(
synced_payload["dependencies"][0]["path"],
synced_payload["dependencies"][-1]["path"],
)

def _make_projects_root(self, files: dict[str, str]) -> Path:
import tempfile

Expand Down
Loading