Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 31 additions & 5 deletions scripts/cloud_run_runtime_guard.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,17 @@ def _load_services() -> list[str]:
return unique


def _cloud_run_log_filter(service: str, since_text: str, region: str = "") -> str:
parts = [
'resource.type="cloud_run_revision"',
f'resource.labels.service_name="{service}"',
f'timestamp >= "{since_text}"',
]
if region:
parts.append(f'resource.labels.location="{region}"')
return " AND ".join(parts)


def _service_job_aliases(service: str) -> list[str]:
service_name = str(service or "").strip()
if not service_name:
Expand Down Expand Up @@ -345,7 +356,26 @@ def _entry_text(entry: dict[str, Any]) -> str:
return " ".join(chunks)


def _request_path(entry: dict[str, Any]) -> str:
request_url = str((entry.get("httpRequest") or {}).get("requestUrl") or "").strip()
if not request_url:
return ""
return urllib.parse.urlparse(request_url).path


def _is_ignorable_monitor_dispatch_capacity_warning(entry: dict[str, Any]) -> bool:
if not _env_bool("RUNTIME_GUARD_IGNORE_MONITOR_DISPATCH_CAPACITY_WARNINGS", True):
return False
return (
_status(entry) == 429
and _request_path(entry) == "/monitor-dispatch"
and "NO AVAILABLE INSTANCE" in _entry_text(entry).upper()
Comment on lines +370 to +372

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Handle scheduler log shape for monitor-dispatch 429s

When RUNTIME_GUARD_SCHEDULER_JOB_PATTERN is broad enough to include the monitor dispatcher job, Cloud Scheduler target failures are logged as URL_ERROR/URL_UNREACHABLE entries (Google docs: https://docs.cloud.google.com/scheduler/docs/troubleshooting#job-fails-due-to-unreachable-destination-target), so they can lack httpRequest.status/requestUrl; this predicate won't match those 429 monitor-dispatch capacity entries, and _is_failure will still alert on the scheduler failure that the new ignore was supposed to suppress.

Useful? React with 👍 / 👎.

)


def _is_failure(entry: dict[str, Any]) -> bool:
if _is_ignorable_monitor_dispatch_capacity_warning(entry):
return False
severity = str(entry.get("severity") or "").upper()
status = _status(entry)
text = _entry_text(entry).upper()
Expand Down Expand Up @@ -491,11 +521,7 @@ def main() -> int:
service_since = _cloud_run_log_since(project, service, since) if ignore_pre_ready_logs else since
service_since_by_name[service] = service_since
service_since_text = _format_timestamp(service_since)
log_filter = (
'resource.type="cloud_run_revision" '
f'AND resource.labels.service_name="{service}" '
f'AND timestamp >= "{service_since_text}"'
)
log_filter = _cloud_run_log_filter(service, service_since_text, _region_for_service(service))

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Skip disabled target regions before scoping log reads

When CLOUD_RUN_SERVICE_TARGETS_JSON contains a disabled target for the same service before an enabled target (or before a service also supplied via RUNTIME_GUARD_CLOUD_RUN_SERVICES), _load_services() ignores that disabled target but _region_for_service() still returns its region. Because this new call now applies that region to the Cloud Run log filter, the guard can query the disabled target's location and miss failures/successes from the actual deployed region.

Useful? React with 👍 / 👎.

try:
entries = _run_gcloud_logging(project, log_filter, limit)
except RuntimeError as exc:
Expand Down
54 changes: 54 additions & 0 deletions tests/test_cloud_run_runtime_guard.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,18 @@
from scripts import cloud_run_runtime_guard as guard


def test_cloud_run_log_filter_includes_region_when_available():
log_filter = guard._cloud_run_log_filter(
"firstrade-service",
"2026-07-01T12:00:00Z",
"asia-east1",
)

assert 'resource.labels.service_name="firstrade-service"' in log_filter
assert 'resource.labels.location="asia-east1"' in log_filter
assert 'timestamp >= "2026-07-01T12:00:00Z"' in log_filter


def test_scheduler_job_pattern_includes_service_alias():
pattern = guard._scheduler_job_pattern_for_services(["firstrade-service"])

Expand Down Expand Up @@ -125,3 +137,45 @@ def test_scheduler_entry_since_uses_matching_service_revision_window():
guard._scheduler_entry_since(entry, {"other-service": service_since}, fallback)
== fallback
)


def test_monitor_dispatch_capacity_warning_is_not_failure_by_default(monkeypatch):
monkeypatch.delenv("RUNTIME_GUARD_IGNORE_MONITOR_DISPATCH_CAPACITY_WARNINGS", raising=False)
entry = {
"severity": "WARNING",
"httpRequest": {
"status": 429,
"requestUrl": "https://example.run.app/monitor-dispatch",
},
"textPayload": "The request was aborted because there was no available instance.",
}

assert guard._is_failure(entry) is False


def test_monitor_dispatch_capacity_warning_can_be_counted(monkeypatch):
monkeypatch.setenv("RUNTIME_GUARD_IGNORE_MONITOR_DISPATCH_CAPACITY_WARNINGS", "false")
entry = {
"severity": "WARNING",
"httpRequest": {
"status": 429,
"requestUrl": "https://example.run.app/monitor-dispatch",
},
"textPayload": "The request was aborted because there was no available instance.",
}

assert guard._is_failure(entry) is True


def test_strategy_request_capacity_warning_still_fails(monkeypatch):
monkeypatch.delenv("RUNTIME_GUARD_IGNORE_MONITOR_DISPATCH_CAPACITY_WARNINGS", raising=False)
entry = {
"severity": "WARNING",
"httpRequest": {
"status": 429,
"requestUrl": "https://example.run.app/dry-run",
},
"textPayload": "The request was aborted because there was no available instance.",
}

assert guard._is_failure(entry) is True
Loading