-
Notifications
You must be signed in to change notification settings - Fork 0
fix: harden cloud run runtime guard #200
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -87,6 +87,17 @@ def _load_services() -> list[str]: | |
| return unique | ||
|
|
||
|
|
||
| def _cloud_run_log_filter(service: str, since_text: str, region: str = "") -> str: | ||
| parts = [ | ||
| 'resource.type="cloud_run_revision"', | ||
| f'resource.labels.service_name="{service}"', | ||
| f'timestamp >= "{since_text}"', | ||
| ] | ||
| if region: | ||
| parts.append(f'resource.labels.location="{region}"') | ||
| return " AND ".join(parts) | ||
|
|
||
|
|
||
| def _service_job_aliases(service: str) -> list[str]: | ||
| service_name = str(service or "").strip() | ||
| if not service_name: | ||
|
|
@@ -345,7 +356,26 @@ def _entry_text(entry: dict[str, Any]) -> str: | |
| return " ".join(chunks) | ||
|
|
||
|
|
||
| def _request_path(entry: dict[str, Any]) -> str: | ||
| request_url = str((entry.get("httpRequest") or {}).get("requestUrl") or "").strip() | ||
| if not request_url: | ||
| return "" | ||
| return urllib.parse.urlparse(request_url).path | ||
|
|
||
|
|
||
| def _is_ignorable_monitor_dispatch_capacity_warning(entry: dict[str, Any]) -> bool: | ||
| if not _env_bool("RUNTIME_GUARD_IGNORE_MONITOR_DISPATCH_CAPACITY_WARNINGS", True): | ||
| return False | ||
| return ( | ||
| _status(entry) == 429 | ||
| and _request_path(entry) == "/monitor-dispatch" | ||
| and "NO AVAILABLE INSTANCE" in _entry_text(entry).upper() | ||
| ) | ||
|
|
||
|
|
||
| def _is_failure(entry: dict[str, Any]) -> bool: | ||
| if _is_ignorable_monitor_dispatch_capacity_warning(entry): | ||
| return False | ||
| severity = str(entry.get("severity") or "").upper() | ||
| status = _status(entry) | ||
| text = _entry_text(entry).upper() | ||
|
|
@@ -491,11 +521,7 @@ def main() -> int: | |
| service_since = _cloud_run_log_since(project, service, since) if ignore_pre_ready_logs else since | ||
| service_since_by_name[service] = service_since | ||
| service_since_text = _format_timestamp(service_since) | ||
| log_filter = ( | ||
| 'resource.type="cloud_run_revision" ' | ||
| f'AND resource.labels.service_name="{service}" ' | ||
| f'AND timestamp >= "{service_since_text}"' | ||
| ) | ||
| log_filter = _cloud_run_log_filter(service, service_since_text, _region_for_service(service)) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When Useful? React with 👍 / 👎. |
||
| try: | ||
| entries = _run_gcloud_logging(project, log_filter, limit) | ||
| except RuntimeError as exc: | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When
RUNTIME_GUARD_SCHEDULER_JOB_PATTERNis broad enough to include the monitor dispatcher job, Cloud Scheduler target failures are logged as URL_ERROR/URL_UNREACHABLE entries (Google docs: https://docs.cloud.google.com/scheduler/docs/troubleshooting#job-fails-due-to-unreachable-destination-target), so they can lackhttpRequest.status/requestUrl; this predicate won't match those 429 monitor-dispatch capacity entries, and_is_failurewill still alert on the scheduler failure that the new ignore was supposed to suppress.Useful? React with 👍 / 👎.