Spaces:
Running
Running
deploy(hf): sync szl-holdings/killinchu@main derived COPY set
Browse filesReusable Dockerfile-COPY-derived deploy from szl-holdings/killinchu main.
Files: 882 Pruned: 0
Derived from Dockerfile COPY sources (NO hand-maintained allowlist).
Signed-off-by: SZL Holdings <noreply@szlholdings.ai>
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
- killinchu_backend.py +117 -6
- killinchu_cot_interop.py +111 -8
- killinchu_elite_console.py +16 -9
- serve.py +34 -61
- szl_connectors/governance.py +70 -12
- szl_connectors/oauth.py +20 -2
- szl_content_address.py +32 -0
- szl_dsse.py +7 -4
- szl_killinchu_cookbook.py +18 -6
- szl_safe_static.py +41 -0
killinchu_backend.py
CHANGED
|
@@ -518,7 +518,7 @@ def _store() -> _Store:
|
|
| 518 |
# ---------------------------------------------------------------------------
|
| 519 |
def _envelope(status: str, data: Dict[str, Any], citations: List[Dict[str, str]]) -> Dict[str, Any]:
|
| 520 |
body = {
|
| 521 |
-
"status": status, # ok | live | cached | degraded | error
|
| 522 |
"doctrine": DOCTRINE,
|
| 523 |
"service": "killinchu",
|
| 524 |
"citations": citations,
|
|
@@ -768,6 +768,20 @@ def _evaluate_watchlists(st: _Store, snap_id: int, event_id: int, count: int,
|
|
| 768 |
# ---------------------------------------------------------------------------
|
| 769 |
_SCHED_STARTED = False
|
| 770 |
_sched_lock = threading.Lock() # non-reentrant: guards against overlapping runs
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 771 |
_sched_state: Dict[str, Any] = {
|
| 772 |
"enabled": None,
|
| 773 |
"interval_seconds": None,
|
|
@@ -775,13 +789,60 @@ _sched_state: Dict[str, Any] = {
|
|
| 775 |
"running": False,
|
| 776 |
"runs": 0,
|
| 777 |
"last_run_at": None,
|
|
|
|
| 778 |
"last_status": None,
|
| 779 |
"last_error": None,
|
| 780 |
"consecutive_failures": 0,
|
| 781 |
"next_run_at": None,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 782 |
}
|
| 783 |
|
| 784 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 785 |
def _env_bool(name: str, default: bool) -> bool:
|
| 786 |
v = os.environ.get(name)
|
| 787 |
if v is None or not v.strip():
|
|
@@ -815,9 +876,11 @@ def run_crawl_guarded(mode: str = "auto") -> Optional[Dict[str, Any]]:
|
|
| 815 |
"""Run one crawl unless one is already in flight (no pile-up).
|
| 816 |
|
| 817 |
Returns the run_crawl() envelope, or None if a run was already in progress
|
| 818 |
-
and this call was skipped
|
| 819 |
-
causes scheduled cycles to queue up.
|
| 820 |
"""
|
|
|
|
|
|
|
| 821 |
if not _sched_lock.acquire(blocking=False):
|
| 822 |
return None
|
| 823 |
try:
|
|
@@ -860,15 +923,37 @@ async def _scheduler_loop() -> None:
|
|
| 860 |
if status == "live":
|
| 861 |
_sched_state["consecutive_failures"] = 0
|
| 862 |
_sched_state["last_error"] = None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 863 |
else:
|
| 864 |
# 'cached'/'degraded' = the scrape did not get fresh data.
|
| 865 |
_sched_state["consecutive_failures"] += 1
|
| 866 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 867 |
except Exception as e: # never let the loop die
|
| 868 |
_sched_state["consecutive_failures"] += 1
|
|
|
|
| 869 |
_sched_state["last_status"] = "error"
|
| 870 |
_sched_state["last_error"] = repr(e)
|
| 871 |
print(f"[killinchu-backend] auto-crawl cycle error: {e!r}", file=sys.stderr)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 872 |
finally:
|
| 873 |
_sched_state["running"] = False
|
| 874 |
|
|
@@ -944,6 +1029,14 @@ def register(app, ns: str = "killinchu") -> str:
|
|
| 944 |
s = _store()
|
| 945 |
if not s.ok():
|
| 946 |
return JSONResponse(_envelope("degraded", {"error": "no durable backend"}, _adsb_citation()), status_code=200)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 947 |
rows = s.query("SELECT id, fetched_at, record_count FROM snapshots ORDER BY id DESC LIMIT 1")
|
| 948 |
if rows:
|
| 949 |
try:
|
|
@@ -967,13 +1060,28 @@ def register(app, ns: str = "killinchu") -> str:
|
|
| 967 |
# -- crawl/run (manual) ------------------------------------------------
|
| 968 |
async def crawl_run(request: Request) -> JSONResponse:
|
| 969 |
"""POST /crawl/run — trigger a manual crawl and return its result envelope."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 970 |
return JSONResponse(run_crawl(mode="crawl"))
|
| 971 |
|
| 972 |
# -- crawl/status (auto-crawl scheduler health) ------------------------
|
| 973 |
async def crawl_status(request: Request) -> JSONResponse:
|
| 974 |
"""GET /crawl/status — report the auto-crawl scheduler configuration + health."""
|
| 975 |
cfg = scheduler_config()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 976 |
data = {
|
|
|
|
|
|
|
| 977 |
"config": {
|
| 978 |
"enabled": cfg["enabled"],
|
| 979 |
"interval_seconds": cfg["interval"],
|
|
@@ -983,9 +1091,12 @@ def register(app, ns: str = "killinchu") -> str:
|
|
| 983 |
},
|
| 984 |
"scheduler": dict(_sched_state),
|
| 985 |
"wired": _SCHED_STARTED,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 986 |
}
|
| 987 |
-
|
| 988 |
-
return JSONResponse(_envelope(status, data, []))
|
| 989 |
|
| 990 |
# -- timeline ----------------------------------------------------------
|
| 991 |
async def timeline(request: Request) -> JSONResponse:
|
|
|
|
| 518 |
# ---------------------------------------------------------------------------
|
| 519 |
def _envelope(status: str, data: Dict[str, Any], citations: List[Dict[str, str]]) -> Dict[str, Any]:
|
| 520 |
body = {
|
| 521 |
+
"status": status, # ok | live | cached | degraded | error | failed
|
| 522 |
"doctrine": DOCTRINE,
|
| 523 |
"service": "killinchu",
|
| 524 |
"citations": citations,
|
|
|
|
| 768 |
# ---------------------------------------------------------------------------
|
| 769 |
_SCHED_STARTED = False
|
| 770 |
_sched_lock = threading.Lock() # non-reentrant: guards against overlapping runs
|
| 771 |
+
_STORAGE_FAILURE_CLASS = "storage_unavailable"
|
| 772 |
+
_STORAGE_FAILURE_PATTERNS = (
|
| 773 |
+
"database or disk is full",
|
| 774 |
+
"disk i/o error",
|
| 775 |
+
"no space left on device",
|
| 776 |
+
"disk quota exceeded",
|
| 777 |
+
"attempt to write a readonly database",
|
| 778 |
+
"read-only file system",
|
| 779 |
+
"enospc",
|
| 780 |
+
)
|
| 781 |
+
_STORAGE_REMEDIATION = (
|
| 782 |
+
"Free writable storage for the Killinchu durable store, then restart the "
|
| 783 |
+
"service. Scheduled and manual crawls remain fail-closed until restart."
|
| 784 |
+
)
|
| 785 |
_sched_state: Dict[str, Any] = {
|
| 786 |
"enabled": None,
|
| 787 |
"interval_seconds": None,
|
|
|
|
| 789 |
"running": False,
|
| 790 |
"runs": 0,
|
| 791 |
"last_run_at": None,
|
| 792 |
+
"last_success_at": None,
|
| 793 |
"last_status": None,
|
| 794 |
"last_error": None,
|
| 795 |
"consecutive_failures": 0,
|
| 796 |
"next_run_at": None,
|
| 797 |
+
"circuit_open": False,
|
| 798 |
+
"failure_class": None,
|
| 799 |
+
"paused_at": None,
|
| 800 |
+
"operator_action": None,
|
| 801 |
}
|
| 802 |
|
| 803 |
|
| 804 |
+
def _scheduler_failure_class(error: Any) -> Optional[str]:
|
| 805 |
+
"""Classify only known non-retriable local-storage failures.
|
| 806 |
+
|
| 807 |
+
Upstream timeouts, rate limits, and malformed responses remain ordinary
|
| 808 |
+
degraded runs with exponential backoff. Storage failures are different:
|
| 809 |
+
retrying the same write forever cannot recover the disk and can flood logs.
|
| 810 |
+
"""
|
| 811 |
+
if isinstance(error, OSError) and getattr(error, "errno", None) in (28, 30, 122):
|
| 812 |
+
return _STORAGE_FAILURE_CLASS
|
| 813 |
+
normalized = str(error or "").casefold()
|
| 814 |
+
if any(pattern in normalized for pattern in _STORAGE_FAILURE_PATTERNS):
|
| 815 |
+
return _STORAGE_FAILURE_CLASS
|
| 816 |
+
return None
|
| 817 |
+
|
| 818 |
+
|
| 819 |
+
def _open_scheduler_circuit(error: Any, paused_at: Optional[str] = None) -> None:
|
| 820 |
+
"""Halt crawl writes after a non-retriable storage failure."""
|
| 821 |
+
_sched_state.update({
|
| 822 |
+
"last_status": "error",
|
| 823 |
+
"last_error": str(error),
|
| 824 |
+
"next_run_at": None,
|
| 825 |
+
"circuit_open": True,
|
| 826 |
+
"failure_class": _STORAGE_FAILURE_CLASS,
|
| 827 |
+
"paused_at": paused_at or _now_iso(),
|
| 828 |
+
"operator_action": _STORAGE_REMEDIATION,
|
| 829 |
+
})
|
| 830 |
+
|
| 831 |
+
|
| 832 |
+
def _scheduler_health(cfg: Dict[str, Any]) -> str:
|
| 833 |
+
"""Return the public scheduler health without equating wiring with health."""
|
| 834 |
+
if _sched_state["circuit_open"]:
|
| 835 |
+
return "failed"
|
| 836 |
+
if not cfg["enabled"] or not _SCHED_STARTED:
|
| 837 |
+
return "disabled"
|
| 838 |
+
last_status = str(_sched_state.get("last_status") or "").lower()
|
| 839 |
+
if last_status == "live":
|
| 840 |
+
return "ok"
|
| 841 |
+
if last_status in ("cached", "degraded", "error", "skipped"):
|
| 842 |
+
return "degraded"
|
| 843 |
+
return "starting"
|
| 844 |
+
|
| 845 |
+
|
| 846 |
def _env_bool(name: str, default: bool) -> bool:
|
| 847 |
v = os.environ.get(name)
|
| 848 |
if v is None or not v.strip():
|
|
|
|
| 876 |
"""Run one crawl unless one is already in flight (no pile-up).
|
| 877 |
|
| 878 |
Returns the run_crawl() envelope, or None if a run was already in progress
|
| 879 |
+
and this call was skipped, or if the storage circuit is open. The lock is
|
| 880 |
+
non-blocking so a slow run never causes scheduled cycles to queue up.
|
| 881 |
"""
|
| 882 |
+
if _sched_state["circuit_open"]:
|
| 883 |
+
return None
|
| 884 |
if not _sched_lock.acquire(blocking=False):
|
| 885 |
return None
|
| 886 |
try:
|
|
|
|
| 923 |
if status == "live":
|
| 924 |
_sched_state["consecutive_failures"] = 0
|
| 925 |
_sched_state["last_error"] = None
|
| 926 |
+
_sched_state["last_success_at"] = started
|
| 927 |
+
_sched_state["failure_class"] = None
|
| 928 |
+
_sched_state["paused_at"] = None
|
| 929 |
+
_sched_state["operator_action"] = None
|
| 930 |
else:
|
| 931 |
# 'cached'/'degraded' = the scrape did not get fresh data.
|
| 932 |
_sched_state["consecutive_failures"] += 1
|
| 933 |
+
error = res.get("error")
|
| 934 |
+
_sched_state["last_error"] = error
|
| 935 |
+
if _scheduler_failure_class(error) == _STORAGE_FAILURE_CLASS:
|
| 936 |
+
_open_scheduler_circuit(error, paused_at=started)
|
| 937 |
+
print(
|
| 938 |
+
"[killinchu-backend] auto-crawl halted: "
|
| 939 |
+
f"{_STORAGE_FAILURE_CLASS} ({error})",
|
| 940 |
+
file=sys.stderr,
|
| 941 |
+
)
|
| 942 |
+
return
|
| 943 |
except Exception as e: # never let the loop die
|
| 944 |
_sched_state["consecutive_failures"] += 1
|
| 945 |
+
_sched_state["last_run_at"] = started
|
| 946 |
_sched_state["last_status"] = "error"
|
| 947 |
_sched_state["last_error"] = repr(e)
|
| 948 |
print(f"[killinchu-backend] auto-crawl cycle error: {e!r}", file=sys.stderr)
|
| 949 |
+
if _scheduler_failure_class(e) == _STORAGE_FAILURE_CLASS:
|
| 950 |
+
_open_scheduler_circuit(e, paused_at=started)
|
| 951 |
+
print(
|
| 952 |
+
"[killinchu-backend] auto-crawl halted fail-closed; "
|
| 953 |
+
f"operator action: {_STORAGE_REMEDIATION}",
|
| 954 |
+
file=sys.stderr,
|
| 955 |
+
)
|
| 956 |
+
return
|
| 957 |
finally:
|
| 958 |
_sched_state["running"] = False
|
| 959 |
|
|
|
|
| 1029 |
s = _store()
|
| 1030 |
if not s.ok():
|
| 1031 |
return JSONResponse(_envelope("degraded", {"error": "no durable backend"}, _adsb_citation()), status_code=200)
|
| 1032 |
+
if _sched_state["circuit_open"]:
|
| 1033 |
+
return JSONResponse(_envelope("failed", {
|
| 1034 |
+
"health": "failed",
|
| 1035 |
+
"error": _sched_state["last_error"],
|
| 1036 |
+
"failure_class": _sched_state["failure_class"],
|
| 1037 |
+
"circuit_open": True,
|
| 1038 |
+
"operator_action": _sched_state["operator_action"],
|
| 1039 |
+
}, _adsb_citation()), status_code=503)
|
| 1040 |
rows = s.query("SELECT id, fetched_at, record_count FROM snapshots ORDER BY id DESC LIMIT 1")
|
| 1041 |
if rows:
|
| 1042 |
try:
|
|
|
|
| 1060 |
# -- crawl/run (manual) ------------------------------------------------
|
| 1061 |
async def crawl_run(request: Request) -> JSONResponse:
|
| 1062 |
"""POST /crawl/run — trigger a manual crawl and return its result envelope."""
|
| 1063 |
+
if _sched_state["circuit_open"]:
|
| 1064 |
+
return JSONResponse(_envelope("failed", {
|
| 1065 |
+
"health": "failed",
|
| 1066 |
+
"error": _sched_state["last_error"],
|
| 1067 |
+
"failure_class": _sched_state["failure_class"],
|
| 1068 |
+
"circuit_open": True,
|
| 1069 |
+
"operator_action": _sched_state["operator_action"],
|
| 1070 |
+
}, _adsb_citation()), status_code=503)
|
| 1071 |
return JSONResponse(run_crawl(mode="crawl"))
|
| 1072 |
|
| 1073 |
# -- crawl/status (auto-crawl scheduler health) ------------------------
|
| 1074 |
async def crawl_status(request: Request) -> JSONResponse:
|
| 1075 |
"""GET /crawl/status — report the auto-crawl scheduler configuration + health."""
|
| 1076 |
cfg = scheduler_config()
|
| 1077 |
+
health = _scheduler_health(cfg)
|
| 1078 |
+
last_status = str(_sched_state.get("last_status") or "").lower()
|
| 1079 |
+
freshness = "fresh" if last_status == "live" else (
|
| 1080 |
+
"stale" if _sched_state.get("last_success_at") else "unverified"
|
| 1081 |
+
)
|
| 1082 |
data = {
|
| 1083 |
+
"health": health,
|
| 1084 |
+
"freshness": freshness,
|
| 1085 |
"config": {
|
| 1086 |
"enabled": cfg["enabled"],
|
| 1087 |
"interval_seconds": cfg["interval"],
|
|
|
|
| 1091 |
},
|
| 1092 |
"scheduler": dict(_sched_state),
|
| 1093 |
"wired": _SCHED_STARTED,
|
| 1094 |
+
"circuit_open": _sched_state["circuit_open"],
|
| 1095 |
+
"failure_class": _sched_state["failure_class"],
|
| 1096 |
+
"paused_at": _sched_state["paused_at"],
|
| 1097 |
+
"operator_action": _sched_state["operator_action"],
|
| 1098 |
}
|
| 1099 |
+
return JSONResponse(_envelope(health, data, []))
|
|
|
|
| 1100 |
|
| 1101 |
# -- timeline ----------------------------------------------------------
|
| 1102 |
async def timeline(request: Request) -> JSONResponse:
|
killinchu_cot_interop.py
CHANGED
|
@@ -53,11 +53,15 @@ from __future__ import annotations
|
|
| 53 |
|
| 54 |
import datetime as _dt
|
| 55 |
import json
|
|
|
|
| 56 |
import os
|
| 57 |
import re
|
| 58 |
from typing import Any, Iterable
|
| 59 |
from xml.etree import ElementTree as ET
|
| 60 |
|
|
|
|
|
|
|
|
|
|
| 61 |
try:
|
| 62 |
from fastapi import APIRouter, Request
|
| 63 |
from fastapi.responses import JSONResponse, Response
|
|
@@ -69,11 +73,26 @@ except Exception: # pragma: no cover
|
|
| 69 |
|
| 70 |
_HERE = os.path.dirname(os.path.abspath(__file__))
|
| 71 |
_VESSELS_PATH = os.path.join(_HERE, "fleet_vessels_data.json")
|
|
|
|
| 72 |
|
| 73 |
COT_VERSION = "2.0"
|
| 74 |
# CoT default multicast SA group (ROADMAP target only — never opened here).
|
| 75 |
COT_DEFAULT_MULTICAST = "239.2.3.1:6969"
|
| 76 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
HONESTY_LABEL = (
|
| 78 |
"Real CoT 2.0 XML export + schema-shape validation + ingest, computed from "
|
| 79 |
"killinchu sample tracks. No live TAK-server / UDP multicast in this build "
|
|
@@ -93,6 +112,71 @@ COT_SOURCES = [
|
|
| 93 |
]
|
| 94 |
|
| 95 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
# ---------------------------------------------------------------------------
|
| 97 |
# CoT type-atom resolution (MIL-STD-2525 / CoT hierarchy).
|
| 98 |
# a = atoms (vs b=bits, t=tasking, etc.)
|
|
@@ -433,9 +517,9 @@ def validate_xml_string(xml: str) -> tuple[bool, list[str]]:
|
|
| 433 |
"""Validate a CoT XML string. Handles both a single <event> and an
|
| 434 |
<events> batch wrapper. Returns (ok, errors)."""
|
| 435 |
try:
|
| 436 |
-
root =
|
| 437 |
-
except
|
| 438 |
-
return (False, [
|
| 439 |
if root.tag == "events":
|
| 440 |
all_errs: list[str] = []
|
| 441 |
children = root.findall("event")
|
|
@@ -457,8 +541,8 @@ def cot_xml_to_track(xml: str) -> dict[str, Any]:
|
|
| 457 |
|
| 458 |
Raises ValueError if the XML is not a valid CoT event."""
|
| 459 |
try:
|
| 460 |
-
root =
|
| 461 |
-
except
|
| 462 |
raise ValueError(f"invalid CoT XML: {exc}") from exc
|
| 463 |
if root.tag != "event":
|
| 464 |
raise ValueError(f"expected root <event>, got <{root.tag}>")
|
|
@@ -623,11 +707,30 @@ def register(app) -> dict[str, Any]:
|
|
| 623 |
|
| 624 |
@router.post(f"{base}/ingest")
|
| 625 |
async def _ingest(request: Request) -> JSONResponse:
|
| 626 |
-
raw = await request.body()
|
| 627 |
try:
|
|
|
|
| 628 |
track = cot_xml_to_track(raw.decode("utf-8"))
|
| 629 |
-
except
|
| 630 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 631 |
return JSONResponse({"ok": True, "track": track, "honesty": HONESTY_LABEL})
|
| 632 |
registered.append(f"{base}/ingest")
|
| 633 |
|
|
|
|
| 53 |
|
| 54 |
import datetime as _dt
|
| 55 |
import json
|
| 56 |
+
import logging
|
| 57 |
import os
|
| 58 |
import re
|
| 59 |
from typing import Any, Iterable
|
| 60 |
from xml.etree import ElementTree as ET
|
| 61 |
|
| 62 |
+
from defusedxml import ElementTree as DefusedET
|
| 63 |
+
from defusedxml.common import DefusedXmlException
|
| 64 |
+
|
| 65 |
try:
|
| 66 |
from fastapi import APIRouter, Request
|
| 67 |
from fastapi.responses import JSONResponse, Response
|
|
|
|
| 73 |
|
| 74 |
_HERE = os.path.dirname(os.path.abspath(__file__))
|
| 75 |
_VESSELS_PATH = os.path.join(_HERE, "fleet_vessels_data.json")
|
| 76 |
+
_LOGGER = logging.getLogger(__name__)
|
| 77 |
|
| 78 |
COT_VERSION = "2.0"
|
| 79 |
# CoT default multicast SA group (ROADMAP target only — never opened here).
|
| 80 |
COT_DEFAULT_MULTICAST = "239.2.3.1:6969"
|
| 81 |
|
| 82 |
+
# CoT events are compact tactical messages. Hard limits bound parser memory and
|
| 83 |
+
# post-parse traversal even after DTD/entity expansion has been disabled.
|
| 84 |
+
MAX_COT_XML_BYTES = 256 * 1024
|
| 85 |
+
MAX_COT_XML_ELEMENTS = 2048
|
| 86 |
+
MAX_COT_XML_DEPTH = 24
|
| 87 |
+
MAX_COT_XML_TEXT_CHARS = 128 * 1024
|
| 88 |
+
MAX_COT_ATTRIBUTES_PER_ELEMENT = 64
|
| 89 |
+
|
| 90 |
+
# Client responses deliberately expose only stable categories. Detailed
|
| 91 |
+
# exception context remains in server logs and never crosses the HTTP boundary.
|
| 92 |
+
_INGEST_ERROR_TOO_LARGE = "CoT XML payload too large"
|
| 93 |
+
_INGEST_ERROR_ENCODING = "CoT XML must be UTF-8"
|
| 94 |
+
_INGEST_ERROR_INVALID = "invalid CoT XML"
|
| 95 |
+
|
| 96 |
HONESTY_LABEL = (
|
| 97 |
"Real CoT 2.0 XML export + schema-shape validation + ingest, computed from "
|
| 98 |
"killinchu sample tracks. No live TAK-server / UDP multicast in this build "
|
|
|
|
| 112 |
]
|
| 113 |
|
| 114 |
|
| 115 |
+
class CotPayloadTooLarge(ValueError):
|
| 116 |
+
"""Raised before parsing when an inbound CoT body exceeds the byte limit."""
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def _parse_untrusted_xml(xml: str) -> ET.Element:
|
| 120 |
+
"""Parse untrusted CoT XML with entity expansion and DTDs disabled.
|
| 121 |
+
|
| 122 |
+
The byte limit is checked before parsing; structural limits are then
|
| 123 |
+
enforced iteratively to avoid recursive traversal of adversarial trees.
|
| 124 |
+
"""
|
| 125 |
+
if not isinstance(xml, str):
|
| 126 |
+
raise ValueError("CoT XML must be text")
|
| 127 |
+
try:
|
| 128 |
+
encoded = xml.encode("utf-8")
|
| 129 |
+
except UnicodeEncodeError as exc:
|
| 130 |
+
raise ValueError(f"invalid CoT XML encoding: {exc}") from exc
|
| 131 |
+
if len(encoded) > MAX_COT_XML_BYTES:
|
| 132 |
+
raise CotPayloadTooLarge(f"CoT XML exceeds {MAX_COT_XML_BYTES} byte limit")
|
| 133 |
+
|
| 134 |
+
try:
|
| 135 |
+
root = DefusedET.fromstring(
|
| 136 |
+
xml,
|
| 137 |
+
forbid_dtd=True,
|
| 138 |
+
forbid_entities=True,
|
| 139 |
+
forbid_external=True,
|
| 140 |
+
)
|
| 141 |
+
except (DefusedXmlException, ET.ParseError, ValueError) as exc:
|
| 142 |
+
raise ValueError(f"XML parse error: {exc}") from exc
|
| 143 |
+
|
| 144 |
+
element_count = 0
|
| 145 |
+
text_chars = 0
|
| 146 |
+
stack: list[tuple[ET.Element, int]] = [(root, 1)]
|
| 147 |
+
while stack:
|
| 148 |
+
element, depth = stack.pop()
|
| 149 |
+
element_count += 1
|
| 150 |
+
if element_count > MAX_COT_XML_ELEMENTS:
|
| 151 |
+
raise ValueError(f"CoT XML exceeds {MAX_COT_XML_ELEMENTS} element limit")
|
| 152 |
+
if depth > MAX_COT_XML_DEPTH:
|
| 153 |
+
raise ValueError(f"CoT XML exceeds {MAX_COT_XML_DEPTH} level depth limit")
|
| 154 |
+
if len(element.attrib) > MAX_COT_ATTRIBUTES_PER_ELEMENT:
|
| 155 |
+
raise ValueError(
|
| 156 |
+
"CoT XML element exceeds "
|
| 157 |
+
f"{MAX_COT_ATTRIBUTES_PER_ELEMENT} attribute limit"
|
| 158 |
+
)
|
| 159 |
+
text_chars += len(element.text or "") + len(element.tail or "")
|
| 160 |
+
if text_chars > MAX_COT_XML_TEXT_CHARS:
|
| 161 |
+
raise ValueError(
|
| 162 |
+
f"CoT XML exceeds {MAX_COT_XML_TEXT_CHARS} text-character limit"
|
| 163 |
+
)
|
| 164 |
+
stack.extend((child, depth + 1) for child in list(element))
|
| 165 |
+
return root
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
async def _read_limited_request_body(request: Request) -> bytes:
|
| 169 |
+
"""Read a streaming request body without buffering past the CoT limit."""
|
| 170 |
+
chunks: list[bytes] = []
|
| 171 |
+
total = 0
|
| 172 |
+
async for chunk in request.stream():
|
| 173 |
+
total += len(chunk)
|
| 174 |
+
if total > MAX_COT_XML_BYTES:
|
| 175 |
+
raise CotPayloadTooLarge(f"CoT XML exceeds {MAX_COT_XML_BYTES} byte limit")
|
| 176 |
+
chunks.append(chunk)
|
| 177 |
+
return b"".join(chunks)
|
| 178 |
+
|
| 179 |
+
|
| 180 |
# ---------------------------------------------------------------------------
|
| 181 |
# CoT type-atom resolution (MIL-STD-2525 / CoT hierarchy).
|
| 182 |
# a = atoms (vs b=bits, t=tasking, etc.)
|
|
|
|
| 517 |
"""Validate a CoT XML string. Handles both a single <event> and an
|
| 518 |
<events> batch wrapper. Returns (ok, errors)."""
|
| 519 |
try:
|
| 520 |
+
root = _parse_untrusted_xml(xml)
|
| 521 |
+
except ValueError as exc:
|
| 522 |
+
return (False, [str(exc)])
|
| 523 |
if root.tag == "events":
|
| 524 |
all_errs: list[str] = []
|
| 525 |
children = root.findall("event")
|
|
|
|
| 541 |
|
| 542 |
Raises ValueError if the XML is not a valid CoT event."""
|
| 543 |
try:
|
| 544 |
+
root = _parse_untrusted_xml(xml)
|
| 545 |
+
except ValueError as exc:
|
| 546 |
raise ValueError(f"invalid CoT XML: {exc}") from exc
|
| 547 |
if root.tag != "event":
|
| 548 |
raise ValueError(f"expected root <event>, got <{root.tag}>")
|
|
|
|
| 707 |
|
| 708 |
@router.post(f"{base}/ingest")
|
| 709 |
async def _ingest(request: Request) -> JSONResponse:
|
|
|
|
| 710 |
try:
|
| 711 |
+
raw = await _read_limited_request_body(request)
|
| 712 |
track = cot_xml_to_track(raw.decode("utf-8"))
|
| 713 |
+
except CotPayloadTooLarge:
|
| 714 |
+
_LOGGER.info(
|
| 715 |
+
"Rejected CoT ingest", extra={"cot_rejection": "payload_too_large"}
|
| 716 |
+
)
|
| 717 |
+
return JSONResponse(
|
| 718 |
+
{"ok": False, "error": _INGEST_ERROR_TOO_LARGE}, status_code=413
|
| 719 |
+
)
|
| 720 |
+
except UnicodeDecodeError:
|
| 721 |
+
_LOGGER.info(
|
| 722 |
+
"Rejected CoT ingest", extra={"cot_rejection": "invalid_encoding"}
|
| 723 |
+
)
|
| 724 |
+
return JSONResponse(
|
| 725 |
+
{"ok": False, "error": _INGEST_ERROR_ENCODING}, status_code=400
|
| 726 |
+
)
|
| 727 |
+
except ValueError:
|
| 728 |
+
_LOGGER.info(
|
| 729 |
+
"Rejected CoT ingest", extra={"cot_rejection": "invalid_xml"}
|
| 730 |
+
)
|
| 731 |
+
return JSONResponse(
|
| 732 |
+
{"ok": False, "error": _INGEST_ERROR_INVALID}, status_code=400
|
| 733 |
+
)
|
| 734 |
return JSONResponse({"ok": True, "track": track, "honesty": HONESTY_LABEL})
|
| 735 |
registered.append(f"{base}/ingest")
|
| 736 |
|
killinchu_elite_console.py
CHANGED
|
@@ -2455,33 +2455,40 @@ async function crawlStatusLoad(id){
|
|
| 2455 |
var j=await getJSON('/api/killinchu/crawl/status');
|
| 2456 |
var cfg=(j&&j.config)||{}, sch=(j&&j.scheduler)||{}, wired=!!(j&&j.wired);
|
| 2457 |
var enabled=(cfg.enabled!==false)&&wired;
|
|
|
|
|
|
|
| 2458 |
var last=String(sch.last_status||'').toLowerCase();
|
| 2459 |
var outcome, oc;
|
| 2460 |
-
if(
|
|
|
|
| 2461 |
else if(last==='live'){ outcome='OK'; oc='#5fb3a3'; }
|
| 2462 |
else if(last==='error'){ outcome='ERROR'; oc='#ff7b7b'; }
|
| 2463 |
else if(last==='skipped'){ outcome='SKIPPED'; oc='#c9a05f'; }
|
| 2464 |
else { outcome='DEGRADED'; oc='#f5b301'; } // cached / degraded — not fresh data
|
| 2465 |
-
var schTxt=enabled?'ENABLED':(wired?'DISABLED':'NOT WIRED');
|
| 2466 |
-
var schC=enabled?'#
|
| 2467 |
var cf=sch.consecutive_failures||0;
|
| 2468 |
-
var boTxt=cf>0?('backing off \u00b7 '+cf+' consecutive miss'+(cf===1?'':'es')):'nominal';
|
| 2469 |
-
var boC=cf>0?'#f5b301':'var(--dim)';
|
| 2470 |
var ivl=cfg.interval_seconds!=null?(cfg.interval_seconds+'s'):'\u2014';
|
|
|
|
|
|
|
| 2471 |
var cell=function(label,val,color){ return '<div class="kpi" style="padding:.5rem .7rem"><div class="k">'+esc(label)+'</div><div class="v" style="font-size:1rem;color:'+(color||'var(--cream)')+'">'+val+'</div></div>'; };
|
| 2472 |
-
var
|
|
|
|
|
|
|
| 2473 |
? '<div class="row mono" style="font-size:11px;color:#f5b301;margin-top:.5rem;word-break:break-word">last issue: '+esc(String(sch.last_error))+'</div>'
|
| 2474 |
: (outcome==='SKIPPED' ? '<div class="row mono dim" style="font-size:11px;margin-top:.5rem">a run was still in progress \u2014 this cycle was skipped to avoid pile-up</div>' : '');
|
| 2475 |
box.innerHTML=
|
| 2476 |
-
'<div class="card-h"><span class="card-t">↻
|
| 2477 |
'<div class="kpis" style="grid-template-columns:repeat(auto-fit,minmax(118px,1fr));gap:.5rem;margin:0">'+
|
| 2478 |
cell('Scheduler',schTxt,schC)+
|
| 2479 |
cell('Last run',_csRel(sch.last_run_at,false),'var(--cream)')+
|
| 2480 |
cell('Outcome',outcome,oc)+
|
| 2481 |
-
cell('Next run',enabled?_csRel(sch.next_run_at,true):'\u2014','var(--cream)')+
|
| 2482 |
cell('Backoff',boTxt,boC)+
|
| 2483 |
cell('Interval',ivl,'var(--dim)')+
|
| 2484 |
-
'</div>'+note;
|
| 2485 |
}catch(e){ box.innerHTML='<div class="row mono" style="color:#ff7b7b">↻ auto-refresh status unavailable: '+esc(e&&e.message||e)+'</div>'; }
|
| 2486 |
}
|
| 2487 |
window.crawlStatusBanner=crawlStatusBanner; window.crawlStatusLoad=crawlStatusLoad;
|
|
|
|
| 2455 |
var j=await getJSON('/api/killinchu/crawl/status');
|
| 2456 |
var cfg=(j&&j.config)||{}, sch=(j&&j.scheduler)||{}, wired=!!(j&&j.wired);
|
| 2457 |
var enabled=(cfg.enabled!==false)&&wired;
|
| 2458 |
+
var health=String((j&&j.health)||(j&&j.status)||'unverified').toLowerCase();
|
| 2459 |
+
var halted=!!(j&&j.circuit_open)||health==='failed';
|
| 2460 |
var last=String(sch.last_status||'').toLowerCase();
|
| 2461 |
var outcome, oc;
|
| 2462 |
+
if(halted){ outcome='HALTED'; oc='#ff7b7b'; }
|
| 2463 |
+
else if(!sch.last_run_at){ outcome='NO RUN YET'; oc='#c9a05f'; }
|
| 2464 |
else if(last==='live'){ outcome='OK'; oc='#5fb3a3'; }
|
| 2465 |
else if(last==='error'){ outcome='ERROR'; oc='#ff7b7b'; }
|
| 2466 |
else if(last==='skipped'){ outcome='SKIPPED'; oc='#c9a05f'; }
|
| 2467 |
else { outcome='DEGRADED'; oc='#f5b301'; } // cached / degraded — not fresh data
|
| 2468 |
+
var schTxt=halted?'CIRCUIT OPEN':(enabled?'ENABLED':(wired?'DISABLED':'NOT WIRED'));
|
| 2469 |
+
var schC=(halted||!enabled)?'#ff7b7b':'#5fb3a3';
|
| 2470 |
var cf=sch.consecutive_failures||0;
|
| 2471 |
+
var boTxt=halted?'fail-closed \u00b7 no retries':(cf>0?('backing off \u00b7 '+cf+' consecutive miss'+(cf===1?'':'es')):'nominal');
|
| 2472 |
+
var boC=halted?'#ff7b7b':(cf>0?'#f5b301':'var(--dim)');
|
| 2473 |
var ivl=cfg.interval_seconds!=null?(cfg.interval_seconds+'s'):'\u2014';
|
| 2474 |
+
var freshness=String((j&&j.freshness)||'unverified').toUpperCase();
|
| 2475 |
+
var title='Intel feed \u00b7 '+(halted?'HALTED':freshness);
|
| 2476 |
var cell=function(label,val,color){ return '<div class="kpi" style="padding:.5rem .7rem"><div class="k">'+esc(label)+'</div><div class="v" style="font-size:1rem;color:'+(color||'var(--cream)')+'">'+val+'</div></div>'; };
|
| 2477 |
+
var remediation=halted&&j.operator_action
|
| 2478 |
+
? '<div class="row mono" style="font-size:11px;color:#ff7b7b;margin-top:.5rem;word-break:break-word">operator action: '+esc(String(j.operator_action))+'</div>' : '';
|
| 2479 |
+
var note=((outcome==='DEGRADED'||outcome==='ERROR'||outcome==='HALTED')&&sch.last_error)
|
| 2480 |
? '<div class="row mono" style="font-size:11px;color:#f5b301;margin-top:.5rem;word-break:break-word">last issue: '+esc(String(sch.last_error))+'</div>'
|
| 2481 |
: (outcome==='SKIPPED' ? '<div class="row mono dim" style="font-size:11px;margin-top:.5rem">a run was still in progress \u2014 this cycle was skipped to avoid pile-up</div>' : '');
|
| 2482 |
box.innerHTML=
|
| 2483 |
+
'<div class="card-h"><span class="card-t">↻ '+esc(title)+'</span><span class="card-ep">GET /api/killinchu/crawl/status \u00b7 adsb.lol military ADS-B</span></div>'+
|
| 2484 |
'<div class="kpis" style="grid-template-columns:repeat(auto-fit,minmax(118px,1fr));gap:.5rem;margin:0">'+
|
| 2485 |
cell('Scheduler',schTxt,schC)+
|
| 2486 |
cell('Last run',_csRel(sch.last_run_at,false),'var(--cream)')+
|
| 2487 |
cell('Outcome',outcome,oc)+
|
| 2488 |
+
cell('Next run',(enabled&&!halted)?_csRel(sch.next_run_at,true):'\u2014','var(--cream)')+
|
| 2489 |
cell('Backoff',boTxt,boC)+
|
| 2490 |
cell('Interval',ivl,'var(--dim)')+
|
| 2491 |
+
'</div>'+note+remediation;
|
| 2492 |
}catch(e){ box.innerHTML='<div class="row mono" style="color:#ff7b7b">↻ auto-refresh status unavailable: '+esc(e&&e.message||e)+'</div>'; }
|
| 2493 |
}
|
| 2494 |
window.crawlStatusBanner=crawlStatusBanner; window.crawlStatusLoad=crawlStatusLoad;
|
serve.py
CHANGED
|
@@ -42,32 +42,14 @@ from starlette.middleware.cors import CORSMiddleware
|
|
| 42 |
|
| 43 |
import killinchu_protocols as kp
|
| 44 |
from killinchu_receipt_export import build_receipt_export
|
|
|
|
| 45 |
|
| 46 |
_APP_ROOT = Path(os.environ.get("KILLINCHU_ROOT", "/app"))
|
| 47 |
STATIC_DIR = _APP_ROOT / "static"
|
| 48 |
ASSETS_DIR = STATIC_DIR / "assets"
|
| 49 |
INDEX_HTML = STATIC_DIR / "index.html"
|
| 50 |
DRONES_DB_PATH = _APP_ROOT / "drones_db.json"
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
def _safe_join_under(base: Path, user_rel: str) -> Path | None:
|
| 54 |
-
"""Resolve `user_rel` under `base` and return it only if it stays inside.
|
| 55 |
-
|
| 56 |
-
Root-cause path-injection guard for user-controlled relative paths reaching
|
| 57 |
-
a filesystem read. We fully resolve with os.path.realpath (following
|
| 58 |
-
symlinks) and require the result to be contained within the canonical base
|
| 59 |
-
directory. Anything that escapes (../, absolute, symlink) returns None and
|
| 60 |
-
the caller serves the SPA/404 fallback. Allowlist-on-resolved-location,
|
| 61 |
-
not a substring denylist.
|
| 62 |
-
"""
|
| 63 |
-
try:
|
| 64 |
-
base_real = os.path.realpath(base)
|
| 65 |
-
cand_real = os.path.realpath(os.path.join(base_real, user_rel))
|
| 66 |
-
if cand_real == base_real or cand_real.startswith(base_real + os.sep):
|
| 67 |
-
return Path(cand_real)
|
| 68 |
-
return None
|
| 69 |
-
except Exception:
|
| 70 |
-
return None
|
| 71 |
|
| 72 |
|
| 73 |
def _wired_ok(status) -> bool:
|
|
@@ -1239,18 +1221,18 @@ try:
|
|
| 1239 |
_F1_SHARED_DIR = _F1_Path("/app/static/shared")
|
| 1240 |
_F1_JS_CT = "application/javascript; charset=utf-8"
|
| 1241 |
_F1_SHARED_ALLOW = {
|
| 1242 |
-
"szl_label_engine.js": _F1_JS_CT,
|
| 1243 |
-
"szl_receipt_cosign.js": _F1_JS_CT,
|
| 1244 |
-
"szl_codename_sanitizer.js": _F1_JS_CT,
|
| 1245 |
-
"szl_holo3d.js": _F1_JS_CT,
|
| 1246 |
}
|
| 1247 |
|
| 1248 |
@app.get("/static/shared/{fname}")
|
| 1249 |
async def _f1_shared_module(fname: str):
|
| 1250 |
-
|
| 1251 |
-
if
|
| 1252 |
return JSONResponse({"error": "shared module not allowlisted", "file": fname}, status_code=404)
|
| 1253 |
-
f =
|
| 1254 |
if not f.is_file():
|
| 1255 |
return JSONResponse({"error": "shared module missing on disk", "file": fname}, status_code=404)
|
| 1256 |
return Response(content=f.read_bytes(), media_type=ct,
|
|
@@ -3430,16 +3412,22 @@ try:
|
|
| 3430 |
from starlette.responses import Response as _OPW_KC_SResp
|
| 3431 |
_OPW_KC_VENDOR = Path("/app/static-vendor")
|
| 3432 |
_OPW_KC_FILES = {
|
| 3433 |
-
"a11oy-operator-widget.js":
|
| 3434 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3435 |
}
|
| 3436 |
|
| 3437 |
@app.get("/vendor/{fname}")
|
| 3438 |
async def _opw_kc_vendor(fname: str):
|
| 3439 |
-
|
| 3440 |
-
if
|
| 3441 |
return JSONResponse({"error": "vendor asset not allowlisted", "file": fname}, status_code=404)
|
| 3442 |
-
f =
|
| 3443 |
if not f.is_file():
|
| 3444 |
return JSONResponse({"error": "vendor asset missing on disk", "file": fname}, status_code=404)
|
| 3445 |
return _OPW_KC_Resp(content=f.read_bytes(), media_type=ct,
|
|
@@ -3456,9 +3444,12 @@ try:
|
|
| 3456 |
_wp = request.url.path
|
| 3457 |
if _wp in ("/vendor/a11oy-operator-widget.js", "/vendor/a11oy-operator-widget.css"):
|
| 3458 |
_fn = _wp.rsplit("/", 1)[-1]
|
| 3459 |
-
|
| 3460 |
-
|
| 3461 |
-
|
|
|
|
|
|
|
|
|
|
| 3462 |
return _OPW_KC_Resp(content=_f.read_bytes(), media_type=_ct,
|
| 3463 |
headers={"Cache-Control": "public, max-age=31536000, immutable"})
|
| 3464 |
return _OPW_KC_Resp(content=b'/* operator widget asset missing on disk */',
|
|
@@ -4620,7 +4611,7 @@ except Exception as _qa6_e: # pragma: no cover — additive; never break the Sp
|
|
| 4620 |
|
| 4621 |
|
| 4622 |
@app.get("/{full_path:path}")
|
| 4623 |
-
async def spa_fallback(full_path: str) -> Response:
|
| 4624 |
# QA6 defense-in-depth: bare data prefixes must NEVER be served the SPA HTML.
|
| 4625 |
# If a bare data path somehow reaches the catch-all (alias not wired), return an
|
| 4626 |
# honest JSON 404 instead of the globe page (which would be silently wrong).
|
|
@@ -4628,11 +4619,9 @@ async def spa_fallback(full_path: str) -> Response:
|
|
| 4628 |
return JSONResponse({"error": "not found"}, status_code=404)
|
| 4629 |
if full_path in ("feeds", "osint", "mesh"):
|
| 4630 |
return JSONResponse({"error": "not found"}, status_code=404)
|
| 4631 |
-
|
| 4632 |
-
if
|
| 4633 |
-
return
|
| 4634 |
-
if candidate.is_file():
|
| 4635 |
-
return FileResponse(candidate)
|
| 4636 |
return FileResponse(INDEX_HTML, media_type="text/html")
|
| 4637 |
|
| 4638 |
|
|
@@ -5063,7 +5052,6 @@ try:
|
|
| 5063 |
from fastapi import Request as _JK_Request
|
| 5064 |
from fastapi.routing import APIRoute as _JK_Route
|
| 5065 |
from fastapi.responses import (
|
| 5066 |
-
FileResponse as _JK_File,
|
| 5067 |
HTMLResponse as _JK_HTML,
|
| 5068 |
RedirectResponse as _JK_Redir,
|
| 5069 |
JSONResponse as _JK_JSON,
|
|
@@ -5072,6 +5060,7 @@ try:
|
|
| 5072 |
|
| 5073 |
_JK_DIR = _JK_Path(__file__).resolve().parent / "static" / "jackin"
|
| 5074 |
_JK_INDEX = _JK_DIR / "index.html"
|
|
|
|
| 5075 |
|
| 5076 |
def _jk_index_html() -> str:
|
| 5077 |
# Serve index.html with a <base href="/jackin/"> injected so the app's
|
|
@@ -5096,11 +5085,9 @@ try:
|
|
| 5096 |
rel = request.path_params.get("jk_path", "") or ""
|
| 5097 |
if rel in ("", "index.html"):
|
| 5098 |
return _JK_HTML(_jk_index_html())
|
| 5099 |
-
|
| 5100 |
-
if
|
| 5101 |
-
return
|
| 5102 |
-
if candidate.is_file():
|
| 5103 |
-
return _JK_File(str(candidate))
|
| 5104 |
# SPA-style fallback to the jackin index for unknown sub-paths.
|
| 5105 |
return _JK_HTML(_jk_index_html())
|
| 5106 |
|
|
@@ -5537,20 +5524,6 @@ except Exception as _kc_wave_e: # pragma: no cover — never break SPA/other or
|
|
| 5537 |
# ============================================================================
|
| 5538 |
|
| 5539 |
|
| 5540 |
-
|
| 5541 |
-
# ============================================================================
|
| 5542 |
-
# READ-ONLY PUBLIC CONTRACTS -- source/deployment attestation, runtime status,
|
| 5543 |
-
# MELT summary, and fail-safe OpenAPI discovery. Registered last and inserted at
|
| 5544 |
-
# the front so explicit machine-readable routes beat the SPA history fallback.
|
| 5545 |
-
# No targeting, collection, decision, or effector behavior is changed here.
|
| 5546 |
-
# ============================================================================
|
| 5547 |
-
try:
|
| 5548 |
-
import killinchu_public_contracts as _kc_public_contracts
|
| 5549 |
-
_kc_public_contract_status = _kc_public_contracts.register(app, ns="killinchu")
|
| 5550 |
-
print(f"[killinchu] public contracts registered: {_kc_public_contract_status}", file=sys.stderr)
|
| 5551 |
-
except Exception as _kc_public_contract_error: # pragma: no cover
|
| 5552 |
-
print(f"[killinchu] public contracts NOT registered: {_kc_public_contract_error!r}", file=sys.stderr)
|
| 5553 |
-
|
| 5554 |
if __name__ == "__main__":
|
| 5555 |
import uvicorn
|
| 5556 |
port = int(os.environ.get("PORT", "7860"))
|
|
|
|
| 42 |
|
| 43 |
import killinchu_protocols as kp
|
| 44 |
from killinchu_receipt_export import build_receipt_export
|
| 45 |
+
from szl_safe_static import RootedStaticFiles
|
| 46 |
|
| 47 |
_APP_ROOT = Path(os.environ.get("KILLINCHU_ROOT", "/app"))
|
| 48 |
STATIC_DIR = _APP_ROOT / "static"
|
| 49 |
ASSETS_DIR = STATIC_DIR / "assets"
|
| 50 |
INDEX_HTML = STATIC_DIR / "index.html"
|
| 51 |
DRONES_DB_PATH = _APP_ROOT / "drones_db.json"
|
| 52 |
+
_SPA_FILES = RootedStaticFiles(STATIC_DIR)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
|
| 54 |
|
| 55 |
def _wired_ok(status) -> bool:
|
|
|
|
| 1221 |
_F1_SHARED_DIR = _F1_Path("/app/static/shared")
|
| 1222 |
_F1_JS_CT = "application/javascript; charset=utf-8"
|
| 1223 |
_F1_SHARED_ALLOW = {
|
| 1224 |
+
"szl_label_engine.js": (_F1_SHARED_DIR / "szl_label_engine.js", _F1_JS_CT),
|
| 1225 |
+
"szl_receipt_cosign.js": (_F1_SHARED_DIR / "szl_receipt_cosign.js", _F1_JS_CT),
|
| 1226 |
+
"szl_codename_sanitizer.js": (_F1_SHARED_DIR / "szl_codename_sanitizer.js", _F1_JS_CT),
|
| 1227 |
+
"szl_holo3d.js": (_F1_SHARED_DIR / "szl_holo3d.js", _F1_JS_CT),
|
| 1228 |
}
|
| 1229 |
|
| 1230 |
@app.get("/static/shared/{fname}")
|
| 1231 |
async def _f1_shared_module(fname: str):
|
| 1232 |
+
asset = _F1_SHARED_ALLOW.get(fname)
|
| 1233 |
+
if asset is None:
|
| 1234 |
return JSONResponse({"error": "shared module not allowlisted", "file": fname}, status_code=404)
|
| 1235 |
+
f, ct = asset
|
| 1236 |
if not f.is_file():
|
| 1237 |
return JSONResponse({"error": "shared module missing on disk", "file": fname}, status_code=404)
|
| 1238 |
return Response(content=f.read_bytes(), media_type=ct,
|
|
|
|
| 3412 |
from starlette.responses import Response as _OPW_KC_SResp
|
| 3413 |
_OPW_KC_VENDOR = Path("/app/static-vendor")
|
| 3414 |
_OPW_KC_FILES = {
|
| 3415 |
+
"a11oy-operator-widget.js": (
|
| 3416 |
+
_OPW_KC_VENDOR / "a11oy-operator-widget.js",
|
| 3417 |
+
"application/javascript; charset=utf-8",
|
| 3418 |
+
),
|
| 3419 |
+
"a11oy-operator-widget.css": (
|
| 3420 |
+
_OPW_KC_VENDOR / "a11oy-operator-widget.css",
|
| 3421 |
+
"text/css; charset=utf-8",
|
| 3422 |
+
),
|
| 3423 |
}
|
| 3424 |
|
| 3425 |
@app.get("/vendor/{fname}")
|
| 3426 |
async def _opw_kc_vendor(fname: str):
|
| 3427 |
+
asset = _OPW_KC_FILES.get(fname)
|
| 3428 |
+
if asset is None:
|
| 3429 |
return JSONResponse({"error": "vendor asset not allowlisted", "file": fname}, status_code=404)
|
| 3430 |
+
f, ct = asset
|
| 3431 |
if not f.is_file():
|
| 3432 |
return JSONResponse({"error": "vendor asset missing on disk", "file": fname}, status_code=404)
|
| 3433 |
return _OPW_KC_Resp(content=f.read_bytes(), media_type=ct,
|
|
|
|
| 3444 |
_wp = request.url.path
|
| 3445 |
if _wp in ("/vendor/a11oy-operator-widget.js", "/vendor/a11oy-operator-widget.css"):
|
| 3446 |
_fn = _wp.rsplit("/", 1)[-1]
|
| 3447 |
+
_asset = _OPW_KC_FILES.get(_fn)
|
| 3448 |
+
if _asset is not None:
|
| 3449 |
+
_f, _ct = _asset
|
| 3450 |
+
else: # defensive: membership above is a fixed allowlist
|
| 3451 |
+
_f, _ct = None, None
|
| 3452 |
+
if _f is not None and _ct is not None and _f.is_file():
|
| 3453 |
return _OPW_KC_Resp(content=_f.read_bytes(), media_type=_ct,
|
| 3454 |
headers={"Cache-Control": "public, max-age=31536000, immutable"})
|
| 3455 |
return _OPW_KC_Resp(content=b'/* operator widget asset missing on disk */',
|
|
|
|
| 4611 |
|
| 4612 |
|
| 4613 |
@app.get("/{full_path:path}")
|
| 4614 |
+
async def spa_fallback(full_path: str, request: Request) -> Response:
|
| 4615 |
# QA6 defense-in-depth: bare data prefixes must NEVER be served the SPA HTML.
|
| 4616 |
# If a bare data path somehow reaches the catch-all (alias not wired), return an
|
| 4617 |
# honest JSON 404 instead of the globe page (which would be silently wrong).
|
|
|
|
| 4619 |
return JSONResponse({"error": "not found"}, status_code=404)
|
| 4620 |
if full_path in ("feeds", "osint", "mesh"):
|
| 4621 |
return JSONResponse({"error": "not found"}, status_code=404)
|
| 4622 |
+
static_response = await _SPA_FILES.get(full_path, request.scope)
|
| 4623 |
+
if static_response is not None:
|
| 4624 |
+
return static_response
|
|
|
|
|
|
|
| 4625 |
return FileResponse(INDEX_HTML, media_type="text/html")
|
| 4626 |
|
| 4627 |
|
|
|
|
| 5052 |
from fastapi import Request as _JK_Request
|
| 5053 |
from fastapi.routing import APIRoute as _JK_Route
|
| 5054 |
from fastapi.responses import (
|
|
|
|
| 5055 |
HTMLResponse as _JK_HTML,
|
| 5056 |
RedirectResponse as _JK_Redir,
|
| 5057 |
JSONResponse as _JK_JSON,
|
|
|
|
| 5060 |
|
| 5061 |
_JK_DIR = _JK_Path(__file__).resolve().parent / "static" / "jackin"
|
| 5062 |
_JK_INDEX = _JK_DIR / "index.html"
|
| 5063 |
+
_JK_FILES = RootedStaticFiles(_JK_DIR)
|
| 5064 |
|
| 5065 |
def _jk_index_html() -> str:
|
| 5066 |
# Serve index.html with a <base href="/jackin/"> injected so the app's
|
|
|
|
| 5085 |
rel = request.path_params.get("jk_path", "") or ""
|
| 5086 |
if rel in ("", "index.html"):
|
| 5087 |
return _JK_HTML(_jk_index_html())
|
| 5088 |
+
static_response = await _JK_FILES.get(rel, request.scope)
|
| 5089 |
+
if static_response is not None:
|
| 5090 |
+
return static_response
|
|
|
|
|
|
|
| 5091 |
# SPA-style fallback to the jackin index for unknown sub-paths.
|
| 5092 |
return _JK_HTML(_jk_index_html())
|
| 5093 |
|
|
|
|
| 5524 |
# ============================================================================
|
| 5525 |
|
| 5526 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5527 |
if __name__ == "__main__":
|
| 5528 |
import uvicorn
|
| 5529 |
port = int(os.environ.get("PORT", "7860"))
|
szl_connectors/governance.py
CHANGED
|
@@ -10,8 +10,8 @@ DOCTRINE (non-negotiable):
|
|
| 10 |
• Every write emits a DSSE-signed Khipu receipt (real ECDSA-P256 over the DSSE
|
| 11 |
PAE when SZL_COSIGN_PRIVATE_PEM is present; an explicit UNSIGNED envelope
|
| 12 |
otherwise — NEVER a fabricated signature). Reuses the live `szl_dsse` module.
|
| 13 |
-
• No credential value is EVER placed in a receipt body — only a
|
| 14 |
-
|
| 15 |
• State-changing writes carry the 2-person Yuyay gate + Khipu 3-of-4 quorum
|
| 16 |
status (the hatun-mcp governance contract). Until a connector is CONNECTED,
|
| 17 |
write() is refused with an honest reason.
|
|
@@ -22,14 +22,22 @@ call directly.
|
|
| 22 |
"""
|
| 23 |
from __future__ import annotations
|
| 24 |
|
| 25 |
-
import hashlib
|
| 26 |
import json
|
| 27 |
-
import
|
| 28 |
from datetime import datetime, timezone
|
| 29 |
from typing import Any
|
| 30 |
|
|
|
|
|
|
|
| 31 |
# Anti-overconfidence floor: Λ is never reported as 1.0. We cap at this ceiling.
|
| 32 |
LAMBDA_CEILING = 0.985
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
|
| 34 |
|
| 35 |
def _now() -> str:
|
|
@@ -67,6 +75,53 @@ def quorum_status(present: list[str] | None = None, n: int = 4, f: int = 1) -> d
|
|
| 67 |
}
|
| 68 |
|
| 69 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
def _dsse_sign(payload: dict[str, Any]) -> dict[str, Any]:
|
| 71 |
"""Sign a receipt payload via the live szl_dsse module if importable; else an
|
| 72 |
honest UNSIGNED envelope (NEVER a fabricated signature)."""
|
|
@@ -98,9 +153,10 @@ def receipt_for_write(*, connector_id: str, action: dict[str, Any],
|
|
| 98 |
NOT secret values), the Λ score, quorum status, credential FINGERPRINT HASHES
|
| 99 |
(never the values), and a result summary. Returns {receipt_hash, dsse, body}.
|
| 100 |
"""
|
| 101 |
-
#
|
| 102 |
-
|
| 103 |
-
|
|
|
|
| 104 |
body = {
|
| 105 |
"kind": "szl.connector.write",
|
| 106 |
"connector_id": connector_id,
|
|
@@ -108,13 +164,15 @@ def receipt_for_write(*, connector_id: str, action: dict[str, Any],
|
|
| 108 |
"lambda_value": lambda_value,
|
| 109 |
"lambda_note": "Λ never 1.0 (conformal anti-overconfidence floor 1/(n+1)); Λ = Conjecture 1",
|
| 110 |
"quorum": quorum or quorum_status(),
|
| 111 |
-
"credential_fingerprints": cred_fingerprints
|
| 112 |
-
"result":
|
| 113 |
"emitted_at": _now(),
|
| 114 |
"doctrine": "v11 — Λ-gate + DSSE/Khipu receipt on every write; no committed keys; trust never 100%",
|
| 115 |
}
|
| 116 |
-
|
| 117 |
-
|
|
|
|
|
|
|
| 118 |
body["receipt_hash"] = receipt_hash
|
| 119 |
dsse = _dsse_sign(body)
|
| 120 |
return {"receipt_hash": receipt_hash, "dsse": dsse, "body": body}
|
|
@@ -135,7 +193,7 @@ def gate_write(*, connector_id: str, connected: bool, action: dict[str, Any],
|
|
| 135 |
"""
|
| 136 |
has_method = bool((action or {}).get("method") or (action or {}).get("object")
|
| 137 |
or (action or {}).get("doctype") or (action or {}).get("sobject"))
|
| 138 |
-
leak =
|
| 139 |
q = quorum_status(present=quorum_present)
|
| 140 |
axes = {
|
| 141 |
"connected": 1.0 if connected else 0.0,
|
|
|
|
| 10 |
• Every write emits a DSSE-signed Khipu receipt (real ECDSA-P256 over the DSSE
|
| 11 |
PAE when SZL_COSIGN_PRIVATE_PEM is present; an explicit UNSIGNED envelope
|
| 12 |
otherwise — NEVER a fabricated signature). Reuses the live `szl_dsse` module.
|
| 13 |
+
• No credential value is EVER placed in a receipt body — only a
|
| 14 |
+
PBKDF2-HMAC-SHA256 credential fingerprint.
|
| 15 |
• State-changing writes carry the 2-person Yuyay gate + Khipu 3-of-4 quorum
|
| 16 |
status (the hatun-mcp governance contract). Until a connector is CONNECTED,
|
| 17 |
write() is refused with an honest reason.
|
|
|
|
| 22 |
"""
|
| 23 |
from __future__ import annotations
|
| 24 |
|
|
|
|
| 25 |
import json
|
| 26 |
+
import re
|
| 27 |
from datetime import datetime, timezone
|
| 28 |
from typing import Any
|
| 29 |
|
| 30 |
+
from szl_content_address import sha256_content_address
|
| 31 |
+
|
| 32 |
# Anti-overconfidence floor: Λ is never reported as 1.0. We cap at this ceiling.
|
| 33 |
LAMBDA_CEILING = 0.985
|
| 34 |
+
_FINGERPRINT_RE = re.compile(r"\Apbkdf2-sha256:[0-9a-f]{32}\Z")
|
| 35 |
+
_FINGERPRINT_LABEL_RE = re.compile(r"\A[A-Za-z0-9_.:-]{1,80}\Z")
|
| 36 |
+
_SENSITIVE_KEY_PARTS = frozenset({"password", "passwd", "secret", "token", "credential"})
|
| 37 |
+
_SENSITIVE_KEY_NAMES = frozenset({
|
| 38 |
+
"api_key", "private_key", "authorization", "proxy_authorization",
|
| 39 |
+
"cookie", "set_cookie",
|
| 40 |
+
})
|
| 41 |
|
| 42 |
|
| 43 |
def _now() -> str:
|
|
|
|
| 75 |
}
|
| 76 |
|
| 77 |
|
| 78 |
+
def _is_sensitive_key(key: object) -> bool:
|
| 79 |
+
normalized = re.sub(r"[^a-z0-9]+", "_", str(key).lower()).strip("_")
|
| 80 |
+
parts = frozenset(part for part in normalized.split("_") if part)
|
| 81 |
+
return normalized in _SENSITIVE_KEY_NAMES or bool(parts & _SENSITIVE_KEY_PARTS)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def _contains_sensitive_field(value: Any) -> bool:
|
| 85 |
+
if isinstance(value, dict):
|
| 86 |
+
return any(
|
| 87 |
+
_is_sensitive_key(key) or _contains_sensitive_field(child)
|
| 88 |
+
for key, child in value.items()
|
| 89 |
+
)
|
| 90 |
+
if isinstance(value, (list, tuple)):
|
| 91 |
+
return any(_contains_sensitive_field(child) for child in value)
|
| 92 |
+
return False
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _scrub_sensitive_fields(value: Any) -> Any:
|
| 96 |
+
"""Recursively remove secret-bearing fields without copying their values."""
|
| 97 |
+
if isinstance(value, dict):
|
| 98 |
+
return {
|
| 99 |
+
key: _scrub_sensitive_fields(child)
|
| 100 |
+
for key, child in value.items()
|
| 101 |
+
if not _is_sensitive_key(key)
|
| 102 |
+
}
|
| 103 |
+
if isinstance(value, list):
|
| 104 |
+
return [_scrub_sensitive_fields(child) for child in value]
|
| 105 |
+
if isinstance(value, tuple):
|
| 106 |
+
return tuple(_scrub_sensitive_fields(child) for child in value)
|
| 107 |
+
return value
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def _validated_fingerprints(values: dict[str, str] | None) -> dict[str, str]:
|
| 111 |
+
"""Allow only the KDF output format emitted by ``cred_fingerprint``."""
|
| 112 |
+
safe: dict[str, str] = {}
|
| 113 |
+
for label, fingerprint in (values or {}).items():
|
| 114 |
+
if not isinstance(label, str) or _FINGERPRINT_LABEL_RE.fullmatch(label) is None:
|
| 115 |
+
continue
|
| 116 |
+
if fingerprint == "absent":
|
| 117 |
+
safe[label] = "absent"
|
| 118 |
+
elif isinstance(fingerprint, str) and _FINGERPRINT_RE.fullmatch(fingerprint):
|
| 119 |
+
safe[label] = fingerprint
|
| 120 |
+
else:
|
| 121 |
+
safe[label] = "invalid-fingerprint"
|
| 122 |
+
return safe
|
| 123 |
+
|
| 124 |
+
|
| 125 |
def _dsse_sign(payload: dict[str, Any]) -> dict[str, Any]:
|
| 126 |
"""Sign a receipt payload via the live szl_dsse module if importable; else an
|
| 127 |
honest UNSIGNED envelope (NEVER a fabricated signature)."""
|
|
|
|
| 153 |
NOT secret values), the Λ score, quorum status, credential FINGERPRINT HASHES
|
| 154 |
(never the values), and a result summary. Returns {receipt_hash, dsse, body}.
|
| 155 |
"""
|
| 156 |
+
# Scrub at every nesting level. Direct callers receive the same protection
|
| 157 |
+
# as gate_write callers, including secrets hidden in list/dict children.
|
| 158 |
+
safe_action = _scrub_sensitive_fields(action or {})
|
| 159 |
+
safe_result = _scrub_sensitive_fields(result_summary or {})
|
| 160 |
body = {
|
| 161 |
"kind": "szl.connector.write",
|
| 162 |
"connector_id": connector_id,
|
|
|
|
| 164 |
"lambda_value": lambda_value,
|
| 165 |
"lambda_note": "Λ never 1.0 (conformal anti-overconfidence floor 1/(n+1)); Λ = Conjecture 1",
|
| 166 |
"quorum": quorum or quorum_status(),
|
| 167 |
+
"credential_fingerprints": _validated_fingerprints(cred_fingerprints),
|
| 168 |
+
"result": safe_result,
|
| 169 |
"emitted_at": _now(),
|
| 170 |
"doctrine": "v11 — Λ-gate + DSSE/Khipu receipt on every write; no committed keys; trust never 100%",
|
| 171 |
}
|
| 172 |
+
canonical_body = json.dumps(body, sort_keys=True, separators=(",", ":")).encode()
|
| 173 |
+
receipt_hash = "sha256:" + sha256_content_address(
|
| 174 |
+
canonical_body, purpose="khipu-receipt"
|
| 175 |
+
)
|
| 176 |
body["receipt_hash"] = receipt_hash
|
| 177 |
dsse = _dsse_sign(body)
|
| 178 |
return {"receipt_hash": receipt_hash, "dsse": dsse, "body": body}
|
|
|
|
| 193 |
"""
|
| 194 |
has_method = bool((action or {}).get("method") or (action or {}).get("object")
|
| 195 |
or (action or {}).get("doctype") or (action or {}).get("sobject"))
|
| 196 |
+
leak = _contains_sensitive_field(action or {})
|
| 197 |
q = quorum_status(present=quorum_present)
|
| 198 |
axes = {
|
| 199 |
"connected": 1.0 if connected else 0.0,
|
szl_connectors/oauth.py
CHANGED
|
@@ -91,6 +91,23 @@ PROVIDER_OAUTH: dict[str, dict[str, str]] = {
|
|
| 91 |
},
|
| 92 |
}
|
| 93 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
|
| 95 |
# Domain-separation salt + work factor for deriving the state-signing key.
|
| 96 |
_STATE_KDF_SALT = b"szl.killinchu.oauth-state.v1"
|
|
@@ -224,18 +241,19 @@ def exchange_code(connector_id: str, *, code: str, state: str, redirect_uri: str
|
|
| 224 |
"provider_detail": str(body)[:200]}
|
| 225 |
refresh = body.get("refresh_token", "")
|
| 226 |
access = body.get("access_token", "")
|
|
|
|
| 227 |
# credential-bound DSSE receipt — fingerprint ONLY, never the token value.
|
| 228 |
from .governance import receipt_for_write
|
| 229 |
rcpt = receipt_for_write(
|
| 230 |
connector_id=connector_id,
|
| 231 |
action={"method": "oauth.credential_bound", "object": "refresh_token",
|
| 232 |
-
"scope":
|
| 233 |
lambda_value=0.9,
|
| 234 |
cred_fingerprints={
|
| 235 |
"refresh_token": cred_fingerprint(refresh) if refresh else "absent",
|
| 236 |
"access_token": cred_fingerprint(access) if access else "absent",
|
| 237 |
},
|
| 238 |
-
result_summary={"granted_scope":
|
| 239 |
"note": "secret persisted to Space secret store only; never committed"},
|
| 240 |
)
|
| 241 |
return {
|
|
|
|
| 91 |
},
|
| 92 |
}
|
| 93 |
|
| 94 |
+
# Keep public scopes in a separate container from endpoint metadata whose
|
| 95 |
+
# ``token`` key is (correctly but over-broadly) treated as sensitive by taint
|
| 96 |
+
# analysis. Receipt content uses this public-only map, so no credential-like
|
| 97 |
+
# container can flow into a protocol content address.
|
| 98 |
+
PROVIDER_SCOPES: dict[str, str] = {
|
| 99 |
+
"salesforce": "api refresh_token",
|
| 100 |
+
"hubspot": "crm.objects.contacts.read crm.objects.companies.read crm.objects.deals.read",
|
| 101 |
+
"zoho_crm": "ZohoCRM.modules.ALL ZohoCRM.org.READ",
|
| 102 |
+
"slack": "channels:read chat:write users:read",
|
| 103 |
+
"okta": "okta.users.read okta.groups.read",
|
| 104 |
+
"entra": "https://graph.microsoft.com/.default offline_access",
|
| 105 |
+
"auth0": "read:users read:logs",
|
| 106 |
+
"dynamics_crm": "https://{org}.api.crm.dynamics.com/.default offline_access",
|
| 107 |
+
"netsuite": "rest_webservices",
|
| 108 |
+
"servicenow": "useraccount",
|
| 109 |
+
}
|
| 110 |
+
|
| 111 |
|
| 112 |
# Domain-separation salt + work factor for deriving the state-signing key.
|
| 113 |
_STATE_KDF_SALT = b"szl.killinchu.oauth-state.v1"
|
|
|
|
| 241 |
"provider_detail": str(body)[:200]}
|
| 242 |
refresh = body.get("refresh_token", "")
|
| 243 |
access = body.get("access_token", "")
|
| 244 |
+
public_scope = PROVIDER_SCOPES.get(connector_id, "")
|
| 245 |
# credential-bound DSSE receipt — fingerprint ONLY, never the token value.
|
| 246 |
from .governance import receipt_for_write
|
| 247 |
rcpt = receipt_for_write(
|
| 248 |
connector_id=connector_id,
|
| 249 |
action={"method": "oauth.credential_bound", "object": "refresh_token",
|
| 250 |
+
"scope": public_scope},
|
| 251 |
lambda_value=0.9,
|
| 252 |
cred_fingerprints={
|
| 253 |
"refresh_token": cred_fingerprint(refresh) if refresh else "absent",
|
| 254 |
"access_token": cred_fingerprint(access) if access else "absent",
|
| 255 |
},
|
| 256 |
+
result_summary={"granted_scope": public_scope,
|
| 257 |
"note": "secret persisted to Space secret store only; never committed"},
|
| 258 |
)
|
| 259 |
return {
|
szl_content_address.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Explicit SHA-256 content addressing for protocol and receipt bytes.
|
| 3 |
+
|
| 4 |
+
This module is deliberately *not* a password or credential derivation API.
|
| 5 |
+
Callers must provide one of the narrow protocol purposes below, and the input
|
| 6 |
+
bytes are hashed exactly as supplied. Keeping this operation separate from
|
| 7 |
+
PBKDF2-based credential fingerprints prevents accidental reuse while
|
| 8 |
+
preserving the byte-for-byte hashes already carried by DSSE envelopes and
|
| 9 |
+
Khipu receipts.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import hashlib
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
_PURPOSES = frozenset({"dsse-pae", "khipu-receipt", "public-key"})
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def sha256_content_address(data: bytes, *, purpose: str) -> str:
|
| 20 |
+
"""Return the protocol-compatible SHA-256 hex address of public content.
|
| 21 |
+
|
| 22 |
+
``purpose`` is intentionally mandatory and allowlisted. Secret values
|
| 23 |
+
belong in a password KDF such as PBKDF2, never in this function.
|
| 24 |
+
"""
|
| 25 |
+
if purpose not in _PURPOSES:
|
| 26 |
+
raise ValueError(f"unsupported content-address purpose: {purpose!r}")
|
| 27 |
+
if not isinstance(data, bytes):
|
| 28 |
+
raise TypeError("content address input must be bytes")
|
| 29 |
+
return hashlib.sha256(data).hexdigest()
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
__all__ = ["sha256_content_address"]
|
szl_dsse.py
CHANGED
|
@@ -66,13 +66,14 @@ for SZL Khipu receipts, backed by the SZLHOLDINGS **Cosign** keypair.
|
|
| 66 |
from __future__ import annotations
|
| 67 |
|
| 68 |
import base64
|
| 69 |
-
import hashlib
|
| 70 |
import json
|
| 71 |
import os
|
| 72 |
import sys
|
| 73 |
from datetime import datetime, timezone
|
| 74 |
from typing import Any
|
| 75 |
|
|
|
|
|
|
|
| 76 |
KEYID = "szlholdings-cosign"
|
| 77 |
KHIPU_PAYLOAD_TYPE = "application/vnd.szl.khipu+json"
|
| 78 |
COSIGN_PUB_FINGERPRINT_ENV = "SZL_COSIGN_PUB_SHA256" # optional pin
|
|
@@ -151,7 +152,9 @@ def signing_available() -> bool:
|
|
| 151 |
|
| 152 |
|
| 153 |
def public_key_fingerprint() -> str:
|
| 154 |
-
return
|
|
|
|
|
|
|
| 155 |
|
| 156 |
|
| 157 |
# ---------------------------------------------------------------------------
|
|
@@ -171,7 +174,7 @@ def sign_payload(payload_obj: Any, payload_type: str = KHIPU_PAYLOAD_TYPE) -> di
|
|
| 171 |
"payloadType": payload_type,
|
| 172 |
"payload": base64.b64encode(body).decode("ascii"),
|
| 173 |
"_dsse": "DSSEv1",
|
| 174 |
-
"_pae_sha256":
|
| 175 |
"_signed_at": datetime.now(timezone.utc).isoformat(),
|
| 176 |
}
|
| 177 |
priv = _load_private_key()
|
|
@@ -210,7 +213,7 @@ def verify_envelope(env: dict[str, Any]) -> dict[str, Any]:
|
|
| 210 |
return {**out, "verified": False, "reason": "no signatures (unsigned envelope)"}
|
| 211 |
body = base64.b64decode(payload_b64)
|
| 212 |
to_verify = pae(payload_type, body)
|
| 213 |
-
out["pae_sha256"] =
|
| 214 |
pub = _load_public_key()
|
| 215 |
from cryptography.hazmat.primitives.asymmetric import ec
|
| 216 |
from cryptography.hazmat.primitives import hashes
|
|
|
|
| 66 |
from __future__ import annotations
|
| 67 |
|
| 68 |
import base64
|
|
|
|
| 69 |
import json
|
| 70 |
import os
|
| 71 |
import sys
|
| 72 |
from datetime import datetime, timezone
|
| 73 |
from typing import Any
|
| 74 |
|
| 75 |
+
from szl_content_address import sha256_content_address
|
| 76 |
+
|
| 77 |
KEYID = "szlholdings-cosign"
|
| 78 |
KHIPU_PAYLOAD_TYPE = "application/vnd.szl.khipu+json"
|
| 79 |
COSIGN_PUB_FINGERPRINT_ENV = "SZL_COSIGN_PUB_SHA256" # optional pin
|
|
|
|
| 152 |
|
| 153 |
|
| 154 |
def public_key_fingerprint() -> str:
|
| 155 |
+
return sha256_content_address(
|
| 156 |
+
COSIGN_PUBLIC_PEM.strip().encode(), purpose="public-key"
|
| 157 |
+
)
|
| 158 |
|
| 159 |
|
| 160 |
# ---------------------------------------------------------------------------
|
|
|
|
| 174 |
"payloadType": payload_type,
|
| 175 |
"payload": base64.b64encode(body).decode("ascii"),
|
| 176 |
"_dsse": "DSSEv1",
|
| 177 |
+
"_pae_sha256": sha256_content_address(to_sign, purpose="dsse-pae"),
|
| 178 |
"_signed_at": datetime.now(timezone.utc).isoformat(),
|
| 179 |
}
|
| 180 |
priv = _load_private_key()
|
|
|
|
| 213 |
return {**out, "verified": False, "reason": "no signatures (unsigned envelope)"}
|
| 214 |
body = base64.b64decode(payload_b64)
|
| 215 |
to_verify = pae(payload_type, body)
|
| 216 |
+
out["pae_sha256"] = sha256_content_address(to_verify, purpose="dsse-pae")
|
| 217 |
pub = _load_public_key()
|
| 218 |
from cryptography.hazmat.primitives.asymmetric import ec
|
| 219 |
from cryptography.hazmat.primitives import hashes
|
szl_killinchu_cookbook.py
CHANGED
|
@@ -25,6 +25,7 @@ import os
|
|
| 25 |
import sys
|
| 26 |
import hashlib
|
| 27 |
import datetime
|
|
|
|
| 28 |
from pathlib import Path
|
| 29 |
|
| 30 |
from fastapi import Request
|
|
@@ -34,6 +35,11 @@ from fastapi.responses import JSONResponse, PlainTextResponse
|
|
| 34 |
# Resolve relative to THIS module so it works regardless of CWD.
|
| 35 |
_HERE = Path(__file__).resolve().parent
|
| 36 |
_DATA = _HERE / "static" / "cookbook"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
|
| 38 |
DOCTRINE = "v11"
|
| 39 |
NUMBERS = {"declarations": 749, "axioms": 14, "sorries": 163}
|
|
@@ -196,9 +202,14 @@ def register_cookbook(app, ns: str = "killinchu", sign_fn=None):
|
|
| 196 |
async def cookbook_get(recipe_id: str) -> JSONResponse:
|
| 197 |
idx = _read_json(_DATA / "recipes" / "_index.json") or []
|
| 198 |
meta = next((r for r in idx if r.get("id") == recipe_id), None)
|
| 199 |
-
#
|
| 200 |
-
|
| 201 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 202 |
if not md or meta is None:
|
| 203 |
return JSONResponse(_base({
|
| 204 |
"error": "recipe not found", "ref": recipe_id,
|
|
@@ -233,15 +244,16 @@ def register_cookbook(app, ns: str = "killinchu", sign_fn=None):
|
|
| 233 |
|
| 234 |
@app.get(p + "/missions/{mission_id}")
|
| 235 |
async def mission_get(mission_id: str) -> JSONResponse:
|
| 236 |
-
|
| 237 |
-
|
|
|
|
| 238 |
if m is None:
|
| 239 |
return JSONResponse(_base({
|
| 240 |
"error": "mission not found", "ref": mission_id,
|
| 241 |
"available": ["P1", "P2", "P3", "P4", "P5", "P6", "P7", "P8"],
|
| 242 |
}), status_code=404)
|
| 243 |
drone = bool(m.get("drone_domain"))
|
| 244 |
-
m["recall_receipt"] = _recall_receipt("mission",
|
| 245 |
# m already carries doctrine/numbers/disclaimer from generation; ensure disclaimer on drone
|
| 246 |
if drone and "disclaimer" not in m:
|
| 247 |
m["disclaimer"] = LEGAL_DISCLAIMER
|
|
|
|
| 25 |
import sys
|
| 26 |
import hashlib
|
| 27 |
import datetime
|
| 28 |
+
import re
|
| 29 |
from pathlib import Path
|
| 30 |
|
| 31 |
from fastapi import Request
|
|
|
|
| 35 |
# Resolve relative to THIS module so it works regardless of CWD.
|
| 36 |
_HERE = Path(__file__).resolve().parent
|
| 37 |
_DATA = _HERE / "static" / "cookbook"
|
| 38 |
+
_SAFE_RECIPE_ID = re.compile(r"\Arecipe-[a-z0-9-]+\Z")
|
| 39 |
+
_MISSION_FILES = {
|
| 40 |
+
"P1": "P1.json", "P2": "P2.json", "P3": "P3.json", "P4": "P4.json",
|
| 41 |
+
"P5": "P5.json", "P6": "P6.json", "P7": "P7.json", "P8": "P8.json",
|
| 42 |
+
}
|
| 43 |
|
| 44 |
DOCTRINE = "v11"
|
| 45 |
NUMBERS = {"declarations": 749, "axioms": 14, "sorries": 163}
|
|
|
|
| 202 |
async def cookbook_get(recipe_id: str) -> JSONResponse:
|
| 203 |
idx = _read_json(_DATA / "recipes" / "_index.json") or []
|
| 204 |
meta = next((r for r in idx if r.get("id") == recipe_id), None)
|
| 205 |
+
# The path component comes from the trusted, committed index entry, not
|
| 206 |
+
# from the route parameter. Validate the index too so a malformed bundle
|
| 207 |
+
# fails closed instead of becoming a filesystem path.
|
| 208 |
+
indexed_id = meta.get("id") if isinstance(meta, dict) else None
|
| 209 |
+
if not isinstance(indexed_id, str) or _SAFE_RECIPE_ID.fullmatch(indexed_id) is None:
|
| 210 |
+
md = ""
|
| 211 |
+
else:
|
| 212 |
+
md = _read_text(_DATA / "recipes" / f"{indexed_id}.md")
|
| 213 |
if not md or meta is None:
|
| 214 |
return JSONResponse(_base({
|
| 215 |
"error": "recipe not found", "ref": recipe_id,
|
|
|
|
| 244 |
|
| 245 |
@app.get(p + "/missions/{mission_id}")
|
| 246 |
async def mission_get(mission_id: str) -> JSONResponse:
|
| 247 |
+
mission_key = mission_id.upper()
|
| 248 |
+
filename = _MISSION_FILES.get(mission_key)
|
| 249 |
+
m = _read_json(_DATA / "missions" / filename) if filename is not None else None
|
| 250 |
if m is None:
|
| 251 |
return JSONResponse(_base({
|
| 252 |
"error": "mission not found", "ref": mission_id,
|
| 253 |
"available": ["P1", "P2", "P3", "P4", "P5", "P6", "P7", "P8"],
|
| 254 |
}), status_code=404)
|
| 255 |
drone = bool(m.get("drone_domain"))
|
| 256 |
+
m["recall_receipt"] = _recall_receipt("mission", mission_key, _digest(m))
|
| 257 |
# m already carries doctrine/numbers/disclaimer from generation; ensure disclaimer on drone
|
| 258 |
if drone and "disclaimer" not in m:
|
| 259 |
m["disclaimer"] = LEGAL_DISCLAIMER
|
szl_safe_static.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Root-confined static-file lookup for application catch-all routes."""
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from os import PathLike
|
| 6 |
+
from typing import Any
|
| 7 |
+
|
| 8 |
+
from starlette.exceptions import HTTPException as StarletteHTTPException
|
| 9 |
+
from starlette.responses import Response
|
| 10 |
+
from starlette.staticfiles import StaticFiles
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class RootedStaticFiles:
|
| 14 |
+
"""Serve a relative URL path through Starlette's containment-checked API.
|
| 15 |
+
|
| 16 |
+
Starlette's ``StaticFiles.lookup_path`` canonicalizes the configured root
|
| 17 |
+
and candidate then requires ``os.path.commonpath`` containment. This thin
|
| 18 |
+
adapter exposes a nullable response so SPA routes can fall back to their
|
| 19 |
+
index without ever constructing a filesystem path from request data.
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
def __init__(self, directory: str | PathLike[str]) -> None:
|
| 23 |
+
self._files = StaticFiles(directory=str(directory), check_dir=False)
|
| 24 |
+
|
| 25 |
+
async def get(self, relative_path: str, scope: dict[str, Any]) -> Response | None:
|
| 26 |
+
if not isinstance(relative_path, str):
|
| 27 |
+
return None
|
| 28 |
+
try:
|
| 29 |
+
response = await self._files.get_response(relative_path, scope)
|
| 30 |
+
except StarletteHTTPException as exc:
|
| 31 |
+
if exc.status_code == 404:
|
| 32 |
+
return None
|
| 33 |
+
raise
|
| 34 |
+
except ValueError:
|
| 35 |
+
# ``os.path.commonpath`` rejects mixed-drive Windows paths. Treat
|
| 36 |
+
# that attacker-controlled shape as an ordinary miss, never a 500.
|
| 37 |
+
return None
|
| 38 |
+
return None if response.status_code == 404 else response
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
__all__ = ["RootedStaticFiles"]
|