Deploy: SCU_v1 default, Sankey, filter reconciliation, nested-expander fixes, deps

Slim deploy on top of the Space's main. Adds the Streamlit feature work and,
critically, the previously-missing runtime deps (stqdm, tqdm, psutil,
transformers, psycopg[binary]) plus the vendored pipeline/ scripts. Fixes the
nested-expander crashes (SCU normalization section + Cramér's pair-comparison).
Excludes frontend_backup/ and the 119 MB cluster-viz HTML (storage limit).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.gitignore

@@ -175,14 +175,17 @@ frontend2/dist/
 # Large source PDFs (kept local, too big for git / HF Space)
 UAP_PDFs/
 
+# Document Preprocessing working dir (PDF corpus + intermediates) — data, not code.
+# The pipeline *scripts* live in pipeline/ and ARE tracked; the data tree is not.
+pipeline_data/
+
 # Windows "downloaded from internet" metadata sidecar files
 *Zone.Identifier
 
-# Scratch notebooks
-Untitled.ipynb
-
-# Superseded frontend backup (kept local, not deployed)
+# Superseded frontend backup + large generated cluster-viz HTML (kept local,
+# excluded from the HF Space deploy to stay within its storage limit).
 frontend_backup/
-
-# Large generated cluster-viz HTML (regenerated at runtime)
 frontend/uap_clusters_llm.html
+
+# Scratch notebooks
+Untitled.ipynb

.python-version

@@ -0,0 +1 @@
+3.12.3

analyzing.py

@@ -43,35 +43,7 @@ def load_data(file_path, key='df'):
     return pd.read_hdf(file_path, key=key)
 
 
-def gemini_query(question, selected_data, gemini_key):
-
-    if question == "":
-        question = "Summarize the following data in relevant bullet points"
-
-    import pathlib
-    import textwrap
-
-    import google.generativeai as genai
-
-    from IPython.display import display
-    from IPython.display import Markdown
-
-
-    def to_markdown(text):
-        text = text.replace('•', '  *')
-        return Markdown(textwrap.indent(text, '> ', predicate=lambda _: True))
-    
-    # selected_data is a list
-    # remove empty
-
-    filtered = [str(x) for x in selected_data if str(x) != '' and x is not None]
-    # make a string
-    context = '\n'.join(filtered)
-
-    genai.configure(api_key=gemini_key)
-    query_model = genai.GenerativeModel('models/gemini-3.1-pro-preview')
-    response = query_model.generate_content([f"{question}\n Answer based on this context: {context}\n\n"])
-    return(response.text)
+# Gemini Q&A moved to rag_search.py (the RAG search page) — see gemini_query there.
 
 def plot_treemap(df, column, top_n=32):
         # Get the value counts and the top N labels
@@ -988,19 +960,26 @@ def render_cramers_v_explorer(df):
         st.markdown("#### Pass 1 — auto-selected categoricals")
         clicked1 = _interactive_cv_heatmap(cv1, "Cramér's V — Pass 1", "cv_hm1")
         pairs1, n_excl1 = _cramers_pairs_table(cv1, exclude_trivial=exclude_trivial)
-        _pair_drilldown(df, cv1, pairs1, "cv_dd1", clicked1, drop_missing)
-        if n_excl1:
-            st.caption(f":grey[Filtered {n_excl1} trivial pair(s) "
-                       "(V≈0 null/constant or V≈1 duplicate).]")
+        # Keep only the download button in the main flow; tuck the pair
+        # comparison drill-down and the ranked table behind a dropdown.
         if not pairs1.empty:
-            st.caption("Strongest associations (pass 1):")
-            st.dataframe(pairs1.head(25), hide_index=True, use_container_width=True)
             st.download_button(
                 "⬇️ Download pass-1 pairs (CSV)", pairs1.to_csv(index=False),
                 "cramers_v_pass1.csv", "text/csv", key="cv_dl_pass1",
             )
-        else:
-            st.info("No non-trivial associations found in pass 1.")
+        # Popover, not an expander: this whole explorer is rendered inside the
+        # "Categorical Association Explorer" expander, and Streamlit forbids
+        # nesting an expander inside another expander.
+        with st.popover("🔎 Pair comparison & ranked table", use_container_width=True):
+            _pair_drilldown(df, cv1, pairs1, "cv_dd1", clicked1, drop_missing)
+            if n_excl1:
+                st.caption(f":grey[Filtered {n_excl1} trivial pair(s) "
+                           "(V≈0 null/constant or V≈1 duplicate).]")
+            if not pairs1.empty:
+                st.caption("Strongest associations (pass 1):")
+                st.dataframe(pairs1.head(25), hide_index=True, use_container_width=True)
+            else:
+                st.info("No non-trivial associations found in pass 1.")
 
         # ── Pass 2 — refine ────────────────────────────────────────────────
         st.divider()
@@ -1047,19 +1026,136 @@ def render_cramers_v_explorer(df):
         if cv2 is not None:
             clicked2 = _interactive_cv_heatmap(cv2, "Cramér's V — Pass 2 (refined)", "cv_hm2")
             pairs2, n_excl2 = _cramers_pairs_table(cv2, exclude_trivial=exclude_trivial)
-            _pair_drilldown(df, cv2, pairs2, "cv_dd2", clicked2, drop_missing)
-            if n_excl2:
-                st.caption(f":grey[Filtered {n_excl2} trivial pair(s) "
-                           "(V≈0 null/constant or V≈1 duplicate).]")
+            # Same as pass 1: only the download button stays in the main flow.
             if not pairs2.empty:
-                st.dataframe(pairs2.head(40), hide_index=True,
-                             use_container_width=True)
                 st.download_button(
                     "⬇️ Download pass-2 pairs (CSV)", pairs2.to_csv(index=False),
                     "cramers_v_pass2.csv", "text/csv", key="cv_dl_pass2",
                 )
-            else:
-                st.info("No non-trivial associations in the refined set.")
+            with st.popover("🔎 Pair comparison & ranked table (refined)", use_container_width=True):
+                _pair_drilldown(df, cv2, pairs2, "cv_dd2", clicked2, drop_missing)
+                if n_excl2:
+                    st.caption(f":grey[Filtered {n_excl2} trivial pair(s) "
+                               "(V≈0 null/constant or V≈1 duplicate).]")
+                if not pairs2.empty:
+                    st.dataframe(pairs2.head(40), hide_index=True,
+                                 use_container_width=True)
+                else:
+                    st.info("No non-trivial associations in the refined set.")
+
+
+def render_categorical_flow_sankey(df):
+    """All-with-all categorical flow Sankey for the Statistical Analysis section.
+
+    Unlike the cross-DB matcher in rag_search.py (which pairs records by
+    similarity), this links every value of each chosen column to every
+    co-occurring value of the next column, with link width = the number of rows
+    sharing that combination (a chained crosstab). No pairwise matching — just
+    all-with-all co-occurrence counts.
+    """
+    st.markdown("### 🌊 Categorical Flow (Sankey)")
+    st.caption(
+        "Pick two or more categorical columns as ordered levels (left → right). "
+        "Every value is linked to every co-occurring value of the next level, "
+        "with link width = the number of rows sharing that combination — no "
+        "pairwise matching, just all-with-all co-occurrence counts."
+    )
+
+    n = len(df)
+    if n == 0:
+        st.info("No rows to chart.")
+        return
+
+    def _safe_nunique(s):
+        # Columns of dicts/lists are unhashable; fall back to string form.
+        try:
+            return s.nunique(dropna=False)
+        except TypeError:
+            return s.astype(str).nunique(dropna=False)
+
+    cat_like = [
+        c for c in df.columns
+        if 1 < _safe_nunique(df[c]) <= max(50, int(n * 0.5))
+    ]
+    if len(cat_like) < 2:
+        st.info("Need at least two categorical-like columns for a flow diagram.")
+        return
+
+    sc1, sc2 = st.columns([3, 1])
+    with sc1:
+        cols = st.multiselect(
+            "Levels (left → right, in order)",
+            options=list(df.columns),
+            default=cat_like[:3],
+            key="sankey_flow_cols",
+        )
+    with sc2:
+        top_n = int(st.number_input(
+            "Top-N / level", min_value=2, max_value=50, value=12, step=1,
+            key="sankey_flow_topn",
+            help="Keep only the most frequent N values per level; the rest are "
+                 "lumped into “Other” so the diagram stays readable.",
+        ))
+
+    if len(cols) < 2:
+        st.info("Select at least two columns.")
+        return
+
+    # Normalise: string-cast, fill missing, cap to top-N per level (+ “Other”).
+    work = pd.DataFrame(index=df.index)
+    for c in cols:
+        s = df[c].astype(str).where(df[c].notna(), "(missing)")
+        keep = s.value_counts().head(top_n).index
+        work[c] = s.where(s.isin(keep), "Other")
+
+    # Namespace nodes by (level, value) so identical values in different levels
+    # never merge into one node.
+    PALETTE = ["#f97316", "#22c55e", "#3b82f6", "#a855f7", "#ec4899",
+               "#14b8a6", "#eab308", "#ef4444"]
+    index, node_label, node_color = {}, [], []
+    L = len(cols)
+    max_per_level = 1
+    for level, c in enumerate(cols):
+        vc = work[c].value_counts()
+        max_per_level = max(max_per_level, len(vc))
+        for v in vc.index:
+            key = (level, v)
+            if key not in index:
+                index[key] = len(node_label)
+                node_label.append(str(v))
+                node_color.append(PALETTE[level % len(PALETTE)])
+
+    # Links: co-occurrence counts between consecutive levels (the "all with all").
+    srcs, tgts, vals = [], [], []
+    for level in range(L - 1):
+        a, b = cols[level], cols[level + 1]
+        counts = work.groupby([a, b]).size().reset_index(name="value")
+        for _, row in counts.iterrows():
+            srcs.append(index[(level, row[a])])
+            tgts.append(index[(level + 1, row[b])])
+            vals.append(int(row["value"]))
+
+    def _rgba(hex_color, alpha=0.35):
+        h = hex_color.lstrip("#")
+        return f"rgba({int(h[0:2], 16)},{int(h[2:4], 16)},{int(h[4:6], 16)},{alpha})"
+
+    fig = go.Figure(go.Sankey(
+        arrangement="snap",
+        node=dict(label=node_label, color=node_color, pad=14, thickness=18),
+        link=dict(source=srcs, target=tgts, value=vals,
+                  color=[_rgba(node_color[s]) for s in srcs]),
+    ))
+    fig.update_layout(
+        template="plotly_dark",
+        paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)",
+        font_size=12, height=max(420, 60 + 24 * max_per_level),
+        margin=dict(l=10, r=10, t=20, b=10),
+    )
+    st.plotly_chart(fig, use_container_width=True)
+    st.caption(
+        f"{' → '.join(cols)} · {len(node_label)} nodes · {len(srcs)} links "
+        "(all-with-all co-occurrence, no matching)."
+    )
 
 
 def analyze_and_predict(data, analyzers, col_names, clusters):
@@ -1154,17 +1250,7 @@ else:
     parsed_responses = filter_dataframe(parsed)
     st.session_state['parsed_responses'] = parsed_responses
     st.dataframe(parsed_responses)
-col1, col2 = st.columns(2)
-with col1:
-    col_parsed = st.selectbox("Which column do you want to query?", st.session_state['parsed_responses'].columns)
-with col2:
-    GEMINI_KEY = st.text_input('Gemini API Key', value=GEMINI_KEY, type='password', help="Enter your Gemini API key")
-
-if col_parsed and GEMINI_KEY:
-    selected_column_data = st.session_state['parsed_responses'][col_parsed].tolist()
-    question = st.text_input("Ask a question or leave empty for summarization")
-    if st.button("Generate Query") and selected_column_data:
-        st.write(gemini_query(question, selected_column_data, GEMINI_KEY))
+# Gemini Q&A over a column moved to the RAG search page (rag_search.py).
 st.session_state['stage'] = 1
 
 # Add enhanced visualization toggle
@@ -1191,6 +1277,8 @@ enable_tfidf_clusters = st.toggle(
 if st.session_state['stage'] > 0 and st.session_state.get('parsed_responses') is not None:
     with st.expander("🎲 Categorical Association Explorer (Cramér's V)", expanded=False):
         render_cramers_v_explorer(st.session_state['parsed_responses'])
+    with st.expander("🌊 Categorical Flow (Sankey)", expanded=False):
+        render_categorical_flow_sankey(st.session_state['parsed_responses'])
 
 if st.session_state['stage'] > 0 :
     # ── High-correlation shortcut ──────────────────────────────────────────────
@@ -1332,45 +1420,7 @@ if st.session_state['stage'] > 0 :
                         st.session_state['analysis_complete'] = True
 
 
-# this will check if the dataframe is not empty
-# if st.session_state['new_data'] is not None:
-#     parsed2 = st.session_state.get('dataset', pd.DataFrame())
-#     parsed2 = filter_dataframe(parsed2)
-#     col1, col2 = st.columns(2)
-#     st.dataframe(parsed2)
-#     with col1:
-#         col_parsed2 = st.selectbox("Which columns do you want to query?", parsed2.columns)
-#     with col2:
-#         GEMINI_KEY = st.text_input('Gemini APIs Key', GEMINI_KEY, type='password', help="Enter your Gemini API key")
-#     if col_parsed and GEMINI_KEY:
-#         selected_column_data2 = parsed2[col_parsed2].tolist()
-#         question2 = st.text_input("Ask a questions or leave empty for summarization")
-#         if st.button("Generate Query") and selected_column_data2:
-#             with st.status(f"Generating Query", expanded=True) as status:
-#                 gemini_answer = gemini_query(question2, selected_column_data2, GEMINI_KEY)
-#                 st.write(gemini_answer)
-#                 st.session_state['gemini_answer'] = gemini_answer
-
-if 'analysis_complete' in st.session_state and st.session_state['analysis_complete']:
-    ticked_analysis = st.checkbox('Query Processed Data')
-    if ticked_analysis:
-        if st.session_state['new_data'] is not None:
-            parsed2 = st.session_state.get('dataset', pd.DataFrame()).copy()
-            parsed2 = filter_dataframe(parsed2)
-            col1, col2 = st.columns(2)
-            st.dataframe(parsed2)
-            with col1:
-                col_parsed2 = st.selectbox("Which columns do you want to query?", parsed2.columns)
-            with col2:
-                GEMINI_KEY = st.text_input('Gemini APIs Key', value=GEMINI_KEY, type='password', help="Enter your Gemini API key")
-            if col_parsed2 and GEMINI_KEY:
-                selected_column_data2 = parsed2[col_parsed2].tolist()
-                question2 = st.text_input("Ask a questions or leave empty for summarization")
-                if st.button("Generate Queries") and selected_column_data2:
-                    with st.status(f"Generating Query", expanded=True) as status:
-                        gemini_answer = gemini_query(question2, selected_column_data2, GEMINI_KEY)
-                        st.write(gemini_answer)
-                        st.session_state['gemini_answer'] = gemini_answer
+# Gemini Q&A over processed/analyzed data moved to the RAG search page (rag_search.py).
 
 # Enhanced visualization section
 if 'analysis_complete' in st.session_state and st.session_state['analysis_complete']:

api/main.py

@@ -121,6 +121,13 @@ class SchemaMergeRequest(BaseModel):
     custom_fields: dict | None = None
 
 
+class SchemaCoverageRequest(BaseModel):
+    labels: list[str]
+    custom_fields: dict | None = None
+    # Dataset columns to diff against; defaults to the uploaded parse source.
+    columns: list[str] | None = None
+
+
 class ParseEstimateRequest(BaseModel):
     columns: list[str]
     format_json: str                   # merged schema template (JSON string)
@@ -173,6 +180,11 @@ class ColumnGroupsRequest(BaseModel):
     source: str = "dataset"            # "dataset" | "parsed"
     high_threshold: int = 30
 
+
+class XgboostRequest(BaseModel):
+    columns: list[str]
+    source: str = "dataset"            # "dataset" | "parsed"
+
 # ---------------------------------------------------------------------------
 # Helpers
 # ---------------------------------------------------------------------------
@@ -668,6 +680,27 @@ def parse_schema_merge(req: SchemaMergeRequest):
         raise HTTPException(status_code=500, detail=str(e))
 
 
+@app.post("/api/parse/schema-coverage")
+def parse_schema_coverage(req: SchemaCoverageRequest, state: SessionState = Depends(get_session)):
+    """Diff the merged schema's leaf fields against the dataset columns (🟢/🔴
+    coverage) and return per-mode extraction schemas (all / missing / database)."""
+    cols = req.columns
+    if cols is None:
+        if state.parse_source_df is None:
+            raise HTTPException(
+                status_code=400,
+                detail="No raw dataset uploaded and no columns provided.",
+            )
+        cols = list(state.parse_source_df.columns)
+    try:
+        return parsing_service.schema_coverage_report(req.labels, cols, req.custom_fields)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Coverage diff failed: {e}")
+
+
 @app.post("/api/parse/upload")
 async def parse_upload(file: UploadFile = File(...), state: SessionState = Depends(get_session)):
     """Upload a raw report dataset to be fed through the LLM extractor."""
@@ -875,6 +908,18 @@ def analysis_column_groups(req: ColumnGroupsRequest, state: SessionState = Depen
         raise HTTPException(status_code=500, detail=f"Column grouping failed: {e}")
 
 
+@app.post("/api/analysis/xgboost")
+def analysis_xgboost(req: XgboostRequest, state: SessionState = Depends(get_session)):
+    """XGBoost feature importance computed directly on the selected categorical
+    columns (no cluster pipeline) — fed by the Cramér's V explorer selection."""
+    df = _assoc_source(state, req.source)
+    try:
+        return analysis_service.xgboost_importance(df, req.columns)
+    except Exception as e:
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"XGBoost feature importance failed: {e}")
+
+
 @app.post("/api/analysis/cramers-v")
 def analysis_cramers_v(req: CramersVRequest, state: SessionState = Depends(get_session)):
     df = _assoc_source(state, req.source)

api/services/analysis_service.py

@@ -226,3 +226,62 @@ def contingency(df: pd.DataFrame, c1: str, c2: str, drop_missing: bool = False,
         "v": round(v, 3),
         "n": int(len(a)),
     }
+
+
+# ── XGBoost feature importance on raw categorical columns ───────────────────
+# Cap on a target column's class count — XGBoost multi:softmax with hundreds of
+# classes is slow and the importances are meaningless. The explorer only feeds
+# binary/low/medium-cardinality columns, so this is just a safety net.
+_XGB_MAX_TARGET_CLASSES = 50
+
+
+def xgboost_importance(df: pd.DataFrame, columns: list[str], *,
+                       test_size: float = 0.2, random_state: int = 42) -> dict:
+    """Per-column XGBoost feature importance computed *directly* on the selected
+    raw categorical columns — predict each column from the others and report the
+    gain-based importance of every other column, plus the test accuracy.
+
+    This mirrors ``analyzing.py``'s ``analyze_and_predict`` loop but runs on the
+    raw values (the same set used by the Cramér's V explorer) instead of cluster
+    labels, so feature importance is available without the embedding/cluster
+    pipeline. Returns ``{results: {col: {feature_importance, accuracy}}, ...}``.
+    """
+    from sklearn.model_selection import train_test_split
+    from uap_analyzer import train_xgboost
+
+    cols = [c for c in columns if c in df.columns]
+    if len(cols) < 2:
+        return {
+            "results": {}, "columns": cols, "skipped": {},
+            "message": "Select at least two categorical columns for feature importance.",
+        }
+
+    # Coalesce missingness the same way Cramér's V does, then category-encode.
+    new_data = pd.DataFrame({c: _coalesce(df[c]) for c in cols}).astype("category")
+    data_nums = new_data.apply(lambda s: s.cat.codes)
+
+    results: dict[str, dict] = {}
+    skipped: dict[str, str] = {}
+    for col in cols:
+        n_classes = len(new_data[col].cat.categories)
+        if n_classes < 2:
+            skipped[col] = "constant column (one class)"
+            continue
+        if n_classes > _XGB_MAX_TARGET_CLASSES:
+            skipped[col] = f"too many classes ({n_classes}) to predict"
+            continue
+        try:
+            x = data_nums.drop(columns=[col])
+            y = data_nums[col]
+            x_train, x_test, y_train, y_test = train_test_split(
+                x, y, test_size=test_size, random_state=random_state,
+            )
+            bst, accuracy, _ = train_xgboost(x_train, y_train, x_test, y_test, n_classes)
+            # Gain-based importance; only features used in a split appear.
+            imp = {k: float(v) for k, v in bst.get_score(importance_type="gain").items()}
+            imp = dict(sorted(imp.items(), key=lambda kv: kv[1], reverse=True))
+            results[col] = {"feature_importance": imp, "accuracy": round(float(accuracy), 3)}
+        except Exception as e:  # noqa: BLE001 — one bad target shouldn't sink the rest
+            skipped[col] = str(e)
+
+    return {"results": results, "columns": cols, "skipped": skipped}

api/services/parsing_service.py

@@ -12,6 +12,7 @@ importing this module never drags in torch or validates secrets.
 from __future__ import annotations
 
 import json
+import re
 from typing import Any, Callable
 
 
@@ -19,6 +20,7 @@ from typing import Any, Callable
 # Rebuilt directly from config.py (no Streamlit dependency) so it stays in
 # sync with parsing.py's SCHEMA_FORMATS / SCHEMA_FORMAT_GROUPS mapping.
 _SCHEMA_SPECS: list[tuple[str, str]] = [
+    ("SCU_v1", "FORMAT_SCU_V1"),
     ("Default UAP Format", "FORMAT_LONG"),
     ("SCU Spreadsheet", "FORMAT_LONG_XLSX"),
     ("SCU_v2", "FORMAT_SCU_V2"),
@@ -43,7 +45,7 @@ _SCHEMA_SPECS: list[tuple[str, str]] = [
 ]
 
 SCHEMA_FORMAT_GROUPS: dict[str, list[str]] = {
-    "Canonical & SCU": ["Default UAP Format", "SCU Spreadsheet", "SCU_v2", "SCU_v3"],
+    "Canonical & SCU": ["SCU_v1", "Default UAP Format", "SCU Spreadsheet", "SCU_v2", "SCU_v3"],
     "Government & official archives": ["Blue Book (USAF)", "UK National Archives"],
     "European national databases": [
         "COBEPS — PAN Notifications (BE)", "COBEPS — COB 2021 (BE)",
@@ -145,6 +147,103 @@ def merge_schema(labels: list[str], custom_fields: dict | None = None) -> dict:
     }
 
 
+# ── Schema ↔ dataset coverage diff ─────────────────────────────────────────
+def _norm_token(s: Any) -> str:
+    """Normalize a name for fuzzy matching: lowercase, alphanumerics only."""
+    return re.sub(r"[^a-z0-9]", "", str(s).lower())
+
+
+def _leaf(path: str) -> str:
+    """Last dotted segment of a (possibly nested) field/column name."""
+    return str(path).split(".")[-1]
+
+
+def schema_coverage(merged: dict, dataset_columns: list[str]) -> dict:
+    """Compare the merged schema's leaf fields against ``dataset_columns``.
+
+    A schema field counts as *present* when some dataset column shares its
+    normalized leaf name (last dotted segment, case/punctuation-insensitive) —
+    e.g. schema ``sightingDetails.objectDescription.shape`` matches a dataset
+    column ``shape`` or ``object.shape``. Returns per-field present flags, the
+    dataset columns that matched, and the dataset-only (unmatched) columns.
+    """
+    flat = _flatten_dotted(merged)
+    col_by_token: dict[str, str] = {}
+    for c in dataset_columns:
+        col_by_token.setdefault(_norm_token(_leaf(c)), c)
+
+    coverage = []
+    matched_cols: set[str] = set()
+    for path in flat:
+        match = col_by_token.get(_norm_token(_leaf(path)))
+        if match is not None:
+            matched_cols.add(match)
+        coverage.append({
+            "path": path,
+            "leaf": _leaf(path),
+            "present": match is not None,
+            "matched_column": match,
+        })
+
+    schema_tokens = {_norm_token(_leaf(p)) for p in flat}
+    db_only = [c for c in dataset_columns if _norm_token(_leaf(c)) not in schema_tokens]
+    n_present = sum(1 for c in coverage if c["present"])
+    return {
+        "coverage": coverage,
+        "summary": {
+            "present": n_present,
+            "missing": len(coverage) - n_present,
+            "total": len(coverage),
+            "db_only": len(db_only),
+        },
+        "matched_columns": sorted(matched_cols),
+        "db_only_columns": db_only,
+    }
+
+
+def prune_schema_to_paths(merged: dict, paths: list[str]) -> dict:
+    """Rebuild a nested schema dict holding only the given dotted leaf paths,
+    preserving each leaf's original description/value and the original nesting
+    (so a FORMAT_LONG ``sightingDetails`` wrapper is kept)."""
+    flat = _flatten_dotted(merged)
+    keep = set(paths)
+    out: dict = {}
+    for path, val in flat.items():
+        if path not in keep:
+            continue
+        parts = path.split(".")
+        node = out
+        for p in parts[:-1]:
+            node = node.setdefault(p, {})
+        node[parts[-1]] = val
+    return out
+
+
+def schema_coverage_report(labels: list[str], dataset_columns: list[str],
+                           custom_fields: dict | None = None) -> dict:
+    """Coverage diff plus ready-to-use extraction-schema variants for the three
+    modes the parsing UI offers:
+
+    - ``all``      → the full merged schema (extract every field).
+    - ``missing``  → schema pruned to fields absent from the dataset (🔴 only).
+    - ``database`` → empty schema; keep the dataset columns as-is (no extraction).
+    """
+    merged = merge_schema(labels, custom_fields)["schema"]
+    cov = schema_coverage(merged, dataset_columns)
+
+    missing_paths = [c["path"] for c in cov["coverage"] if not c["present"]]
+    all_paths = [c["path"] for c in cov["coverage"]]
+    missing_schema = prune_schema_to_paths(merged, missing_paths)
+
+    variants = {
+        "all": {"schema_json": json.dumps(merged, indent=2), "n_fields": len(all_paths)},
+        "missing": {"schema_json": json.dumps(missing_schema, indent=2),
+                    "n_fields": len(missing_paths)},
+        "database": {"schema_json": "{}", "n_fields": 0},
+    }
+    return {**cov, "variants": variants}
+
+
 # ── Cost estimation ────────────────────────────────────────────────────────
 def estimate(descriptions: list[str], schema_json: str, model: str,
              use_cache: bool = True, use_batch: bool = False) -> dict:

config.py

@@ -761,6 +761,22 @@ FORMAT_SCU_V3 = {
     }
 }
 
+# ---------------------------------------------------------------------------
+# FORMAT_SCU_V1 — "SCU v1" parsing schema (the default extraction format)
+#
+# The same field set as FORMAT_SCU_V3, minus two blocks:
+#   • sightingDetails — the verbose nested narrative group
+#   • manifest        — the all-null downstream-join placeholder columns the
+#                       parser must always leave null
+# Everything else (source, assessment, anomaly, date_time, location, witness,
+# investigation, craft, performance, military, effects, engagement_flags,
+# engagement_type, case_text) is inherited unchanged. Derived from
+# FORMAT_SCU_V3 so it never drifts out of sync; _deep_merge treats schemas as
+# read-only, so the shared nested references are safe.
+# ---------------------------------------------------------------------------
+_SCU_V1_DROP = ("sightingDetails", "manifest")
+FORMAT_SCU_V1 = {k: v for k, v in FORMAT_SCU_V3.items() if k not in _SCU_V1_DROP}
+
 # ---------------------------------------------------------------------------
 # Merged compact schema — used by the Markdown-folder ingestion agent.
 # Covers the essential fields from both FORMAT_LONG (sightingDetails) and

frontend/src/api/client.ts

@@ -4,6 +4,7 @@ import type {
   DashboardSummary,
   SchemaListResponse,
   SchemaMergeResponse,
+  SchemaCoverageResponse,
   ParseUploadResponse,
   CostEstimate,
   ParseRunResponse,
@@ -14,6 +15,7 @@ import type {
   CramersVResponse,
   ContingencyResponse,
   ColumnGroupsResponse,
+  XgboostImportanceResponse,
 } from '../types';
 
 // API origin is configurable for split deployments (e.g. frontend on Vercel,
@@ -120,6 +122,17 @@ export const api = {
     });
   },
 
+  schemaCoverage(
+    labels: string[],
+    columns: string[],
+    customFields?: Record<string, unknown>
+  ): Promise<SchemaCoverageResponse> {
+    return request('/parse/schema-coverage', {
+      method: 'POST',
+      body: JSON.stringify({ labels, columns, custom_fields: customFields ?? null }),
+    });
+  },
+
   uploadParseFile(file: File): Promise<ParseUploadResponse> {
     const form = new FormData();
     form.append('file', file);
@@ -227,6 +240,13 @@ export const api = {
     });
   },
 
+  xgboostImportance(columns: string[], source = 'dataset'): Promise<XgboostImportanceResponse> {
+    return request('/analysis/xgboost', {
+      method: 'POST',
+      body: JSON.stringify({ columns, source }),
+    });
+  },
+
   queryGemini(question: string, columns: string[], geminiKey: string): Promise<{ status: string; response: string; context_rows_used?: number; columns_used?: string[] }> {
     return request('/query/gemini', {
       method: 'POST',

frontend/src/components/analysis/AnalysisPage.tsx

@@ -9,7 +9,7 @@ import { CorrelationHeatmap } from './CorrelationHeatmap';
 import { XGBoostResults } from './XGBoostResults';
 import { DistributionChart } from './DistributionChart';
 import { CramersVExplorer } from './CramersVExplorer';
-import type { AnalysisResponse } from '../../types';
+import type { AnalysisResponse, XGBoostResult } from '../../types';
 
 type TabId = 'clusters' | 'correlation' | 'xgboost' | 'distribution' | 'association';
 
@@ -18,6 +18,13 @@ export function AnalysisPage() {
   const [selected, setSelected] = useState<string[]>([]);
   const [error, setError] = useState<string | null>(null);
   const [activeTab, setActiveTab] = useState<TabId>('clusters');
+  // XGBoost feature importance handed over from the Cramér's V explorer.
+  const [assocXgboost, setAssocXgboost] = useState<Record<string, XGBoostResult> | null>(null);
+
+  const handleAssocXgboost = (r: Record<string, XGBoostResult>) => {
+    setAssocXgboost(r);
+    setActiveTab('xgboost');
+  };
 
   // Cluster pipeline tuning (mirrors analyzing.py controls)
   const [showParams, setShowParams] = useState(false);
@@ -213,11 +220,48 @@ export function AnalysisPage() {
         ))}
       </div>
 
-      {/* Association explorer — independent of the cluster pipeline */}
-      {activeTab === 'association' && <CramersVExplorer source="dataset" />}
+      {/* Association explorer — independent of the cluster pipeline. Its XGBoost
+          run is handed to the Feature Importance tab via handleAssocXgboost. */}
+      {activeTab === 'association' && (
+        <CramersVExplorer source="dataset" onXgboost={handleAssocXgboost} />
+      )}
+
+      {/* Feature importance — independent of the cluster pipeline: it shows the
+          explorer-driven results when present, otherwise the pipeline's. */}
+      {activeTab === 'xgboost' && (() => {
+        const xgb =
+          assocXgboost && Object.keys(assocXgboost).length > 0
+            ? assocXgboost
+            : results?.xgboost ?? null;
+        if (xgb && Object.keys(xgb).length > 0) {
+          return (
+            <div className="space-y-3">
+              {assocXgboost && Object.keys(assocXgboost).length > 0 && (
+                <div className="flex items-center gap-2 rounded-md border border-purple/30 bg-purple/10 px-4 py-2.5 text-xs text-text-secondary">
+                  <Network className="h-4 w-4 text-purple" />
+                  Computed directly from your Cramér's V column selection — each column predicted
+                  from the others.
+                </div>
+              )}
+              <XGBoostResults results={xgb} />
+            </div>
+          );
+        }
+        return (
+          <Panel title="Feature Importance">
+            <p className="text-sm text-text-muted">
+              No feature-importance results yet. Run the cluster pipeline above, or open the{' '}
+              <button onClick={() => setActiveTab('association')} className="text-accent hover:underline">
+                Cramér's V Explorer
+              </button>
+              , select columns, and click <span className="text-accent">Feature Importance →</span>.
+            </p>
+          </Panel>
+        );
+      })()}
 
       {/* Result-dependent tabs */}
-      {activeTab !== 'association' && !results && (
+      {activeTab !== 'association' && activeTab !== 'xgboost' && !results && (
         <Panel title="Run the analysis pipeline">
           <p className="text-sm text-text-muted">
             Select columns above and click <span className="text-accent">Run Analysis</span> to
@@ -257,18 +301,6 @@ export function AnalysisPage() {
             </Panel>
           )}
 
-          {/* XGBoost */}
-          {activeTab === 'xgboost' && results.xgboost && Object.keys(results.xgboost).length > 0 && (
-            <XGBoostResults results={results.xgboost} />
-          )}
-          {activeTab === 'xgboost' && (!results.xgboost || Object.keys(results.xgboost).length === 0) && (
-            <Panel title="Feature Importance">
-              <p className="text-sm text-text-muted">
-                Select at least 2 columns to run feature importance analysis.
-              </p>
-            </Panel>
-          )}
-
           {/* Distribution */}
           {activeTab === 'distribution' && (
             <div className="grid grid-cols-1 gap-4 lg:grid-cols-2 xl:grid-cols-3">

frontend/src/components/analysis/CramersVExplorer.tsx

@@ -1,16 +1,21 @@
 import { useState, useEffect, useMemo } from 'react';
 import Plot from 'react-plotly.js';
-import { Play, AlertTriangle, Grid3x3, Layers } from 'lucide-react';
+import { Play, AlertTriangle, Grid3x3, Layers, BarChart3 } from 'lucide-react';
 import { api } from '../../api/client';
 import { Panel } from '../common/Panel';
 import { LoadingSpinner } from '../common/LoadingSpinner';
-import type { CramersVResponse, ContingencyResponse, ColumnGroup } from '../../types';
+import { XGBoostResults } from './XGBoostResults';
+import type { CramersVResponse, ContingencyResponse, ColumnGroup, XGBoostResult } from '../../types';
 
 interface Props {
   source: 'dataset' | 'parsed';
+  // When provided, XGBoost feature importance computed from the selected columns
+  // is handed off to the parent (e.g. the Feature Importance tab) instead of
+  // rendering inline.
+  onXgboost?: (results: Record<string, XGBoostResult>) => void;
 }
 
-export function CramersVExplorer({ source }: Props) {
+export function CramersVExplorer({ source, onXgboost }: Props) {
   const [report, setReport] = useState<CramersVResponse | null>(null);
   const [contingency, setContingency] = useState<ContingencyResponse | null>(null);
   const [pair, setPair] = useState<{ a: string; b: string } | null>(null);
@@ -22,6 +27,10 @@ export function CramersVExplorer({ source }: Props) {
   const [loading, setLoading] = useState(false);
   const [error, setError] = useState<string | null>(null);
 
+  // XGBoost feature importance run directly on the selected columns.
+  const [xgbLoading, setXgbLoading] = useState(false);
+  const [localXgb, setLocalXgb] = useState<Record<string, XGBoostResult> | null>(null);
+
   // Eligible categorical columns, grouped by their dotted parent (e.g. craft.*).
   const [groups, setGroups] = useState<ColumnGroup[]>([]);
   const [selected, setSelected] = useState<Set<string>>(new Set());
@@ -101,6 +110,29 @@ export function CramersVExplorer({ source }: Props) {
     }
   };
 
+  const runXgboost = async () => {
+    const cols = orderedEligible.filter((c) => selected.has(c));
+    if (cols.length < 2) {
+      setError('Select at least two columns to run feature importance.');
+      return;
+    }
+    setXgbLoading(true);
+    setError(null);
+    try {
+      const res = await api.xgboostImportance(cols, source);
+      if (!Object.keys(res.results).length) {
+        setError(res.message || 'No feature-importance results (need ≥2 non-constant columns).');
+        return;
+      }
+      if (onXgboost) onXgboost(res.results);
+      else setLocalXgb(res.results);
+    } catch (e) {
+      setError(e instanceof Error ? e.message : 'Feature importance failed');
+    } finally {
+      setXgbLoading(false);
+    }
+  };
+
   const loadContingency = async (a: string, b: string) => {
     setPair({ a, b });
     try {
@@ -122,14 +154,25 @@ export function CramersVExplorer({ source }: Props) {
         title="Categorical Association Explorer (Cramér's V)"
         subtitle="Pairwise association across the selected categorical columns"
         actions={
-          <button
-            onClick={run}
-            disabled={loading}
-            className="flex items-center gap-2 rounded-md bg-accent-dim px-4 py-1.5 text-xs font-medium text-white transition-colors hover:bg-accent disabled:opacity-50"
-          >
-            <Play className="h-3.5 w-3.5" />
-            {loading ? 'Computing…' : 'Compute'}
-          </button>
+          <div className="flex items-center gap-2">
+            <button
+              onClick={runXgboost}
+              disabled={xgbLoading || selected.size < 2}
+              title="Train XGBoost on the selected columns and send the result to Feature Importance"
+              className="flex items-center gap-1.5 rounded-md border border-border bg-raised px-3 py-1.5 text-xs font-medium text-text-secondary transition-colors hover:border-accent hover:text-accent disabled:opacity-50"
+            >
+              <BarChart3 className="h-3.5 w-3.5" />
+              {xgbLoading ? 'Training…' : 'Feature Importance →'}
+            </button>
+            <button
+              onClick={run}
+              disabled={loading}
+              className="flex items-center gap-2 rounded-md bg-accent-dim px-4 py-1.5 text-xs font-medium text-white transition-colors hover:bg-accent disabled:opacity-50"
+            >
+              <Play className="h-3.5 w-3.5" />
+              {loading ? 'Computing…' : 'Compute'}
+            </button>
+          </div>
         }
       >
         <div className="flex flex-wrap items-center gap-4">
@@ -268,6 +311,16 @@ export function CramersVExplorer({ source }: Props) {
       )}
 
       {loading && <LoadingSpinner text="Computing Cramér's V matrix..." />}
+      {xgbLoading && <LoadingSpinner text="Training XGBoost on the selected columns..." />}
+
+      {localXgb && (
+        <Panel
+          title="Feature Importance (XGBoost)"
+          subtitle="Each selected column predicted from the others — gain-based importance"
+        >
+          <XGBoostResults results={localXgb} />
+        </Panel>
+      )}
 
       {report && report.labels.length < 2 && (
         <Panel title="Not enough categorical columns">

frontend/src/components/parsing/ParsingPage.tsx

@@ -9,6 +9,9 @@ import {
   Key,
   Layers,
   ShieldCheck,
+  GitCompare,
+  Database,
+  Plus,
 } from 'lucide-react';
 import { api } from '../../api/client';
 import { useStore } from '../../store/useStore';
@@ -20,8 +23,16 @@ import type {
   CostEstimate,
   ParseRunResponse,
   DataResponse,
+  SchemaCoverageResponse,
+  CoverageMode,
 } from '../../types';
 
+const COVERAGE_MODES: { id: CoverageMode; label: string; hint: string }[] = [
+  { id: 'missing', label: 'Add missing only', hint: 'Extract only schema fields the dataset is missing (🔴)' },
+  { id: 'all', label: 'All fields', hint: 'Extract every schema field, present or not' },
+  { id: 'database', label: 'Database only', hint: 'No extraction — keep the uploaded columns as-is' },
+];
+
 export function ParsingPage() {
   const {
     openaiKey,
@@ -37,6 +48,10 @@ export function ParsingPage() {
   const [mergedFormat, setMergedFormat] = useState('');
   const [fieldCount, setFieldCount] = useState(0);
 
+  // Schema ↔ dataset coverage diff + extraction mode (missing / all / database)
+  const [coverage, setCoverage] = useState<SchemaCoverageResponse | null>(null);
+  const [coverageMode, setCoverageMode] = useState<CoverageMode>('all');
+
   const [source, setSource] = useState<DataResponse | null>(null);
   const [sourceColumns, setSourceColumns] = useState<string[]>([]);
   const [textColumns, setTextColumns] = useState<string[]>([]);
@@ -71,21 +86,41 @@ export function ParsingPage() {
     if (list.length && !list.includes(model)) setModel(list[0]);
   }, [provider, schemas]); // eslint-disable-line react-hooks/exhaustive-deps
 
-  // Re-merge schemas whenever the selection changes
+  // Re-merge schemas / recompute the coverage diff whenever the schema
+  // selection or the uploaded dataset columns change. With a dataset present we
+  // fetch the diff (which also carries the per-mode extraction schemas); the
+  // effective mergedFormat is then derived from the active mode below.
   useEffect(() => {
     if (selectedSchemas.length === 0) {
       setMergedFormat('');
       setFieldCount(0);
+      setCoverage(null);
       return;
     }
-    api
-      .mergeSchema(selectedSchemas)
-      .then((m) => {
-        setMergedFormat(m.schema_json);
-        setFieldCount(m.fields.length);
-      })
-      .catch((e) => setError(e instanceof Error ? e.message : 'Schema merge failed'));
-  }, [selectedSchemas]);
+    if (sourceColumns.length) {
+      api
+        .schemaCoverage(selectedSchemas, sourceColumns)
+        .then(setCoverage)
+        .catch((e) => setError(e instanceof Error ? e.message : 'Coverage diff failed'));
+    } else {
+      setCoverage(null);
+      api
+        .mergeSchema(selectedSchemas)
+        .then((m) => {
+          setMergedFormat(m.schema_json);
+          setFieldCount(m.fields.length);
+        })
+        .catch((e) => setError(e instanceof Error ? e.message : 'Schema merge failed'));
+    }
+  }, [selectedSchemas, sourceColumns]);
+
+  // Derive the effective extraction schema from the chosen coverage mode.
+  useEffect(() => {
+    if (!coverage) return;
+    const variant = coverage.variants[coverageMode];
+    setMergedFormat(variant.schema_json);
+    setFieldCount(variant.n_fields);
+  }, [coverage, coverageMode]);
 
   const apiKey = provider === 'openai' ? openaiKey : deepseekKey;
   const setApiKey = provider === 'openai' ? setOpenaiKey : setDeepseekKey;
@@ -95,6 +130,13 @@ export function ParsingPage() {
       : schemas.models.deepseek
     : [];
 
+  // "Database only" mode yields an empty schema ({}), so there is nothing to
+  // estimate or extract — the uploaded columns are kept as-is.
+  const extractionReady = !!mergedFormat && mergedFormat.trim() !== '{}';
+
+  const addToKeep = (cols: string[]) =>
+    setKeepColumns((prev) => Array.from(new Set([...prev, ...cols])));
+
   const handleUpload = async (e: React.ChangeEvent<HTMLInputElement>) => {
     const file = e.target.files?.[0];
     if (!file) return;
@@ -119,7 +161,7 @@ export function ParsingPage() {
     list.includes(v) ? list.filter((x) => x !== v) : [...list, v];
 
   const runEstimate = async () => {
-    if (!textColumns.length || !mergedFormat) return;
+    if (!textColumns.length || !extractionReady) return;
     setError(null);
     try {
       const est = await api.estimateParse(textColumns, mergedFormat, model);
@@ -130,8 +172,8 @@ export function ParsingPage() {
   };
 
   const runParse = async () => {
-    if (!textColumns.length || !mergedFormat) {
-      setError('Select at least one text column and one schema.');
+    if (!textColumns.length || !extractionReady) {
+      setError('Select at least one text column and a schema with fields to extract.');
       return;
     }
     if (!apiKey) {
@@ -195,6 +237,130 @@ export function ParsingPage() {
         </Panel>
       )}
 
+      {source && coverage && (
+        <Panel
+          title="Schema ↔ Dataset Coverage"
+          subtitle="How many merged-schema fields the uploaded dataset already provides"
+          actions={
+            <div className="flex items-center gap-3 text-[11px]">
+              <span className="flex items-center gap-1 text-success">
+                <span className="h-2 w-2 rounded-full bg-success" /> {coverage.summary.present} present
+              </span>
+              <span className="flex items-center gap-1 text-danger">
+                <span className="h-2 w-2 rounded-full bg-danger" /> {coverage.summary.missing} missing
+              </span>
+              <span className="flex items-center gap-1 text-text-muted">
+                <Database className="h-3 w-3" /> {coverage.summary.db_only} DB-only
+              </span>
+            </div>
+          }
+        >
+          {/* Extraction-mode selector */}
+          <div className="mb-2 flex flex-wrap gap-1.5">
+            {COVERAGE_MODES.map((m) => {
+              const active = coverageMode === m.id;
+              const n = coverage.variants[m.id].n_fields;
+              const Icon = m.id === 'missing' ? Plus : m.id === 'all' ? Layers : Database;
+              return (
+                <button
+                  key={m.id}
+                  onClick={() => setCoverageMode(m.id)}
+                  title={m.hint}
+                  className={`flex items-center gap-1.5 rounded-md border px-3 py-1.5 text-xs font-medium transition-colors ${
+                    active
+                      ? 'border-accent bg-accent-dim/30 text-accent-bright'
+                      : 'border-border bg-raised text-text-secondary hover:border-border-bright'
+                  }`}
+                >
+                  <Icon className="h-3.5 w-3.5" />
+                  {m.label}
+                  {m.id !== 'database' && <span className="text-text-muted">({n})</span>}
+                </button>
+              );
+            })}
+          </div>
+          <p className="mb-3 flex items-center gap-1.5 text-[11px] text-text-muted">
+            <GitCompare className="h-3.5 w-3.5" />
+            {COVERAGE_MODES.find((m) => m.id === coverageMode)?.hint}
+          </p>
+
+          <div className="grid grid-cols-1 gap-3 lg:grid-cols-2">
+            {/* Schema fields (🟢 present / 🔴 missing) */}
+            <div>
+              <p className="mb-1.5 text-[11px] font-semibold text-text-secondary">Schema fields</p>
+              <div className="max-h-56 overflow-y-auto rounded border border-border/40 bg-deep/40 p-2">
+                <div className="flex flex-wrap gap-1.5">
+                  {coverage.coverage.map((f) => {
+                    // Which fields get extracted in the active mode.
+                    const extracted =
+                      coverageMode === 'all' ? true : coverageMode === 'missing' ? !f.present : false;
+                    return (
+                      <span
+                        key={f.path}
+                        title={`${f.path}${f.matched_column ? `  ←  ${f.matched_column}` : ''}`}
+                        className={`flex items-center gap-1 rounded border px-2 py-0.5 text-[11px] ${
+                          f.present
+                            ? 'border-success/40 bg-success/10 text-success'
+                            : 'border-danger/40 bg-danger/10 text-danger'
+                        } ${extracted ? '' : 'opacity-40'}`}
+                      >
+                        {f.present ? '🟢' : '🔴'} {f.leaf}
+                      </span>
+                    );
+                  })}
+                </div>
+              </div>
+              <p className="mt-1 text-[10px] text-text-muted">
+                Dimmed chips aren’t extracted in this mode · matched by field (leaf) name.
+              </p>
+            </div>
+
+            {/* Database-only columns (in dataset, not in schema) */}
+            <div>
+              <div className="mb-1.5 flex items-center justify-between">
+                <p className="text-[11px] font-semibold text-text-secondary">
+                  Database-only columns ({coverage.db_only_columns.length})
+                </p>
+                {coverage.db_only_columns.length > 0 && (
+                  <button
+                    onClick={() => addToKeep(coverage.db_only_columns)}
+                    className="flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[10px] text-text-secondary hover:border-accent hover:text-accent"
+                    title="Append these to the carry-through columns"
+                  >
+                    <Plus className="h-3 w-3" /> Carry through
+                  </button>
+                )}
+              </div>
+              <div className="max-h-56 overflow-y-auto rounded border border-border/40 bg-deep/40 p-2">
+                {coverage.db_only_columns.length ? (
+                  <div className="flex flex-wrap gap-1.5">
+                    {coverage.db_only_columns.map((c) => (
+                      <span
+                        key={c}
+                        className="rounded border border-border bg-raised px-2 py-0.5 text-[11px] text-text-secondary"
+                      >
+                        {c}
+                      </span>
+                    ))}
+                  </div>
+                ) : (
+                  <p className="text-[11px] text-text-muted">Every dataset column maps to a schema field.</p>
+                )}
+              </div>
+              {coverage.matched_columns.length > 0 && (
+                <button
+                  onClick={() => addToKeep(coverage.matched_columns)}
+                  className="mt-1.5 flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[10px] text-text-secondary hover:border-accent hover:text-accent"
+                  title="Append the dataset columns that already cover schema fields to carry-through"
+                >
+                  <Plus className="h-3 w-3" /> Carry through {coverage.matched_columns.length} matched column(s)
+                </button>
+              )}
+            </div>
+          </div>
+        </Panel>
+      )}
+
       {source && (
         <div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
           {/* Left: column + schema config */}
@@ -330,7 +496,7 @@ export function ParsingPage() {
               actions={
                 <button
                   onClick={runEstimate}
-                  disabled={!textColumns.length || !mergedFormat}
+                  disabled={!textColumns.length || !extractionReady}
                   className="flex items-center gap-1.5 rounded-md border border-border bg-raised px-3 py-1 text-xs text-text-secondary hover:border-accent hover:text-accent disabled:opacity-50"
                 >
                   <DollarSign className="h-3.5 w-3.5" /> Estimate Cost
@@ -364,15 +530,22 @@ export function ParsingPage() {
             <Panel title="4 · Run Extraction">
               <button
                 onClick={runParse}
-                disabled={running || !textColumns.length || !mergedFormat || !apiKey}
+                disabled={running || !textColumns.length || !extractionReady || !apiKey}
                 className="flex w-full items-center justify-center gap-2 rounded-md bg-accent-dim px-4 py-2 text-sm font-medium text-white transition-colors hover:bg-accent disabled:opacity-50"
               >
                 <Play className="h-4 w-4" />
                 {running ? 'Parsing…' : 'Parse Dataset'}
               </button>
-              <p className="mt-2 text-[11px] text-text-muted">
-                {textColumns.length} text column(s), {selectedSchemas.length} schema(s) selected.
-              </p>
+              {!extractionReady && coverageMode === 'database' ? (
+                <p className="mt-2 text-[11px] text-warning">
+                  Database-only mode: no extraction needed — your uploaded columns are kept as-is.
+                  Switch to “Add missing only” or “All fields” to run the LLM.
+                </p>
+              ) : (
+                <p className="mt-2 text-[11px] text-text-muted">
+                  {textColumns.length} text column(s), {fieldCount} field(s) to extract ({coverageMode}).
+                </p>
+              )}
             </Panel>
 
             {result && (

frontend/src/types/index.ts

@@ -129,6 +129,23 @@ export interface ParseUploadResponse {
   total_rows: number;
 }
 
+export type CoverageMode = 'all' | 'missing' | 'database';
+
+export interface SchemaCoverageField {
+  path: string;
+  leaf: string;
+  present: boolean;
+  matched_column: string | null;
+}
+
+export interface SchemaCoverageResponse {
+  coverage: SchemaCoverageField[];
+  summary: { present: number; missing: number; total: number; db_only: number };
+  matched_columns: string[];
+  db_only_columns: string[];
+  variants: Record<CoverageMode, { schema_json: string; n_fields: number }>;
+}
+
 export interface CostEstimate {
   total_usd: number;
   cost_input_usd: number;
@@ -227,3 +244,10 @@ export interface ContingencyResponse {
   v: number;
   n: number;
 }
+
+export interface XgboostImportanceResponse {
+  results: Record<string, XGBoostResult>;
+  columns: string[];
+  skipped: Record<string, string>;
+  message?: string;
+}

parsing.py

@@ -24,7 +24,7 @@ import openai
 from openai import OpenAI
 import os
 import json
-# this is a test comment
+from utils.data_processing import DataProcessor
 import plotly.graph_objects as go
 
 # st.set_option('deprecation.showPyplotGlobalUse', False)
@@ -389,8 +389,20 @@ def attach_kept_columns(df):
     return out
 
 
+def _parsed_responses_dict():
+    """The parsing dict ({description: parsed_json}) from session state, or None.
+
+    Other pages (analyzing.py, map.py, …) store a *DataFrame* under the shared
+    'parsed_responses' session key. Returning None for any non-dict value keeps
+    parsing.py's dict-only paths from raising "The truth value of a DataFrame is
+    ambiguous" when the user navigates back here after using another page.
+    """
+    pr = st.session_state.get('parsed_responses')
+    return pr if isinstance(pr, dict) and pr else None
+
+
 def convert_cached_data_to_df(parsed_responses):
-    if not parsed_responses:
+    if not isinstance(parsed_responses, dict) or not parsed_responses:
         st.warning("No cached data available. Please parse the dataset first.")
         return
     try:
@@ -428,7 +440,7 @@ def render_scu_normalization(parsed_responses):
     then surfaces the SCU five-criterion eligibility funnel and download buttons.
     Toggled on/off via the global "Apply SCU normalization" switch in app.py.
     """
-    if not parsed_responses:
+    if not isinstance(parsed_responses, dict) or not parsed_responses:
         st.info("No parsed data to normalize yet.")
         return
     try:
@@ -762,6 +774,17 @@ def convert_df(df):
 
 
 def filter_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """Shared filtering UI — delegates to the DataProcessor so the parsing and
+    analysis pages use one and the same filter system (mirrors
+    ``analyzing.py``'s ``filter_dataframe``). Binary 0/1 columns get a value
+    picker; other numerics get range/percentile/std-dev modes.
+    """
+    return DataProcessor.filter_dataframe_enhanced(
+        df, enable_quick_filters=False, enable_advanced_filters=True
+    )
+
+
+def filter_dataframe_legacy(df: pd.DataFrame) -> pd.DataFrame:
     """
     Adds a UI on top of a dataframe to let viewers filter columns
 
@@ -824,24 +847,47 @@ def filter_dataframe(df: pd.DataFrame) -> pd.DataFrame:
                 st.pyplot(plot_treemap(df_, column))
 
         elif is_numeric_dtype(df_[column]):
-            _min = float(df_[column].min())
-            _max = float(df_[column].max())
-            step = (_max - _min) / 100
-            user_num_input = right.slider(
-                f"Values for {column}",
-                min_value=_min,
-                max_value=_max,
-                value=(_min, _max),
-                step=step,
-            )
-            df_ = df_[df_[column].between(*user_num_input)]
-            filtered_columns.append(column)
+            _nonnull = df_[column].dropna()
+            _uniq = set(_nonnull.unique())
+            # Binary boolean columns (values ⊆ {0, 1}) — a slider is meaningless
+            # here, so offer a value picker (0/1) instead of a min–max range.
+            if _uniq and _uniq.issubset({0, 1}):
+                _opts = sorted(_uniq)
+                user_bin_input = right.multiselect(
+                    f"Values for {column}",
+                    _opts,
+                    default=_opts,
+                    format_func=lambda v: f"{int(v)} — {'true' if int(v) == 1 else 'false'}",
+                    key=f"filt_bin_{column}",
+                )
+                df_ = df_[df_[column].isin(user_bin_input)]
+                filtered_columns.append(column)
+
+                with st.status(f"Binary Distribution: {column}", expanded=False):
+                    _vc = df_[column].value_counts().sort_index()
+                    _vc.index = _vc.index.map(
+                        lambda v: f"{int(v)} ({'true' if int(v) == 1 else 'false'})"
+                    )
+                    st.bar_chart(_vc)
+            else:
+                _min = float(df_[column].min())
+                _max = float(df_[column].max())
+                step = (_max - _min) / 100
+                user_num_input = right.slider(
+                    f"Values for {column}",
+                    min_value=_min,
+                    max_value=_max,
+                    value=(_min, _max),
+                    step=step,
+                )
+                df_ = df_[df_[column].between(*user_num_input)]
+                filtered_columns.append(column)
 
-            # Chart_GPT = ChartGPT(df_, title_font, body_font, title_size,
-            #      colors, interpretation, extract_docx, img_path)
+                # Chart_GPT = ChartGPT(df_, title_font, body_font, title_size,
+                #      colors, interpretation, extract_docx, img_path)
 
-            with st.status(f"Numerical Distribution: {column}", expanded=False) as stat_:
-                st.pyplot(plot_hist(df_, column, bins=int(round(len(df_[column].unique())-1)/2)))
+                with st.status(f"Numerical Distribution: {column}", expanded=False) as stat_:
+                    st.pyplot(plot_hist(df_, column, bins=int(round(len(df_[column].unique())-1)/2)))
 
         elif is_object_dtype(df_[column]):
             _orig_col = df_[column].copy()
@@ -945,7 +991,7 @@ def filter_dataframe(df: pd.DataFrame) -> pd.DataFrame:
 
 
 from config import (
-    FORMAT_LONG, FORMAT_LONG_XLSX, FORMAT_SCU_V2, FORMAT_SCU_V3, FORMAT_MERGED, FORMAT_UFOSETI_RU,
+    FORMAT_LONG, FORMAT_LONG_XLSX, FORMAT_SCU_V1, FORMAT_SCU_V2, FORMAT_SCU_V3, FORMAT_MERGED, FORMAT_UFOSETI_RU,
     FORMAT_NUFORC, FORMAT_BLUE_BOOK, FORMAT_UK_NATIONAL_ARCHIVES,
     FORMAT_COBEPS_NOTIFICATIONS_PAN, FORMAT_COBEPS_COB_2021,
     FORMAT_GEP, FORMAT_UPDB_NICAP, FORMAT_OVNIBASE, FORMAT_UFOSETI,
@@ -964,6 +1010,7 @@ DEEPSEEK_KEY = st.secrets.get("DEEPSEEK_KEY", "")
 # Both the format selector and the schema↔dataset comparison read from here so
 # the two never drift apart.
 SCHEMA_FORMATS = {
+    "SCU_v1":                          FORMAT_SCU_V1,
     "Default UAP Format":              FORMAT_LONG,
     "SCU Spreadsheet":                 FORMAT_LONG_XLSX,
     "SCU_v2":                          FORMAT_SCU_V2,
@@ -993,7 +1040,7 @@ SCHEMA_FORMATS = {
 # the schema picker presents them grouped so the right standard is easy to find.
 SCHEMA_FORMAT_GROUPS = {
     "Canonical & SCU": [
-        "Default UAP Format", "SCU Spreadsheet", "SCU_v2", "SCU_v3",
+        "SCU_v1", "Default UAP Format", "SCU Spreadsheet", "SCU_v2", "SCU_v3",
     ],
     "Government & official archives": [
         "Blue Book (USAF)", "UK National Archives",
@@ -1019,6 +1066,11 @@ SCHEMA_FORMAT_GROUPS = {
 # Short provenance blurb per schema (drawn from the config.py header comments)
 # so the picker explains where each dataset/standard comes from at a glance.
 SCHEMA_FORMAT_ORIGINS = {
+    "SCU_v1":
+        "The default extraction schema — the full SCU_v3 field set minus the "
+        "verbose `sightingDetails` narrative block and the all-null `manifest` "
+        "join placeholders. Keeps source, assessment, anomaly, location, "
+        "witness, craft, performance, military, effects and engagement fields.",
     "Default UAP Format":
         "The project's canonical nested narrative schema (`sightingDetails`). "
         "General-purpose GPT extraction template covering location, craft, "
@@ -1128,7 +1180,7 @@ def _current_dataset_columns() -> pd.DataFrame:
     df = st.session_state.get('parsed_responses_df')
     if isinstance(df, pd.DataFrame) and not df.empty:
         return df
-    pr = st.session_state.get('parsed_responses')
+    pr = _parsed_responses_dict()
     if pr:
         try:
             return pd.json_normalize(list(pr.values()))
@@ -1208,6 +1260,49 @@ def render_schema_dataset_diff(schema_dict: dict, df: pd.DataFrame,
         "schema_dataset_diff.csv", "text/csv", key="dl_schema_dataset_diff",
     )
 
+    # ── Send a field subset to the extraction JSON editor ─────────────────────
+    # Each button builds a pruned JSON schema from one side of the diff and
+    # stashes it in session_state; the "UAP Feature Extraction" JSON editor
+    # (rendered later in the same page) picks it up and fills its text area.
+    flat_schema = _flatten_dotted(schema_dict)  # {dotted schema path: description}
+    src_payload  = dict(flat_schema)                                                   # every schema field
+    both_payload = {s_map[k]: flat_schema.get(s_map[k], "string") for k in matched_k}  # ✓ in both
+    miss_payload = {s_map[k]: flat_schema.get(s_map[k], "string") for k in missing_k}  # 🔴 missing
+    dest_payload = {d_map[k]: flat_schema.get(s_map[k], "string") for k in matched_k}  # dataset columns
+    dest_payload.update({d_map[k]: "string — in dataset, not defined by the schema"
+                         for k in extra_k})
+
+    st.caption(
+        "**Send to extract** — fill the JSON Format editor on the **UAP Feature "
+        "Extraction** step with one side of this diff:"
+    )
+
+    def _send_to_extract(note: str, payload: dict) -> None:
+        n = len(payload)
+        st.session_state["_extract_seed_json"] = json.dumps(
+            _unflatten_dotted(payload), indent=2
+        )
+        st.session_state["_extract_seed_note"] = f"{note} ({n} field{'' if n == 1 else 's'})"
+        st.toast(f"📤 Sent {n} field{'' if n == 1 else 's'} to the extraction JSON editor.")
+
+    sb1, sb2, sb3, sb4 = st.columns(4)
+    if sb1.button(f"📤 Source · {len(src_payload)}", key="send_extract_source",
+                  use_container_width=True,
+                  help="All fields defined by the selected schema."):
+        _send_to_extract("Source / schema fields", src_payload)
+    if sb2.button(f"📤 Destination · {len(dest_payload)}", key="send_extract_dest",
+                  use_container_width=True,
+                  help="Every column in the uploaded dataset (matched + 🟢 extra)."):
+        _send_to_extract("Destination / dataset columns", dest_payload)
+    if sb3.button(f"📤 Missing · {len(miss_payload)}", key="send_extract_missing",
+                  use_container_width=True, disabled=not miss_payload,
+                  help="🔴 Schema fields the dataset lacks — extract these to fill the gaps."):
+        _send_to_extract("Missing fields", miss_payload)
+    if sb4.button(f"📤 Both · {len(both_payload)}", key="send_extract_both",
+                  use_container_width=True, disabled=not both_payload,
+                  help="✓ Fields present in both the schema and the dataset."):
+        _send_to_extract("Matched fields", both_payload)
+
     return {
         "matched": [s_map[k] for k in matched_k],
         "missing": [s_map[k] for k in missing_k],
@@ -1405,7 +1500,7 @@ if parsed_table_restore is not None:
         st.error(f"Could not read CSV/XLSX: {e}")
 
 # ── JSON-only export controls (shown when JSON loaded but no CSV uploaded) ─────
-if json_restore is not None and unparsed is None and st.session_state.get('parsed_responses'):
+if json_restore is not None and unparsed is None and _parsed_responses_dict():
     st.divider()
     st.markdown("**Loaded from JSON — export options**")
 
@@ -1641,7 +1736,7 @@ if False:
                         st.error("No files were processed successfully.")
 
     # ── Inline export controls (available after processing) ───────────────
-    if st.session_state.get('parsed_responses') and md_files:
+    if _parsed_responses_dict() and md_files:
         _pr = st.session_state['parsed_responses']
         _json_dl = json.dumps(_pr, indent=2)
         st.download_button(
@@ -1838,6 +1933,21 @@ if unparsed is not None:
             _merged_dict = _deep_merge(_merged_dict, _custom_extra)
             active_format = json.dumps(_merged_dict, indent=2)
 
+    # A "Send to extract" action from the schema↔dataset diff (above) seeds this
+    # editor with a chosen field subset. Drop the persisted widget value so the
+    # new payload becomes the text area's content (setting the keyed value
+    # directly would clash with the value= arg passed to the widget below).
+    _seed = st.session_state.pop("_extract_seed_json", None)
+    if _seed is not None:
+        active_format = _seed
+        st.session_state.pop("preview_json_text", None)
+        _seed_note = st.session_state.pop("_extract_seed_note", "")
+        st.success(
+            f"📥 Loaded {_seed_note} from the schema diff into the editor."
+            if _seed_note else
+            "📥 Loaded fields from the schema diff into the editor."
+        )
+
     st.markdown("### Preview / Edit JSON Format")
     _tab_tree, _tab_fields, _tab_raw = st.tabs(["🌳 Tree", "📋 Fields", "✏️ Raw"])
 
@@ -2222,7 +2332,7 @@ if unparsed is not None:
                 st.error(f"An error occurred: {str(e)}")
 
     # ── Download / export ──────────────────────────────────────────────────────
-    if st.session_state['parsed_responses'] is not None:
+    if _parsed_responses_dict() is not None:
         json_str = download_json(st.session_state['parsed_responses'])
         st.download_button(
             label="Download Parsed Data as JSON",
@@ -2248,7 +2358,7 @@ if unparsed is not None:
 
         # Allow loading a previously saved parsed_responses.json when there is
         # no live session (e.g. user reloaded the page after parsing).
-        if st.session_state['parsed_responses'] is None:
+        if _parsed_responses_dict() is None:
             st.info(
                 "No parsed data in session. Upload a previously saved "
                 "`parsed_responses.json` **or** an SCU-format `.csv` to continue."
@@ -2277,7 +2387,7 @@ if unparsed is not None:
                 st.session_state['_scu_csv_df'] = pd.read_csv(csv_upload)
                 st.success(f"Loaded CSV: {len(st.session_state['_scu_csv_df'])} rows.")
 
-        if st.session_state['parsed_responses'] is not None or '_scu_csv_df' in st.session_state:
+        if _parsed_responses_dict() is not None or '_scu_csv_df' in st.session_state:
             col_t, col_n = st.columns(2)
             xlsx_template = col_t.file_uploader(
                 "Upload template spreadsheet (.xlsx)",
@@ -2439,15 +2549,18 @@ with st.expander("🧬 Compute Embeddings → HDF5", expanded=False):
 # Markdown ingestion) and expose the SCU v2 normalized CSV + audit report.
 if st.session_state.get('scu_normalize_enabled'):
     st.divider()
-    if st.session_state.get('parsed_responses'):
-        with st.expander("🧹 SCU Normalization (SCU v2)", expanded=True):
-            st.markdown(
-                "Canonicalises the parsed data — country ISO codes, US-state "
-                "codes, witness roles, craft shape/size bands — and derives the "
-                "**SCU five-criterion eligibility gate** (`scu_eligible` plus the "
-                "per-criterion columns). Turn this off in the app sidebar to hide."
-            )
-            render_scu_normalization(st.session_state['parsed_responses'])
+    if _parsed_responses_dict():
+        # Titled section, not an expander: render_scu_normalization opens its own
+        # "Normalization audit report" expander, and Streamlit forbids nesting
+        # expanders inside expanders.
+        st.markdown("#### 🧹 SCU Normalization (SCU v2)")
+        st.markdown(
+            "Canonicalises the parsed data — country ISO codes, US-state "
+            "codes, witness roles, craft shape/size bands — and derives the "
+            "**SCU five-criterion eligibility gate** (`scu_eligible` plus the "
+            "per-criterion columns). Turn this off in the app sidebar to hide."
+        )
+        render_scu_normalization(st.session_state['parsed_responses'])
     else:
         st.caption(
             "🧹 SCU Normalization is enabled — parse a dataset or load a "

pipeline/README.md

@@ -0,0 +1,43 @@
+# `pipeline/` — Document Preprocessing scripts (vendored)
+
+These are the standalone scripts that turn raw government PDFs into the report
+table consumed by the **UAP Feature Extraction** page. They used to live in an
+external directory (`D:\divided` / `/mnt/d/divided`); they are now vendored here
+so the **Document Preprocessing Pipeline** Streamlit page (`preprocessing.py`)
+is self-contained — no external script folder, no separate interpreter.
+
+## How they're run
+
+`preprocessing.py` invokes each script with the **app's own interpreter**
+(`sys.executable`) and an absolute path into this folder, while the **working
+directory** (the configurable data dir, default `pipeline_data/`) is the cwd.
+So the scripts read/write `raw/`, `pages_out/`, `concat/`, `extracted/`, … in
+the working dir, and their code ships with the app. Each script is unmodified
+from the original pipeline and remains runnable directly:
+
+```bash
+python pipeline/page_coverage.py --root /path/to/workdir
+```
+
+## Stages
+
+| Stage | Scripts |
+|-------|---------|
+| 0 · Scrape | `download_uap_pdfs.py` |
+| 1 · Layout | `split_pages.py`, `restructure_pages.py`, `reorganize.py` |
+| 2 · OCR | `stamp_pages.py`, `find_ocr_targets.py`, `run_ocr.py`, `destamp_pages.py` |
+| 3 · Assembly | `page_coverage.py`, `concat_pages.py` |
+| 4 · Extraction | `extract_reports.py`, `pdf_to_reports.py`, `reconcile.py` |
+| 5 · Table | `add_source_agency.py`, `analyze_reports.py`, `join_reports.py`, `map_yaml_to_reports.py` |
+| Audit | `audit.py`, `find_missing_concat.py` |
+
+## Dependencies
+
+Pinned in the project's `pyproject.toml` / `requirements.txt`, so the app's
+interpreter already has them: `pyyaml`, `google-genai`, `pypdf`, `mistralai`,
+`openai`, `pdfplumber`, `reportlab`, `pdf2image`, `pillow`, `pymupdf`,
+`requests`. `pdf2image` additionally needs the system **poppler** binary
+(`apt-get install poppler-utils`).
+
+API keys (entered in the page or via `st.secrets`): `MISTRAL_API_KEY` (OCR),
+`GEMINI_API_KEY` (extraction), `NVIDIA_API_KEY` (optional NIM extraction).

pipeline/add_source_agency.py

@@ -0,0 +1,343 @@
+"""
+add_source_agency.py
+────────────────────
+Reads an Excel file and adds two columns after 'may_release_source_file':
+  • agency     — top-level agency folder  (DOD, FBI, NASA, DOS, NARA-CIA, MISC)
+  • collection — sub-folder within agency (mission-reports, series-38, transcripts …)
+
+Path/slug resolution order for both columns:
+  1. Full or relative path  → parse path parts directly
+  2. Slug prefix map        → e.g. "dow-uap-*" → DOD / mission-reports
+  3. Substring scan         → look for known tokens anywhere in the value
+
+Usage
+-----
+    python add_source_agency.py
+    python add_source_agency.py --input my_file.xlsx --col file_path --out out.xlsx
+"""
+
+import argparse
+import re
+import sys
+from pathlib import Path, PurePosixPath, PureWindowsPath
+
+import pandas as pd
+
+# ── Agency tokens (longer first to avoid sub-match, e.g. NARA-CIA before NASA) ─
+AGENCY_TOKENS = ["NARA-CIA", "NARA_CIA", "NASA", "DOD", "FBI", "DOS", "MISC"]
+
+# ── Fallback: map free-text agency names (from YAML) → canonical token ────────
+YAML_AGENCY_FALLBACK: dict[str, str] = {
+    "department of war":          "DOD",
+    "department of defense":      "DOD",
+    "dod":                        "DOD",
+    "fbi":                        "FBI",
+    "federal bureau of investigation": "FBI",
+    "nasa":                       "NASA",
+    "national aeronautics and space administration": "NASA",
+    "dos":                        "DOS",
+    "department of state":        "DOS",
+    "nara-cia":                   "NARA-CIA",
+    "nara_cia":                   "NARA-CIA",
+    "misc":                       "MISC",
+}
+
+# ── Normalised agency set for fast membership tests ───────────────────────────
+_AGENCY_SET = {"NARA-CIA", "NASA", "DOD", "FBI", "DOS", "MISC"}
+
+# ── Known collection sub-folders per agency ────────────────────────────────────
+# Values are the canonical collection name returned by the script.
+COLLECTION_TOKENS = {
+    # DOD
+    "mission-reports": "mission-reports",
+    "mission_reports": "mission-reports",
+    "range-fouler-debriefs": "range-fouler-debriefs",
+    "range_fouler_debriefs": "range-fouler-debriefs",
+    "email-correspondence": "email-correspondence",
+    "email_correspondence": "email-correspondence",
+    "reports-other": "reports-other",
+    "reports_other": "reports-other",
+    # FBI
+    "photo-collections": "photo-collections",
+    "photo_collections": "photo-collections",
+    # NASA
+    "transcripts": "transcripts",
+    "crew-debriefings": "crew-debriefings",
+    "crew_debriefings": "crew-debriefings",
+    # DOS
+    "cables": "cables",
+    # NARA-CIA series / collections
+    "hs1-834228961": "hs1-834228961",
+    "hs1-101634279": "hs1-101634279",
+    "series-18": "series-18",
+    "series-38": "series-38",
+    "series-59": "series-59",
+    "series-255": "series-255",
+    "series-331": "series-331",
+    "series-341": "series-341",
+    "series-342": "series-342",
+    # MISC
+    "statements-redacted": "statements-redacted",
+    "statements_redacted": "statements-redacted",
+    "visuals": "visuals",
+    "presentations": "presentations",
+    "unclassified": "unclassified",
+}
+
+# ── Slug-prefix → (agency, collection) ────────────────────────────────────────
+# Keyed by lowercase slug prefix; longest prefixes first.
+PREFIX_MAP: list[tuple[str, str, str | None]] = [
+    # DOD mission reports
+    ("dow-uap-d",           "DOD",      "mission-reports"),
+    ("dow-uap-",            "DOD",      "mission-reports"),
+    ("dod-range-fouler",    "DOD",      "range-fouler-debriefs"),
+    ("dod-email",           "DOD",      "email-correspondence"),
+    ("dod-",                "DOD",      None),
+    ("pr-",                 "DOD",      "mission-reports"),
+    # FBI
+    ("fbi-",                "FBI",      "photo-collections"),
+    # NASA
+    ("nasa-transcript",     "NASA",     "transcripts"),
+    ("nasa-crew",           "NASA",     "crew-debriefings"),
+    ("nasa-",               "NASA",     None),
+    # DOS
+    ("dos-",                "DOS",      "cables"),
+    # NARA-CIA series (slug starts with series number)
+    ("65_hs1-834228961",    "NARA-CIA", "hs1-834228961"),
+    ("65_hs1-101634279",    "NARA-CIA", "hs1-101634279"),
+    ("18_",                 "NARA-CIA", "series-18"),
+    ("38_",                 "NARA-CIA", "series-38"),
+    ("59_",                 "NARA-CIA", "series-59"),
+    ("255_",                "NARA-CIA", "series-255"),
+    ("331_",                "NARA-CIA", "series-331"),
+    ("341_",                "NARA-CIA", "series-341"),
+    ("342_",                "NARA-CIA", "series-342"),
+    ("series-",             "NARA-CIA", None),
+]
+
+AGENCY_RE = re.compile(
+    r"\b(" + "|".join(re.escape(a) for a in AGENCY_TOKENS) + r")\b",
+    re.IGNORECASE,
+)
+COLLECTION_RE = re.compile(
+    r"\b(" + "|".join(re.escape(k) for k in sorted(COLLECTION_TOKENS, key=len, reverse=True)) + r")\b",
+    re.IGNORECASE,
+)
+
+
+def _norm_agency(token: str) -> str:
+    return token.upper().replace("_", "-")
+
+
+def _path_parts(value: str) -> list[str]:
+    """Return path parts trying both Windows and Posix interpretations."""
+    for PathCls in (PureWindowsPath, PurePosixPath):
+        try:
+            parts = list(PathCls(value).parts)
+            if len(parts) > 1:
+                return parts
+        except Exception:
+            pass
+    return [value]
+
+
+def extract_both(value) -> tuple[str | None, str | None]:
+    """Return (agency, collection) for a single cell value."""
+    if not isinstance(value, str) or not value.strip():
+        return None, None
+
+    v = value.strip()
+    lower = v.lower()
+
+    # 1 ── Path-based: walk parts, find agency then take the next part ──────────
+    parts = _path_parts(v)
+    for i, part in enumerate(parts):
+        norm = _norm_agency(part)
+        if norm in _AGENCY_SET:
+            agency = norm
+            # collection = next path segment after agency, if it exists and is known
+            collection = None
+            if i + 1 < len(parts):
+                nxt = parts[i + 1].lower().replace("_", "-")
+                collection = COLLECTION_TOKENS.get(nxt) or COLLECTION_TOKENS.get(parts[i + 1])
+            return agency, collection
+
+    # 2 ── Slug prefix map ─────────────────────────────────────────────────────
+    for prefix, agency, collection in PREFIX_MAP:
+        if lower.startswith(prefix):
+            # Try to refine collection from slug body if prefix gave None
+            if collection is None:
+                m = COLLECTION_RE.search(v)
+                if m:
+                    collection = COLLECTION_TOKENS.get(m.group(1).lower().replace("_", "-"))
+            return agency, collection
+
+    # 3 ── Substring scan for agency ──────────────────────────────────────────
+    m_agency = AGENCY_RE.search(v)
+    if m_agency:
+        agency = _norm_agency(m_agency.group(1))
+        m_coll = COLLECTION_RE.search(v)
+        collection = COLLECTION_TOKENS.get(m_coll.group(1).lower().replace("_", "-")) if m_coll else None
+        return agency, collection
+
+    return None, None
+
+
+def _find_path_column(df: pd.DataFrame) -> str | None:
+    """Heuristic: find the column most likely to contain file paths or slugs."""
+    candidates = []
+    for col in df.columns:
+        sample = df[col].dropna().astype(str).head(30)
+        hits = sum(
+            1 for s in sample
+            if extract_both(s)[0] is not None
+            or any(sep in s for sep in ("/", "\\"))
+        )
+        if hits > 0:
+            candidates.append((hits, col))
+    if not candidates:
+        return None
+    candidates.sort(reverse=True)
+    return candidates[0][1]
+
+
+def _read_file(path: Path) -> pd.DataFrame:
+    """Read CSV or Excel, detecting format from content (not just extension)."""
+    # Try CSV first by sniffing the first bytes
+    try:
+        with open(path, "rb") as fh:
+            header = fh.read(8)
+        is_excel = header[:4] in (b"\xd0\xcf\x11\xe0", b"PK\x03\x04")  # XLS or XLSX magic
+    except OSError:
+        is_excel = path.suffix.lower() in (".xlsx", ".xls", ".xlsm")
+
+    if is_excel:
+        return pd.read_excel(path)
+    else:
+        # CSV — try common separators
+        for sep in (",", ";", "\t", "|"):
+            try:
+                df = pd.read_csv(path, sep=sep, dtype=str, encoding="utf-8-sig")
+                if len(df.columns) > 1:
+                    return df
+            except Exception:
+                pass
+        # Last resort: let pandas infer
+        return pd.read_csv(path, dtype=str, encoding="utf-8-sig")
+
+
+def process(input_path: Path, col_name: str | None, output_path: Path) -> None:
+    print(f"Reading: {input_path}")
+    df = _read_file(input_path)
+    print(f"Columns found: {list(df.columns)}")
+
+    if col_name:
+        if col_name not in df.columns:
+            sys.exit(f"ERROR: Column '{col_name}' not found. Available: {list(df.columns)}")
+        path_col = col_name
+    else:
+        path_col = _find_path_column(df)
+        if path_col is None:
+            sys.exit(
+                "ERROR: Could not auto-detect a path/slug column. "
+                "Re-run with --col <column_name>."
+            )
+        print(f"Auto-detected path column: '{path_col}'")
+
+    pairs = df[path_col].apply(extract_both)
+    agency_values     = pairs.apply(lambda x: x[0])
+    collection_values = pairs.apply(lambda x: x[1])
+
+    # ── Fallback 1: may_release_agency_yaml column ────────────────────────────
+    # Rows where path extraction failed can borrow from the pre-existing YAML
+    # agency column (e.g. "Department of War" → "DOD").
+    YAML_COL = "may_release_agency_yaml"
+    if YAML_COL in df.columns:
+        null_mask = agency_values.isna()
+        if null_mask.any():
+            yaml_mapped = (
+                df.loc[null_mask, YAML_COL]
+                .fillna("")
+                .str.strip()
+                .str.lower()
+                .map(YAML_AGENCY_FALLBACK)
+            )
+            agency_values = agency_values.copy()
+            agency_values[null_mask] = yaml_mapped
+            fallback_filled = yaml_mapped.notna().sum()
+            print(f"Fallback from '{YAML_COL}': filled {fallback_filled} additional agency values")
+
+    # ── Fallback 2: updb_source column ───────────────────────────────────────
+    # Some rows have an updb_source that contains the source document slug.
+    for extra_col in ("updb_source", "overmeire_reference", "overmeire_Reference"):
+        if extra_col not in df.columns:
+            continue
+        null_mask = agency_values.isna()
+        if not null_mask.any():
+            break
+        extra_pairs = df.loc[null_mask, extra_col].apply(extract_both)
+        extra_agency = extra_pairs.apply(lambda x: x[0])
+        extra_coll   = extra_pairs.apply(lambda x: x[1])
+        filled = extra_agency.notna().sum()
+        if filled:
+            agency_values = agency_values.copy()
+            collection_values = collection_values.copy()
+            agency_values[null_mask]     = extra_agency.values
+            collection_values[null_mask] = extra_coll.values
+            print(f"Fallback from '{extra_col}': filled {filled} additional agency values")
+
+    # Insert both columns right after 'may_release_source_file' if it exists
+    if "may_release_source_file" in df.columns:
+        insert_pos = df.columns.get_loc("may_release_source_file") + 1
+        df.insert(insert_pos,     "agency",     agency_values)
+        df.insert(insert_pos + 1, "collection", collection_values)
+    else:
+        df["agency"]     = agency_values
+        df["collection"] = collection_values
+
+    found_agency = df["agency"].notna().sum()
+    found_coll   = df["collection"].notna().sum()
+    total        = len(df)
+    print(f"agency resolved:     {found_agency}/{total}")
+    print(f"collection resolved: {found_coll}/{total}")
+
+    if found_agency < total:
+        # Show a sample of what's still missing to guide further fixes
+        still_missing = df[df["agency"].isna()]
+        print(f"\nStill unresolved ({len(still_missing)} rows) — first 10 '{path_col}' values:")
+        for v in still_missing[path_col].head(10):
+            print(f"  {str(v)[:100]!r}")
+
+    print(f"\nagency distribution:\n{df['agency'].value_counts(dropna=False).to_string()}")
+    print(f"\ncollection distribution:\n{df['collection'].value_counts(dropna=False).to_string()}\n")
+
+    if output_path.suffix.lower() in (".xlsx", ".xls", ".xlsm"):
+        df.to_excel(output_path, index=False)
+    else:
+        df.to_csv(output_path, index=False)
+    print(f"Saved → {output_path}")
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Add agency + collection columns to an Excel file"
+    )
+    ap.add_argument("--input", default="duplicates_UPDB_overmeire_reviewed_2.xlsx",
+                    help="Input Excel file")
+    ap.add_argument("--col",   default=None,
+                    help="Column with file paths/slugs (auto-detected if omitted)")
+    ap.add_argument("--out",   default=None,
+                    help="Output path (default: <stem>_with_agency.xlsx)")
+    args = ap.parse_args()
+
+    input_path  = Path(args.input)
+    if args.out:
+        output_path = Path(args.out)
+    else:
+        # Preserve the original extension so CSV in → CSV out
+        output_path = input_path.with_name(input_path.stem + "_with_agency" + input_path.suffix)
+    process(input_path, args.col, output_path)
+
+
+if __name__ == "__main__":
+    main()

pipeline/analyze_reports.py

@@ -0,0 +1,332 @@
+"""
+analyze_reports.py
+──────────────────
+Reads _all_reports.json and flags structural and content anomalies.
+
+Usage:
+    python analyze_reports.py
+    python analyze_reports.py --input D:/divided/extracted/_all_reports.json
+    python analyze_reports.py --input _all_reports.json --out anomalies.md
+"""
+
+import json
+import re
+import argparse
+from pathlib import Path
+from collections import defaultdict
+
+DEFAULT_INPUT = "D:/divided/extracted/_all_reports.json"
+DEFAULT_OUT   = "D:/divided/anomaly_report.md"
+
+# ── thresholds ─────────────────────────────────────────────────────────────────
+SHORT_TEXT_CHARS    = 200    # raw_text shorter than this is suspicious
+VERY_SHORT_CHARS    = 50     # almost certainly truncated / empty
+LONG_TEXT_CHARS     = 60000  # single report longer than this may be two merged
+ASSESS_DUP_RATIO    = 0.95   # assessment/raw_text similarity threshold (len ratio)
+MIN_REPORTS_PER_CHUNK = 0.2  # if reports / chunks < this, extraction may have failed
+
+
+def load_json(path: Path) -> list:
+    with open(path, encoding="utf-8") as f:
+        return json.load(f)
+
+
+def page_label_to_ints(label: str) -> list[int]:
+    """Extract all page numbers from a label like 'page_0004-page_0006'."""
+    return [int(m) for m in re.findall(r"(\d+)", label)]
+
+
+def check_assessment_duplicate(raw: str, assess: str) -> bool:
+    """Return True if assessment is essentially a copy of the end of raw_text."""
+    if not assess:
+        return False
+    assess_stripped = assess.strip()
+    raw_stripped    = raw.strip()
+    # Check if assess appears verbatim in raw_text (common tail duplication)
+    if assess_stripped in raw_stripped:
+        return True
+    # Check length ratio — if assessment is almost as long as raw_text
+    if len(assess_stripped) / max(len(raw_stripped), 1) > ASSESS_DUP_RATIO:
+        return True
+    return False
+
+
+def analyze(data: list) -> dict:
+    anomalies = defaultdict(list)   # category → [items]
+    stats = {
+        "total_files":   len(data),
+        "total_reports": 0,
+        "files_with_errors": 0,
+        "files_zero_reports": 0,
+    }
+
+    for entry in data:
+        src   = entry.get("source_file", "UNKNOWN")
+        chunks = entry.get("chunk_count", 1)
+        reports = entry.get("reports", [])
+        errors  = entry.get("parse_errors", [])
+
+        stats["total_reports"] += len(reports)
+
+        # ── A1: API / parse errors ─────────────────────────────────────────────
+        if errors:
+            stats["files_with_errors"] += 1
+            for e in errors:
+                anomalies["A1_api_parse_error"].append({
+                    "file":  src,
+                    "chunk": e.get("chunk"),
+                    "error": e.get("error", ""),
+                })
+
+        # ── A2: Zero reports extracted ─────────────────────────────────────────
+        if len(reports) == 0:
+            stats["files_zero_reports"] += 1
+            anomalies["A2_zero_reports"].append({
+                "file":   src,
+                "chunks": chunks,
+                "note":   "No reports extracted — possible all-cover-page doc, "
+                          "truly empty, or extraction failure",
+            })
+            continue  # nothing to inspect inside reports
+
+        # ── A3: Low reports-per-chunk ratio ────────────────────────────────────
+        ratio = len(reports) / chunks
+        if ratio < MIN_REPORTS_PER_CHUNK and chunks > 2:
+            anomalies["A3_low_report_yield"].append({
+                "file":    src,
+                "chunks":  chunks,
+                "reports": len(reports),
+                "ratio":   round(ratio, 2),
+                "note":    "Very few reports relative to chunk count — "
+                           "some chunks may have been silent failures",
+            })
+
+        # ── Per-report checks ──────────────────────────────────────────────────
+        seen_pages = []
+
+        for i, rep in enumerate(reports):
+            raw      = rep.get("raw_text") or ""
+            assess   = rep.get("assessment") or ""
+            pages    = rep.get("pages")
+            rep_id   = f"{src} · report {i+1}"
+
+            # ── A4: Null or missing pages label ───────────────────────────────
+            if not pages:
+                anomalies["A4_null_pages"].append({
+                    "file":   src,
+                    "report": i + 1,
+                    "note":   "pages field is null — page attribution lost",
+                })
+
+            # ── A5: Very short raw_text ────────────────────────────────────────
+            if len(raw) < VERY_SHORT_CHARS:
+                anomalies["A5_very_short_text"].append({
+                    "file":       src,
+                    "report":     i + 1,
+                    "pages":      pages,
+                    "char_count": len(raw),
+                    "preview":    raw[:80].replace("\n", " "),
+                    "note":       "raw_text < 50 chars — likely truncation or "
+                                  "cover-page / separator only",
+                })
+            elif len(raw) < SHORT_TEXT_CHARS:
+                anomalies["A6_short_text"].append({
+                    "file":       src,
+                    "report":     i + 1,
+                    "pages":      pages,
+                    "char_count": len(raw),
+                    "preview":    raw[:120].replace("\n", " "),
+                    "note":       "raw_text < 200 chars — possibly incomplete",
+                })
+
+            # ── A7: Suspiciously long raw_text (possible merge of 2+ reports) ─
+            if len(raw) > LONG_TEXT_CHARS:
+                anomalies["A7_very_long_text"].append({
+                    "file":       src,
+                    "report":     i + 1,
+                    "pages":      pages,
+                    "char_count": len(raw),
+                    "note":       f"raw_text > {LONG_TEXT_CHARS} chars — "
+                                  "may be two reports merged into one",
+                })
+
+            # ── A8: Assessment duplicates raw_text ─────────────────────────────
+            if assess and check_assessment_duplicate(raw, assess):
+                anomalies["A8_assessment_duplication"].append({
+                    "file":   src,
+                    "report": i + 1,
+                    "pages":  pages,
+                    "note":   "assessment appears to be a verbatim copy of (part of) raw_text",
+                })
+
+            # ── A9: Non-standard pages label format ───────────────────────────
+            if pages and pages not in ("all-pages",):
+                if not re.match(r"^page_\d{4}(-page_\d{4})?$", pages):
+                    anomalies["A9_nonstandard_pages_label"].append({
+                        "file":   src,
+                        "report": i + 1,
+                        "pages":  pages,
+                        "note":   "pages label doesn't match expected pattern "
+                                  "(page_XXXX or page_XXXX-page_XXXX)",
+                    })
+
+            # Accumulate page numbers for overlap check
+            if pages:
+                seen_pages.append((i + 1, pages, page_label_to_ints(pages)))
+
+        # ── A10: Overlapping page ranges within same document ─────────────────
+        for idx_a, (ri_a, lbl_a, nums_a) in enumerate(seen_pages):
+            for ri_b, lbl_b, nums_b in seen_pages[idx_a + 1:]:
+                overlap = set(nums_a) & set(nums_b)
+                if overlap:
+                    anomalies["A10_overlapping_pages"].append({
+                        "file":      src,
+                        "report_a":  ri_a,
+                        "pages_a":   lbl_a,
+                        "report_b":  ri_b,
+                        "pages_b":   lbl_b,
+                        "overlap":   sorted(overlap),
+                        "note":      "Two reports claim the same page(s) — "
+                                     "possible split boundary error",
+                    })
+
+    return {"stats": stats, "anomalies": dict(anomalies)}
+
+
+CATEGORY_LABELS = {
+    "A1_api_parse_error":      "A1 — API / parse errors",
+    "A2_zero_reports":         "A2 — Zero reports extracted",
+    "A3_low_report_yield":     "A3 — Low reports-per-chunk ratio",
+    "A4_null_pages":           "A4 — Null pages label",
+    "A5_very_short_text":      "A5 — Very short raw_text (< 50 chars)",
+    "A6_short_text":           "A6 — Short raw_text (50–200 chars)",
+    "A7_very_long_text":       "A7 — Very long raw_text (> 60 000 chars)",
+    "A8_assessment_duplication": "A8 — Assessment duplicates raw_text",
+    "A9_nonstandard_pages_label": "A9 — Non-standard pages label",
+    "A10_overlapping_pages":   "A10 — Overlapping page ranges",
+}
+
+
+def render_markdown(result: dict, input_path: Path) -> str:
+    stats     = result["stats"]
+    anomalies = result["anomalies"]
+
+    lines = [
+        "# _all_reports.json — Anomaly Report",
+        "",
+        f"**Source:** `{input_path}`  ",
+        "",
+        "## Summary",
+        "",
+        f"| Metric | Count |",
+        f"|--------|-------|",
+        f"| Total files | {stats['total_files']} |",
+        f"| Total reports extracted | {stats['total_reports']} |",
+        f"| Files with API/parse errors | {stats['files_with_errors']} |",
+        f"| Files with zero reports | {stats['files_zero_reports']} |",
+        f"| Avg reports per file | {stats['total_reports'] / max(stats['total_files'], 1):.1f} |",
+        "",
+        "## Anomaly Counts",
+        "",
+        "| Code | Category | Count |",
+        "|------|----------|-------|",
+    ]
+
+    total_issues = 0
+    for cat, label in CATEGORY_LABELS.items():
+        n = len(anomalies.get(cat, []))
+        total_issues += n
+        lines.append(f"| {cat.split('_')[0]} | {label.split(' — ')[1]} | {n} |")
+
+    lines += [
+        f"| | **Total issues** | **{total_issues}** |",
+        "",
+    ]
+
+    if total_issues == 0:
+        lines.append("✅ No anomalies detected.")
+        return "\n".join(lines)
+
+    lines.append("## Detailed Findings")
+
+    for cat, label in CATEGORY_LABELS.items():
+        items = anomalies.get(cat, [])
+        if not items:
+            continue
+        lines += [
+            "",
+            f"### {label} ({len(items)})",
+            "",
+        ]
+        for item in items:
+            file_  = item.get("file", "")
+            report = item.get("report")
+            pages  = item.get("pages")
+            note   = item.get("note", "")
+            loc    = f"report {report}" if report else ""
+            pg     = f"pages `{pages}`" if pages else ""
+            detail = ", ".join(filter(None, [loc, pg]))
+            lines.append(f"- **`{file_}`**{' — ' + detail if detail else ''}")
+            lines.append(f"  {note}")
+            # Extra fields
+            for k in ("error", "preview", "char_count", "chunks", "reports",
+                      "ratio", "overlap", "report_a", "pages_a", "report_b",
+                      "pages_b"):
+                if k in item:
+                    lines.append(f"  _{k}:_ `{item[k]}`")
+            lines.append("")
+
+    return "\n".join(lines)
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Analyze _all_reports.json for anomalies")
+    ap.add_argument("--input", default=DEFAULT_INPUT,
+                    help="Path to _all_reports.json")
+    ap.add_argument("--out",   default=DEFAULT_OUT,
+                    help="Output Markdown report path")
+    ap.add_argument("--json-out", default=None,
+                    help="Also write raw anomaly data as JSON")
+    args = ap.parse_args()
+
+    input_path = Path(args.input)
+    if not input_path.exists():
+        raise SystemExit(f"✗  File not found: {input_path}")
+
+    print(f"Loading {input_path} …")
+    data = load_json(input_path)
+    print(f"  {len(data)} file entries loaded")
+
+    result = analyze(data)
+    stats  = result["stats"]
+    print(f"\n  Total reports  : {stats['total_reports']}")
+    print(f"  Zero-report files : {stats['files_zero_reports']}")
+    print(f"  Files with errors : {stats['files_with_errors']}")
+
+    total_issues = sum(len(v) for v in result["anomalies"].values())
+    print(f"  Total anomaly flags : {total_issues}")
+
+    # Markdown report
+    md = render_markdown(result, input_path)
+    out_path = Path(args.out)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    out_path.write_text(md, encoding="utf-8")
+    print(f"\n  ✓  Markdown report → {out_path}")
+
+    # Optional JSON dump
+    if args.json_out:
+        jout = Path(args.json_out)
+        jout.write_text(json.dumps(result, indent=2, ensure_ascii=False),
+                        encoding="utf-8")
+        print(f"  ✓  JSON data      → {jout}")
+
+    # Print top anomalies to console
+    print("\n── Top anomalies ────────────────────────────────────────────────")
+    for cat, label in CATEGORY_LABELS.items():
+        items = result["anomalies"].get(cat, [])
+        if items:
+            print(f"  {label}: {len(items)}")
+
+
+if __name__ == "__main__":
+    main()

pipeline/audit.py

@@ -0,0 +1,339 @@
+"""
+audit.py
+────────
+Produces a statistical summary of the UAP archive corpus.
+
+Metrics
+-------
+  • Total documents (PDF + VID) in CSV
+  • Released vs pending (CSV rows with/without matching local folder)
+  • Redacted — CSV flag + OCR keyword scan per document
+  • Contains embedded images / photos in .md files
+  • Insufficient evidence (heavy redaction or very thin content)
+  • Previously known / reviewed (keyword scan of blurb + OCR)
+
+Output
+------
+  Prints a summary to stdout.
+  Writes a detailed per-document markdown report to --out (default: audit_report.md).
+
+Usage
+-----
+    python audit.py
+    python audit.py --csv uap-csv.csv --root D:/divided --out audit_report.md
+"""
+
+import re
+import csv
+import argparse
+from pathlib import Path
+from collections import defaultdict
+from datetime import datetime
+
+DEFAULT_CSV  = "uap-csv.csv"
+DEFAULT_ROOT = "D:/divided"
+DEFAULT_OUT  = "audit_report.md"
+
+# ── keyword patterns ──────────────────────────────────────────────────────────
+
+RE_REDACTED   = re.compile(r"\[?redacted\]?|\bblacked.out\b|\bcensored\b", re.I)
+RE_IMAGE      = re.compile(r"!\[.*?\]\(.*?\)")
+RE_INSUFF     = re.compile(
+    r"insufficient\s+(?:information|evidence|data)|no\s+(?:further\s+)?detail|"
+    r"unable\s+to\s+(?:determine|assess|identify)|not\s+enough\s+(?:data|evidence)|"
+    r"inconclusive",
+    re.I
+)
+RE_KNOWN      = re.compile(
+    r"previously\s+(?:reported|documented|known|reviewed|identified)|"
+    r"prior\s+(?:report|incident|case)|known\s+(?:case|incident)|"
+    r"already\s+(?:reported|documented)|follow.up\s+to",
+    re.I
+)
+
+# ── helpers ───────────────────────────────────────────────────────────────────
+
+PAGE_DIR_RE = re.compile(r"^page_(\d+)$", re.IGNORECASE)
+
+SKIP_NAMES = {
+    "reorganize.py", "restructure_pages.py", "stamp_pages.py",
+    "reconcile.py", "pdf_to_reports.py", "concat_pages.py", "audit.py",
+    "uap_record_schema.yaml", "uap-csv.csv",
+    "move_log.json", "page_move_log.json",
+    "records", "reports_out", "pages_out", "wiki", "scripts",
+    "MISC", "DOD", "NASA", "FBI", "DOS", "NARA-CIA",
+    "CLAUDE.md", "Untitled.md", "README.md",
+}
+
+
+def _slug(title: str) -> str:
+    s = title.lower().strip()
+    s = re.sub(r"[\s,]+", "-", s)
+    s = re.sub(r"[^\w\-]", "", s)
+    return re.sub(r"-+", "-", s).strip("-")
+
+
+def _index_docs(root: Path) -> dict[str, Path]:
+    """Map doc-slug → Path for every folder containing page_XXXX subfolders."""
+    index = {}
+    for d in root.rglob("*"):
+        if not d.is_dir() or d.name in SKIP_NAMES:
+            continue
+        if any(PAGE_DIR_RE.match(sub.name) for sub in d.iterdir() if sub.is_dir()):
+            index[d.name.lower()] = d
+    return index
+
+
+def _collect_md_texts(doc_dir: Path) -> list[tuple[str, str]]:
+    """Return [(page_name, text), ...] sorted by page number."""
+    pages = []
+    for sub in doc_dir.iterdir():
+        if not sub.is_dir():
+            continue
+        m = PAGE_DIR_RE.match(sub.name)
+        if not m:
+            continue
+        md = sub / f"{sub.name}.md"
+        text = md.read_text(encoding="utf-8", errors="replace") if md.exists() else ""
+        pages.append((int(m.group(1)), sub.name, text))
+    return [(name, text) for _, name, text in sorted(pages)]
+
+
+def _find_doc_dir(index: dict[str, Path], title: str) -> Path | None:
+    slug = _slug(title)
+    if slug in index:
+        return index[slug]
+    prefix = slug[:20]
+    for key, path in index.items():
+        if key.startswith(prefix):
+            return path
+    words = set(slug.split("-")[:5])
+    for key, path in index.items():
+        kwords = set(key.split("-")[:5])
+        if len(words & kwords) >= min(3, len(words)):
+            return path
+    return None
+
+
+# ── per-document analysis ─────────────────────────────────────────────────────
+
+def analyse_document(title: str, csv_row: dict, doc_dir: Path | None) -> dict:
+    result = {
+        "title":           title,
+        "agency":          csv_row.get("Agency", "").strip(),
+        "doc_type":        csv_row.get("Type", "PDF").strip().upper(),
+        "csv_redacted":    csv_row.get("Redaction", "").strip().upper() == "TRUE",
+        "release_date":    csv_row.get("Release Date", "").strip(),
+        "incident_date":   csv_row.get("Incident Date", "").strip(),
+        "location":        csv_row.get("Incident Location", "").strip(),
+        "blurb":           csv_row.get("Description Blurb", "").replace("\n", " ").strip(),
+        # page-level findings
+        "found_locally":   doc_dir is not None,
+        "page_count":      0,
+        "ocr_redacted":    False,
+        "redacted_pages":  [],
+        "has_images":      False,
+        "image_pages":     [],
+        "insuff_evidence": False,
+        "previously_known":False,
+        "total_chars":     0,
+    }
+
+    blurb = result["blurb"]
+    if RE_INSUFF.search(blurb):
+        result["insuff_evidence"] = True
+    if RE_KNOWN.search(blurb):
+        result["previously_known"] = True
+
+    if doc_dir is None:
+        return result
+
+    pages = _collect_md_texts(doc_dir)
+    result["page_count"] = len(pages)
+
+    all_text = ""
+    for page_name, text in pages:
+        all_text += text
+        if RE_REDACTED.search(text):
+            result["ocr_redacted"] = True
+            result["redacted_pages"].append(page_name)
+        if RE_IMAGE.search(text):
+            result["has_images"] = True
+            result["image_pages"].append(page_name)
+        if RE_INSUFF.search(text):
+            result["insuff_evidence"] = True
+        if RE_KNOWN.search(text):
+            result["previously_known"] = True
+
+    result["total_chars"] = len(all_text)
+
+    # Insufficient evidence heuristic: very thin content or mostly redacted
+    if result["page_count"] > 0:
+        redact_ratio = len(result["redacted_pages"]) / result["page_count"]
+        avg_chars    = result["total_chars"] / result["page_count"]
+        if redact_ratio > 0.5 or avg_chars < 200:
+            result["insuff_evidence"] = True
+
+    return result
+
+
+# ── report writer ─────────────────────────────────────────────────────────────
+
+def write_report(docs: list[dict], out_path: Path) -> None:
+    total       = len(docs)
+    found       = sum(1 for d in docs if d["found_locally"])
+    pending     = total - found
+    pdfs        = sum(1 for d in docs if d["doc_type"] == "PDF")
+    vids        = sum(1 for d in docs if d["doc_type"] == "VID")
+    csv_redact  = sum(1 for d in docs if d["csv_redacted"])
+    ocr_redact  = sum(1 for d in docs if d["ocr_redacted"])
+    has_images  = sum(1 for d in docs if d["has_images"])
+    insuff      = sum(1 for d in docs if d["insuff_evidence"])
+    known       = sum(1 for d in docs if d["previously_known"])
+
+    lines = [
+        f"# UAP Archive Audit Report",
+        f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M')}",
+        "",
+        "---",
+        "",
+        "## Summary",
+        "",
+        f"| Metric | Count |",
+        f"|--------|-------|",
+        f"| Total documents in CSV | {total} |",
+        f"| — PDFs | {pdfs} |",
+        f"| — Videos (VID) | {vids} |",
+        f"| Found locally (pages on disk) | {found} |",
+        f"| **Pending / not yet processed** | **{pending}** |",
+        f"| Redacted (CSV flag) | {csv_redact} |",
+        f"| Redacted (detected in OCR text) | {ocr_redact} |",
+        f"| Contains embedded photos/images | {has_images} |",
+        f"| Insufficient evidence | {insuff} |",
+        f"| Previously known / reviewed | {known} |",
+        "",
+        "---",
+        "",
+        "## Pending (no local folder found)",
+        "",
+    ]
+
+    pending_docs = [d for d in docs if not d["found_locally"]]
+    if pending_docs:
+        for d in pending_docs:
+            lines.append(f"- {d['title']}  _(agency: {d['agency']}, type: {d['doc_type']})_")
+    else:
+        lines.append("_None — all CSV rows matched a local folder._")
+
+    lines += ["", "---", "", "## Redacted documents (OCR-detected)", ""]
+    redact_docs = [d for d in docs if d["ocr_redacted"]]
+    if redact_docs:
+        lines.append("| Document | Pages with redactions | Total pages |")
+        lines.append("|----------|-----------------------|-------------|")
+        for d in redact_docs:
+            pages_str = ", ".join(d["redacted_pages"][:5])
+            if len(d["redacted_pages"]) > 5:
+                pages_str += f" … +{len(d['redacted_pages'])-5} more"
+            lines.append(f"| {d['title'][:60]} | {pages_str} | {d['page_count']} |")
+    else:
+        lines.append("_None detected._")
+
+    lines += ["", "---", "", "## Documents with embedded photos/images", ""]
+    img_docs = [d for d in docs if d["has_images"]]
+    if img_docs:
+        lines.append("| Document | Pages with images |")
+        lines.append("|----------|-------------------|")
+        for d in img_docs:
+            lines.append(f"| {d['title'][:60]} | {', '.join(d['image_pages'])} |")
+    else:
+        lines.append("_None detected._")
+
+    lines += ["", "---", "", "## Insufficient evidence", ""]
+    insuff_docs = [d for d in docs if d["insuff_evidence"]]
+    if insuff_docs:
+        for d in insuff_docs:
+            lines.append(f"- {d['title']}  _(pages: {d['page_count']}, redacted pages: {len(d['redacted_pages'])})_")
+    else:
+        lines.append("_None flagged._")
+
+    lines += ["", "---", "", "## Previously known / reviewed", ""]
+    known_docs = [d for d in docs if d["previously_known"]]
+    if known_docs:
+        for d in known_docs:
+            lines.append(f"- {d['title']}")
+    else:
+        lines.append("_None detected._")
+
+    lines += ["", "---", "", "## Full document inventory", ""]
+    lines.append("| # | Title | Agency | Type | Local | Redacted | Images | Insuff | Known |")
+    lines.append("|---|-------|--------|------|-------|----------|--------|--------|-------|")
+    for i, d in enumerate(docs, 1):
+        lines.append(
+            f"| {i} | {d['title'][:50]} | {d['agency']} | {d['doc_type']} "
+            f"| {'✓' if d['found_locally'] else '✗'} "
+            f"| {'✓' if d['csv_redacted'] or d['ocr_redacted'] else '—'} "
+            f"| {'✓' if d['has_images'] else '—'} "
+            f"| {'✓' if d['insuff_evidence'] else '—'} "
+            f"| {'✓' if d['previously_known'] else '—'} |"
+        )
+
+    out_path.write_text("\n".join(lines), encoding="utf-8")
+
+
+# ── main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description="Audit the UAP archive corpus")
+    ap.add_argument("--csv",  default=DEFAULT_CSV)
+    ap.add_argument("--root", default=DEFAULT_ROOT)
+    ap.add_argument("--out",  default=DEFAULT_OUT)
+    args = ap.parse_args()
+
+    root     = Path(args.root)
+    out_path = Path(args.out)
+
+    print(f"\n  Reading CSV: {args.csv}")
+    with open(args.csv, newline="", encoding="utf-8-sig") as f:
+        rows = list(csv.DictReader(f))
+    print(f"  {len(rows)} rows found")
+
+    print(f"  Indexing local folders under {root} …")
+    doc_index = _index_docs(root)
+    print(f"  {len(doc_index)} document folders found\n")
+
+    docs = []
+    for i, row in enumerate(rows):
+        title = row.get("Title", "").replace("\n", " ").strip()
+        if not title:
+            continue
+        doc_dir = _find_doc_dir(doc_index, title)
+        result  = analyse_document(title, row, doc_dir)
+        docs.append(result)
+        status = "✓" if doc_dir else "✗"
+        if (i + 1) % 10 == 0 or not doc_dir:
+            print(f"  {status}  [{i+1:>3d}] {title[:60]}")
+
+    # ── print summary ─────────────────────────────────────────────────────────
+    total      = len(docs)
+    found      = sum(1 for d in docs if d["found_locally"])
+    print(f"\n{'─'*60}")
+    print(f"  AUDIT SUMMARY")
+    print(f"{'─'*60}")
+    print(f"  Total in CSV          : {total}")
+    print(f"  PDFs                  : {sum(1 for d in docs if d['doc_type']=='PDF')}")
+    print(f"  Videos                : {sum(1 for d in docs if d['doc_type']=='VID')}")
+    print(f"  Found locally         : {found}")
+    print(f"  Pending (not on disk) : {total - found}")
+    print(f"  Redacted (CSV)        : {sum(1 for d in docs if d['csv_redacted'])}")
+    print(f"  Redacted (OCR scan)   : {sum(1 for d in docs if d['ocr_redacted'])}")
+    print(f"  Has photos/images     : {sum(1 for d in docs if d['has_images'])}")
+    print(f"  Insufficient evidence : {sum(1 for d in docs if d['insuff_evidence'])}")
+    print(f"  Previously known      : {sum(1 for d in docs if d['previously_known'])}")
+    print(f"{'─'*60}\n")
+
+    write_report(docs, out_path)
+    print(f"  ✅  Full report → {out_path}\n")
+
+
+if __name__ == "__main__":
+    main()

pipeline/concat_pages.py

@@ -0,0 +1,223 @@
+"""
+concat_pages.py
+───────────────
+Concatenates all page_XXXX.md files for each document into a single
+<document-slug>.md at the document folder root AND copies a flat version
+into <root>/concat/<document-slug>.md for easy bulk review.
+
+Source (unchanged):
+    <doc-slug>/page_0001/page_0001.md
+    <doc-slug>/page_0002/page_0002.md
+    ...
+
+Output:
+    <doc-slug>/<doc-slug>.md          ← all pages joined in order (next to source)
+    concat/<doc-slug>.md              ← flat copy for bulk review / sharing
+
+Pipeline with stamp_pages in the middle
+---------------------------------------
+    python stamp_pages.py --src raw --out pages_out     # stamp individual pages
+    python concat_pages.py --src pages_out --execute    # concat stamped pages
+
+Usage
+-----
+    python concat_pages.py                              # dry-run: list what would be written
+    python concat_pages.py --execute                    # write concatenated files
+    python concat_pages.py --root D:/divided            # explicit root (also controls concat/ location)
+    python concat_pages.py --src pages_out --execute    # read from pages_out/, write concat/ to root
+    python concat_pages.py --execute --force            # overwrite existing concatenated files
+    python concat_pages.py --no-inline                  # skip writing next to source, only write concat/
+"""
+
+import os
+import re
+import argparse
+from pathlib import Path
+
+DEFAULT_ROOT = "D:/divided"
+
+# Directories to never descend into during os.walk / rglob
+# NOTE: agency folder names (DOD, NASA, FBI, DOS, NARA-CIA, MISC) are intentionally
+# NOT here — we need to descend into them when --src points at pages_out/ or raw/.
+SKIP_TRAVERSE = {
+    "records", "reports_out", "wiki",
+    "concat", "extracted", "scripts", "raw",
+    ".git", "__pycache__",
+}
+
+# Directory/file names to skip as document-folder *candidates* (not as traversal roots)
+SKIP_CANDIDATE = {
+    "reorganize.py", "restructure_pages.py", "stamp_pages.py",
+    "reconcile.py", "pdf_to_reports.py", "concat_pages.py",
+    "find_missing_concat.py", "find_ocr_targets.py", "run_ocr.py",
+    "audit.py", "page_coverage.py", "destamp_pages.py",
+    "uap_record_schema.yaml", "uap-csv.csv",
+    "move_log.json", "page_move_log.json",
+    "CLAUDE.md", "Untitled.md", "README.md",
+}
+
+PAGE_DIR_RE = re.compile(r"^page_(\d+)$", re.IGNORECASE)
+
+PAGE_SEP = "\n\n---\n\n"   # separator inserted between pages
+
+CONCAT_DIR = "concat"       # flat output folder name (always at root level)
+
+
+def find_document_dirs(src: Path) -> list[Path]:
+    """Return all document folders (those containing page_XXXX/ subfolders)."""
+    doc_dirs = []
+    for dirpath, dirnames, _ in os.walk(str(src)):
+        dirnames[:] = [d for d in dirnames if d not in SKIP_TRAVERSE]
+        p = Path(dirpath)
+        if p == src or p.name in SKIP_CANDIDATE:
+            continue
+        try:
+            has_pages = any(
+                PAGE_DIR_RE.match(sub.name)
+                for sub in p.iterdir() if sub.is_dir()
+            )
+        except (PermissionError, OSError):
+            continue
+        if has_pages:
+            doc_dirs.append(p)
+    return sorted(doc_dirs)
+
+
+def collect_pages(doc_dir: Path) -> list[Path]:
+    """Return page .md files sorted by page number."""
+    pages = []
+    for sub in doc_dir.iterdir():
+        if not sub.is_dir():
+            continue
+        m = PAGE_DIR_RE.match(sub.name)
+        if not m:
+            continue
+        md = sub / f"{sub.name}.md"
+        if md.exists():
+            pages.append((int(m.group(1)), md))
+    return [md for _, md in sorted(pages)]
+
+
+def build_concat(doc_dir: Path, pages: list[Path]) -> str:
+    """Build the full concatenated markdown string."""
+    chunks = []
+    for md in pages:
+        page_num = md.parent.name          # e.g. "page_0001"
+        header   = f"## {page_num}\n\n"
+        body     = md.read_text(encoding="utf-8", errors="replace").strip()
+        chunks.append(header + body)
+    return PAGE_SEP.join(chunks)
+
+
+def process(src: Path, root: Path, execute: bool, force: bool, inline: bool = True) -> None:
+    doc_dirs = find_document_dirs(src)
+
+    if not doc_dirs:
+        print("  No document folders found.")
+        return
+
+    concat_dir = root / CONCAT_DIR
+
+    if execute:
+        concat_dir.mkdir(exist_ok=True)
+
+    written = skipped = errors = 0
+
+    for doc_dir in doc_dirs:
+        pages = collect_pages(doc_dir)
+        if not pages:
+            continue
+
+        inline_path = doc_dir / f"{doc_dir.name}.md"
+        flat_path   = concat_dir / f"{doc_dir.name}.md"
+
+        # In dry-run mode, report both destinations
+        if not execute:
+            destinations = []
+            if inline and (force or not inline_path.exists()):
+                try:
+                    destinations.append(str(inline_path.relative_to(root)))
+                except ValueError:
+                    destinations.append(str(inline_path))
+            if force or not flat_path.exists():
+                destinations.append(f"{CONCAT_DIR}/{doc_dir.name}.md")
+            if destinations:
+                for dest in destinations:
+                    print(f"  would write  {dest}  ({len(pages)} pages)")
+                written += 1
+            else:
+                skipped += 1
+            continue
+
+        # Build content once, write to both destinations
+        try:
+            content = build_concat(doc_dir, pages)
+            wrote_any = False
+
+            if inline:
+                if force or not inline_path.exists():
+                    inline_path.write_text(content, encoding="utf-8")
+                    print(f"  ✓  {inline_path.relative_to(root)}  ({len(pages)} pages)")
+                    wrote_any = True
+
+            if force or not flat_path.exists():
+                flat_path.write_text(content, encoding="utf-8")
+                print(f"  ✓  {CONCAT_DIR}/{doc_dir.name}.md  ({len(pages)} pages)")
+                wrote_any = True
+
+            if wrote_any:
+                written += 1
+            else:
+                skipped += 1
+
+        except Exception as exc:
+            print(f"  ✗  {doc_dir.name}  ERROR: {exc}")
+            errors += 1
+
+    label = "would write" if not execute else "written"
+    print(f"\n  {written} {label},  {skipped} skipped (already exist — use --force to overwrite),  {errors} errors\n")
+    if execute:
+        print(f"  Flat copies → {concat_dir}\n")
+        print(f"  Source read → {src}\n")
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Concatenate per-page .md files into one .md per document"
+    )
+    ap.add_argument("--root",      default=DEFAULT_ROOT,
+                    help="Root folder — controls where concat/ is written (default: D:/divided)")
+    ap.add_argument("--src",       default=None,
+                    help="Source folder to scan for documents (default: same as --root). "
+                         "Set to pages_out/ when stamp_pages is in the pipeline.")
+    ap.add_argument("--execute",   action="store_true",  help="Write files (default: dry-run)")
+    ap.add_argument("--force",     action="store_true",  help="Overwrite existing concatenated files")
+    ap.add_argument("--no-inline", action="store_true",  help="Skip writing next to source; only write to concat/")
+    args = ap.parse_args()
+
+    root = Path(args.root)
+    src  = Path(args.src) if args.src else root
+
+    if not root.exists():
+        print(f"  ✗  Root not found: {root}")
+        return
+    if not src.exists():
+        print(f"  ✗  Source not found: {src}")
+        return
+
+    if not args.execute:
+        print(f"\n{'─'*60}")
+        print(f"  DRY RUN — nothing will be written")
+        print(f"  Source:      {src}")
+        print(f"  Root:        {root}")
+        print(f"  Flat copies: {root / CONCAT_DIR}")
+        print(f"{'─'*60}\n")
+
+    process(src, root, args.execute, args.force, inline=not args.no_inline)
+
+    if not args.execute:
+        print("  Run with --execute to apply.\n")
+
+
+if __name__ == "__main__":
+    main()

pipeline/destamp_pages.py

@@ -0,0 +1,144 @@
+"""
+destamp_pages.py
+────────────────
+Strips the YAML frontmatter stamp prepended by stamp_pages.py from every
+page_XXXX.md file.
+
+The stamp always looks like this at the top of the file:
+    ---
+    document: ...
+    page: ...
+    of: ...
+    agency: ...
+    subtype: ...
+    region: ...
+    src_path: ...
+    ---
+    <blank line>
+    <original OCR content>
+
+Modes
+-----
+  --inplace   (default)  Strip stamps from --src in place.
+  --out DIR              Write destamped copies to DIR, keeping the same
+                         folder hierarchy. Source files are not modified.
+
+Usage
+-----
+    python destamp_pages.py                         # strip pages_out/ in place
+    python destamp_pages.py --src pages_out         # same, explicit
+    python destamp_pages.py --out clean_pages       # write copies, don't touch originals
+    python destamp_pages.py --src pages_out --dry-run  # preview without writing
+"""
+
+import re
+import argparse
+from pathlib import Path
+
+DEFAULT_SRC = "pages_out"
+
+# Matches the stamp: starts with ---, ends with --- followed by optional blank line
+# Anchored to the very start of the file (re.DOTALL so . matches newlines)
+STAMP_RE = re.compile(
+    r"^---\r?\n(?:[^\n]+\r?\n)*?---\r?\n\r?\n?",
+    re.DOTALL,
+)
+
+
+def strip_stamp(text: str) -> tuple[str, bool]:
+    """
+    Remove the leading YAML frontmatter stamp from text.
+    Returns (stripped_text, was_stamped).
+    """
+    m = STAMP_RE.match(text)
+    if m:
+        return text[m.end():], True
+    return text, False
+
+
+def process(src_root: Path, out_root: Path | None, dry_run: bool) -> tuple[int, int, int]:
+    """
+    Walk src_root, strip stamps, write results.
+    If out_root is None → write back to the same file (in-place).
+    Returns (stripped, skipped, errors).
+    """
+    stripped = skipped = errors = 0
+
+    md_files = sorted(src_root.rglob("page_*.md"))
+    total = len(md_files)
+
+    if total == 0:
+        print(f"  No page_*.md files found under {src_root}")
+        return 0, 0, 0
+
+    print(f"  Found {total} page_*.md files\n")
+
+    for md_file in md_files:
+        try:
+            text = md_file.read_text(encoding="utf-8", errors="replace")
+        except Exception as exc:
+            print(f"  ✗  READ  {md_file.name}  {exc}")
+            errors += 1
+            continue
+
+        clean, was_stamped = strip_stamp(text)
+
+        if not was_stamped:
+            skipped += 1
+            continue
+
+        if out_root is None:
+            dst = md_file          # in-place
+        else:
+            rel = md_file.relative_to(src_root)
+            dst = out_root / rel
+
+        if dry_run:
+            rel_display = md_file.relative_to(src_root)
+            print(f"  [dry-run] would strip  {rel_display}")
+            stripped += 1
+            continue
+
+        try:
+            if out_root is not None:
+                dst.parent.mkdir(parents=True, exist_ok=True)
+            dst.write_text(clean, encoding="utf-8")
+            stripped += 1
+        except Exception as exc:
+            print(f"  ✗  WRITE {dst}  {exc}")
+            errors += 1
+
+    return stripped, skipped, errors
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Remove YAML frontmatter stamps from page .md files")
+    ap.add_argument("--src",     default=DEFAULT_SRC,
+                    help=f"Source folder to scan (default: {DEFAULT_SRC})")
+    ap.add_argument("--out",     default=None,
+                    help="Write destamped copies here instead of modifying in place")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="Print what would be stripped without writing anything")
+    args = ap.parse_args()
+
+    src_root = Path(args.src)
+    if not src_root.exists():
+        raise SystemExit(f"  ✗  Source not found: {src_root}")
+
+    out_root = Path(args.out) if args.out else None
+
+    mode = "dry-run" if args.dry_run else ("in-place" if out_root is None else f"→ {out_root}")
+    print(f"\n  Source : {src_root}")
+    print(f"  Mode   : {mode}\n")
+
+    stripped, skipped, errors = process(src_root, out_root, args.dry_run)
+
+    print(f"\n  {'─'*50}")
+    print(f"  Stripped : {stripped}")
+    print(f"  Skipped  : {skipped}  (no stamp found — already clean or not stamped)")
+    print(f"  Errors   : {errors}")
+    print(f"  {'─'*50}\n")
+
+
+if __name__ == "__main__":
+    main()

pipeline/download_uap_pdfs.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+"""UAP PDF Downloader — fixed for war.gov hotlink protection
+
+Downloads all PDFs into a UAP_PDFs folder beside this script.
+Run: python3 download_uap_pdfs.py
+"""
+import urllib.request
+import urllib.parse
+import os
+import time
+
+PDF_URLS = [
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_section_10.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_section_2.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_section_3.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_section_4.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_section_5.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_section_6.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_section_7.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_section_9.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_serial_130.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_serial_164.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_serial_220.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_serial_403.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_serial_438.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_serial_449.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_sub_a.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/18_100754_ general 1946-7_vol_2.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/18_6369445_general_1948_vol_1.pdf',
+    "https://www.war.gov/medialink/ufo/release_1/255_413270_ufo's_and_defense_what_should_we_prepare_for.pdf",
+    'https://www.war.gov/medialink/ufo/release_1/255_t_763_r1b_transcripts.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/331_120752_numeric_files_1944–1945_37153_german_armament_equipment_documents.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/341_110448_records_relating_to_the_collection_and_dissemination_of_intelligence_1948-1955-ts_cont_no.2_2-5300-2-5399.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/341_110677_numerical_file_5-2500.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/342_hs1-416511228_box186_319.1-flying-discs-1949.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/38_143685_box7_incident_summaries_101-172.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/38_143685_box7_incident_summaries_173-233.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/38_143685_box7_incident_summaries_1-100.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/59_214434_sp_16_[7.18.1963].pdf',
+    'https://www.war.gov/medialink/ufo/release_1/59_214434_sp_16_7.18.1963.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-101634279_100-de-18221_serial_844.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-101634279_100-de-26505.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_section_1.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/65_hs1-834228961_62-hq-83894_section_8.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d10-mission-report-middle-east-may-2022.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d12-mission-report-iraq-may-2022.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d14-mission-report-iraq-may-2022.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d16-mission-report-syria-july-2022.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d18-mission-report-iraq-december-2022.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d19-mission-report-syria-february-21-2023.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d20-mission-report-southern-united-states-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d23-mission-report-united-arab-emirates-october-2023.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d25-mission-report-greece-january-2024.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d27-mission-report-united-arab-emirates-october-2023.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d28-mission-report-east-china-sea-2024.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d3-mission-report-arabian-gulf-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d32-mission-report,-syria-october-2024.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d33-mission-report-greece-october-2023.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d35-mission-report-greece-october-2023.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d38-range-fouler-debrief-middle-east-may-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d4-mission-report-arabian-gulf-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d42-range-fouler-debrief-japan-2023.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d44-range-fouler-arabian-sea-october-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d48-report-september-1996.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d49-launch-summary-february-2000.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d5-mission-report-arabian-gulf-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d50-email-correspondence-indopacom-april-2025.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d51-email-correspondence-pacific-time-zone-march-2023.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d52-email-correspondance-na-august-2024.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d54-mission-report-mediterranean-sea-na.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d55-mission-report-syria-november-2016.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d56-range-fouler-debrief-arabian-sea-august-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d57-mission-report-gulf-of-aden-september-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d58-range-fouler-debrief-na-october-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d6-mission-report-arabian-gulf-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d60-mission-report-persian-gulf-august-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d61-mission-report-persian-gulf-august-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d62-mission-report-strait-of-hormuz-september-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d63-mission-report-strait-of-hormuz-october-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d64-mission-report-iran-november-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d65-mission-report-persian-gulf-july-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d7-mission-report-arabian-gulf-2020.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d74-mission-report-syria-november-2023.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d75-mission-report-gulf-of-aden-july-2024.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-d8-mission-report-djibouti-2025.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dow-uap-pr20.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b1.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b10.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b11.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b12.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b13.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b14.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b15.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b16.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b17.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b18.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b19.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b2.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b20.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b21.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b22.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b23.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b24.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b3.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b4.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b5.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b6.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b7.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b8.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/fbi-photo-b9.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/nasa-uap-d1-apollo-12-transcript-1969.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/nasa-uap-d2-apollo-17-transcript-1972.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/nasa-uap-d3-gemini-7-transcript-1965.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/nasa-uap-d4-apollo-11-technical-crew-debriefing-1969.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/nasa-uap-d5-apollo-17-crew-debriefing-for-science-1973.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/nasa-uap-d6-apollo-17-technical-crew-debriefing-1973.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/nasa-uap-d7-skylab-technical-crew-debriefing-1973.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dos-uap-d1-cable-1-papua-new-guinea-january-1985.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/dos-uap-d2-cable-2-kazakhstan-january-1994.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/059uap00011.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/059uap00012.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/059uap00013.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/usper-statement-redacted.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/2024-04-30-composite-sketch.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/serial 5 redacted_redacted.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/serial-3_redacted.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/serial-4-redacted_redacted.pdf',
+    'https://www.war.gov/medialink/ufo/release_1/western_us_event_slides_5.08.2026.pdf',
+]
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/124.0.0.0 Safari/537.36"
+    ),
+    "Accept": "application/pdf,application/octet-stream,*/*;q=0.9",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Referer": "https://www.war.gov/",
+    "Origin": "https://www.war.gov",
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "same-origin",
+    "Connection": "keep-alive",
+}
+
+
+def download_all():
+    output_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "UAP_PDFs")
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Prime a session cookie by visiting the homepage first
+    opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor())
+    urllib.request.install_opener(opener)
+    try:
+        print("Priming session on war.gov...")
+        req0 = urllib.request.Request("https://www.war.gov/", headers=HEADERS)
+        opener.open(req0, timeout=15)
+    except Exception as e:
+        print(f"  (homepage pre-fetch skipped: {e})")
+
+    success, failed = [], []
+    for i, url in enumerate(PDF_URLS):
+        filename = os.path.basename(urllib.parse.unquote(url))
+        filepath = os.path.join(output_dir, filename)
+        if os.path.exists(filepath) and os.path.getsize(filepath) > 1024:
+            print(f"[{i+1}/{len(PDF_URLS)}] SKIP (exists): {filename}")
+            success.append(filename)
+            continue
+        try:
+            req = urllib.request.Request(url, headers=HEADERS)
+            with opener.open(req, timeout=30) as resp:
+                data = resp.read()
+            if len(data) < 100:
+                raise ValueError(f"Response too small ({len(data)} bytes) — likely an error page")
+            with open(filepath, "wb") as f:
+                f.write(data)
+            size_kb = len(data) // 1024
+            print(f"[{i+1}/{len(PDF_URLS)}] OK ({size_kb} KB): {filename}")
+            success.append(filename)
+        except Exception as e:
+            print(f"[{i+1}/{len(PDF_URLS)}] FAIL: {filename} — {e}")
+            failed.append((url, str(e)))
+        time.sleep(0.4)
+
+    print(f"\n{'='*60}")
+    print(f"Downloaded: {len(success)}/{len(PDF_URLS)}")
+    if failed:
+        print(f"\nFailed ({len(failed)}):")
+        for url, err in failed:
+            print(f"  {os.path.basename(url)}: {err}")
+
+
+if __name__ == "__main__":
+    download_all()

pipeline/extract_reports.py

@@ -0,0 +1,593 @@
+"""
+extract_reports.py
+──────────────────
+For every concat/*.md whose filename does NOT start with a digit, sends the
+full text to Gemini and asks it to extract each individual UAP/UFO sighting
+report as structured JSON.
+
+Output per file:
+  {
+    "source_file": "dow-uap-d33-mission-report-greece-october-2023.md",
+    "document_id": "dow-uap-d33-mission-report-greece-october-2023",
+    "agency":      "DOD",
+    "collection":  "mission-reports",
+    "region":      "greece",
+    "manifest":    { ... war.gov CSV row, if --csv given ... },
+    "reports": [
+      {
+        "document_id": "dow-uap-d33-mission-report-greece-october-2023",
+        "agency":      "DOD",
+        "collection":  "mission-reports",
+        "pages":       "page_0004-page_0006",
+        "raw_text":    "<verbatim text block for this report>",
+        "assessment":  "<verbatim assessment / analysis block, or null>"
+      },
+      ...
+    ]
+  }
+
+No summarisation — raw text only. Page numbers, agency and collection are derived
+deterministically from the document path and the '## page_XXXX' markers — NOT from
+the LLM, which only identifies report boundaries (raw_text + assessment).
+
+Chunking
+--------
+Large documents are split into page-blocks before sending to avoid the 65,536
+output-token ceiling.  Thinking is disabled (pure extraction — no reasoning
+needed) to maximise the usable output budget.  Use --chunk-pages to tune block
+size (default 40 pages).  Chunk results are merged into a single JSON file.
+
+Parallelism
+-----------
+--workers N  runs N files concurrently (default 1 = sequential).
+Gemini API handles concurrent requests; stay within your rate-limit quota.
+
+Usage
+-----
+    set GEMINI_API_KEY=your_key_here          (Windows CMD)
+    export GEMINI_API_KEY=your_key_here       (bash)
+
+    python extract_reports.py
+    python extract_reports.py --concat D:/divided/concat --out D:/divided/extracted
+    python extract_reports.py --file dow-uap-d33-mission-report-greece-october-2023.md
+    python extract_reports.py --workers 4                    # parallel files
+    python extract_reports.py --chunk-pages 30               # smaller chunks
+    python extract_reports.py --no-skip                      # re-process existing
+"""
+
+import os
+import re
+import csv
+import json
+import time
+import argparse
+import threading
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from google import genai
+from google.genai import types
+from google.api_core import exceptions as google_exceptions
+
+DEFAULT_CONCAT       = "D:/divided/concat"
+DEFAULT_OUT          = "D:/divided/extracted"
+DEFAULT_CHUNK_PAGES  = 40     # pages per API call; tune down if still truncating
+DEFAULT_WORKERS      = 1      # concurrent files; increase for throughput
+MAX_RETRIES          = 6      # retries on 429 / 503 before giving up
+RETRY_BASE_SECS      = 5      # first wait; doubles each attempt (5, 10, 20, 40, 80, 160)
+MODEL                = "gemini-3.1-pro-preview"
+
+# Thread-safe print lock
+_print_lock = threading.Lock()
+
+SYSTEM_PROMPT = """\
+You are a document analyst processing declassified U.S. government UAP/UFO records.
+Your job is to identify every distinct UAP/UFO sighting report within the document.
+No summaries, no paraphrasing — raw verbatim text only.
+
+Rules:
+1. Each "report" is one coherent UAP/UFO sighting description (may span one or more pages).
+2. "raw_text" must contain the verbatim text of that sighting report exactly as it appears
+   in the source, including headers, field labels, and any partially redacted content.
+   Do NOT shorten, paraphrase, or omit anything.
+3. "assessment" must contain the verbatim text of any analyst assessment, conclusion,
+   classification, or recommendation block associated with that report.
+   Use null if none is present.
+4. Return the reports in the order they appear in the document.
+5. If the whole document is a single report, return one entry.
+6. If this is a chunk of a larger document, extract only what appears in THIS chunk.
+
+Do NOT output page numbers. Page ranges are assigned deterministically afterwards
+from the document's '## page_XXXX' markers — that is not your job.
+"""
+
+# JSON schema for structured output — enforced at API level, no manual parsing needed.
+# Google GenAI uses OpenAPI 3.0 style: nullable fields use {"type": "string", "nullable": True}
+# NOT JSON Schema draft-07 union types like {"type": ["string", "null"]}.
+RESPONSE_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "reports": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "raw_text":   {"type": "string"},
+                    "assessment": {"type": "string", "nullable": True},
+                },
+                "required": ["raw_text", "assessment"],
+            },
+        },
+    },
+    "required": ["reports"],
+}
+
+# ── page splitting ─────────────────────────────────────────────────────────────
+
+PAGE_HEADER_RE = re.compile(r"^## (page_\d+)\s*$", re.MULTILINE)
+
+
+def split_into_chunks(text: str, chunk_pages: int) -> list[tuple[str, str]]:
+    """
+    Split a concat .md (pages separated by '## page_XXXX' headers) into
+    blocks of at most chunk_pages pages.
+
+    Returns list of (label, chunk_text) where label is e.g. "page_0001-page_0040".
+    If the document has no page headers (single-page or unstamped), returns one chunk.
+    """
+    # Find all page header positions
+    headers = [(m.group(1), m.start()) for m in PAGE_HEADER_RE.finditer(text)]
+
+    if not headers:
+        return [("all-pages", text)]
+
+    # Build page boundaries
+    chunks = []
+    for i in range(0, len(headers), chunk_pages):
+        block_headers = headers[i : i + chunk_pages]
+        start_pos     = block_headers[0][1]
+        end_pos       = headers[i + chunk_pages][1] if (i + chunk_pages) < len(headers) else len(text)
+        first_page    = block_headers[0][0]
+        last_page     = block_headers[-1][0]
+        label         = first_page if first_page == last_page else f"{first_page}-{last_page}"
+        chunks.append((label, text[start_pos:end_pos]))
+
+    return chunks
+
+
+# ── deterministic metadata — page numbers, agency, collection ──────────────────
+# These come from the document path and the '## page_XXXX' markers, NOT the LLM.
+# The LLM only identifies report boundaries (raw_text + assessment).
+
+_AGENCY_PREFIXES = [
+    ("dow-uap-d",        ("DOD", "mission-reports")),
+    ("dow-uap-",         ("DOD", "mission-reports")),
+    ("dod-range-fouler", ("DOD", "range-fouler-debriefs")),
+    ("dod-email",        ("DOD", "email-correspondence")),
+    ("dod-",             ("DOD", None)),
+    ("pr-",              ("DOD", "mission-reports")),
+    ("fbi-",             ("FBI", "photo-collections")),
+    ("nasa-transcript",  ("NASA", "transcripts")),
+    ("nasa-crew",        ("NASA", "crew-debriefings")),
+    ("nasa-",            ("NASA", None)),
+    ("dos-",             ("DOS", "cables")),
+    ("65_hs1-834228961", ("NARA-CIA", "hs1-834228961")),
+    ("65_hs1-101634279", ("NARA-CIA", "hs1-101634279")),
+    ("18_",              ("NARA-CIA", "series-18")),
+    ("38_",              ("NARA-CIA", "series-38")),
+    ("59_",              ("NARA-CIA", "series-59")),
+    ("255_",             ("NARA-CIA", "series-255")),
+    ("331_",             ("NARA-CIA", "series-331")),
+    ("341_",             ("NARA-CIA", "series-341")),
+    ("342_",             ("NARA-CIA", "series-342")),
+    ("series-",          ("NARA-CIA", None)),
+]
+
+
+def _first_frontmatter(text: str) -> str:
+    """Body of the first YAML frontmatter block (stamp_pages stamp), or ''."""
+    m = re.search(r"(?ms)^---\r?\n(.*?)\r?\n---\r?\n", text)
+    return m.group(1) if m else ""
+
+
+def _fm_value(block: str, key: str):
+    """Read a single `key: value` line from a frontmatter block."""
+    m = re.search(rf"(?m)^{re.escape(key)}:\s*(.*)$", block)
+    if not m:
+        return None
+    v = m.group(1).strip().strip('"').strip("'")
+    return v if v and v.lower() != "null" else None
+
+
+def resolve_doc_meta(text: str, document_id: str):
+    """(agency, collection, region) — from the stamped frontmatter (path-derived)
+    when present, else from the document slug prefix. Never from the LLM."""
+    fm = _first_frontmatter(text)
+    agency  = _fm_value(fm, "agency")
+    subtype = _fm_value(fm, "subtype")
+    region  = _fm_value(fm, "region")
+    if agency:
+        return agency, subtype, region
+    low = document_id.lower()
+    for prefix, (ag, coll) in _AGENCY_PREFIXES:
+        if low.startswith(prefix):
+            return ag, coll, region
+    return None, None, region
+
+
+def _page_spans(chunk_text: str):
+    """[(page_label, start, end)] character spans for each '## page_XXXX' section."""
+    hdrs = [(m.group(1), m.start()) for m in PAGE_HEADER_RE.finditer(chunk_text)]
+    spans = []
+    for i, (label, start) in enumerate(hdrs):
+        end = hdrs[i + 1][1] if i + 1 < len(hdrs) else len(chunk_text)
+        spans.append((label, start, end))
+    return spans
+
+
+def attach_pages(reports: list, chunk_text: str, chunk_label: str) -> list:
+    """Deterministically set each report's 'pages' from the document's
+    '## page_XXXX' markers.
+
+    A multi-page report's raw_text has the page markers/separators stripped, so
+    the full block won't match the chunk verbatim. Instead we anchor on the
+    report's first and last non-empty *lines* — a single line never spans a
+    page break — and map that [start, end] span onto the page markers it
+    covers. Reports are assumed to be in document order (the prompt requires it).
+    """
+    spans = _page_spans(chunk_text)
+    full_range = chunk_label
+    if spans:
+        full_range = (spans[0][0] if spans[0][0] == spans[-1][0]
+                      else f"{spans[0][0]}-{spans[-1][0]}")
+    cursor = 0
+    for rep in reports:
+        lines = [ln.strip() for ln in (rep.get("raw_text") or "").splitlines()
+                 if ln.strip()]
+        if not spans or not lines:
+            rep["pages"] = None if chunk_label == "all-pages" else full_range
+            continue
+        first, last = lines[0][:120], lines[-1][:120]
+        start_idx = chunk_text.find(first, cursor)
+        if start_idx < 0:
+            start_idx = chunk_text.find(first, 0)
+        if start_idx < 0:
+            rep["pages"] = full_range          # could not locate — whole chunk
+            continue
+        end_hit = chunk_text.find(last, start_idx)
+        end_idx = (end_hit + len(last)) if end_hit >= 0 else start_idx + 1
+        covered = [lbl for (lbl, s, e) in spans if s < end_idx and e > start_idx]
+        rep["pages"] = (
+            covered[0] if len(covered) == 1
+            else f"{covered[0]}-{covered[-1]}" if covered
+            else full_range
+        )
+        cursor = start_idx + 1
+    return reports
+
+
+def _slug_from_title(title: str) -> str:
+    """Normalise a CSV Title to a filesystem slug — matches reconcile.py."""
+    s = (title or "").lower().strip()
+    s = re.sub(r"[\s,]+", "-", s)
+    s = re.sub(r"[^\w\-]", "", s)
+    s = re.sub(r"-+", "-", s).strip("-")
+    return s
+
+
+def load_manifest(csv_path: Path) -> dict:
+    """Index the war.gov release CSV by document slug → row dict."""
+    index = {}
+    with open(csv_path, encoding="utf-8-sig", newline="") as fh:
+        for row in csv.DictReader(fh):
+            slug = _slug_from_title((row.get("Title") or "").strip())
+            if slug:
+                index.setdefault(slug, row)
+    return index
+
+
+def manifest_block(manifest_index, document_id: str):
+    """Deterministic slug-join of one document to its war.gov manifest row."""
+    if manifest_index is None:
+        return None
+    row = manifest_index.get(_slug_from_title(document_id))
+    if not row:
+        return {"matched": False}
+    g = lambda k: (row.get(k) or "").strip()
+    return {
+        "matched":           True,
+        "csv_title":         " ".join(g("Title").split()),
+        "release_date":      g("Release Date"),
+        "redacted":          g("Redaction").upper() == "TRUE",
+        "csv_agency":        g("Agency"),
+        "incident_date":     g("Incident Date"),
+        "incident_location": g("Incident Location"),
+        "pdf_link":          g("PDF | Image Link"),
+        "dvids_video_id":    g("DVIDS Video ID"),
+        "type":              g("Type"),
+    }
+
+
+# ── API call ───────────────────────────────────────────────────────────────────
+
+def _call_gemini(client: genai.Client, model: str,
+                 filename: str, chunk_label: str, chunk_text: str,
+                 verbose: bool = True) -> dict:
+    """
+    Send one chunk to Gemini using structured JSON output (response_mime_type).
+    The API guarantees valid JSON matching RESPONSE_SCHEMA — no manual parsing needed.
+    Thinking is disabled (budget=0) — verbatim extraction needs no reasoning.
+
+    Returns dict with key 'reports' (list), guaranteed by the schema.
+    On API error returns {'reports': [], 'api_error': str}.
+    """
+    prompt = (
+        f"Document filename: {filename}\n"
+        f"Pages in this chunk: {chunk_label}\n\n"
+        f"Document content:\n\n{chunk_text}"
+    )
+
+    # Blocking call — response is returned in full once complete.
+    # Retries with exponential backoff on 429 (rate limit) and 503 (overload).
+    full_response = ""
+    for attempt in range(1, MAX_RETRIES + 1):
+        try:
+            response = client.models.generate_content(
+                model=model,
+                contents=[
+                    types.Content(role="user", parts=[types.Part.from_text(text=prompt)])
+                ],
+                config=types.GenerateContentConfig(
+                    system_instruction=SYSTEM_PROMPT,
+                    response_mime_type="application/json",
+                    response_json_schema=RESPONSE_SCHEMA,
+                    thinking_config=types.ThinkingConfig(thinking_budget=-1),
+                ),
+            )
+            full_response = response.text or ""
+
+            # Success — break out of retry loop
+            break
+
+        except google_exceptions.ResourceExhausted as exc:
+            # 429 — rate limited; back off and retry
+            wait = RETRY_BASE_SECS * (2 ** (attempt - 1))
+            with _print_lock:
+                print(f"\n  ⏳ 429 rate limit ({filename} chunk {chunk_label}) "
+                      f"attempt {attempt}/{MAX_RETRIES} — waiting {wait}s …")
+            if attempt == MAX_RETRIES:
+                return {"reports": [], "api_error": f"429 after {MAX_RETRIES} retries: {exc}"}
+            time.sleep(wait)
+
+        except google_exceptions.ServiceUnavailable as exc:
+            # 503 — transient overload; same backoff
+            wait = RETRY_BASE_SECS * (2 ** (attempt - 1))
+            with _print_lock:
+                print(f"\n  ⏳ 503 unavailable ({filename} chunk {chunk_label}) "
+                      f"attempt {attempt}/{MAX_RETRIES} — waiting {wait}s …")
+            if attempt == MAX_RETRIES:
+                return {"reports": [], "api_error": f"503 after {MAX_RETRIES} retries: {exc}"}
+            time.sleep(wait)
+
+        except Exception as exc:
+            # Non-retryable error
+            with _print_lock:
+                print(f"\n  ✗  API error ({filename} chunk {chunk_label}): {exc}")
+            return {"reports": [], "api_error": str(exc)}
+
+    if verbose:
+        with _print_lock:
+            print(f"    {filename} [{chunk_label}] ✓  {len(full_response)} chars")
+
+    # Guaranteed valid JSON from the API; defensive parse handles edge cases
+    try:
+        return json.loads(full_response)
+    except json.JSONDecodeError as e:
+        with _print_lock:
+            print(f"\n  ⚠  Unexpected JSON parse error ({filename} chunk {chunk_label}): {e}")
+        return {"reports": [], "parse_error": str(e), "raw_response": full_response}
+
+
+# ── per-file processing ────────────────────────────────────────────────────────
+
+def files_to_process(concat_dir: Path, single: str | None,
+                     out_dir: Path, skip_existing: bool) -> list[Path]:
+    if single:
+        p = concat_dir / single
+        return [p] if p.exists() else []
+    all_md = sorted(concat_dir.glob("*.md"))
+    # Exclude only the combined output file and other underscore-prefixed files.
+    # Do NOT exclude digit-prefixed files (65_hs1-..., 18_..., 255_... are valid docs).
+    candidates = [f for f in all_md if not f.name.startswith("_")]
+    if skip_existing:
+        candidates = [f for f in candidates
+                      if not (out_dir / (f.stem + ".json")).exists()]
+    return candidates
+
+
+def extract_file(client: genai.Client, model: str,
+                 md_path: Path, out_dir: Path, chunk_pages: int,
+                 file_index: int, file_total: int,
+                 manifest_index: dict | None = None,
+                 multi_worker: bool = False) -> dict:
+    """
+    Process one concat .md file: chunk it, call Gemini per chunk, merge reports.
+    Writes the per-file JSON immediately and returns the result dict.
+
+    multi_worker=True suppresses per-chunk noise and prints only one summary line
+    per file — keeps output readable when many workers run concurrently.
+    """
+    t0 = time.time()
+
+    # Announce start only in single-worker or low-worker mode
+    if not multi_worker:
+        with _print_lock:
+            print(f"\n{'─'*70}")
+            print(f"  [{file_index:>3}/{file_total}]  {md_path.name}")
+
+    text   = md_path.read_text(encoding="utf-8", errors="replace")
+    chunks = split_into_chunks(text, chunk_pages)
+    document_id = md_path.stem
+    agency, collection, region = resolve_doc_meta(text, document_id)
+
+    if not multi_worker:
+        with _print_lock:
+            print(f"  Chunks: {len(chunks)}  (chunk_pages={chunk_pages})")
+            print(f"{'─'*70}\n")
+
+    all_reports  = []
+    parse_errors = []
+
+    for ci, (label, chunk_text) in enumerate(chunks, 1):
+        if not multi_worker and len(chunks) > 1:
+            with _print_lock:
+                print(f"\n  ── chunk {ci}/{len(chunks)}  [{label}] ──\n")
+
+        # Pass verbose=False when running multi-worker to suppress per-call noise
+        inner = _call_gemini(client, model, md_path.name, label, chunk_text,
+                             verbose=not multi_worker)
+        # Page numbers + agency/collection are derived from the path, not the LLM.
+        # document_id / agency / collection are NOT repeated here — they already
+        # live in the document-level envelope.  Repeating them causes a pandas
+        # column-overlap error when json_normalize flattens records + meta together.
+        for _rep in attach_pages(inner.get("reports", []), chunk_text, label):
+            all_reports.append({
+                "pages":      _rep.get("pages"),
+                "raw_text":   _rep.get("raw_text", ""),
+                "assessment": _rep.get("assessment"),
+            })
+        if "parse_error" in inner:
+            parse_errors.append({"chunk": label, "error": inner["parse_error"]})
+        if "api_error" in inner:
+            parse_errors.append({"chunk": label, "error": inner["api_error"]})
+
+    result = {
+        "source_file":  md_path.name,
+        "document_id":  document_id,
+        "agency":       agency,
+        "collection":   collection,
+        "region":       region,
+        "chunk_count":  len(chunks),
+        "manifest":     manifest_block(manifest_index, document_id),
+        "reports":      all_reports,
+    }
+    if parse_errors:
+        result["parse_errors"] = parse_errors
+
+    out_file = out_dir / (md_path.stem + ".json")
+    out_file.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
+
+    with _print_lock:
+        n = len(all_reports)
+        errs = f"  ⚠ {len(parse_errors)} chunk error(s)" if parse_errors else ""
+        print(f"\n  ✓  {n} report(s) → {out_file.name}{errs}")
+
+    return result
+
+
+# ── main ───────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Extract UAP reports from concat .md files via Gemini"
+    )
+    ap.add_argument("--concat",       default=DEFAULT_CONCAT,
+                    help="Path to concat/ folder")
+    ap.add_argument("--out",          default=DEFAULT_OUT,
+                    help="Output folder for JSON files")
+    ap.add_argument("--file",         default=None,
+                    help="Process a single file by name")
+    ap.add_argument("--model",        default=MODEL,
+                    help=f"Gemini model (default: {MODEL})")
+    ap.add_argument("--chunk-pages",  type=int, default=DEFAULT_CHUNK_PAGES,
+                    help=f"Max pages per API call (default: {DEFAULT_CHUNK_PAGES}). "
+                         f"Reduce to 20-25 if still truncating.")
+    ap.add_argument("--workers",      type=int, default=DEFAULT_WORKERS,
+                    help=f"Concurrent files (default: {DEFAULT_WORKERS}). "
+                         f"Increase for throughput; mind rate limits.")
+    ap.add_argument("--no-skip",      action="store_true",
+                    help="Re-process files that already have a JSON output")
+    ap.add_argument("--csv",          default=None,
+                    help="war.gov manifest CSV (e.g. uap-csv.csv). When given, each "
+                         "document's release metadata (agency, release date, "
+                         "redaction, PDF link, incident date/location) is joined in "
+                         "deterministically by document slug.")
+    args = ap.parse_args()
+
+    api_key = os.environ.get("GEMINI_API_KEY")
+    if not api_key:
+        raise SystemExit("  ✗  GEMINI_API_KEY environment variable not set.")
+
+    concat_dir = Path(args.concat)
+    out_dir    = Path(args.out)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    manifest_index = None
+    if args.csv:
+        try:
+            manifest_index = load_manifest(Path(args.csv))
+        except Exception as exc:
+            raise SystemExit(f"  ✗  Could not read manifest CSV {args.csv}: {exc}")
+
+    files = files_to_process(concat_dir, args.file, out_dir,
+                              skip_existing=not args.no_skip)
+    if not files:
+        already = len(list(out_dir.glob("*.json")))
+        print(f"  No new files to process.  "
+              f"({already} already extracted — use --no-skip to reprocess)")
+        return
+
+    print(f"\n  Files to process : {len(files)}")
+    print(f"  Model            : {args.model}")
+    print(f"  Chunk size       : {args.chunk_pages} pages / call")
+    print(f"  Workers          : {args.workers}")
+    print(f"  Thinking         : none (budget=-1)")
+    if manifest_index is not None:
+        print(f"  Manifest CSV     : {len(manifest_index)} documents indexed")
+    print(f"  Output           : {out_dir}\n")
+
+    client      = genai.Client(api_key=api_key)
+    total       = len(files)
+    all_results = [None] * total
+
+    if args.workers <= 1:
+        for i, md_path in enumerate(files, 1):
+            all_results[i - 1] = extract_file(
+                client, args.model, md_path, out_dir, args.chunk_pages, i, total,
+                manifest_index=manifest_index,
+            )
+    else:
+        futures = {}
+        with ThreadPoolExecutor(max_workers=args.workers) as pool:
+            for i, md_path in enumerate(files, 1):
+                fut = pool.submit(
+                    extract_file,
+                    client, args.model, md_path, out_dir, args.chunk_pages, i, total,
+                    manifest_index,
+                )
+                futures[fut] = i - 1
+            for fut in as_completed(futures):
+                idx = futures[fut]
+                try:
+                    all_results[idx] = fut.result()
+                except Exception as exc:
+                    with _print_lock:
+                        print(f"\n  ✗  Worker error for {files[idx].name}: {exc}")
+                    all_results[idx] = {"source_file": files[idx].name,
+                                        "reports": [], "error": str(exc)}
+
+    # Combined output (merge with any pre-existing entries)
+    combined_path = out_dir / "_all_reports.json"
+    combined = [r for r in all_results if r is not None]
+    combined_path.write_text(
+        json.dumps(combined, indent=2, ensure_ascii=False), encoding="utf-8"
+    )
+    total_reports = sum(len(r.get("reports", [])) for r in combined)
+    print(f"\n{'═'*70}")
+    print(f"  Done. {len(combined)} files  →  {total_reports} total reports")
+    print(f"  Combined → {combined_path}")
+    print(f"{'═'*70}\n")
+
+
+if __name__ == "__main__":
+    main()

pipeline/find_missing_concat.py

@@ -0,0 +1,120 @@
+"""
+find_missing_concat.py
+──────────────────────
+Compares document folders (those with page_XXXX subfolders) against the
+files already written to concat/, and reports what's missing and why.
+
+Usage
+-----
+    python find_missing_concat.py
+    python find_missing_concat.py --root D:/divided
+"""
+
+import re
+import os
+import argparse
+from pathlib import Path
+
+DEFAULT_ROOT = "D:/divided"
+
+PAGE_DIR_RE = re.compile(r"^page_(\d+)$", re.IGNORECASE)
+SKIP_NAMES = {
+    "reorganize.py", "restructure_pages.py", "stamp_pages.py",
+    "reconcile.py", "pdf_to_reports.py", "concat_pages.py",
+    "find_missing_concat.py", "audit.py",
+    "uap_record_schema.yaml", "uap-csv.csv",
+    "move_log.json", "page_move_log.json",
+    "records", "reports_out", "pages_out", "wiki",
+    "MISC", "DOD", "NASA", "FBI", "DOS", "NARA-CIA",
+    "CLAUDE.md", "Untitled.md", "README.md",
+    "scripts", "raw", "concat",
+}
+
+
+def find_document_dirs(root: Path) -> list[Path]:
+    doc_dirs = []
+    for dirpath, dirnames, _ in os.walk(str(root)):
+        # Prune skip names in-place so os.walk doesn't descend into them
+        dirnames[:] = [d for d in dirnames if d not in SKIP_NAMES]
+        p = Path(dirpath)
+        if p == root:
+            continue
+        try:
+            page_subs = [
+                sub for sub in p.iterdir()
+                if sub.is_dir() and PAGE_DIR_RE.match(sub.name)
+            ]
+        except (PermissionError, OSError):
+            continue
+        if page_subs:
+            doc_dirs.append(p)
+    return sorted(doc_dirs)
+
+
+def count_md_pages(doc_dir: Path) -> int:
+    count = 0
+    try:
+        for sub in doc_dir.iterdir():
+            if sub.is_dir() and PAGE_DIR_RE.match(sub.name):
+                md = sub / f"{sub.name}.md"
+                if md.exists():
+                    count += 1
+    except (PermissionError, OSError):
+        pass
+    return count
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Find document folders missing from concat/")
+    ap.add_argument("--root", default=DEFAULT_ROOT)
+    args = ap.parse_args()
+
+    root = Path(args.root)
+    concat_dir = root / "concat"
+
+    print(f"\n  Scanning {root} …")
+    doc_dirs = find_document_dirs(root)
+    print(f"  Found {len(doc_dirs)} document folders with page_XXXX subfolders")
+
+    existing = set()
+    if concat_dir.exists():
+        existing = {p.stem for p in concat_dir.glob("*.md")}
+    print(f"  Files in concat/: {len(existing)}")
+
+    # Categorise missing docs
+    no_md    = []   # page dirs exist but no .md files in them
+    has_md   = []   # has .md pages but not in concat/
+
+    for d in doc_dirs:
+        if d.name in existing:
+            continue
+        md_count = count_md_pages(d)
+        if md_count == 0:
+            no_md.append((d.name, str(d.relative_to(root))))
+        else:
+            has_md.append((d.name, md_count, str(d.relative_to(root))))
+
+    print(f"\n{'─'*65}")
+    print(f"  Missing from concat/: {len(no_md) + len(has_md)}")
+    print(f"{'─'*65}")
+
+    if has_md:
+        print(f"\n  ❌  Have .md pages but NOT in concat/ ({len(has_md)}):")
+        print(f"      (concat_pages.py should have caught these — investigate)")
+        for name, cnt, rel in has_md:
+            print(f"      [{cnt:>3d} pages]  {rel}")
+
+    if no_md:
+        print(f"\n  ⚠️   No .md files in any page subfolder ({len(no_md)}):")
+        print(f"      (OCR not yet run, or pages are PDF-only)")
+        for name, rel in no_md:
+            print(f"                   {rel}")
+
+    if not has_md and not no_md:
+        print("\n  ✅  All document folders are represented in concat/.")
+
+    print()
+
+
+if __name__ == "__main__":
+    main()

pipeline/find_ocr_targets.py

@@ -0,0 +1,175 @@
+"""
+find_ocr_targets.py
+───────────────────
+Walks every document folder and finds page_XXXX subfolders where a .pdf
+exists but the matching .md does not — these are your OCR targets.
+
+Output
+------
+  stdout  — grouped list per document
+  ocr_targets.txt  — one absolute PDF path per line (pipe-friendly)
+  ocr_targets.json — structured: {doc, page, pdf_path, expected_md_path}
+
+Usage
+-----
+    python find_ocr_targets.py
+    python find_ocr_targets.py --root D:/divided
+"""
+
+import re
+import os
+import json
+import argparse
+from pathlib import Path
+
+DEFAULT_ROOT = "D:/divided"
+
+PAGE_DIR_RE = re.compile(r"^page_(\d+)$", re.IGNORECASE)
+
+# Folders to skip entirely during traversal (never descend into them)
+SKIP_TRAVERSE = {
+    "records", "reports_out", "pages_out", "wiki",
+    "concat", "extracted", "scripts", "raw",
+    ".git", "__pycache__",
+}
+
+# Folder/file names that are not document dirs (skip as candidates only)
+SKIP_CANDIDATE = {
+    "reorganize.py", "restructure_pages.py", "stamp_pages.py",
+    "reconcile.py", "pdf_to_reports.py", "concat_pages.py",
+    "find_missing_concat.py", "audit.py", "page_coverage.py",
+    "find_ocr_targets.py", "extract_reports.py",
+    "uap_record_schema.yaml", "uap-csv.csv",
+    "move_log.json", "page_move_log.json",
+    "CLAUDE.md", "Untitled.md", "README.md",
+    # agency folders are valid traversal roots — don't skip them here
+}
+
+
+def find_document_dirs(root: Path) -> list[Path]:
+    doc_dirs = []
+    for dirpath, dirnames, _ in os.walk(str(root)):
+        # Prune traversal — never descend into output/tool folders
+        dirnames[:] = [d for d in dirnames if d not in SKIP_TRAVERSE]
+        p = Path(dirpath)
+        if p == root or p.name in SKIP_CANDIDATE:
+            continue
+        try:
+            has_pages = any(
+                PAGE_DIR_RE.match(sub.name)
+                for sub in p.iterdir() if sub.is_dir()
+            )
+        except (PermissionError, OSError):
+            continue
+        if has_pages:
+            doc_dirs.append(p)
+    return sorted(doc_dirs)
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Find pages with PDF but no .md (OCR targets)")
+    ap.add_argument("--root", default=DEFAULT_ROOT)
+    args = ap.parse_args()
+
+    root = Path(args.root)
+    print(f"\n  Scanning {root} …\n")
+
+    doc_dirs = find_document_dirs(root)
+
+    targets = []          # {doc, page, pdf_path, expected_md_path}
+    docs_with_gaps = []
+
+    for doc_dir in doc_dirs:
+        doc_targets = []
+        try:
+            subs = sorted(
+                (sub for sub in doc_dir.iterdir()
+                 if sub.is_dir() and PAGE_DIR_RE.match(sub.name)),
+                key=lambda s: s.name
+            )
+        except (PermissionError, OSError):
+            continue
+
+        for sub in subs:
+            md  = sub / f"{sub.name}.md"
+            if md.exists():
+                continue   # OCR already done — skip
+
+            # .md is missing — find whatever source file is present
+            # Check for PDF with matching name, then any PDF, then any file at all
+            pdf_match = sub / f"{sub.name}.pdf"
+            if pdf_match.exists():
+                source = pdf_match
+            else:
+                # Look for any PDF or image in this page folder
+                others = [f for f in sub.iterdir() if f.is_file()
+                          and f.suffix.lower() in {".pdf", ".png", ".jpg", ".jpeg", ".tiff", ".tif"}]
+                source = others[0] if others else None
+
+            doc_targets.append({
+                "doc":              doc_dir.name,
+                "page":             sub.name,
+                "pdf_path":         str(source) if source else "NOT FOUND",
+                "expected_md_path": str(md),
+                "has_source_file":  source is not None,
+            })
+
+        if doc_targets:
+            docs_with_gaps.append((doc_dir.name, doc_targets))
+            targets.extend(doc_targets)
+
+    # ── stdout ────────────────────────────────────────────────────────────────
+    if not targets:
+        print("  ✅  No gaps — every page_XXXX.pdf has a matching .md file.\n")
+        return
+
+    no_source = sum(1 for t in targets if not t["has_source_file"])
+
+    print(f"  {'─'*65}")
+    print(f"  Documents with missing .md  : {len(docs_with_gaps)}")
+    print(f"  Total pages needing OCR     : {len(targets)}")
+    print(f"    — have a source PDF/image : {len(targets) - no_source}")
+    print(f"    — NO source file found    : {no_source}  ← page folder is empty")
+    print(f"  {'─'*65}\n")
+
+    for doc_name, doc_targets in docs_with_gaps:
+        pages = [t["page"] for t in doc_targets]
+        nums = sorted(int(re.search(r"\d+", p).group()) for p in pages)
+        runs = []
+        start = prev = nums[0]
+        for n in nums[1:]:
+            if n == prev + 1:
+                prev = n
+            else:
+                runs.append(f"{start}" if start == prev else f"{start}–{prev}")
+                start = prev = n
+        runs.append(f"{start}" if start == prev else f"{start}–{prev}")
+
+        missing_src = sum(1 for t in doc_targets if not t["has_source_file"])
+        src_note = f"  ⚠ {missing_src} page folder(s) have no source file" if missing_src else ""
+        print(f"  {doc_name}")
+        print(f"    Missing {len(doc_targets)} page(s): {', '.join(runs)}{src_note}")
+        for t in doc_targets:
+            src_flag = "  ← NO SOURCE FILE" if not t["has_source_file"] else ""
+            print(f"    → {t['pdf_path']}{src_flag}")
+        print()
+
+    # ── ocr_targets.txt ───────────────────────────────────────────────────────
+    txt_path = root / "ocr_targets.txt"
+    txt_path.write_text(
+        "\n".join(t["pdf_path"] for t in targets) + "\n",
+        encoding="utf-8"
+    )
+    print(f"  Flat list  → {txt_path}")
+
+    # ── ocr_targets.json ──────────────────────────────────────────────────────
+    json_path = root / "ocr_targets.json"
+    json_path.write_text(
+        json.dumps(targets, indent=2, ensure_ascii=False),
+        encoding="utf-8"
+    )
+    print(f"  JSON list  → {json_path}\n")
+
+
+if __name__ == "__main__":
+    main()

pipeline/join_reports.py

@@ -0,0 +1,124 @@
+"""
+join_reports.py
+───────────────
+Left-join parsed_reports_with_agency (left) onto raw_reports_table (right)
+on:  left["Unnamed: 0"]  ==  right["raw_text"]
+
+All rows from the parsed file are kept; matching columns from raw_reports_table
+(source_file, chunk_count, pages, parse_errors, assessment) are appended.
+The redundant raw_text column from the right side is dropped after the join.
+
+Usage
+-----
+    python join_reports.py
+    python join_reports.py --left parsed_reports_with_agency.xlsx \
+                           --right raw_reports_table.csv \
+                           --out joined_reports.xlsx
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+import pandas as pd
+
+
+def read_any(path: Path) -> pd.DataFrame:
+    """Read CSV or Excel, sniffing format from magic bytes."""
+    with open(path, "rb") as fh:
+        magic = fh.read(4)
+    is_excel = magic[:4] in (b"\xd0\xcf\x11\xe0", b"PK\x03\x04")
+    if is_excel or path.suffix.lower() in (".xlsx", ".xls", ".xlsm"):
+        return pd.read_excel(path, dtype=str)
+    for sep in (",", ";", "\t", "|"):
+        try:
+            df = pd.read_csv(path, sep=sep, dtype=str, encoding="utf-8-sig")
+            if len(df.columns) > 1:
+                return df
+        except Exception:
+            pass
+    return pd.read_csv(path, dtype=str, encoding="utf-8-sig")
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Left-join parsed reports with raw reports table")
+    ap.add_argument("--left",  default="parsed_reports_with_agency.xlsx")
+    ap.add_argument("--right", default="raw_reports_table.csv")
+    ap.add_argument("--out",   default="joined_reports.xlsx")
+    args = ap.parse_args()
+
+    left_path  = Path(args.left)
+    right_path = Path(args.right)
+    out_path   = Path(args.out)
+
+    for p in (left_path, right_path):
+        if not p.exists():
+            sys.exit(f"ERROR: file not found: {p}")
+
+    print(f"Reading left  : {left_path}")
+    df_left = read_any(left_path)
+    print(f"  {len(df_left):,} rows × {len(df_left.columns)} cols")
+    print(f"  Columns: {df_left.columns.tolist()}\n")
+
+    print(f"Reading right : {right_path}")
+    df_right = read_any(right_path)
+    print(f"  {len(df_right):,} rows × {len(df_right.columns)} cols")
+    print(f"  Columns: {df_right.columns.tolist()}\n")
+
+    # Validate join keys
+    for col, df, label in [
+        ("Unnamed: 0", df_left,  "left (parsed_reports_with_agency)"),
+        ("raw_text",   df_right, "right (raw_reports_table)"),
+    ]:
+        if col not in df.columns:
+            sys.exit(
+                f"ERROR: column '{col}' not found in {label}.\n"
+                f"  Available: {df.columns.tolist()}"
+            )
+
+    # Strip whitespace from join keys to avoid invisible mismatches
+    df_left["Unnamed: 0"]  = df_left["Unnamed: 0"].str.strip()
+    df_right["raw_text"]   = df_right["raw_text"].str.strip()
+
+    # Left join
+    merged = df_left.merge(
+        df_right,
+        left_on="Unnamed: 0",
+        right_on="raw_text",
+        how="left",
+        suffixes=("", "_right"),
+    )
+
+    # Drop the redundant raw_text column from the right side
+    if "raw_text" in merged.columns:
+        merged = merged.drop(columns=["raw_text"])
+
+    # Drop any duplicate unnamed index columns brought in from the right CSV
+    unnamed_right = [c for c in merged.columns if c.startswith("Unnamed:") and c != "Unnamed: 0"]
+    if unnamed_right:
+        merged = merged.drop(columns=unnamed_right)
+
+    # Rename the key column to something meaningful
+    merged = merged.rename(columns={"Unnamed: 0": "raw_text"})
+
+    # Stats
+    matched   = merged["source_file"].notna().sum() if "source_file" in merged.columns else "?"
+    unmatched = len(merged) - (matched if isinstance(matched, int) else 0)
+    print(f"Join complete: {len(merged):,} rows total  |  matched: {matched}  |  unmatched: {unmatched}")
+
+    if unmatched and isinstance(unmatched, int) and unmatched > 0:
+        miss = merged[merged["source_file"].isna()]["raw_text"].str[:80]
+        print(f"\nFirst 5 unmatched raw_text previews:")
+        for s in miss.head(5):
+            print(f"  {s!r}")
+
+    # Save
+    if out_path.suffix.lower() in (".xlsx", ".xls", ".xlsm"):
+        merged.to_excel(out_path, index=False)
+    else:
+        merged.to_csv(out_path, index=False)
+    print(f"\nSaved → {out_path.resolve()}")
+
+
+if __name__ == "__main__":
+    main()

pipeline/map_yaml_to_reports.py

@@ -0,0 +1,222 @@
+"""
+map_yaml_to_reports.py
+──────────────────────
+Joins YAML records in records/ with rows in raw_reports_table.csv.
+
+Normalization rule (both sides):
+  source_file  →  strip .md  →  lowercase  →  spaces → hyphens  →  remove apostrophes
+  record_id    →  already normalized (but may differ by apostrophe / trailing underscore)
+
+Three output groups:
+  A  matched        — YAML record + CSV report row joined
+  B  yaml-only      — YAML record with no CSV counterpart (e.g. dow-uap-*)
+  C  csv-only       — CSV row with no YAML record       (e.g. 059uap* UPDB files)
+
+Usage
+-----
+    python map_yaml_to_reports.py
+    python map_yaml_to_reports.py --records records --csv raw_reports_table.csv --out enriched_reports.xlsx
+"""
+
+import argparse
+import re
+import sys
+from pathlib import Path
+
+import pandas as pd
+import yaml
+
+
+# ── Normalization ─────────────────────────────────────────────────────────────
+
+def normalize_key(s: str) -> str:
+    """
+    Canonical join key:
+      1. Strip extension (.md, .yaml, …)
+      2. Lowercase
+      3. Collapse runs of whitespace → single hyphen
+      4. Remove apostrophes
+      5. Collapse multiple consecutive hyphens or underscores into one hyphen
+    """
+    s = Path(s).stem          # drop extension
+    s = s.lower()
+    s = re.sub(r"[\s]+", "-", s)          # whitespace → hyphen
+    s = s.replace("'", "")               # apostrophes gone
+    s = re.sub(r"[_]+", "_", s)          # keep underscores single
+    s = re.sub(r"[-]+", "-", s)          # keep hyphens single
+    return s.strip("-_")
+
+
+# ── YAML loading ──────────────────────────────────────────────────────────────
+
+# Fields we want to pull out of each YAML record
+_YAML_FIELDS = [
+    ("record_id",        lambda y: y.get("record_id")),
+    ("agency_yaml",      lambda y: y.get("csv", {}).get("agency")),
+    ("report_subtype",   lambda y: y.get("csv", {}).get("report_subtype")),
+    ("release_date",     lambda y: y.get("csv", {}).get("release_date")),
+    ("incident_date",    lambda y: y.get("csv", {}).get("incident_date")),
+    ("location_csv",     lambda y: y.get("csv", {}).get("location_csv")),
+    ("uap_shape",        lambda y: y.get("observation", {}).get("morphology", {}).get("shape")),
+    ("uap_maneuver",     lambda y: y.get("observation", {}).get("kinematics", {}).get("maneuver_type")),
+    ("uap_speed_mph",    lambda y: y.get("observation", {}).get("kinematics", {}).get("speed_mph")),
+    ("uap_altitude_ft",  lambda y: y.get("observation", {}).get("kinematics", {}).get("altitude_ft")),
+    ("sensor_types",     lambda y: "; ".join(y.get("observation", {}).get("sensor_types") or []) or None),
+    ("platform",         lambda y: y.get("observation", {}).get("platform")),
+    ("threat_assessment",lambda y: y.get("observation", {}).get("threat_assessment")),
+    ("region",           lambda y: y.get("csv", {}).get("region")),
+    ("page_count",       lambda y: len(y.get("files", {}).get("pages") or [])),
+    ("has_nim_report",   lambda y: bool(y.get("reports"))),
+]
+
+
+def load_yaml_records(records_dir: Path) -> pd.DataFrame:
+    rows = []
+    for yaml_path in sorted(records_dir.glob("*.yaml")):
+        try:
+            with open(yaml_path, encoding="utf-8") as fh:
+                data = yaml.safe_load(fh)
+        except Exception as exc:
+            print(f"  WARN: could not parse {yaml_path.name}: {exc}")
+            continue
+        if not isinstance(data, dict):
+            continue
+        row = {"yaml_file": yaml_path.name}
+        for field_name, extractor in _YAML_FIELDS:
+            try:
+                row[field_name] = extractor(data)
+            except Exception:
+                row[field_name] = None
+        rows.append(row)
+
+    if not rows:
+        sys.exit(f"ERROR: no .yaml files found in {records_dir}")
+
+    df = pd.DataFrame(rows)
+    df["join_key"] = df["record_id"].fillna("").apply(normalize_key)
+    return df
+
+
+# ── CSV loading ───────────────────────────────────────────────────────────────
+
+def load_csv_reports(csv_path: Path) -> pd.DataFrame:
+    for sep in (",", ";", "\t", "|"):
+        try:
+            df = pd.read_csv(csv_path, sep=sep, dtype=str, encoding="utf-8-sig")
+            if len(df.columns) > 1:
+                break
+        except Exception:
+            pass
+    else:
+        df = pd.read_csv(csv_path, dtype=str, encoding="utf-8-sig")
+
+    if "source_file" not in df.columns:
+        sys.exit(
+            f"ERROR: 'source_file' column not found in {csv_path}.\n"
+            f"  Available: {df.columns.tolist()}"
+        )
+
+    df["join_key"] = df["source_file"].fillna("").apply(normalize_key)
+    return df
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Join YAML records with raw_reports_table.csv"
+    )
+    ap.add_argument("--records", default="records",        help="records/ directory")
+    ap.add_argument("--csv",     default="raw_reports_table.csv")
+    ap.add_argument("--out",     default="enriched_reports.xlsx")
+    args = ap.parse_args()
+
+    records_dir = Path(args.records)
+    csv_path    = Path(args.csv)
+    out_path    = Path(args.out)
+
+    for p in (records_dir, csv_path):
+        if not p.exists():
+            sys.exit(f"ERROR: path not found: {p}")
+
+    # ── Load ──────────────────────────────────────────────────────────────────
+    print(f"Loading YAML records from {records_dir} …")
+    df_yaml = load_yaml_records(records_dir)
+    print(f"  {len(df_yaml):,} YAML records  |  {df_yaml['join_key'].nunique():,} unique keys")
+
+    print(f"\nLoading CSV from {csv_path} …")
+    df_csv = load_csv_reports(csv_path)
+    print(f"  {len(df_csv):,} CSV rows  |  {df_csv['join_key'].nunique():,} unique keys")
+
+    # ── Join ──────────────────────────────────────────────────────────────────
+    # Outer join so we can see all three groups
+    merged = df_csv.merge(
+        df_yaml,
+        on="join_key",
+        how="outer",
+        suffixes=("_csv", "_yaml"),
+        indicator=True,
+    )
+
+    # Group labels
+    merged["group"] = merged["_merge"].map({
+        "both":       "A_matched",
+        "right_only": "B_yaml_only",
+        "left_only":  "C_csv_only",
+    })
+    merged = merged.drop(columns=["_merge"])
+
+    # ── Stats ─────────────────────────────────────────────────────────────────
+    counts = merged["group"].value_counts().sort_index()
+    total = len(merged)
+
+    print("\n" + "─" * 60)
+    print(f"{'Group':<20}  {'Count':>6}  {'%':>6}")
+    print("─" * 60)
+    for grp, cnt in counts.items():
+        print(f"{grp:<20}  {cnt:>6,}  {cnt/total*100:>5.1f}%")
+    print("─" * 60)
+    print(f"{'TOTAL':<20}  {total:>6,}")
+
+    # Breakdown of yaml-only by agency
+    yaml_only = merged[merged["group"] == "B_yaml_only"]
+    if len(yaml_only):
+        print(f"\nYAML-only agency breakdown ({len(yaml_only):,} records):")
+        for agency, cnt in yaml_only["agency_yaml"].value_counts(dropna=False).items():
+            print(f"  {agency!s:<20} {cnt:>4,}")
+
+    # Breakdown of csv-only by prefix
+    csv_only = merged[merged["group"] == "C_csv_only"]
+    if len(csv_only):
+        print(f"\nCSV-only source_file examples (first 15):")
+        for sf in csv_only["source_file"].dropna().head(15):
+            print(f"  {sf}")
+
+    # ── Duplicate key check ───────────────────────────────────────────────────
+    dup_yaml = df_yaml[df_yaml.duplicated("join_key", keep=False)]
+    if len(dup_yaml):
+        print(f"\nWARN: {len(dup_yaml):,} YAML records share a join_key with another YAML record:")
+        print(dup_yaml[["yaml_file", "record_id", "join_key"]].to_string(index=False))
+
+    dup_csv = df_csv[df_csv.duplicated("join_key", keep=False)]
+    if len(dup_csv):
+        print(f"\nWARN: {len(dup_csv):,} CSV rows share a join_key with another CSV row:")
+        print(dup_csv[["source_file", "join_key"]].head(10).to_string(index=False))
+
+    # ── Save ──────────────────────────────────────────────────────────────────
+    # Sort: matched first, then yaml-only, then csv-only; alpha within each group
+    merged = merged.sort_values(["group", "join_key"]).reset_index(drop=True)
+
+    if out_path.suffix.lower() in (".xlsx", ".xls", ".xlsm"):
+        merged.to_excel(out_path, index=False)
+    else:
+        merged.to_csv(out_path, index=False)
+
+    print(f"\nSaved → {out_path.resolve()}")
+    print("Columns in output:")
+    for col in merged.columns:
+        print(f"  {col}")
+
+
+if __name__ == "__main__":
+    main()

pipeline/page_coverage.py

@@ -0,0 +1,156 @@
+"""
+page_coverage.py
+────────────────
+For every document folder in the archive, counts:
+  - Total page_XXXX subfolders  (pages that exist as folders)
+  - Pages WITH a matching .md file  (OCR complete)
+  - Pages WITHOUT a .md file         (gap — PDF only or missing)
+
+Outputs:
+  - Summary table to stdout
+  - page_coverage.json  for downstream use / graphing
+  - page_coverage.csv   for spreadsheet / manual review
+
+Usage
+-----
+    python page_coverage.py
+    python page_coverage.py --root D:/divided
+    python page_coverage.py --root D:/divided --out-dir .
+"""
+
+import re
+import os
+import csv
+import json
+import argparse
+from pathlib import Path
+
+DEFAULT_ROOT    = "D:/divided"
+DEFAULT_OUT_DIR = "D:/divided"
+
+PAGE_DIR_RE = re.compile(r"^page_(\d+)$", re.IGNORECASE)
+
+SKIP_TRAVERSE = {
+    "records", "reports_out", "pages_out", "wiki",
+    "concat", "extracted", "scripts", "raw",
+    ".git", "__pycache__",
+}
+
+SKIP_CANDIDATE = {
+    "reorganize.py", "restructure_pages.py", "stamp_pages.py",
+    "reconcile.py", "pdf_to_reports.py", "concat_pages.py",
+    "find_missing_concat.py", "audit.py", "page_coverage.py",
+    "uap_record_schema.yaml", "uap-csv.csv",
+    "move_log.json", "page_move_log.json",
+    "CLAUDE.md", "Untitled.md", "README.md",
+}
+
+
+def find_document_dirs(root: Path) -> list[Path]:
+    doc_dirs = []
+    for dirpath, dirnames, _ in os.walk(str(root)):
+        dirnames[:] = [d for d in dirnames if d not in SKIP_TRAVERSE]
+        p = Path(dirpath)
+        if p == root or p.name in SKIP_CANDIDATE:
+            continue
+        try:
+            page_subs = [
+                sub for sub in p.iterdir()
+                if sub.is_dir() and PAGE_DIR_RE.match(sub.name)
+            ]
+        except (PermissionError, OSError):
+            continue
+        if page_subs:
+            doc_dirs.append(p)
+    return sorted(doc_dirs)
+
+
+def analyse_doc(doc_dir: Path) -> dict:
+    total_pages = 0
+    pages_with_md = 0
+    missing_pages = []
+
+    try:
+        subs = [sub for sub in doc_dir.iterdir() if sub.is_dir()]
+    except (PermissionError, OSError):
+        subs = []
+
+    for sub in subs:
+        m = PAGE_DIR_RE.match(sub.name)
+        if not m:
+            continue
+        total_pages += 1
+        md = sub / f"{sub.name}.md"
+        if md.exists():
+            pages_with_md += 1
+        else:
+            missing_pages.append(sub.name)
+
+    return {
+        "doc":          doc_dir.name,
+        "total_pages":  total_pages,
+        "with_md":      pages_with_md,
+        "missing_md":   total_pages - pages_with_md,
+        "coverage_pct": round(100 * pages_with_md / total_pages, 1) if total_pages else 0,
+        "missing_list": missing_pages,
+    }
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Count page coverage (.md vs total) per document")
+    ap.add_argument("--root",    default=DEFAULT_ROOT)
+    ap.add_argument("--out-dir", default=DEFAULT_OUT_DIR)
+    args = ap.parse_args()
+
+    root    = Path(args.root)
+    out_dir = Path(args.out_dir)
+
+    print(f"\n  Scanning {root} …")
+    doc_dirs = find_document_dirs(root)
+    print(f"  Found {len(doc_dirs)} document folders\n")
+
+    results = [analyse_doc(d) for d in doc_dirs]
+
+    # ── stdout summary ────────────────────────────────────────────────────────
+    total_pages   = sum(r["total_pages"]  for r in results)
+    total_with_md = sum(r["with_md"]      for r in results)
+    total_missing = sum(r["missing_md"]   for r in results)
+    full_coverage = sum(1 for r in results if r["missing_md"] == 0)
+    partial       = sum(1 for r in results if 0 < r["missing_md"] < r["total_pages"])
+    no_md         = sum(1 for r in results if r["with_md"] == 0)
+
+    print(f"{'─'*72}")
+    print(f"  {'Document':<55} {'Pages':>5}  {'MD':>5}  {'Gap':>5}  {'Cov%':>5}")
+    print(f"{'─'*72}")
+    for r in results:
+        gap_flag = "  ⚠" if r["missing_md"] > 0 else ""
+        print(f"  {r['doc'][:55]:<55} {r['total_pages']:>5}  {r['with_md']:>5}  {r['missing_md']:>5}{gap_flag}")
+    print(f"{'─'*72}")
+    print(f"  {'TOTAL':<55} {total_pages:>5}  {total_with_md:>5}  {total_missing:>5}")
+    print(f"{'─'*72}")
+    print(f"\n  Docs with full MD coverage : {full_coverage}")
+    print(f"  Docs with partial coverage : {partial}  ← pages missing .md")
+    print(f"  Docs with NO .md at all    : {no_md}")
+    print()
+
+    # ── JSON output ───────────────────────────────────────────────────────────
+    json_path = out_dir / "page_coverage.json"
+    with open(json_path, "w", encoding="utf-8") as f:
+        json.dump(results, f, indent=2)
+    print(f"  JSON → {json_path}")
+
+    # ── CSV output ────────────────────────────────────────────────────────────
+    csv_path = out_dir / "page_coverage.csv"
+    with open(csv_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(
+            f,
+            fieldnames=["doc", "total_pages", "with_md", "missing_md", "coverage_pct"],
+            extrasaction="ignore",
+        )
+        writer.writeheader()
+        writer.writerows(results)
+    print(f"  CSV  → {csv_path}\n")
+
+
+if __name__ == "__main__":
+    main()

pipeline/pdf_to_reports.py

@@ -0,0 +1,457 @@
+"""
+pdf_to_reports.py
+-----------------
+Multithreaded PDF → per-report .md extractor via NVIDIA NIM.
+
+TWO-PASS PIPELINE
+  Pass 1 (parallel)  — every non-blank page is sent to NIM concurrently for text
+                        clean-up and metadata extraction.
+  Pass 2 (single)    — a compact "page manifest" (page number + header snippet +
+                        detected signals) is sent to NIM in ONE call so it can
+                        see the full document structure and authoritatively decide
+                        which pages start new reports.  For PDFs > SEGMENT_CHUNK
+                        pages the manifest is split into overlapping chunks and
+                        the boundary lists are merged.
+
+Usage:
+    NVIDIA_API_KEY=<key> python pdf_to_reports.py path/to/file.pdf
+    NVIDIA_API_KEY=<key> python pdf_to_reports.py path/to/file.pdf --out my_reports --workers 6
+    NVIDIA_API_KEY=<key> python pdf_to_reports.py path/to/file.pdf --no-segment
+"""
+
+import os
+import re
+import time
+import json
+import random
+import argparse
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from textwrap import shorten
+
+import pdfplumber
+from openai import OpenAI, RateLimitError, APIStatusError
+
+# ── client ────────────────────────────────────────────────────────────────────
+
+client = OpenAI(
+    base_url="https://integrate.api.nvidia.com/v1",
+    api_key=os.getenv("NVIDIA_API_KEY"),
+)
+
+MODEL            = "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning"
+MAX_RETRIES      = 6
+BASE_WAIT        = 1.0    # seconds; doubles per retry attempt
+MAX_CHARS_PAGE   = 8_000  # chars sent to NIM per page in pass 1
+HEADER_CHARS     = 300    # chars per page sent in the pass-2 manifest
+SEGMENT_CHUNK    = 80     # pages per segmentation call (fits well within context)
+SEGMENT_OVERLAP  = 5      # pages of overlap between chunks to catch cross-boundary reports
+
+# ── NIM call with exponential backoff ─────────────────────────────────────────
+
+def call_nim(messages: list[dict], *, max_retries: int = MAX_RETRIES) -> str:
+    """
+    Call the NIM chat endpoint.  Retries on 429 and 5xx with
+    exponential back-off + full jitter to prevent thundering herd.
+    """
+    for attempt in range(max_retries):
+        try:
+            resp = client.chat.completions.create(
+                model=MODEL,
+                messages=messages,
+                temperature=0.3,
+                top_p=0.95,
+                max_tokens=8192,
+                stream=False,
+                extra_body={
+                    "chat_template_kwargs": {"enable_thinking": True},
+                    "reasoning_budget": 4096,
+                },
+            )
+            return resp.choices[0].message.content or ""
+
+        except RateLimitError:
+            wait = BASE_WAIT * (2 ** attempt) + random.uniform(0, BASE_WAIT)
+            print(f"    [rate-limit] attempt {attempt + 1}/{max_retries} — waiting {wait:.1f}s …")
+            time.sleep(wait)
+
+        except APIStatusError as exc:
+            if exc.status_code >= 500:
+                wait = BASE_WAIT * (2 ** attempt) + random.uniform(0, BASE_WAIT)
+                print(f"    [server-error {exc.status_code}] attempt {attempt + 1}/{max_retries} — waiting {wait:.1f}s …")
+                time.sleep(wait)
+            else:
+                raise   # 4xx other than 429 are not retryable
+
+    raise RuntimeError(f"NIM API failed after {max_retries} attempts")
+
+
+def _parse_json(raw: str) -> dict | list:
+    """Strip markdown fences and parse JSON; raise on failure."""
+    raw = re.sub(r"^```(?:json)?\s*", "", raw.strip())
+    raw = re.sub(r"\s*```$", "", raw)
+    return json.loads(raw)
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# PASS 1 — per-page extraction (parallel)
+# ══════════════════════════════════════════════════════════════════════════════
+
+PAGE_SYSTEM = (
+    "You are a precise document-analysis assistant. "
+    "Output ONLY valid JSON — no prose, no markdown fences."
+)
+
+PAGE_PROMPT = """\
+Analyse the page below (from a government / agency report PDF).
+
+Return valid JSON matching this exact schema (no extra keys):
+{{
+  "report_id": <str|null>,
+  "metadata": {{
+    "date":               <str|null>,
+    "location":           <str|null>,
+    "agency":             <str|null>,
+    "classification":     <str|null>,
+    "object_description": <str|null>,
+    "witnesses":          <str|null>,
+    "redacted_sections":  [<str>]
+  }},
+  "page_text": <str>,
+  "header_line": <str>   // first meaningful heading or first 120 chars of text
+}}
+
+[PAGE {page_num}]
+{text}
+"""
+
+
+def process_page(page_num: int, raw_text: str) -> dict:
+    """Send one page to NIM; return structured dict."""
+    print(f"  → page {page_num:>4d}")
+    messages = [
+        {"role": "system", "content": PAGE_SYSTEM},
+        {"role": "user",   "content": PAGE_PROMPT.format(
+            page_num=page_num,
+            text=raw_text[:MAX_CHARS_PAGE],
+        )},
+    ]
+    raw = call_nim(messages)
+    try:
+        result = _parse_json(raw)
+    except (json.JSONDecodeError, ValueError):
+        result = {
+            "report_id":   None,
+            "metadata":    {"redacted_sections": []},
+            "page_text":   raw_text,
+            "header_line": raw_text[:120],
+        }
+    result["page_num"] = page_num
+    return result
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# PASS 2 — document-level segmentation (single / chunked call)
+# ══════════════════════════════════════════════════════════════════════════════
+
+SEGMENT_SYSTEM = (
+    "You are a document-segmentation assistant. "
+    "Output ONLY valid JSON — no prose, no markdown fences."
+)
+
+SEGMENT_PROMPT = """\
+Below is a page manifest for a multi-report PDF.  Each entry shows the page
+number, the report/case ID found on that page (if any), and the opening text.
+
+Your task: identify which pages START a new, distinct report or case.
+A new report typically begins with a new case number, incident number, new
+header/title, or a clear section break.  Continuation pages (cover sheets,
+attachments, exhibits that belong to the same report) are NOT new reports.
+
+Return a JSON array — one object per page — in page order:
+[
+  {{"page": <int>, "starts_report": <bool>, "report_id": <str|null>}},
+  ...
+]
+
+PAGE MANIFEST:
+{manifest}
+"""
+
+
+def _build_manifest(pages: list[dict]) -> str:
+    """Build a compact text manifest for the segmentation call."""
+    lines = []
+    for p in sorted(pages, key=lambda x: x["page_num"]):
+        rid    = p.get("report_id") or "—"
+        header = shorten(p.get("header_line") or p.get("page_text") or "", HEADER_CHARS, placeholder="…")
+        lines.append(f"p{p['page_num']:>4d}  id={rid:<20s}  {header}")
+    return "\n".join(lines)
+
+
+def _segment_chunk(pages: list[dict]) -> list[dict]:
+    """
+    Call NIM once with a manifest for `pages` and return boundary info.
+    Returns: [{"page": int, "starts_report": bool, "report_id": str|None}, ...]
+    """
+    manifest = _build_manifest(pages)
+    messages = [
+        {"role": "system", "content": SEGMENT_SYSTEM},
+        {"role": "user",   "content": SEGMENT_PROMPT.format(manifest=manifest)},
+    ]
+    raw = call_nim(messages)
+    try:
+        result = _parse_json(raw)
+        if isinstance(result, list):
+            return result
+    except (json.JSONDecodeError, ValueError):
+        pass
+    # fallback: first page of chunk starts a report, rest continue
+    return [
+        {"page": p["page_num"], "starts_report": (i == 0), "report_id": p.get("report_id")}
+        for i, p in enumerate(sorted(pages, key=lambda x: x["page_num"]))
+    ]
+
+
+def segment_document(pages: list[dict]) -> dict[int, dict]:
+    """
+    Run pass-2 segmentation over the full page list, chunking if needed.
+
+    Returns a dict keyed by page_num:
+      {page_num: {"starts_report": bool, "report_id": str|None}}
+    """
+    sorted_pages = sorted(pages, key=lambda p: p["page_num"])
+    n            = len(sorted_pages)
+    boundary_map: dict[int, dict] = {}
+
+    if n <= SEGMENT_CHUNK:
+        # entire document fits in one call
+        print("  → segmentation: single call")
+        results = _segment_chunk(sorted_pages)
+        for r in results:
+            boundary_map[r["page"]] = r
+    else:
+        # chunk with overlap so reports that straddle chunk edges are detected
+        step  = SEGMENT_CHUNK - SEGMENT_OVERLAP
+        start = 0
+        chunk_idx = 1
+        seen: set[int] = set()
+
+        while start < n:
+            end   = min(start + SEGMENT_CHUNK, n)
+            chunk = sorted_pages[start:end]
+            print(f"  → segmentation chunk {chunk_idx}  (pages {chunk[0]['page_num']}–{chunk[-1]['page_num']})")
+            results = _segment_chunk(chunk)
+            for r in results:
+                pn = r["page"]
+                if pn not in seen:
+                    boundary_map[pn] = r
+                    seen.add(pn)
+                else:
+                    # overlap zone: OR the starts_report flags
+                    # (if either pass said "new report", trust it)
+                    boundary_map[pn]["starts_report"] = (
+                        boundary_map[pn]["starts_report"] or r["starts_report"]
+                    )
+                    # prefer the more specific report_id
+                    if not boundary_map[pn].get("report_id") and r.get("report_id"):
+                        boundary_map[pn]["report_id"] = r["report_id"]
+            start     += step
+            chunk_idx += 1
+
+    return boundary_map
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# Grouping, metadata merge, markdown output
+# ══════════════════════════════════════════════════════════════════════════════
+
+def apply_segmentation(pages: list[dict], boundary_map: dict[int, dict]) -> list[dict]:
+    """
+    Override each page's report_id with the authoritative value from pass 2,
+    and set a 'new_report_start' flag according to the boundary map.
+    """
+    for p in pages:
+        pn   = p["page_num"]
+        info = boundary_map.get(pn, {})
+        p["new_report_start"] = info.get("starts_report", False)
+        if info.get("report_id"):
+            p["report_id"] = info["report_id"]
+    return pages
+
+
+def group_into_reports(pages: list[dict]) -> list[list[dict]]:
+    """Split sorted pages into report groups on new_report_start boundaries."""
+    groups:  list[list[dict]] = []
+    current: list[dict]       = []
+
+    for page in sorted(pages, key=lambda p: p["page_num"]):
+        if page.get("new_report_start") and current:
+            groups.append(current)
+            current = [page]
+        else:
+            current.append(page)
+
+    if current:
+        groups.append(current)
+
+    return groups
+
+
+def merge_metadata(pages: list[dict]) -> dict:
+    """
+    Merge metadata across all pages: first non-null scalar value wins;
+    redacted_sections lists are concatenated and de-duplicated.
+    """
+    merged: dict = {"redacted_sections": []}
+    seen_redacted: set[str] = set()
+
+    for page in pages:
+        m = page.get("metadata") or {}
+        for key, val in m.items():
+            if key == "redacted_sections":
+                for item in val or []:
+                    if item not in seen_redacted:
+                        merged["redacted_sections"].append(item)
+                        seen_redacted.add(item)
+            elif val and key not in merged:
+                merged[key] = val
+
+    merged["report_id"] = next(
+        (p["report_id"] for p in pages if p.get("report_id")),
+        f"report_p{pages[0]['page_num']}",
+    )
+    merged["pages"] = [p["page_num"] for p in pages]
+    return merged
+
+
+def write_report_md(report_pages: list[dict], out_dir: Path, index: int) -> Path:
+    """Write one .md file for a single logical report."""
+    meta  = merge_metadata(report_pages)
+    slug  = re.sub(r"[^\w\-]", "_", str(meta["report_id"]))
+    path  = out_dir / f"{slug}.md"
+
+    lines = [
+        f"# {meta['report_id']}",
+        "",
+        "## Metadata",
+        "",
+        "| Field | Value |",
+        "|-------|-------|",
+        f"| Pages | {meta['pages']} |",
+        f"| Date | {meta.get('date', '—')} |",
+        f"| Location | {meta.get('location', '—')} |",
+        f"| Agency | {meta.get('agency', '—')} |",
+        f"| Classification | {meta.get('classification', '—')} |",
+        f"| Object / Phenomenon | {meta.get('object_description', '—')} |",
+        f"| Witnesses | {meta.get('witnesses', '—')} |",
+    ]
+    if meta["redacted_sections"]:
+        lines.append(f"| Redacted sections | {'; '.join(meta['redacted_sections'])} |")
+
+    lines += ["", "---", "", "## Content", ""]
+
+    for page in sorted(report_pages, key=lambda p: p["page_num"]):
+        lines.append(f"### Page {page['page_num']}")
+        lines.append("")
+        lines.append((page.get("page_text") or "").strip())
+        lines.append("")
+
+    path.write_text("\n".join(lines), encoding="utf-8")
+    return path
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# Main pipeline
+# ══════════════════════════════════════════════════════════════════════════════
+
+def parse_pdf_to_reports(
+    pdf_path:    str | Path,
+    out_dir:     str | Path = "reports_out",
+    max_workers: int        = 4,
+    use_segment: bool       = True,
+) -> list[Path]:
+    """
+    Full two-pass pipeline.
+
+    Pass 1 (parallel):  extract + clean every page via NIM.
+    Pass 2 (single):    segment the full document via NIM to find report
+                        boundaries, chunking every SEGMENT_CHUNK pages for
+                        very long PDFs and merging results across overlaps.
+
+    Set use_segment=False to skip pass 2 and rely on per-page heuristics only.
+    """
+    pdf_path = Path(pdf_path)
+    out_dir  = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    # ── 0. extract raw text ───────────────────────────────────────────────────
+    print(f"\n📄  {pdf_path.name}")
+    with pdfplumber.open(pdf_path) as pdf:
+        pages_raw = [
+            (i + 1, page.extract_text() or "")
+            for i, page in enumerate(pdf.pages)
+        ]
+    non_blank = [(n, t) for n, t in pages_raw if t.strip()]
+    print(f"    {len(pages_raw)} pages total, {len(non_blank)} non-blank")
+
+    # ── 1. per-page NIM calls (threaded) ─────────────────────────────────────
+    print(f"\n🧵  Pass 1 — page extraction  (workers={max_workers}) …")
+    results: list[dict] = []
+    errors:  list[int]  = []
+
+    with ThreadPoolExecutor(max_workers=max_workers) as pool:
+        futures = {pool.submit(process_page, num, text): num for num, text in non_blank}
+        for fut in as_completed(futures):
+            pn = futures[fut]
+            try:
+                results.append(fut.result())
+            except Exception as exc:
+                print(f"  ✗ page {pn} permanently failed: {exc}")
+                errors.append(pn)
+
+    if errors:
+        print(f"  ⚠  {len(errors)} page(s) failed: {errors}")
+
+    # ── 2. document-level segmentation (single / chunked) ────────────────────
+    if use_segment:
+        print(f"\n🔍  Pass 2 — document segmentation …")
+        boundary_map = segment_document(results)
+        results = apply_segmentation(results, boundary_map)
+    else:
+        print("\n⏭  Skipping pass 2 (--no-segment)")
+
+    # ── 3. group + write ──────────────────────────────────────────────────────
+    groups = group_into_reports(results)
+    print(f"\n📊  {len(groups)} report(s) found across {len(results)} pages\n")
+
+    written: list[Path] = []
+    for i, group in enumerate(groups, start=1):
+        path = write_report_md(group, out_dir, i)
+        print(f"  ✓  {path.name}  ({len(group)} page(s))")
+        written.append(path)
+
+    print(f"\n✅  Done — {len(written)} file(s) → {out_dir}/\n")
+    return written
+
+
+# ── CLI ───────────────────────────────────────────────────────────────────────
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser(
+        description="Parse a PDF of reports into per-report .md files via NVIDIA NIM (two-pass)"
+    )
+    ap.add_argument("pdf",                           help="Path to input PDF")
+    ap.add_argument("--out",      default="reports_out", help="Output directory  [reports_out]")
+    ap.add_argument("--workers",  type=int, default=4,   help="Parallel NIM threads for pass 1  [4]")
+    ap.add_argument("--no-segment", dest="segment",
+                    action="store_false", default=True,
+                    help="Skip pass-2 segmentation (use per-page heuristics only)")
+    args = ap.parse_args()
+
+    parse_pdf_to_reports(
+        args.pdf,
+        out_dir=args.out,
+        max_workers=args.workers,
+        use_segment=args.segment,
+    )

pipeline/reconcile.py

@@ -0,0 +1,615 @@
+"""
+reconcile.py
+------------
+Reads the three independent layers of the UAP document archive and emits
+one YAML record file per document into --out-dir.
+
+Layers joined:
+  1. CSV row         ← uap-csv.csv
+  2. Page files      ← D:/divided/<slug>/page_XXXX/page_XXXX.{md,pdf}
+  3. Extracted JSON  ← extracted/*.json  (extract_reports.py output)  ← primary
+     NIM report .md  ← reports_out/*.md  (pdf_to_reports.py output)  ← fallback
+  4. Blurb regex     ← speed / altitude / heading / count parsed inline
+
+extract_reports.py replaces pdf_to_reports.py as the Tier 3 source.
+If an extracted/*.json file exists for a record, it is used in preference
+to any matching reports_out/*.md file.
+
+Usage:
+    python reconcile.py
+    python reconcile.py --csv uap-csv.csv --divided D:/divided --out-dir records
+    python reconcile.py --csv uap-csv.csv --divided D:/divided --extracted extracted --out-dir records
+    python reconcile.py --csv uap-csv.csv --divided D:/divided --reports reports_out --out-dir records
+"""
+
+import re
+import csv
+import json
+import argparse
+from pathlib import Path
+from datetime import datetime
+
+import yaml          # pip install pyyaml --break-system-packages
+
+# ── tuneable paths ────────────────────────────────────────────────────────────
+DEFAULT_CSV        = "uap-csv.csv"
+DEFAULT_DIVIDED    = "D:/divided"
+DEFAULT_REPORTS    = "reports_out"
+DEFAULT_EXTRACTED  = "extracted"
+DEFAULT_OUT        = "records"
+
+SCHEMA_VERSION   = "1.0"
+
+# ── regex patterns (all blurb-parseable without NIM) ─────────────────────────
+RE_SPEED_KNOTS  = re.compile(r"(\d[\d,]+)\s*kn(?:ots?)?", re.I)
+RE_SPEED_MPH    = re.compile(r"(\d[\d,]*)\s*mph", re.I)
+RE_ALT_FEET     = re.compile(r"([\d,]+)\s*(?:ft|feet)", re.I)
+RE_HEADING_DEG  = re.compile(r"(\d{1,3})\s*degrees?", re.I)
+RE_ZULU         = re.compile(r"\b(\d{4}Z)\b")
+RE_OBJ_COUNT    = re.compile(
+    r"(\d+)\s*(?:x\s*)?UAP|"
+    r"(?:one|two|three|four|five|six|seven|eight|nine|ten)\s+(?:separate\s+)?UAP|"
+    r"a\s+(?:formation|group)\s+of\s+(\d+)|"
+    r"(two|three|four|five)\s+UAP",
+    re.I
+)
+COUNT_WORDS = {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5,
+               "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10}
+
+# Shape vocabulary – order matters (more specific first)
+SHAPE_PATTERNS = [
+    ("diamond",    re.compile(r"diamond", re.I)),
+    ("triangle",   re.compile(r"triangul", re.I)),
+    ("sphere",     re.compile(r"sphere|spherical|round|bouncy ball", re.I)),
+    ("disc",       re.compile(r"\bdisc\b|\bsaucer\b", re.I)),
+    ("balloon",    re.compile(r"balloon", re.I)),
+    ("cylinder",   re.compile(r"cylindr", re.I)),
+    ("orb",        re.compile(r"\borb\b", re.I)),
+    ("elongated",  re.compile(r"elongated|cigar", re.I)),
+    ("amorphous",  re.compile(r"ball of (?:white |bright )?light|glare|halo", re.I)),
+    ("formation",  re.compile(r"formation|line of dots", re.I)),
+]
+
+MANEUVER_PATTERNS = [
+    ("right_angle_turns", re.compile(r"90.degree|right.angle", re.I)),
+    ("circling",          re.compile(r"circling|orbiting", re.I)),
+    ("erratic",           re.compile(r"erratic|irregular", re.I)),
+    ("sea_skim",          re.compile(r"sea.skim|surface.*skim", re.I)),
+    ("accelerating",      re.compile(r"increased speed|accelerat", re.I)),
+    ("straight",          re.compile(r"straight|consistent.*course|same.*altitude", re.I)),
+    ("abrupt_turns",      re.compile(r"abrupt.*direction|directional change", re.I)),
+]
+
+THERMAL_PATTERNS = [
+    ("white_hot",   re.compile(r"white.hot",   re.I)),
+    ("black_hot",   re.compile(r"black.hot",   re.I)),
+    ("bright_white",re.compile(r"bright white", re.I)),
+    ("cold",        re.compile(r"\bcold\b",     re.I)),
+]
+
+SUBTYPE_MAP = {
+    "MISREP":                       re.compile(r"Mission Report|MISREP", re.I),
+    "Range_Fouler_Debrief":         re.compile(r"Range Fouler Debrief", re.I),
+    "Range_Fouler_Reporting_Form":  re.compile(r"Range Fouler Reporting Form", re.I),
+    "Email_Correspondence":         re.compile(r"email correspondence", re.I),
+    "Mission_Briefing":             re.compile(r"mission briefing", re.I),
+    "Intelligence_Report":          re.compile(r"intelligence report|air intelligence", re.I),
+    "Transcript":                   re.compile(r"transcript", re.I),
+    "Crew_Debriefing":              re.compile(r"crew.debriefing|technical.debriefing", re.I),
+    "Case_File":                    re.compile(r"case file", re.I),
+    "Launch_Summary":               re.compile(r"launch summary", re.I),
+    "Cable":                        re.compile(r"\bcable\b", re.I),
+    "Policy_Memo":                  re.compile(r"memorandum|memo\b", re.I),
+    "Photo_Collection":             re.compile(r"photo|image|picture", re.I),
+}
+
+
+# ── helpers ────────────────────���──────────────────────────────────────────────
+
+def _first_int(pattern: re.Pattern, text: str) -> int | None:
+    m = pattern.search(text)
+    if not m:
+        return None
+    raw = m.group(1).replace(",", "")
+    try:
+        return int(raw)
+    except ValueError:
+        return None
+
+
+def _all_matches(pattern: re.Pattern, text: str) -> list[str]:
+    return pattern.findall(text)
+
+
+def _first_vocab(pairs: list[tuple[str, re.Pattern]], text: str) -> str | None:
+    for label, pat in pairs:
+        if pat.search(text):
+            return label
+    return None
+
+
+def _parse_object_count(blurb: str) -> int | None:
+    """Extract explicit UAP count from blurb."""
+    m = RE_OBJ_COUNT.search(blurb)
+    if not m:
+        return None
+    for g in m.groups():
+        if g:
+            word = g.lower()
+            if word.isdigit():
+                return int(word)
+            return COUNT_WORDS.get(word)
+    return None
+
+
+def _parse_subtype(blurb: str) -> str:
+    for label, pat in SUBTYPE_MAP.items():
+        if pat.search(blurb):
+            return label
+    return "Other"
+
+
+def _parse_date(raw: str) -> str | None:
+    """Normalise messy CSV dates to ISO 8601 (best effort)."""
+    if not raw or raw.strip().upper() in ("N/A", ""):
+        return None
+    raw = raw.strip()
+    # Handle ranges like "4/10/2025-4/11/2025" → take first
+    raw = raw.split("-")[0].strip()
+    for fmt in ("%m/%d/%y", "%m/%d/%Y", "%Y-%m-%d"):
+        try:
+            return datetime.strptime(raw, fmt).strftime("%Y-%m-%d")
+        except ValueError:
+            pass
+    return raw  # fallback: return as-is
+
+
+def _slug_from_title(title: str) -> str:
+    """Derive a lowercase filesystem slug from a CSV title."""
+    s = title.lower().strip()
+    s = re.sub(r"[\s,]+", "-", s)
+    s = re.sub(r"[^\w\-]", "", s)
+    s = re.sub(r"-+", "-", s).strip("-")
+    return s
+
+
+def _index_document_dirs(divided_root: Path) -> list[Path]:
+    """
+    Return every directory under divided_root that looks like a document folder
+    (contains at least one page_XXXX/ subfolder).  Builds once, reused for all rows.
+    """
+    doc_dirs = []
+    for d in divided_root.rglob("*"):
+        if not d.is_dir():
+            continue
+        if any(re.match(r"page_\d+$", sub.name, re.I)
+               for sub in d.iterdir() if sub.is_dir()):
+            doc_dirs.append(d)
+    return doc_dirs
+
+
+def _find_document_dir(divided_root: Path, title: str,
+                       doc_index: list[Path] | None = None) -> Path | None:
+    """
+    Match a CSV title to a document folder anywhere under divided_root.
+    Pass a pre-built doc_index (from _index_document_dirs) to avoid
+    re-walking the tree for every row.
+    """
+    slug = _slug_from_title(title)
+
+    if doc_index is None:
+        doc_index = _index_document_dirs(divided_root)
+
+    # 1. Exact slug match
+    for d in doc_index:
+        if d.name.lower() == slug.lower():
+            return d
+
+    # 2. Prefix match (handles abbreviated folder names)
+    prefix = slug[:20].lower()
+    for d in doc_index:
+        if d.name.lower().startswith(prefix):
+            return d
+
+    # 3. Word-intersection fallback
+    words = set(slug.split("-")[:5])
+    for d in doc_index:
+        dwords = set(d.name.lower().split("-")[:5])
+        if len(words & dwords) >= min(3, len(words)):
+            return d
+
+    return None
+
+
+def _scan_pages(doc_dir: Path) -> list[dict]:
+    """Return sorted list of page dicts from the document folder."""
+    pages = []
+    for sub in sorted(doc_dir.iterdir()):
+        if not sub.is_dir():
+            continue
+        m = re.match(r"page_(\d+)$", sub.name, re.I)
+        if not m:
+            continue
+        idx = int(m.group(1))
+        md_file  = sub / f"{sub.name}.md"
+        pdf_file = sub / f"{sub.name}.pdf"
+
+        md_text = ""
+        if md_file.exists():
+            md_text = md_file.read_text(encoding="utf-8", errors="replace")
+
+        has_images = bool(re.search(r"!\[", md_text))
+        image_assets = re.findall(r"!\[.*?\]\((.*?)\)", md_text)
+
+        pages.append({
+            "index": idx,
+            "subdir": sub.name,
+            "md":     str(md_file.relative_to(doc_dir)) if md_file.exists() else None,
+            "pdf":    str(pdf_file.relative_to(doc_dir)) if pdf_file.exists() else None,
+            "has_images": has_images,
+            "image_assets": image_assets,
+        })
+    return pages
+
+
+def _find_report_md(reports_root: Path, record_id: str) -> Path | None:
+    """Locate the best matching .md file in reports_out for this record."""
+    if not reports_root.exists():
+        return None
+    slug_upper = record_id.upper().replace("-", "_")
+    for md in reports_root.glob("*.md"):
+        name = md.stem.upper().replace("-", "_")
+        if name == slug_upper or name.startswith(slug_upper[:20]):
+            return md
+    return None
+
+
+def _parse_nim_md(md_path: Path) -> dict:
+    """Extract the metadata table from a pdf_to_reports.py .md file."""
+    text = md_path.read_text(encoding="utf-8", errors="replace")
+    result = {k: None for k in
+              ["date", "location", "agency", "classification",
+               "object_description", "witnesses", "redacted_sections"]}
+    result["redacted_sections"] = []
+
+    for line in text.splitlines():
+        m = re.match(r"\|\s*(.+?)\s*\|\s*(.+?)\s*\|", line)
+        if not m:
+            continue
+        key, val = m.group(1).strip().lower(), m.group(2).strip()
+        if val in ("—", "N/A", ""):
+            val = None
+        if "date"        in key: result["date"]               = val
+        elif "location"  in key: result["location"]           = val
+        elif "agency"    in key: result["agency"]             = val
+        elif "classif"   in key: result["classification"]     = val
+        elif "object"    in key: result["object_description"] = val
+        elif "witness"   in key: result["witnesses"]          = val
+        elif "redacted"  in key and val:
+            result["redacted_sections"] = [s.strip() for s in val.split(";")]
+
+    return result
+
+
+def _find_extracted_json(extracted_root: Path, record_id: str) -> Path | None:
+    """
+    Locate the best matching .json file in extracted/ for this record.
+    Mirrors _find_report_md but for extract_reports.py JSON output.
+    Skips the combined _all_reports.json file.
+    """
+    if not extracted_root.exists():
+        return None
+    slug_upper = record_id.upper().replace("-", "_")
+    for jf in sorted(extracted_root.glob("*.json")):
+        if jf.name.startswith("_"):          # skip _all_reports.json
+            continue
+        name = jf.stem.upper().replace("-", "_")
+        if name == slug_upper or name.startswith(slug_upper[:20]):
+            return jf
+    return None
+
+
+# Date patterns used by _parse_extracted_json
+_RE_DATE_ISO    = re.compile(r"\b(\d{4}-\d{2}-\d{2})\b")
+_RE_DATE_US     = re.compile(r"\b(\d{1,2}/\d{1,2}/\d{2,4})\b")
+_RE_DATE_LONG   = re.compile(
+    r"\b((?:January|February|March|April|May|June|July|August|"
+    r"September|October|November|December)\s+\d{1,2},?\s+\d{4})\b", re.I
+)
+_RE_CLASSIF     = re.compile(
+    r"\b(UNCLASSIFIED|SECRET|CONFIDENTIAL|TOP\s+SECRET|FOUO|"
+    r"FOR\s+OFFICIAL\s+USE\s+ONLY)\b", re.I
+)
+_RE_REDACTED    = re.compile(r"\[REDACTED\]|\(b\)\(\d\)", re.I)
+
+
+def _parse_extracted_json(json_path: Path) -> list[dict]:
+    """
+    Read an extract_reports.py JSON file and return a list of nim_extracted
+    dicts — one per report entry found in the file.
+
+    Strategy:
+      1. Try markdown-table parsing first (handles docs where verbatim text
+         preserved a table layout from pdf_to_reports.py output).
+      2. Fall back to regex patterns for date, classification, and redactions.
+
+    Each dict has the standard nim_extracted keys plus a bonus 'pages' key
+    (the page-range label from the JSON).  The caller pops 'pages' before
+    storing into the YAML record.
+    """
+    try:
+        data = json.loads(json_path.read_text(encoding="utf-8", errors="replace"))
+    except Exception as exc:
+        return []                            # unreadable JSON — skip silently
+
+    results = []
+    for rep in data.get("reports", []):
+        raw_text   = rep.get("raw_text") or ""
+        assessment = rep.get("assessment") or ""
+        combined   = raw_text + "\n" + assessment
+
+        nim = {k: None for k in
+               ["date", "location", "agency", "classification",
+                "object_description", "witnesses", "redacted_sections"]}
+        nim["redacted_sections"] = []
+
+        # ── pass 1: markdown table rows (verbatim tables from NIM-style docs) ──
+        for line in combined.splitlines():
+            m = re.match(r"\|\s*(.+?)\s*\|\s*(.+?)\s*\|", line)
+            if not m:
+                continue
+            key, val = m.group(1).strip().lower(), m.group(2).strip()
+            if val in ("—", "N/A", ""):
+                val = None
+            if "date"       in key: nim["date"]               = val
+            elif "location" in key: nim["location"]           = val
+            elif "agency"   in key: nim["agency"]             = val
+            elif "classif"  in key: nim["classification"]     = val
+            elif "object"   in key: nim["object_description"] = val
+            elif "witness"  in key: nim["witnesses"]          = val
+            elif "redacted" in key and val:
+                nim["redacted_sections"] = [s.strip() for s in val.split(";")]
+
+        # ── pass 2: plain-text regex fallbacks ────────────────────────────────
+        if nim["date"] is None:
+            for pat in (_RE_DATE_ISO, _RE_DATE_US, _RE_DATE_LONG):
+                dm = pat.search(combined)
+                if dm:
+                    nim["date"] = dm.group(1)
+                    break
+
+        if nim["classification"] is None:
+            cm = _RE_CLASSIF.search(combined)
+            if cm:
+                nim["classification"] = cm.group(1).upper()
+
+        n_redacted = len(_RE_REDACTED.findall(combined))
+        if n_redacted > 0 and not nim["redacted_sections"]:
+            nim["redacted_sections"] = [f"{n_redacted} redaction marker(s) detected"]
+
+        nim["pages"] = rep.get("pages")      # bonus field — popped by caller
+        results.append(nim)
+
+    return results
+
+
+# ── per-row record builder ────────────────────────────────────────────────────
+
+def build_record(row: dict, divided_root: Path, reports_root: Path,
+                 doc_index: list[Path] | None = None,
+                 extracted_root: Path | None = None) -> dict:
+    title  = row.get("Title", "").replace("\n", " ").strip()
+    blurb  = row.get("Description Blurb", "").replace("\n", " ").strip()
+    agency = row.get("Agency", "").strip()
+    loc    = row.get("Incident Location", "").strip()
+    if loc.upper() == "N/A":
+        loc = None
+
+    record_id = _slug_from_title(title) or "unknown"
+
+    # ── Tier 1: CSV ──────────────────────────────────────────────────────────
+    csv_block = {
+        "title":                title,
+        "agency":               agency,
+        "release_date":         _parse_date(row.get("Release Date", "")),
+        "redacted":             row.get("Redaction", "").strip().upper() == "TRUE",
+        "document_type":        row.get("Type", "PDF").strip(),
+        "report_subtype":       _parse_subtype(blurb),
+        "incident_date":        _parse_date(row.get("Incident Date", "")),
+        "incident_date_precision": "day",   # refined by enrich.py
+        "location_csv":         loc,
+        "dvids_video_id":       row.get("DVIDS Video ID", "").strip() or None,
+        "video_pairing":        row.get("Video Pairing", "").strip() or None,
+        "pdf_pairing":          row.get("PDF Pairing", "").strip() or None,
+        "pdf_url":              row.get("PDF | Image Link", "").strip() or None,
+        "thumbnail_url":        row.get("Modal Image", "").strip() or None,
+        "description_blurb":    blurb,
+    }
+
+    # ── Tier 2: Page files ───────────────────────────────────────────────────
+    doc_dir = _find_document_dir(divided_root, title, doc_index)
+    if doc_dir:
+        pages = _scan_pages(doc_dir)
+    else:
+        pages = []
+
+    files_block = {
+        "document_dir": str(doc_dir) if doc_dir else None,
+        "page_count":   len(pages),
+        "pages":        pages,
+    }
+
+    # ── Tier 3: Extracted JSON (primary) or NIM report .md (fallback) ───────
+    json_path = (
+        _find_extracted_json(extracted_root, record_id)
+        if extracted_root else None
+    )
+
+    if json_path:
+        # ── primary: extract_reports.py JSON ──────────────────────────────────
+        extracted_reports = _parse_extracted_json(json_path)
+
+        if extracted_reports:
+            reports_block = []
+            for idx, nim_data in enumerate(extracted_reports, 1):
+                page_label = nim_data.pop("pages", None)  # remove bonus field
+                reports_block.append({
+                    "report_id":  f"{record_id}_r{idx:02d}" if len(extracted_reports) > 1
+                                  else record_id,
+                    "json_path":  str(json_path),
+                    "page_range": page_label or ([1, len(pages)] if pages else [1, None]),
+                    "nim_extracted": nim_data,
+                })
+        else:
+            # JSON found but empty / parse error
+            reports_block = [{
+                "report_id":  record_id,
+                "json_path":  str(json_path),
+                "page_range": [1, len(pages)] if pages else [1, None],
+                "nim_extracted": {k: None for k in
+                                  ["date", "location", "agency", "classification",
+                                   "object_description", "witnesses", "redacted_sections"]},
+            }]
+    else:
+        # ── fallback: pdf_to_reports.py .md ───────────────────────────────────
+        report_md = _find_report_md(reports_root, record_id)
+        nim_data  = _parse_nim_md(report_md) if report_md else {
+            k: None for k in ["date", "location", "agency",
+                               "classification", "object_description",
+                               "witnesses", "redacted_sections"]
+        }
+        if "redacted_sections" not in nim_data:
+            nim_data["redacted_sections"] = []
+
+        reports_block = [{
+            "report_id":     record_id,
+            "md_path":       str(report_md) if report_md else None,
+            "page_range":    [1, len(pages)] if pages else [1, None],
+            "nim_extracted": nim_data,
+        }]
+
+    # ── Tier 4: Observation (regex from blurb) ───────────────────────────────
+    observation_block = {
+        "location_precise": loc,
+        "morphology": {
+            "shape":            _first_vocab(SHAPE_PATTERNS, blurb),
+            "color":            None,
+            "thermal_appearance": _first_vocab(THERMAL_PATTERNS, blurb),
+            "material": "metallic" if re.search(r"metallic", blurb, re.I) else None,
+            "size_estimate":    None,
+        },
+        "kinematics": {
+            "object_count":     _parse_object_count(blurb),
+            "speed_knots":      _first_int(RE_SPEED_KNOTS, blurb),
+            "speed_mph":        _first_int(RE_SPEED_MPH,   blurb),
+            "altitude_ft":      _first_int(RE_ALT_FEET,    blurb),
+            "heading_degrees":  _first_int(RE_HEADING_DEG, blurb),
+            "direction_cardinal": None,
+            "duration_seconds": None,
+            "maneuver_type":    _first_vocab(MANEUVER_PATTERNS, blurb),
+            "flight_profile":   None,
+        },
+        "detection": {
+            "sensor_types":         [],
+            "platform":             "P8A" if re.search(r"P-8A", blurb) else None,
+            "tracking_outcome":     None,
+            "environmental_factors": (
+                ["cloud_cover"] if re.search(r"cloud", blurb, re.I) else []
+            ),
+        },
+        "assessment": {
+            "observer_label":    None,
+            "threat_assessment": "benign" if re.search(r"\bbenign\b", blurb, re.I) else None,
+            "pursuit_status":    (
+                "not_pursued" if re.search(r"did not pursue", blurb, re.I) else None
+            ),
+            "zulu_timestamps":   list(set(RE_ZULU.findall(blurb))),
+        },
+    }
+
+    # ── Tier 5: Relations ────────────────────────────────────────────────────
+    # Detect parent case / sibling logic for HS1 records
+    case_match = re.search(r"(\d{2}-HQ-\d+|\d{2}-[A-Z]+-\d+)", title)
+    relations_block = {
+        "part_of_case":      case_match.group(1) if case_match else None,
+        "parent_document":   None,
+        "sibling_documents": [],
+        "paired_video":      csv_block["video_pairing"],
+        "paired_pdf":        csv_block["pdf_pairing"],
+        "related_incidents": [],
+    }
+
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "record_id":      record_id,
+        "csv":            csv_block,
+        "files":          files_block,
+        "reports":        reports_block,
+        "observation":    observation_block,
+        "relations":      relations_block,
+    }
+
+
+# ── main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description="Reconcile CSV + pages + reports → YAML records")
+    ap.add_argument("--csv",       default=DEFAULT_CSV,       help="Path to uap-csv.csv")
+    ap.add_argument("--divided",   default=DEFAULT_DIVIDED,   help="Root of divided/ folder")
+    ap.add_argument("--extracted", default=DEFAULT_EXTRACTED,
+                    help="extracted/ directory with JSON from extract_reports.py (primary Tier 3)")
+    ap.add_argument("--reports",   default=DEFAULT_REPORTS,
+                    help="reports_out/ directory with .md from pdf_to_reports.py (fallback Tier 3)")
+    ap.add_argument("--out-dir",   default=DEFAULT_OUT,       help="Output directory for YAML files")
+    ap.add_argument("--limit",     type=int, default=0,       help="Process only first N rows (0 = all)")
+    args = ap.parse_args()
+
+    divided_root   = Path(args.divided)
+    extracted_root = Path(args.extracted)
+    reports_root   = Path(args.reports)
+    out_dir        = Path(args.out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    # Informative summary of which Tier 3 sources are present
+    ext_count = len([f for f in extracted_root.glob("*.json")
+                     if not f.name.startswith("_")]) if extracted_root.exists() else 0
+    rpt_count = len(list(reports_root.glob("*.md"))) if reports_root.exists() else 0
+    print(f"  Tier 3 — extracted JSON : {ext_count} files  ({extracted_root})")
+    print(f"  Tier 3 — reports .md    : {rpt_count} files  ({reports_root})  [fallback]")
+
+    with open(args.csv, newline="", encoding="utf-8-sig") as f:
+        reader = csv.DictReader(f)
+        rows = list(reader)
+
+    print(f"\n  {len(rows)} CSV rows  →  {out_dir}/")
+    print(f"  Indexing document folders under {divided_root} …")
+    doc_index = _index_document_dirs(divided_root)
+    print(f"  Found {len(doc_index)} document folders\n")
+
+    written, skipped = 0, 0
+    for i, row in enumerate(rows):
+        if args.limit and i >= args.limit:
+            break
+        title = row.get("Title", "").replace("\n", " ").strip()
+        if not title:
+            skipped += 1
+            continue
+
+        record = build_record(row, divided_root, reports_root, doc_index,
+                               extracted_root=extracted_root)
+        slug   = record["record_id"]
+        out_path = out_dir / f"{slug}.yaml"
+
+        with open(out_path, "w", encoding="utf-8") as f:
+            yaml.dump(record, f, allow_unicode=True, sort_keys=False,
+                      default_flow_style=False, width=100)
+
+        written += 1
+        if written <= 5 or written % 20 == 0:
+            print(f"  ✓  {out_path.name}")
+
+    print(f"\n✅  {written} records written, {skipped} skipped  →  {out_dir}/")
+
+
+if __name__ == "__main__":
+    main()

pipeline/reorganize.py

@@ -0,0 +1,273 @@
+"""
+reorganize.py
+─────────────
+Classifies every folder in D:/divided by name pattern and moves it into
+a logical agency/type subfolder hierarchy.
+
+Step 1 — dry run (default):  prints every planned move, nothing changes.
+Step 2 — execute:            pass --execute to actually move folders.
+Step 3 — undo:               pass --undo to reverse (reads the move log).
+
+Usage
+-----
+    python reorganize.py                     # dry-run, print plan
+    python reorganize.py --execute           # move everything
+    python reorganize.py --undo              # reverse all moves (uses move_log.json)
+    python reorganize.py --execute --root D:/somewhere_else
+"""
+
+import re
+import json
+import shutil
+import argparse
+from pathlib import Path
+from datetime import datetime
+
+# ── Target root (all new subfolders are created inside here) ─────────────────
+DEFAULT_ROOT = "D:/divided"
+
+# ── Classification rules  (first match wins — order matters) ─────────────────
+# Each tuple: (regex_on_folder_name_lowercase, destination_relative_to_root)
+RULES: list[tuple[str, str]] = [
+
+    # ── Duplicates (catch before anything else) ───────────────────────────────
+    (r"- copy$",                                    "MISC/_duplicates"),
+
+    # ── DOD / Department of War ───────────────────────────────────────────────
+    # Range Fouler reports (name contains "range-fouler" or "range_fouler")
+    (r"dow-uap-d\d+.*range.fouler.*arabian",        "DOD/range-fouler-debriefs/arabian"),
+    (r"dow-uap-d\d+.*range.fouler.*japan",          "DOD/range-fouler-debriefs/japan"),
+    (r"dow-uap-d\d+.*range.fouler.*aden",           "DOD/range-fouler-debriefs/gulf-of-aden"),
+    (r"dow-uap-d\d+.*range.fouler.*middle",         "DOD/range-fouler-debriefs/middle-east"),
+    (r"dow-uap-d\d+.*range.fouler",                 "DOD/range-fouler-debriefs/other"),
+
+    # Email correspondence
+    (r"dow-uap-d\d+.*email",                        "DOD/email-correspondence"),
+
+    # Mission reports by region
+    (r"dow-uap-d\d+.*mission.*arabian",             "DOD/mission-reports/arabian-gulf"),
+    (r"dow-uap-d\d+.*mission.*iraq",                "DOD/mission-reports/iraq"),
+    (r"dow-uap-d\d+.*mission.*syria",               "DOD/mission-reports/syria"),
+    (r"dow-uap-d\d+.*mission.*persian",             "DOD/mission-reports/persian-gulf"),
+    (r"dow-uap-d\d+.*mission.*hormuz",              "DOD/mission-reports/strait-of-hormuz"),
+    (r"dow-uap-d\d+.*mission.*greece",              "DOD/mission-reports/greece"),
+    (r"dow-uap-d\d+.*mission.*emirates",            "DOD/mission-reports/uae"),
+    (r"dow-uap-d\d+.*mission.*aden",                "DOD/mission-reports/gulf-of-aden"),
+    (r"dow-uap-d\d+.*mission.*mediterranean",       "DOD/mission-reports/mediterranean"),
+    (r"dow-uap-d\d+.*mission.*iran",                "DOD/mission-reports/iran"),
+    (r"dow-uap-d\d+.*mission.*djibouti",            "DOD/mission-reports/djibouti"),
+    (r"dow-uap-d\d+.*mission.*southern",            "DOD/mission-reports/southern-us"),
+    (r"dow-uap-d\d+.*mission.*middle",              "DOD/mission-reports/middle-east"),
+    (r"dow-uap-d\d+.*mission.*china",               "DOD/mission-reports/east-china-sea"),
+    (r"dow-uap-d\d+.*mission.*gulf.of.aden",        "DOD/mission-reports/gulf-of-aden"),
+    (r"dow-uap-d\d+.*mission",                      "DOD/mission-reports/other"),
+
+    # DOD catch-alls
+    (r"dow-uap-pr\d+",                              "DOD/reports-other"),
+    (r"dow-uap-d\d+",                               "DOD/reports-other"),
+
+    # ── NASA ──────────────────────────────────────────────────────────────────
+    (r"nasa-uap-d\d+.*transcript",                  "NASA/transcripts"),
+    (r"nasa-uap-d\d+.*debriefing",                  "NASA/crew-debriefings"),
+    (r"nasa-uap",                                   "NASA/other"),
+
+    # ── FBI ───────────────────────────────────────────────────────────────────
+    (r"fbi-photo",                                  "FBI/photo-collections"),
+
+    # ── Department of State ───────────────────────────────────────────────────
+    (r"dos-uap",                                    "DOS/cables"),
+
+    # ── NARA / CIA archives ───────────────────────────────────────────────────
+    (r"65_hs1-834228961",                           "NARA-CIA/hs1-834228961"),
+    (r"65_hs1-101634279",                           "NARA-CIA/hs1-101634279"),
+    (r"^341_",                                      "NARA-CIA/series-341"),
+    (r"^342_",                                      "NARA-CIA/series-342"),
+    (r"^331_",                                      "NARA-CIA/series-331"),
+    (r"^38_",                                       "NARA-CIA/series-38"),
+    (r"^59_",                                       "NARA-CIA/series-59"),
+    (r"^18_",                                       "NARA-CIA/series-18"),
+    (r"^255_",                                      "NARA-CIA/series-255"),
+
+    # ── MISC ──────────────────────────────────────────────────────────────────
+    (r"serial.*redacted|usper.*statement",          "MISC/statements-redacted"),
+    (r"sketch|composite",                           "MISC/visuals"),
+    (r"slides",                                     "MISC/visuals"),
+    (r"059uap",                                     "MISC/unclassified"),
+    (r"western_us",                                 "MISC/presentations"),
+]
+
+# Folders (and files) that should never be moved
+SKIP_NAMES = {
+    "reorganize.py",
+    "reconcile.py",
+    "pdf_to_reports.py",
+    "uap_record_schema.yaml",
+    "uap-csv.csv",
+    "move_log.json",
+    "records",
+    "reports_out",
+    "MISC",
+    "DOD",
+    "NASA",
+    "FBI",
+    "DOS",
+    "NARA-CIA",
+    "Untitled.md",
+}
+
+
+# ── core logic ────────────────────────────────────────────────────────────────
+
+def classify(name: str) -> str | None:
+    """Return destination subpath (relative to root) for a folder name, or None to skip."""
+    lower = name.lower()
+    for pattern, dest in RULES:
+        if re.search(pattern, lower):
+            return dest
+    return None
+
+
+def plan_moves(root: Path) -> list[dict]:
+    """
+    Walk the immediate children of root (folders only) and return a list of
+    planned move operations: {src, dst, dest_category}.
+    """
+    moves = []
+    unmatched = []
+
+    for item in sorted(root.iterdir()):
+        if not item.is_dir():
+            continue
+        if item.name in SKIP_NAMES:
+            continue
+
+        dest_rel = classify(item.name)
+        if dest_rel is None:
+            unmatched.append(item.name)
+            continue
+
+        dst_dir = root / dest_rel
+        dst     = dst_dir / item.name
+
+        moves.append({
+            "src":           str(item),
+            "dst":           str(dst),
+            "dest_category": dest_rel,
+        })
+
+    return moves, unmatched
+
+
+def print_plan(moves: list[dict], unmatched: list[str]) -> None:
+    current_cat = None
+    for m in moves:
+        cat = m["dest_category"]
+        if cat != current_cat:
+            print(f"\n  📁  {cat}/")
+            current_cat = cat
+        src_name = Path(m["src"]).name
+        print(f"       {src_name}")
+
+    if unmatched:
+        print(f"\n  ⚠  {len(unmatched)} folder(s) did not match any rule:")
+        for u in unmatched:
+            print(f"       {u}")
+
+    print(f"\n  Total: {len(moves)} moves planned,  {len(unmatched)} unmatched\n")
+
+
+def execute_moves(moves: list[dict], root: Path, log_path: Path) -> None:
+    done    = []
+    errors  = []
+
+    for m in moves:
+        src = Path(m["src"])
+        dst = Path(m["dst"])
+
+        try:
+            dst.parent.mkdir(parents=True, exist_ok=True)
+            if dst.exists():
+                print(f"  ⚠  destination exists, skipping: {dst.name}")
+                continue
+            shutil.move(str(src), str(dst))
+            done.append(m)
+            print(f"  ✓  {src.name}  →  {m['dest_category']}/")
+        except Exception as exc:
+            errors.append({"move": m, "error": str(exc)})
+            print(f"  ✗  {src.name}  ERROR: {exc}")
+
+    # Write undo log
+    log = {
+        "timestamp": datetime.now().isoformat(),
+        "root":      str(root),
+        "moves":     done,
+    }
+    log_path.write_text(json.dumps(log, indent=2), encoding="utf-8")
+    print(f"\n  ✅  {len(done)} moved,  {len(errors)} errors")
+    print(f"  📝  Undo log written to {log_path}")
+
+
+def undo_moves(log_path: Path) -> None:
+    if not log_path.exists():
+        print(f"  ✗  No undo log found at {log_path}")
+        return
+
+    log   = json.loads(log_path.read_text(encoding="utf-8"))
+    moves = log.get("moves", [])
+
+    print(f"  Reversing {len(moves)} moves from {log['timestamp']} …\n")
+    errors = 0
+    for m in reversed(moves):
+        src = Path(m["dst"])   # swap: dst is where it ended up
+        dst = Path(m["src"])   # src is where it came from
+        try:
+            dst.parent.mkdir(parents=True, exist_ok=True)
+            shutil.move(str(src), str(dst))
+            print(f"  ↩  {src.name}  →  {dst.parent.name}/")
+        except Exception as exc:
+            print(f"  ✗  {src.name}  ERROR: {exc}")
+            errors += 1
+
+    print(f"\n  ✅  Undo complete.  {len(moves) - errors} restored,  {errors} errors")
+    log_path.unlink(missing_ok=True)
+
+
+# ── CLI ───────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Reorganise D:/divided into a logical folder hierarchy"
+    )
+    ap.add_argument("--root",    default=DEFAULT_ROOT, help="Root folder to reorganise")
+    ap.add_argument("--execute", action="store_true",  help="Actually move folders (default: dry-run)")
+    ap.add_argument("--undo",    action="store_true",  help="Reverse previous run using move_log.json")
+    args = ap.parse_args()
+
+    root     = Path(args.root)
+    log_path = root / "move_log.json"
+
+    if args.undo:
+        undo_moves(log_path)
+        return
+
+    moves, unmatched = plan_moves(root)
+
+    if not args.execute:
+        print(f"\n{'─'*60}")
+        print(f"  DRY RUN — nothing will be moved")
+        print(f"  Root: {root}")
+        print(f"{'─'*60}")
+        print_plan(moves, unmatched)
+        print("  Run with --execute to apply.\n")
+    else:
+        print(f"\n  Moving folders under {root} …\n")
+        execute_moves(moves, root, log_path)
+
+        if unmatched:
+            print(f"\n  ⚠  {len(unmatched)} folder(s) left in place (no rule matched):")
+            for u in unmatched:
+                print(f"       {u}")
+
+
+if __name__ == "__main__":
+    main()

pipeline/restructure_pages.py

@@ -0,0 +1,209 @@
+"""
+restructure_pages.py
+────────────────────
+Migrates every document folder in D:/divided from Variant B to Variant A.
+
+Variant B (current):
+    <doc-slug>/
+        page_0001.pdf          ← PDF at document root
+        page_0001/
+            page_0001.md       ← OCR in subfolder
+
+Variant A (target):
+    <doc-slug>/
+        page_0001/
+            page_0001.pdf      ← PDF moved into subfolder
+            page_0001.md       ← OCR already here
+
+Usage
+-----
+    python restructure_pages.py                     # dry-run, print plan
+    python restructure_pages.py --execute           # move PDFs
+    python restructure_pages.py --undo              # reverse (reads page_move_log.json)
+    python restructure_pages.py --root D:/somewhere # different root
+"""
+
+import re
+import json
+import shutil
+import argparse
+from pathlib import Path
+from datetime import datetime
+
+DEFAULT_ROOT = "D:/divided"
+LOG_FILENAME = "page_move_log.json"
+
+# Names that are not document folders
+SKIP_NAMES = {
+    "reorganize.py",
+    "restructure_pages.py",
+    "reconcile.py",
+    "pdf_to_reports.py",
+    "uap_record_schema.yaml",
+    "uap-csv.csv",
+    "move_log.json",
+    "page_move_log.json",
+    "records",
+    "reports_out",
+    "MISC",
+    "DOD",
+    "NASA",
+    "FBI",
+    "DOS",
+    "NARA-CIA",
+    "CLAUDE.md",
+    "Untitled.md",
+    "README.md",
+    "wiki",
+    "raw",
+    "scripts",
+}
+
+PAGE_PDF_RE = re.compile(r"^(page_\d+)\.pdf$", re.IGNORECASE)
+
+
+def find_moves(root: Path) -> list[dict]:
+    """
+    Walk every document folder and find root-level page PDFs that need
+    to move into their matching page subfolder.
+    """
+    moves = []
+
+    for doc_dir in sorted(root.iterdir()):
+        if not doc_dir.is_dir():
+            continue
+        if doc_dir.name in SKIP_NAMES:
+            continue
+
+        for item in sorted(doc_dir.iterdir()):
+            if not item.is_file():
+                continue
+            m = PAGE_PDF_RE.match(item.name)
+            if not m:
+                continue
+
+            page_slug = m.group(1)          # e.g. "page_0001"
+            subfolder = doc_dir / page_slug  # e.g. .../page_0001/
+            dst = subfolder / item.name      # e.g. .../page_0001/page_0001.pdf
+
+            moves.append({
+                "doc":  doc_dir.name,
+                "src":  str(item),
+                "dst":  str(dst),
+                "subfolder_exists": subfolder.is_dir(),
+            })
+
+    return moves
+
+
+def print_plan(moves: list[dict]) -> None:
+    current_doc = None
+    missing_subfolder = []
+
+    for m in moves:
+        if m["doc"] != current_doc:
+            print(f"\n  📁  {m['doc']}/")
+            current_doc = m["doc"]
+        src_name = Path(m["src"]).name
+        note = "" if m["subfolder_exists"] else "  ⚠ subfolder will be created"
+        print(f"       {src_name}  →  {Path(m['dst']).parent.name}/{src_name}{note}")
+        if not m["subfolder_exists"]:
+            missing_subfolder.append(m["src"])
+
+    print(f"\n  Total: {len(moves)} PDFs to move")
+    if missing_subfolder:
+        print(f"  ⚠  {len(missing_subfolder)} subfolder(s) will be created (no .md yet):")
+        for s in missing_subfolder:
+            print(f"       {Path(s).parent.name}/{Path(s).name}")
+    print()
+
+
+def execute_moves(moves: list[dict], log_path: Path) -> None:
+    done   = []
+    errors = []
+
+    for m in moves:
+        src = Path(m["src"])
+        dst = Path(m["dst"])
+
+        try:
+            dst.parent.mkdir(parents=True, exist_ok=True)
+            if dst.exists():
+                print(f"  ⚠  already in place, skipping: {dst}")
+                continue
+            shutil.move(str(src), str(dst))
+            done.append(m)
+            print(f"  ✓  {src.parent.name}/{src.name}  →  {dst.parent.name}/")
+        except Exception as exc:
+            errors.append({"move": m, "error": str(exc)})
+            print(f"  ✗  {src.name}  ERROR: {exc}")
+
+    log = {
+        "timestamp": datetime.now().isoformat(),
+        "moves": done,
+    }
+    log_path.write_text(json.dumps(log, indent=2), encoding="utf-8")
+    print(f"\n  ✅  {len(done)} moved,  {len(errors)} errors")
+    print(f"  📝  Undo log → {log_path}\n")
+
+
+def undo_moves(log_path: Path) -> None:
+    if not log_path.exists():
+        print(f"  ✗  No undo log found at {log_path}")
+        return
+
+    log   = json.loads(log_path.read_text(encoding="utf-8"))
+    moves = log.get("moves", [])
+    print(f"  Reversing {len(moves)} moves from {log['timestamp']} …\n")
+
+    errors = 0
+    for m in reversed(moves):
+        src = Path(m["dst"])   # where it ended up
+        dst = Path(m["src"])   # where it came from
+        try:
+            shutil.move(str(src), str(dst))
+            print(f"  ↩  {src.name}  →  {dst.parent.name}/")
+        except Exception as exc:
+            print(f"  ✗  {src.name}  ERROR: {exc}")
+            errors += 1
+
+    print(f"\n  ✅  Undo complete.  {len(moves) - errors} restored,  {errors} errors")
+    log_path.unlink(missing_ok=True)
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Move root-level page PDFs into their page subfolders (Variant B → A)"
+    )
+    ap.add_argument("--root",    default=DEFAULT_ROOT)
+    ap.add_argument("--execute", action="store_true", help="Actually move files")
+    ap.add_argument("--undo",    action="store_true", help="Reverse using page_move_log.json")
+    args = ap.parse_args()
+
+    root     = Path(args.root)
+    log_path = root / LOG_FILENAME
+
+    if args.undo:
+        undo_moves(log_path)
+        return
+
+    moves = find_moves(root)
+
+    if not moves:
+        print("\n  ✅  Nothing to do — all page PDFs are already in their subfolders.\n")
+        return
+
+    if not args.execute:
+        print(f"\n{'─'*60}")
+        print(f"  DRY RUN — nothing will be moved")
+        print(f"  Root: {root}")
+        print(f"{'─'*60}")
+        print_plan(moves)
+        print("  Run with --execute to apply.\n")
+    else:
+        print(f"\n  Restructuring pages under {root} …\n")
+        execute_moves(moves, log_path)
+
+
+if __name__ == "__main__":
+    main()

pipeline/run_ocr.py

@@ -0,0 +1,407 @@
+"""
+run_ocr.py
+──────────
+Batch OCR via Mistral with two modes:
+
+  --mode parallel  (default)
+      Sends N concurrent requests using ThreadPoolExecutor.
+      Results land immediately. Use --workers to tune concurrency.
+
+  --mode batch
+      Builds a JSONL file, submits a Mistral batch job, polls until done,
+      then writes all .md files. ~50% cheaper, but async (takes minutes+).
+
+Output always goes to the correct location:
+    <doc-folder>/page_XXXX/page_XXXX.md
+    <doc-folder>/page_XXXX/page_XXXX_img_0.png  (if images present)
+
+Safe to re-run — pages that already have .md are skipped.
+
+Usage
+-----
+    pip install mistralai
+    set MISTRAL_API_KEY=...
+
+    python run_ocr.py                          # parallel, all targets
+    python run_ocr.py --mode batch             # batch API (cheaper, async)
+    python run_ocr.py --limit 20               # test on first 20
+    python run_ocr.py --workers 20             # more concurrency
+    python run_ocr.py --file path/to/page.pdf  # single file
+    python run_ocr.py --mode batch --poll      # submit + wait for results
+"""
+
+import base64
+import os
+import re
+import sys
+import time
+import json
+import argparse
+import tempfile
+from pathlib import Path
+from datetime import datetime
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+DEFAULT_TARGETS = "D:/divided/ocr_targets.txt"
+MODEL           = "mistral-ocr-latest"
+MAX_RETRIES     = 3
+RETRY_WAIT      = 10
+BATCH_POLL_SECS = 30   # how often to poll for batch completion
+
+
+# ── helpers ───────────────────────────────────────────────────────────────────
+
+def encode_pdf(pdf_path: Path) -> str:
+    with open(pdf_path, "rb") as f:
+        return base64.b64encode(f.read()).decode("utf-8")
+
+
+def save_images(page_dir: Path, page_name: str, images) -> dict:
+    saved = {}
+    for idx, img in enumerate(images):
+        data = getattr(img, "image_base64", None)
+        if not data:
+            continue
+        if "," in data:
+            data = data.split(",", 1)[1]
+        fname = f"{page_name}_img_{idx}.png"
+        (page_dir / fname).write_bytes(base64.b64decode(data))
+        saved[getattr(img, "id", str(idx))] = fname
+    return saved
+
+
+def write_md(pdf_path: Path, response) -> Path:
+    """Write OCR response to the .md file next to the PDF. Returns md_path."""
+    page_dir  = pdf_path.parent
+    page_name = pdf_path.stem
+    md_path   = page_dir / f"{page_name}.md"
+
+    chunks = []
+    for page in response.pages:
+        text   = page.markdown or ""
+        images = getattr(page, "images", []) or []
+        if images:
+            saved = save_images(page_dir, page_name, images)
+            for img_id, fname in saved.items():
+                text = text.replace(img_id, fname)
+        chunks.append(text)
+
+    md_path.write_text("\n\n".join(chunks).strip(), encoding="utf-8")
+    return md_path
+
+
+def load_targets(targets_file: Path, single: str | None, limit: int | None) -> list[Path]:
+    if single:
+        return [Path(single)]
+    if not targets_file.exists():
+        raise SystemExit(
+            f"  ✗  Targets file not found: {targets_file}\n"
+            f"     Run:  python find_ocr_targets.py"
+        )
+    lines = targets_file.read_text(encoding="utf-8").splitlines()
+    paths = [
+        Path(l.strip()) for l in lines
+        if l.strip() and l.strip() != "NOT FOUND"
+    ]
+    # Skip already-done
+    paths = [p for p in paths if not (p.parent / f"{p.stem}.md").exists()]
+    if limit:
+        paths = paths[:limit]
+    return paths
+
+
+# ── parallel mode ─────────────────────────────────────────────────────────────
+
+def ocr_one(client, pdf_path: Path) -> tuple[Path, bool, str]:
+    """OCR a single PDF. Returns (pdf_path, success, message)."""
+    for attempt in range(1, MAX_RETRIES + 1):
+        try:
+            b64 = encode_pdf(pdf_path)
+            response = client.ocr.process(
+                model=MODEL,
+                document={
+                    "type": "document_url",
+                    "document_url": f"data:application/pdf;base64,{b64}",
+                },
+                table_format=None,
+                include_image_base64=True,
+            )
+            md_path = write_md(pdf_path, response)
+            return pdf_path, True, f"{md_path.stat().st_size:,} bytes"
+        except Exception as e:
+            msg = str(e).lower()
+            if any(x in msg for x in ("rate", "429", "503", "502")) and attempt < MAX_RETRIES:
+                time.sleep(RETRY_WAIT * attempt)
+                continue
+            return pdf_path, False, str(e)
+    return pdf_path, False, "max retries exceeded"
+
+
+def run_parallel(client, targets: list[Path], workers: int) -> tuple[int, int]:
+    ok = failed = 0
+    total = len(targets)
+
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        futures = {pool.submit(ocr_one, client, p): p for p in targets}
+        for i, future in enumerate(as_completed(futures), 1):
+            pdf_path, success, msg = future.result()
+            label = "✓" if success else "✗"
+            rel   = "/".join(pdf_path.parts[-3:])
+            print(f"  [{i:>4}/{total}]  {label}  {rel}  {msg}")
+            if success:
+                ok += 1
+            else:
+                failed += 1
+
+    return ok, failed
+
+
+# ── batch mode ────────────────────────────────────────────────────────────────
+
+def build_jsonl(targets: list[Path]) -> str:
+    """Build a JSONL string — one OCR request per line."""
+    lines = []
+    for pdf_path in targets:
+        b64 = encode_pdf(pdf_path)
+        record = {
+            "custom_id": str(pdf_path),   # used to map results back to paths
+            "body": {
+                "model": MODEL,
+                "document": {
+                    "type": "document_url",
+                    "document_url": f"data:application/pdf;base64,{b64}",
+                },
+                "table_format": None,
+                "include_image_base64": True,
+            },
+        }
+        lines.append(json.dumps(record, ensure_ascii=False))
+    return "\n".join(lines)
+
+
+def run_batch(client, targets: list[Path], poll: bool) -> tuple[int, int]:
+    total = len(targets)
+    print(f"  Building JSONL for {total} files …")
+    jsonl_content = build_jsonl(targets)
+
+    # Upload JSONL file
+    print("  Uploading JSONL to Mistral Files API …")
+    with tempfile.NamedTemporaryFile(
+        mode="w", suffix=".jsonl", delete=False, encoding="utf-8"
+    ) as tmp:
+        tmp.write(jsonl_content)
+        tmp_path = tmp.name
+
+    try:
+        with open(tmp_path, "rb") as f:
+            uploaded = client.files.upload(
+                file={"file_name": "ocr_batch.jsonl", "content": f},
+                purpose="batch",
+            )
+        file_id = uploaded.id
+        print(f"  Uploaded → file_id: {file_id}")
+
+        # Submit batch job
+        batch_job = client.batch.jobs.create(
+            input_files=[file_id],
+            model=MODEL,
+            endpoint="/v1/ocr",
+            metadata={"description": "UAP archive OCR batch"},
+        )
+        job_id = batch_job.id
+        print(f"  Batch job submitted → job_id: {job_id}")
+
+        # Save job ID for later retrieval if not polling
+        job_log = Path(DEFAULT_TARGETS).parent / "ocr_batch_jobs.json"
+        jobs = []
+        if job_log.exists():
+            try:
+                jobs = json.loads(job_log.read_text(encoding="utf-8"))
+            except Exception:
+                pass
+        jobs.append({
+            "job_id":    job_id,
+            "file_id":   file_id,
+            "submitted": datetime.now().isoformat(timespec="seconds"),
+            "files":     [str(p) for p in targets],
+        })
+        job_log.write_text(json.dumps(jobs, indent=2), encoding="utf-8")
+        print(f"  Job info saved → {job_log}")
+
+        if not poll:
+            print(
+                f"\n  Job running asynchronously. To fetch results later:\n"
+                f"    python run_ocr.py --mode batch-fetch --job-id {job_id}\n"
+            )
+            return 0, 0
+
+        # Poll for completion
+        print("\n  Polling for completion …")
+        while True:
+            status = client.batch.jobs.get(job_id=job_id)
+            pct    = ""
+            if hasattr(status, "request_counts") and status.request_counts:
+                rc   = status.request_counts
+                done = getattr(rc, "completed", 0) + getattr(rc, "failed", 0)
+                pct  = f"  ({done}/{total})"
+            print(f"  status: {status.status}{pct}", end="\r")
+            if status.status in ("SUCCESS", "FAILED", "TIMEOUT_EXCEEDED", "EXPIRED"):
+                print()
+                break
+            time.sleep(BATCH_POLL_SECS)
+
+        if status.status != "SUCCESS":
+            print(f"  ✗  Batch ended with status: {status.status}")
+            return 0, total
+
+        return fetch_batch_results(client, job_id, targets)
+
+    finally:
+        os.unlink(tmp_path)
+
+
+def fetch_batch_results(client, job_id: str, targets: list[Path] | None = None) -> tuple[int, int]:
+    """Download results for a completed batch job and write .md files."""
+    # Build path lookup from job log if targets not provided
+    path_map: dict[str, Path] = {}
+    if targets:
+        path_map = {str(p): p for p in targets}
+    else:
+        job_log = Path(DEFAULT_TARGETS).parent / "ocr_batch_jobs.json"
+        if job_log.exists():
+            jobs = json.loads(job_log.read_text(encoding="utf-8"))
+            for job in jobs:
+                if job["job_id"] == job_id:
+                    path_map = {p: Path(p) for p in job.get("files", [])}
+                    break
+
+    job    = client.batch.jobs.get(job_id=job_id)
+    out_id = job.output_file
+
+    print(f"  Downloading results (output file: {out_id}) …")
+    result_bytes = client.files.download(file_id=out_id)
+    results_text = result_bytes.read().decode("utf-8")
+
+    ok = failed = 0
+    for line in results_text.splitlines():
+        if not line.strip():
+            continue
+        record = json.loads(line)
+        custom_id = record.get("custom_id", "")
+        pdf_path  = path_map.get(custom_id)
+
+        if record.get("error"):
+            print(f"  ✗  {custom_id}: {record['error']}")
+            failed += 1
+            continue
+
+        if pdf_path is None:
+            print(f"  ⚠  Unknown custom_id: {custom_id}")
+            continue
+
+        # Reconstruct a response-like object from the JSON result
+        body = record.get("response", {}).get("body", {})
+        pages_data = body.get("pages", [])
+
+        page_dir  = pdf_path.parent
+        page_name = pdf_path.stem
+        md_path   = page_dir / f"{page_name}.md"
+
+        chunks = []
+        for page in pages_data:
+            text   = page.get("markdown", "") or ""
+            images = page.get("images", []) or []
+            for idx, img in enumerate(images):
+                data = img.get("image_base64", "")
+                if data:
+                    if "," in data:
+                        data = data.split(",", 1)[1]
+                    fname = f"{page_name}_img_{idx}.png"
+                    (page_dir / fname).write_bytes(base64.b64decode(data))
+                    text = text.replace(img.get("id", str(idx)), fname)
+            chunks.append(text)
+
+        md_path.write_text("\n\n".join(chunks).strip(), encoding="utf-8")
+        print(f"  ✓  {page_dir.name}/{md_path.name}  ({md_path.stat().st_size:,} bytes)")
+        ok += 1
+
+    return ok, failed
+
+
+# ── CLI ───────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description="OCR page PDFs via Mistral")
+    ap.add_argument("--mode",    default="parallel",
+                    choices=["parallel", "batch", "batch-fetch"],
+                    help="parallel=immediate, batch=async submit, batch-fetch=fetch existing job")
+    ap.add_argument("--targets", default=DEFAULT_TARGETS)
+    ap.add_argument("--file",    default=None,   help="Single PDF path")
+    ap.add_argument("--limit",   type=int,        help="Max files to process")
+    ap.add_argument("--workers", type=int, default=10,
+                    help="Concurrent workers for parallel mode (default 10)")
+    ap.add_argument("--poll",    action="store_true",
+                    help="In batch mode: wait and download results when done")
+    ap.add_argument("--job-id",  default=None,
+                    help="In batch-fetch mode: the job_id to retrieve")
+    args = ap.parse_args()
+
+    # Strip stray whitespace and surrounding quotes — a common cause of 401s
+    # (e.g. Windows  set MISTRAL_API_KEY="sk-..."  keeps the quotes in the value).
+    api_key = (os.environ.get("MISTRAL_API_KEY") or "").strip().strip('"').strip("'").strip()
+    if not api_key:
+        raise SystemExit("  ✗  MISTRAL_API_KEY not set.")
+
+    from mistralai import Mistral
+    client = Mistral(api_key=api_key)
+
+    # Preflight — verify the key now, so a bad key fails immediately with a clear
+    # message instead of after every page returns 401.
+    try:
+        client.models.list()
+    except Exception as exc:
+        if "401" in str(exc) or "unauthor" in str(exc).lower():
+            raise SystemExit(
+                "  ✗  MISTRAL_API_KEY rejected by Mistral — 401 Unauthorized.\n"
+                "     The key is set but not valid. Check that:\n"
+                "       • it is a current key from https://console.mistral.ai/api-keys\n"
+                "       • it was set WITHOUT surrounding quotes or stray whitespace\n"
+                "         (Windows:  set MISTRAL_API_KEY=sk-...   — no quotes)\n"
+                f"       • the key in use is {len(api_key)} characters long\n"
+            )
+        print(f"  ⚠  Could not pre-verify the API key ({exc}) — continuing anyway.")
+
+    # batch-fetch mode — retrieve a previously submitted job
+    if args.mode == "batch-fetch":
+        if not args.job_id:
+            raise SystemExit("  ✗  --job-id required for batch-fetch mode.")
+        ok, failed = fetch_batch_results(client, args.job_id)
+        print(f"\n  Done.  ✓ {ok}  ✗ {failed}\n")
+        return
+
+    targets = load_targets(Path(args.targets), args.file, args.limit)
+    if not targets:
+        print("  ✅  Nothing to do — all targets already have .md files.")
+        return
+
+    print(f"\n  Mode    : {args.mode}")
+    print(f"  Targets : {len(targets)} pages")
+    if args.mode == "parallel":
+        print(f"  Workers : {args.workers}")
+    print()
+
+    start = time.time()
+
+    if args.mode == "parallel":
+        ok, failed = run_parallel(client, targets, args.workers)
+    else:
+        ok, failed = run_batch(client, targets, poll=args.poll)
+
+    elapsed = time.time() - start
+    print(f"\n{'─'*60}")
+    print(f"  Done in {elapsed:.1f}s   ✓ {ok}   ✗ {failed}")
+    print(f"{'─'*60}\n")
+
+
+if __name__ == "__main__":
+    main()

pipeline/split_pages.py

@@ -0,0 +1,137 @@
+"""
+split_pages.py
+──────────────
+Splits every multi-page PDF in a folder into single-page PDFs, creating the
+standard archive structure:
+
+    <out>/<doc-slug>/
+        page_0001/
+            page_0001.pdf
+        page_0002/
+            page_0002.pdf
+        ...
+
+By default the split folders are created *inside* the source folder, so:
+
+    FBI/reports/ufo1.pdf  →  FBI/reports/ufo1/page_0001/page_0001.pdf
+
+Safe to re-run — existing page folders are skipped unless --force is used.
+
+Install dependency
+------------------
+    pip install pypdf
+
+Usage
+-----
+    python split_pages.py --src D:/divided/FBI/reports
+    python split_pages.py --src D:/divided/FBI/reports --out D:/divided/FBI/reports
+    python split_pages.py --src D:/divided/FBI/reports --force   # re-split existing
+    python split_pages.py --file D:/divided/FBI/reports/ufo1.pdf  # single file
+"""
+
+import argparse
+from pathlib import Path
+
+def split_pdf(pdf_path: Path, out_root: Path, force: bool = False) -> tuple[int, int]:
+    """
+    Split a single PDF into per-page PDFs.
+    Returns (pages_written, pages_skipped).
+    """
+    try:
+        from pypdf import PdfReader, PdfWriter
+    except ImportError:
+        raise SystemExit("  ✗  pypdf not installed. Run: pip install pypdf")
+
+    doc_slug = pdf_path.stem                     # e.g. "ufo1"
+    doc_dir  = out_root / doc_slug
+
+    reader = PdfReader(str(pdf_path))
+    n = len(reader.pages)
+
+    written = skipped = 0
+
+    for i, page in enumerate(reader.pages, start=1):
+        page_name = f"page_{i:04d}"
+        page_dir  = doc_dir / page_name
+        page_file = page_dir / f"{page_name}.pdf"
+
+        if page_file.exists() and not force:
+            skipped += 1
+            continue
+
+        page_dir.mkdir(parents=True, exist_ok=True)
+
+        writer = PdfWriter()
+        writer.add_page(page)
+        with open(page_file, "wb") as f:
+            writer.write(f)
+
+        written += 1
+
+    return written, skipped
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Split multi-page PDFs into per-page archive structure")
+    ap.add_argument("--src",   default=None,
+                    help="Folder containing PDFs to split")
+    ap.add_argument("--out",   default=None,
+                    help="Output root (default: same as --src)")
+    ap.add_argument("--file",  default=None,
+                    help="Split a single PDF instead of a whole folder")
+    ap.add_argument("--force", action="store_true",
+                    help="Re-split even if page folders already exist")
+    args = ap.parse_args()
+
+    if not args.src and not args.file:
+        raise SystemExit("  ✗  Provide --src <folder> or --file <pdf>")
+
+    if args.file:
+        pdf_path = Path(args.file)
+        if not pdf_path.exists():
+            raise SystemExit(f"  ✗  File not found: {pdf_path}")
+        out_root = Path(args.out) if args.out else pdf_path.parent
+        print(f"\n  Splitting {pdf_path.name} → {out_root / pdf_path.stem}/\n")
+        w, s = split_pdf(pdf_path, out_root, args.force)
+        print(f"  ✓  {w} pages written,  {s} skipped\n")
+        return
+
+    src = Path(args.src)
+    if not src.exists():
+        raise SystemExit(f"  ✗  Source not found: {src}")
+
+    out_root = Path(args.out) if args.out else src
+    pdfs = sorted(src.glob("*.pdf"))
+
+    if not pdfs:
+        print(f"  No PDFs found in {src}")
+        return
+
+    print(f"\n  Source  : {src}")
+    print(f"  Output  : {out_root}")
+    print(f"  PDFs    : {len(pdfs)}\n")
+
+    total_written = total_skipped = total_errors = 0
+
+    for pdf_path in pdfs:
+        try:
+            w, s = split_pdf(pdf_path, out_root, args.force)
+            from pypdf import PdfReader
+            n = len(PdfReader(str(pdf_path)).pages)
+            label = "✓" if w > 0 else "–"
+            print(f"  {label}  {pdf_path.name:<30} {n:>4} pages  ({w} written, {s} skipped)")
+            total_written += w
+            total_skipped += s
+        except Exception as exc:
+            print(f"  ✗  {pdf_path.name}  ERROR: {exc}")
+            total_errors += 1
+
+    print(f"\n  {'─'*50}")
+    print(f"  Written : {total_written}")
+    print(f"  Skipped : {total_skipped}")
+    print(f"  Errors  : {total_errors}")
+    print(f"  {'─'*50}\n")
+
+
+if __name__ == "__main__":
+    main()

pipeline/stamp_pages.py

@@ -0,0 +1,211 @@
+"""
+stamp_pages.py
+──────────────
+Creates enriched copies of every page_XXXX.md, with a minimal YAML frontmatter
+block prepended. The originals in raw/ (or D:/divided) are never modified.
+
+Output mirrors the source folder hierarchy under --out-dir (default: pages_out/).
+
+Example
+-------
+    Source:  raw/DOD/mission-reports/greece/dow-uap-d33-.../page_0001/page_0001.md
+    Output:  pages_out/DOD/mission-reports/greece/dow-uap-d33-.../page_0001/page_0001.md
+
+    Frontmatter prepended to each copy:
+
+        ---
+        document: dow-uap-d33-mission-report-greece-october-2023
+        page: 1
+        of: 5
+        agency: DOD
+        subtype: mission-reports
+        region: greece
+        src_path: raw/DOD/mission-reports/greece/dow-uap-d33-.../page_0001/page_0001.md
+        ---
+
+Usage
+-----
+    python stamp_pages.py                                # uses raw/ as source
+    python stamp_pages.py --src D:/divided               # pre-reorganize flat layout
+    python stamp_pages.py --src raw --out pages_out
+    python stamp_pages.py --src raw --out pages_out --force   # overwrite existing copies
+"""
+
+import argparse
+import re
+from pathlib import Path
+from collections import defaultdict
+
+DEFAULT_SRC = "raw"
+DEFAULT_OUT = "pages_out"
+
+# Depth hints for extracting agency / subtype / region from the path
+# Works for both the organised raw/ tree AND the flat D:/divided layout.
+AGENCY_NAMES = {"DOD", "FBI", "NASA", "DOS", "NARA-CIA", "MISC"}
+
+PAGE_MD_RE = re.compile(r"^page_(\d+)\.md$", re.IGNORECASE)
+
+
+# ── path parsing ──────────────────────────────────────────────────────────────
+
+def parse_path_context(doc_dir: Path, src_root: Path) -> dict:
+    """
+    Extract agency / subtype / region from the path relative to src_root.
+
+    Organised tree:   src_root/DOD/mission-reports/greece/<doc-slug>/
+    Flat layout:      src_root/<doc-slug>/
+
+    Returns a dict with keys: agency, subtype, region (all may be None).
+    """
+    try:
+        rel_parts = doc_dir.relative_to(src_root).parts
+    except ValueError:
+        rel_parts = (doc_dir.name,)
+
+    agency  = None
+    subtype = None
+    region  = None
+
+    if len(rel_parts) >= 4:
+        # organised: agency / subtype / region / doc-slug
+        agency  = rel_parts[0] if rel_parts[0] in AGENCY_NAMES else None
+        subtype = rel_parts[1]
+        region  = rel_parts[2]
+    elif len(rel_parts) >= 2:
+        agency  = rel_parts[0] if rel_parts[0] in AGENCY_NAMES else None
+
+    return {"agency": agency, "subtype": subtype, "region": region}
+
+
+# ── discovery ─────────────────────────────────────────────────────────────────
+
+def collect_documents(src_root: Path) -> list[dict]:
+    """
+    Walk src_root and group page .md files by document folder.
+    Returns a list of dicts, one per document.
+    """
+    # Map: doc_dir_path → [page paths sorted]
+    doc_pages: dict[Path, list[Path]] = defaultdict(list)
+
+    for md_file in src_root.rglob("page_*.md"):
+        # The document folder is the grandparent: doc_dir/page_XXXX/page_XXXX.md
+        if md_file.parent.parent == src_root:
+            # flat layout: src_root/doc_dir/page_XXXX/page_XXXX.md
+            doc_dir = md_file.parent.parent / md_file.parent.name
+            # Actually: md_file.parent IS the page subfolder,
+            # its parent IS the doc dir
+        doc_dir = md_file.parent.parent
+        if doc_dir == src_root:
+            continue  # skip .md files sitting directly in root
+        doc_pages[doc_dir].append(md_file)
+
+    documents = []
+    for doc_dir, pages in sorted(doc_pages.items()):
+        pages_sorted = sorted(pages, key=lambda p: p.name)
+        ctx = parse_path_context(doc_dir, src_root)
+        documents.append({
+            "doc_dir":   doc_dir,
+            "doc_slug":  doc_dir.name,
+            "page_count": len(pages_sorted),
+            "pages":     pages_sorted,
+            **ctx,
+        })
+
+    return documents
+
+
+# ── frontmatter builder ───────────────────────────────────────────────────────
+
+def build_frontmatter(doc: dict, md_file: Path, page_index: int, src_root: Path) -> str:
+    try:
+        src_path = md_file.relative_to(src_root).as_posix()
+    except ValueError:
+        src_path = md_file.as_posix()
+
+    def _yaml_str(v) -> str:
+        return f'"{v}"' if v is not None else "null"
+
+    lines = [
+        "---",
+        f"document: {doc['doc_slug']}",
+        f"page: {page_index}",
+        f"of: {doc['page_count']}",
+        f"agency: {_yaml_str(doc['agency'])}",
+        f"subtype: {_yaml_str(doc['subtype'])}",
+        f"region: {_yaml_str(doc['region'])}",
+        f"src_path: {src_path}",
+        "---",
+        "",
+    ]
+    return "\n".join(lines)
+
+
+# ── stamping ──────────────────────────────────────────────────────────────────
+
+def stamp_documents(documents: list[dict], src_root: Path, out_root: Path,
+                    force: bool = False) -> tuple[int, int, int]:
+    written = skipped = errors = 0
+
+    for doc in documents:
+        for idx, md_file in enumerate(doc["pages"], start=1):
+            try:
+                rel = md_file.relative_to(src_root)
+            except ValueError:
+                rel = Path(doc["doc_slug"]) / md_file.parent.name / md_file.name
+
+            dst = out_root / rel
+
+            if dst.exists() and not force:
+                skipped += 1
+                continue
+
+            try:
+                original = md_file.read_text(encoding="utf-8", errors="replace")
+                frontmatter = build_frontmatter(doc, md_file, idx, src_root)
+                dst.parent.mkdir(parents=True, exist_ok=True)
+                dst.write_text(frontmatter + original, encoding="utf-8")
+                written += 1
+            except Exception as exc:
+                print(f"  ✗  {md_file}  ERROR: {exc}")
+                errors += 1
+
+    return written, skipped, errors
+
+
+# ── CLI ───────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Copy page .md files with YAML frontmatter into pages_out/"
+    )
+    ap.add_argument("--src",   default=DEFAULT_SRC,
+                    help="Source root (raw/ after reorganize, or D:/divided before)")
+    ap.add_argument("--out",   default=DEFAULT_OUT,
+                    help="Output root for stamped copies (default: pages_out/)")
+    ap.add_argument("--force", action="store_true",
+                    help="Overwrite existing output files")
+    args = ap.parse_args()
+
+    src_root = Path(args.src)
+    out_root = Path(args.out)
+
+    if not src_root.exists():
+        print(f"  ✗  Source not found: {src_root}")
+        return
+
+    print(f"\n  Scanning {src_root} …")
+    documents = collect_documents(src_root)
+
+    total_pages = sum(d["page_count"] for d in documents)
+    print(f"  Found {len(documents)} documents, {total_pages} pages total")
+    print(f"  Output → {out_root}/")
+    if not args.force:
+        print(f"  (existing files will be skipped — use --force to overwrite)\n")
+
+    written, skipped, errors = stamp_documents(documents, src_root, out_root, args.force)
+
+    print(f"\n  ✅  {written} written,  {skipped} skipped,  {errors} errors\n")
+
+
+if __name__ == "__main__":
+    main()

preprocessing.py

@@ -1,10 +1,12 @@
 """
 Document Preprocessing Pipeline — Streamlit page.
 
-Orchestrates the standalone preprocessing scripts (scrape → layout → OCR →
-concat → report extraction → table creation) that live in a separate pipeline
-directory (e.g. D:/divided). Each step is run as a subprocess with its output
-streamed live; the scripts themselves are not modified.
+Self-contained: the preprocessing scripts (scrape → layout → OCR → concat →
+report extraction → table creation) are vendored in-repo under ``pipeline/`` and
+run with the app's own interpreter, so this page no longer depends on an
+external script directory (the old ``D:/divided``) or a separate Python env.
+Only a *working directory* (where the PDFs and intermediate outputs live) is
+configurable. Each step is run as a subprocess with its output streamed live.
 
 The end product (extracted reports / parsed_reports.xlsx) is what the
 "UAP Feature Extraction" page consumes — this page is the stage that produces
@@ -12,6 +14,7 @@ that table from raw government PDFs.
 """
 
 import os
+import sys
 import shlex
 import subprocess
 from pathlib import Path
@@ -48,20 +51,26 @@ def _secret(*names):
     return ""
 
 
+# Pipeline scripts are vendored beside this page, so there is no external
+# script-directory dependency anymore.
+SCRIPTS_DIR = Path(__file__).resolve().parent / "pipeline"
+DEFAULT_WORKDIR = Path(__file__).resolve().parent / "pipeline_data"
+
 # ── Configuration ──────────────────────────────────────────────────────────
 with st.expander("⚙️ Configuration", expanded=True):
-    pipeline_dir = st.text_input(
-        "Pipeline directory (where the scripts and data live)",
-        value=st.session_state.get("prep_dir", "/mnt/d/divided"),
-        key="prep_dir",
-        help="On WSL the Windows D: drive is /mnt/d/divided; on native Windows "
-             "use D:/divided. Every command runs with this as the working dir.",
+    work_dir = st.text_input(
+        "Working directory (where the PDFs and intermediate outputs live)",
+        value=st.session_state.get("prep_workdir", str(DEFAULT_WORKDIR)),
+        key="prep_workdir",
+        help="Every step runs with this as its working directory — the scripts "
+             "read/write raw/, pages_out/, concat/, extracted/, … here. The "
+             "pipeline *scripts* themselves now ship with the app in pipeline/.",
     )
     py_exe = st.text_input(
-        "Python executable", value=st.session_state.get("prep_py", "python"),
+        "Python executable", value=st.session_state.get("prep_py", sys.executable),
         key="prep_py",
-        help="Must be a Python that has the pipeline dependencies installed "
-             "(see the dependencies note below).",
+        help="Defaults to the app's own interpreter. Override only if the "
+             "pipeline dependencies are installed in a different environment.",
     )
     k1, k2, k3 = st.columns(3)
     mistral_key = k1.text_input(
@@ -77,27 +86,40 @@ with st.expander("⚙️ Configuration", expanded=True):
         help="Optional — used by pdf_to_reports.py (NVIDIA NIM extraction).",
     )
 
-    with st.expander("Pipeline dependencies", expanded=False):
+    # Popover (not a nested expander — Streamlit forbids expander-in-expander).
+    with st.popover("Pipeline dependencies"):
         st.code(
             f"{py_exe or 'python'} -m pip install pyyaml google-genai pypdf "
-            "mistralai reportlab pdf2image pillow pymupdf",
+            "mistralai openai pdfplumber reportlab pdf2image pillow pymupdf requests",
             language="bash",
         )
+        st.caption(
+            "`pdf2image` also needs the system **poppler** binary "
+            "(`apt-get install poppler-utils`). These deps are also pinned in "
+            "the project's requirements, so the app's own interpreter has them."
+        )
 
-_pdir = Path(pipeline_dir) if pipeline_dir else None
-if _pdir is None or not _pdir.is_dir():
-    st.error(f"Pipeline directory not found: `{pipeline_dir}`")
-    st.stop()
-
-_present = [s for s in PIPELINE_SCRIPTS if (_pdir / s).is_file()]
-if len(_present) == len(PIPELINE_SCRIPTS):
-    st.success(f"All {len(PIPELINE_SCRIPTS)} pipeline scripts found in `{pipeline_dir}`.")
-else:
-    _missing = [s for s in PIPELINE_SCRIPTS if s not in _present]
-    st.warning(
-        f"{len(_present)}/{len(PIPELINE_SCRIPTS)} pipeline scripts found. "
-        f"Missing: {', '.join(_missing)}"
+# Sanity check on the vendored scripts (should always pass).
+_missing = [s for s in PIPELINE_SCRIPTS if not (SCRIPTS_DIR / s).is_file()]
+if _missing:
+    st.error(
+        f"{len(PIPELINE_SCRIPTS) - len(_missing)}/{len(PIPELINE_SCRIPTS)} "
+        f"bundled scripts found in `pipeline/`. Missing: {', '.join(_missing)}"
     )
+else:
+    st.success(f"All {len(PIPELINE_SCRIPTS)} pipeline scripts bundled in `pipeline/`.")
+
+# Resolve (and create) the working directory — no external folder required.
+_wdir = Path(work_dir).expanduser() if work_dir else None
+if _wdir is None:
+    st.error("Set a working directory above.")
+    st.stop()
+try:
+    _wdir.mkdir(parents=True, exist_ok=True)
+except Exception as e:
+    st.error(f"Could not create working directory `{work_dir}`: {e}")
+    st.stop()
+st.caption(f"Working directory: `{_wdir}`")
 
 with st.expander("📋 Recommended sequence", expanded=False):
     st.markdown(
@@ -133,6 +155,15 @@ def run_command(cmd_str: str, key: str) -> None:
     # Route a leading `python` / `python3` to the configured interpreter.
     if parts and parts[0] in ("python", "python3") and py_exe:
         parts[0] = py_exe
+    # Resolve a bare pipeline-script filename (e.g. `run_ocr.py`) to its vendored
+    # absolute path so the script runs from pipeline/ while its working dir stays
+    # the configured data directory.
+    if len(parts) >= 2 and parts[1].endswith(".py") and not any(
+        sep in parts[1] for sep in ("/", "\\")
+    ):
+        _candidate = SCRIPTS_DIR / parts[1]
+        if _candidate.is_file():
+            parts[1] = str(_candidate)
 
     env = os.environ.copy()
     if mistral_key:
@@ -147,7 +178,7 @@ def run_command(cmd_str: str, key: str) -> None:
         out_box = st.empty()
         try:
             proc = subprocess.Popen(
-                parts, cwd=str(_pdir), env=env,
+                parts, cwd=str(_wdir), env=env,
                 stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                 text=True, bufsize=1,
             )
@@ -185,7 +216,9 @@ def step(key: str, label: str, desc: str, default_cmd: str) -> None:
     if run:
         run_command(cmd, key)
     elif st.session_state.get(f"prep_log_{key}"):
-        with st.expander("Last run output", expanded=False):
+        # Popover, not an expander: step() runs inside a stage expander, and
+        # Streamlit forbids nesting an expander inside another expander.
+        with st.popover("Last run output"):
             st.code(st.session_state[f"prep_log_{key}"][-8000:], language="text")
     st.divider()
 
@@ -323,7 +356,7 @@ with st.expander("🔍 Audit tools", expanded=False):
     )
 
 st.caption(
-    "Each step shells out to the script in the pipeline directory — the scripts "
-    "themselves are unchanged. Long steps (OCR, extraction) keep this tab busy "
-    "until they finish; output streams live above."
+    "Each step runs a bundled `pipeline/` script with the working directory as "
+    "its cwd — no external script folder required. Long steps (OCR, extraction) "
+    "keep this tab busy until they finish; output streams live above."
 )

pyproject.toml

@@ -56,6 +56,7 @@ dependencies = [
     "mistralai<2",
     "reportlab>=4.5.1",
     "pdf2image>=1.17.0",
+    "pdfplumber>=0.11.0",
     "runpod-flash>=1.16.0",
     "psycopg[binary]>=3.3.4",
     "pgvector>=0.4.2",

rag_search.py

@@ -23,6 +23,7 @@ import matplotlib.colors as mcolors
 import textwrap
 import datamapplot
 import json
+from utils.data_processing import DataProcessor
 
 # st.set_option('deprecation.showPyplotGlobalUse', False)
 
@@ -385,7 +386,39 @@ def plot_date_distribution(df, date_column, figsize=(14, 8), color='orange', tit
         return fig
 
 
+def gemini_query(question, selected_data, gemini_key):
+    """Ask Gemini a question over a column's values (blank question → summarize).
+
+    Moved here from analyzing.py so all natural-language search/Q&A over a
+    dataset lives on the RAG search page.
+    """
+    import google.generativeai as genai
+
+    if not question:
+        question = "Summarize the following data in relevant bullet points"
+
+    filtered = [str(x) for x in selected_data if str(x) != '' and x is not None]
+    context = '\n'.join(filtered)
+
+    genai.configure(api_key=gemini_key)
+    query_model = genai.GenerativeModel('models/gemini-3.1-pro-preview')
+    response = query_model.generate_content(
+        [f"{question}\n Answer based on this context: {context}\n\n"]
+    )
+    return response.text
+
+
 def filter_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """Shared filtering UI — delegates to the DataProcessor so the RAG search
+    page uses the same filter system as the parsing and analysis pages. Binary
+    0/1 columns get a value picker; other numerics get range/percentile/std-dev.
+    """
+    return DataProcessor.filter_dataframe_enhanced(
+        df, enable_quick_filters=False, enable_advanced_filters=True
+    )
+
+
+def filter_dataframe_legacy(df: pd.DataFrame) -> pd.DataFrame:
     """
     Adds a UI on top of a dataframe to let viewers filter columns
 
@@ -2423,6 +2456,35 @@ with tab_rag:
                 except Exception as e:
                     st.error(f"An error occurred during reranking: {e}")
 
+            # ── Gemini Q&A (moved from analyzing.py) ───────────────────────────────
+            st.divider()
+            st.markdown("#### 🔮 Gemini Q&A")
+            st.caption(
+                "Ask Gemini a question over one column's values, or leave the "
+                "question empty to get a bullet-point summary."
+            )
+            gq1, gq2 = st.columns(2)
+            with gq1:
+                gemini_col = st.selectbox(
+                    "Which column do you want to query?",
+                    list(st.session_state['parsed_responses'].columns),
+                    key="gemini_qa_col",
+                )
+            with gq2:
+                gemini_key_input = st.text_input(
+                    "Gemini API Key", value=GEMINI_KEY, type="password",
+                    help="Defaults to GEMINI_KEY in secrets.", key="gemini_qa_key",
+                )
+            if gemini_col and gemini_key_input:
+                gemini_question = st.text_input(
+                    "Ask a question or leave empty for summarization",
+                    key="gemini_qa_question",
+                )
+                selected_column_data = st.session_state['parsed_responses'][gemini_col].tolist()
+                if st.button("Generate Query", key="gemini_qa_run") and selected_column_data:
+                    with st.status("Generating answer with Gemini…", expanded=True):
+                        st.write(gemini_query(gemini_question, selected_column_data, gemini_key_input))
+
         else:
             # ── Multi-DB Cross Similarity ─────────────────────────────────────────
             import plotly.graph_objects as go

requirements.txt

@@ -34,3 +34,27 @@ torch
 umap_learn
 xgboost
 tf-keras
+
+# Imported by the app but previously missing from requirements (present in the
+# uv env / uv.lock). stqdm, tqdm and psutil are hard top-level imports
+# (uap_analyzer.py, the pages, and utils/memory_manager.py via utils/__init__),
+# so the pages fail to import without them. psycopg (Neon semantic search) and
+# transformers (pdf_ocr LightOnOCR) are lazy feature deps.
+# Note: GPU acceleration (cupy/cuml/cuvs/rmm) and tiktoken are optional and
+# guarded by try/except, so they are intentionally NOT pinned here.
+stqdm
+tqdm
+psutil
+transformers
+psycopg[binary]
+
+# Document Preprocessing pipeline (pipeline/) — vendored scripts' deps.
+# (openai, Pillow, pymupdf, Requests are already listed above.)
+# Note: pdf2image also needs the system `poppler` binary (poppler-utils).
+pyyaml
+google-genai
+pypdf
+mistralai
+reportlab
+pdf2image
+pdfplumber

utils/data_processing.py

@@ -641,9 +641,34 @@ class DataProcessor:
         col_info = next((col for col in profile['numeric_columns'] if col['name'] == column), None)
         if not col_info:
             return df
-        
+
+        # Binary boolean columns (values ⊆ {0, 1}) — Range/Percentile/StdDev are
+        # meaningless here, so offer a 0/1 value picker instead of a slider.
+        _nonnull = df[column].dropna()
+        _uniq = set(_nonnull.unique())
+        if _uniq and _uniq.issubset({0, 1}):
+            bcol1, bcol2 = st.columns([1, 2])
+            with bcol1:
+                _opts = sorted(_uniq)
+                picked = st.multiselect(
+                    f"Values for {column}",
+                    _opts,
+                    default=_opts,
+                    format_func=lambda v: f"{int(v)} — {'true' if int(v) == 1 else 'false'}",
+                    key=f"binary_{column}",
+                )
+                df_filtered = df[df[column].isin(picked)]
+            with bcol2:
+                if len(df_filtered) > 0:
+                    _vc = df_filtered[column].value_counts().sort_index()
+                    _vc.index = _vc.index.map(
+                        lambda v: f"{int(v)} ({'true' if int(v) == 1 else 'false'})"
+                    )
+                    st.bar_chart(_vc)
+            return df_filtered
+
         col1, col2 = st.columns([1, 2])
-        
+
         with col1:
             filter_mode = st.radio(
                 f"Filter mode for {column}",

uv.lock

@@ -2958,6 +2958,33 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/62/33/61766ae033518957f877ab246f87ca30a85b778ebaad65b7f74fa7e52988/pdf2image-1.17.0-py3-none-any.whl", hash = "sha256:ecdd58d7afb810dffe21ef2b1bbc057ef434dabbac6c33778a38a3f7744a27e2", size = 11618, upload-time = "2024-01-07T20:32:59.957Z" },
 ]
 
+[[package]]
+name = "pdfminer-six"
+version = "20251230"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "charset-normalizer" },
+    { name = "cryptography" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/46/9a/d79d8fa6d47a0338846bb558b39b9963b8eb2dfedec61867c138c1b17eeb/pdfminer_six-20251230.tar.gz", hash = "sha256:e8f68a14c57e00c2d7276d26519ea64be1b48f91db1cdc776faa80528ca06c1e", size = 8511285, upload-time = "2025-12-30T15:49:13.104Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/65/d7/b288ea32deb752a09aab73c75e1e7572ab2a2b56c3124a5d1eb24c62ceb3/pdfminer_six-20251230-py3-none-any.whl", hash = "sha256:9ff2e3466a7dfc6de6fd779478850b6b7c2d9e9405aa2a5869376a822771f485", size = 6591909, upload-time = "2025-12-30T15:49:10.76Z" },
+]
+
+[[package]]
+name = "pdfplumber"
+version = "0.11.9"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pdfminer-six" },
+    { name = "pillow" },
+    { name = "pypdfium2" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/38/37/9ca3519e92a8434eb93be570b131476cc0a4e840bb39c62ddb7813a39d53/pdfplumber-0.11.9.tar.gz", hash = "sha256:481224b678b2bbdbf376e2c39bf914144eef7c3d301b4a28eebf0f7f6109d6dc", size = 102768, upload-time = "2026-01-05T08:10:29.072Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8b/c8/cdbc975f5b634e249cfa6597e37c50f3078412474f21c015e508bfbfe3c3/pdfplumber-0.11.9-py3-none-any.whl", hash = "sha256:33ec5580959ba524e9100138746e090879504c42955df1b8a997604dd326c443", size = 60045, upload-time = "2026-01-05T08:10:27.512Z" },
+]
+
 [[package]]
 name = "pexpect"
 version = "4.9.0"
@@ -3527,6 +3554,35 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/f4/fa/3597fb3fb28f40bf8291fdddbc4dcd51ce52fccaf1cbfca10ee9db09c69a/pypdf-6.12.0-py3-none-any.whl", hash = "sha256:a8e104ab950e655d0bcf5fa5e71317c06474bc707987335da44a210f73a8883b", size = 343457, upload-time = "2026-05-21T09:21:40.852Z" },
 ]
 
+[[package]]
+name = "pypdfium2"
+version = "5.9.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b0/98/6b44bf82ddb3c7a3e0249203772aad8981b4491d6227f182685f310faeff/pypdfium2-5.9.0.tar.gz", hash = "sha256:db1274bd27844db6fda17ef1dbcd0026c47d357437058d838e98060c0da9e92e", size = 272455, upload-time = "2026-06-01T15:43:38.08Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8b/d9/59630cb40e5f37e7712e6ea65e9cac633f4195e8b737bb3a46054aa63340/pypdfium2-5.9.0-py3-none-android_23_arm64_v8a.whl", hash = "sha256:91914837c4a4285b3e0724a84eca8079363db7475acbcab405933d1807785664", size = 3407817, upload-time = "2026-06-01T15:42:58.426Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/3d/e205708835a3730d5242652b6577ac06ad4721e6fcef77cc7c9d3541c686/pypdfium2-5.9.0-py3-none-android_23_armeabi_v7a.whl", hash = "sha256:90610d352f050b065b703f3a46602a852fce7dd8787300c8c7a472485b644d8f", size = 2862706, upload-time = "2026-06-01T15:43:00.581Z" },
+    { url = "https://files.pythonhosted.org/packages/01/47/e843fb895a891438b3f8c6d834fdc9c19183cd60980fc9325429d5c01505/pypdfium2-5.9.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6c4fbe3a7190b329c526358fb2855d797f7b74b5ecfc61d19657ef20bcebc108", size = 3489945, upload-time = "2026-06-01T15:43:02.542Z" },
+    { url = "https://files.pythonhosted.org/packages/35/bd/f5e6afd556f97fcaa2bec4cb04669664c166028fc2a059bd65447c852b43/pypdfium2-5.9.0-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:e93f0cf440169a3e445e6fbd06c803877e7418f3e13254287875cb67f208bb5a", size = 3674186, upload-time = "2026-06-01T15:43:04.496Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/4d/5286812216a292d51dfba8e7bff276da198f126508f8c2afa3630bf701dc/pypdfium2-5.9.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d902e03dff5efd51d93cd23d3e55bde53802fa6207bcd0e455239518859a069", size = 3669571, upload-time = "2026-06-01T15:43:06.571Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/c8/822db2c89baa13e6cee321d587fcd42df463a1fc2f7520b3f6814768bc71/pypdfium2-5.9.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6cf38d7ad3575947b82384869f2ab69ba345eb21d83118d25db3e83f967b0421", size = 3400412, upload-time = "2026-06-01T15:43:08.35Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/dd/7d09d8cdc28383df13f739a97ac4f1215a704a97a29506dee2bf89d8a350/pypdfium2-5.9.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:77f7479a28b43aa658735e3ce79cfd1fccd5d42db035c21bb4c26e8bd7e280e5", size = 3803326, upload-time = "2026-06-01T15:43:10.054Z" },
+    { url = "https://files.pythonhosted.org/packages/99/58/3f4e04ffe1ae62b437de07a96da672091cef62b619d0dc78207c1af442e6/pypdfium2-5.9.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:07e6ba170d577eabf60dbba701d051c64318dd029d38ca5907d83ae1a66fe779", size = 4216890, upload-time = "2026-06-01T15:43:11.701Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/f6/2dde4656750c4a6da99e1f070ca09d2b5a9d68186b42e711a1a3e5b1cb32/pypdfium2-5.9.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ce3a3dd23ec0adaa079d8be54565ba2aa2f6060e76a4989cd42dabc163d74ee", size = 3728830, upload-time = "2026-06-01T15:43:13.329Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/ca/f2ff8b9200c7dfc5aee85126edc856eb93c7056085da2454a75ef1e4dbc4/pypdfium2-5.9.0-py3-none-manylinux_2_27_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ae177938f5cf95a275db25a4f8553e2ebd954ecda2f9bc84848ba4b027ce438f", size = 4063322, upload-time = "2026-06-01T15:43:15.158Z" },
+    { url = "https://files.pythonhosted.org/packages/64/88/0b587de03c873c28adc59f6ac959de4032d3f3bc946094523b14a192d9c3/pypdfium2-5.9.0-py3-none-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ffe49edde2ac86f28ca7e58f565255a442f38a7508fff31b79a55f508f25a31e", size = 4039738, upload-time = "2026-06-01T15:43:16.975Z" },
+    { url = "https://files.pythonhosted.org/packages/83/4c/fa627f00a954e66465e929077cf43bd012595091fff82758d989486e7bdc/pypdfium2-5.9.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:b7b760bc2957ecf73c274af6ed8b168a2dcb328ac0a0f7ed6123cd92f6e7c9c9", size = 4997259, upload-time = "2026-06-01T15:43:18.915Z" },
+    { url = "https://files.pythonhosted.org/packages/32/f0/1736d80c5d12d931f74ca6b4213b006ee016ec33c6325fad870234cc240c/pypdfium2-5.9.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7cdc8e5d2f8d82add1e4f70a4fbe5f3b33c17f301ebde38c669fd7f78a7d032c", size = 4537061, upload-time = "2026-06-01T15:43:20.879Z" },
+    { url = "https://files.pythonhosted.org/packages/01/00/aa8890dfd385b2e7365034231987029cff15cc7eb4f06e8380da5608738a/pypdfium2-5.9.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:38a058dbd4929acaf0ab9171179eb86c24d8c6655a6836006796105a9f200890", size = 5232786, upload-time = "2026-06-01T15:43:23.73Z" },
+    { url = "https://files.pythonhosted.org/packages/65/12/8f45ea698781a0bed96ac4fbde440060790863273943461f0f160a993d52/pypdfium2-5.9.0-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:1894511a0e862e7ec5679f3a6dc43ac72c4ef92c7ca438357203913e8634a643", size = 5170121, upload-time = "2026-06-01T15:43:25.858Z" },
+    { url = "https://files.pythonhosted.org/packages/25/bd/9bb6ba375796e1de1d6c1af8d8303dd1781190346871c81a94d4e09eddfd/pypdfium2-5.9.0-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:040f5513b808db705d4878f57e2bf0b9dc6e6a0ad8d765c36cf62febf3933b28", size = 4663540, upload-time = "2026-06-01T15:43:27.677Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/4a/fd103bac197f22038bf70be1f7507ced7519f1214ea0dae137f37803ab8a/pypdfium2-5.9.0-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:f4991ae39bcea757552579bba4aebfaedb71c96dd35c2292f957b8ac9132f1ff", size = 5090619, upload-time = "2026-06-01T15:43:29.522Z" },
+    { url = "https://files.pythonhosted.org/packages/22/89/9531fa1e6e004fe522cdca0cd945cd6a9d7338e7125e6b0734d632d31fa6/pypdfium2-5.9.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:25ff1a5abd08ff9e87f62e5dac114ea95647c257fbbdbe029be8db71a6d7650b", size = 5050806, upload-time = "2026-06-01T15:43:31.322Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/d0/e53c68555ff128b2470e4a468762b320d9c6ae2c914decea3487d923982f/pypdfium2-5.9.0-py3-none-win32.whl", hash = "sha256:b0057dc8c2033584dc3e61afb5f23a135dab52b081695b435e27f9b7b074c605", size = 3670966, upload-time = "2026-06-01T15:43:32.991Z" },
+    { url = "https://files.pythonhosted.org/packages/da/0c/22e5fc035ad1594b44f265bc0a59ae34d377bc2ea74a92793e7a674bf96d/pypdfium2-5.9.0-py3-none-win_amd64.whl", hash = "sha256:06508c33b9772cf3878e48364c6e14c70cefc18a3abd6983ac9f338da9305275", size = 3800959, upload-time = "2026-06-01T15:43:34.536Z" },
+    { url = "https://files.pythonhosted.org/packages/11/e3/cf1711add7add22a17f7c7633cd795edc92f17ab7bdf1930493ae0f56680/pypdfium2-5.9.0-py3-none-win_arm64.whl", hash = "sha256:565ddfc98795fd2f6054b544ee9791d7b9032f9cf77a57891b6e501fafd0ef3f", size = 3585718, upload-time = "2026-06-01T15:43:36.521Z" },
+]
+
 [[package]]
 name = "pyproj"
 version = "3.7.1"
@@ -5021,6 +5077,7 @@ dependencies = [
     { name = "openpyxl" },
     { name = "pandas" },
     { name = "pdf2image" },
+    { name = "pdfplumber" },
     { name = "pgvector" },
     { name = "pillow" },
     { name = "plotly" },
@@ -5079,6 +5136,7 @@ requires-dist = [
     { name = "openpyxl", specifier = ">=3.1.5" },
     { name = "pandas", specifier = ">=2.3.1" },
     { name = "pdf2image", specifier = ">=1.17.0" },
+    { name = "pdfplumber", specifier = ">=0.11.0" },
     { name = "pgvector", specifier = ">=0.4.2" },
     { name = "pillow", specifier = ">=11.3.0" },
     { name = "plotly", specifier = ">=6.3.0" },