fix: Resolve slow page loads and self-ping failure

- Fix self-ping default port (7860→8000) to match local uvicorn
- Lazy-initialize all backend services to speed up startup
- Cap Canadian scraper to 40 DDG queries (was 240+) with endpoint-level caching
- Add 30s fetch timeout to frontend API wrapper with abort handling
- Reduce Opportunities max_per_source from 250 to 20
- Unify Dashboard API_BASE_URL with shared config and add timeout
- Downgrade tiptap extensions from v3 to v2 to fix peer dependency conflict

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

backend/app/main.py

@@ -80,12 +80,12 @@ import httpx
 
 async def self_ping():
     """Ping the service periodically to prevent sleeping."""
-    port = os.getenv("PORT", "7860")
+    # PORT env is set by HF Spaces (7860); locally uvicorn defaults to 8000
+    port = os.getenv("PORT", "8000")
     while True:
         try:
             await asyncio.sleep(600)  # Ping every 10 minutes
-            async with httpx.AsyncClient() as client:
-                # Use localhost for internal keep-alive
+            async with httpx.AsyncClient(timeout=10) as client:
                 await client.get(f"http://127.0.0.1:{port}/api/v1/health")
         except Exception as e:
             logger.warning(f"Self-ping failed: {e}")
@@ -151,17 +151,33 @@ async def log_requests(request: Request, call_next):
     logger.info(f"Response: {request.method} {request.url.path} -> {response.status_code}")
     return response
 
-# Initialize Services
-ingestion_service = IngestionService()
-shredding_service = ShreddingService()
-knowledge_engine = KnowledgeEngine()
-analysis_engine = AnalysisEngine()
-assembly_engine = AssemblyEngine()
-visual_engine = VisualEngine()
-nli_detector = NLIContradictionDetector(use_transformers=settings.LOAD_TRANSFORMER_MODELS)
-cross_encoder_reranker = CrossEncoderReranker(model_preference="balanced")
-
-# Services are initialized below
+# Lazy-initialized services — created on first use to speed up startup
+class _LazyService:
+    """Proxy that defers service initialization until first attribute access."""
+    def __init__(self, factory):
+        object.__setattr__(self, '_factory', factory)
+        object.__setattr__(self, '_instance', None)
+
+    def _ensure(self):
+        if object.__getattribute__(self, '_instance') is None:
+            factory = object.__getattribute__(self, '_factory')
+            object.__setattr__(self, '_instance', factory())
+        return object.__getattribute__(self, '_instance')
+
+    def __getattr__(self, name):
+        return getattr(self._ensure(), name)
+
+    def __call__(self, *args, **kwargs):
+        return self._ensure()(*args, **kwargs)
+
+ingestion_service = _LazyService(IngestionService)
+shredding_service = _LazyService(ShreddingService)
+knowledge_engine = _LazyService(KnowledgeEngine)
+analysis_engine = _LazyService(AnalysisEngine)
+assembly_engine = _LazyService(AssemblyEngine)
+visual_engine = _LazyService(VisualEngine)
+nli_detector = _LazyService(lambda: NLIContradictionDetector(use_transformers=settings.LOAD_TRANSFORMER_MODELS))
+cross_encoder_reranker = _LazyService(lambda: CrossEncoderReranker(model_preference="balanced"))
 
 
 # ==================== Request/Response Models ====================

backend/app/services/canadian_rfp_scraper.py

@@ -794,12 +794,14 @@ class CanadianRFPScraper:
         self._set_cached(cache_key, enriched)
         return enriched
 
-    def _search_canadian_tenders_bulk(self, max_results: int = 5000) -> List[Dict]:
+    def _search_canadian_tenders_bulk(self, max_results: int = 500) -> List[Dict]:
         """
-        High-volume bulk search for Canadian tenders using Heavenkeys service keywords.
-        Generates hundreds of targeted queries across all service areas, provinces,
-        and procurement portals to maximize the number of unique opportunities found.
+        Bulk search for Canadian tenders using Heavenkeys service keywords.
+        Uses targeted queries across service areas, provinces, and procurement portals.
+        Capped at a reasonable max to avoid excessive search engine queries.
         """
+        # Cap to avoid running hundreds of DDG queries (each takes 1-5 seconds)
+        max_results = min(max_results, 500)
         year = datetime.now().year
         prev_year = year - 1
 
@@ -934,7 +936,18 @@ class CanadianRFPScraper:
         queries.append(f"Canada government bid deadline {year}")
         queries.append(f"Canadian tender closing date {year}")
 
-        logger.info(f"Bulk search: {len(queries)} queries to execute for up to {max_results} results")
+        # Limit the number of queries to avoid extremely long scrape times.
+        # Each DDG query takes 1-5 seconds; 40 queries ≈ 40-200 seconds max.
+        import random
+        MAX_QUERIES = 40
+        if len(queries) > MAX_QUERIES:
+            # Always keep the first few broad queries, shuffle the rest
+            priority_queries = queries[:10]
+            remaining = queries[10:]
+            random.shuffle(remaining)
+            queries = priority_queries + remaining[:MAX_QUERIES - 10]
+
+        logger.info(f"Bulk search: running {len(queries)} queries (capped) for up to {max_results} results")
 
         all_results = []
         seen_urls = set()
@@ -1074,13 +1087,21 @@ class CanadianRFPScraper:
             max_per_source: Max results per source
             min_relevance: Minimum relevance score (0-100) to include
         """
+        # Check endpoint-level cache first (keyed on the full query params)
+        sorted_keys = sorted(source_keys) if source_keys else ["__all__"]
+        cache_key = f"all_sources_{'_'.join(sorted_keys)}_{max_per_source}_{min_relevance}"
+        cached = self._get_cached(cache_key)
+        if cached is not None:
+            logger.info(f"Returning {len(cached)} cached results for Canadian search")
+            return cached
+
         all_results = []
         seen_urls = set()
 
         # For "all sources" or large source lists, use bulk search approach
         if source_keys is None or len(source_keys) > 3:
             total_wanted = max_per_source * len(source_keys or CANADIAN_SOURCES)
-            bulk_results = self._search_canadian_tenders_bulk(max_results=max(total_wanted, 5000))
+            bulk_results = self._search_canadian_tenders_bulk(max_results=min(total_wanted, 500))
 
             for r in bulk_results:
                 url = r.get("url", "")
@@ -1112,6 +1133,10 @@ class CanadianRFPScraper:
         # Sort by relevance score descending
         all_results.sort(key=lambda x: -x.get("relevance_score", 0))
 
+        # Cache the results for subsequent requests
+        self._set_cached(cache_key, all_results)
+        logger.info(f"Cached {len(all_results)} results for Canadian search")
+
         return all_results
 
     def _enrich_result(self, result: Dict, source_key: str) -> Optional[Dict]:

package-lock.json

@@ -8,12 +8,12 @@
       "name": "contravaulthvnkz",
       "version": "0.0.0",
       "dependencies": {
-        "@tiptap/extension-color": "^3.17.1",
-        "@tiptap/extension-highlight": "^3.17.1",
-        "@tiptap/extension-placeholder": "^3.17.1",
-        "@tiptap/extension-text-align": "^3.17.1",
-        "@tiptap/extension-text-style": "^3.17.1",
-        "@tiptap/extension-underline": "^3.17.1",
+        "@tiptap/extension-color": "^2.10.2",
+        "@tiptap/extension-highlight": "^2.10.2",
+        "@tiptap/extension-placeholder": "^2.10.2",
+        "@tiptap/extension-text-align": "^2.10.2",
+        "@tiptap/extension-text-style": "^2.10.2",
+        "@tiptap/extension-underline": "^2.10.2",
         "@tiptap/pm": "^2.10.2",
         "@tiptap/react": "^2.10.2",
         "@tiptap/starter-kit": "^2.10.2",
@@ -2651,16 +2651,17 @@
       }
     },
     "node_modules/@tiptap/extension-color": {
-      "version": "3.17.1",
-      "resolved": "https://registry.npmjs.org/@tiptap/extension-color/-/extension-color-3.17.1.tgz",
-      "integrity": "sha512-QVlzpzGB+QcZgHgvFMRPckZutpkOLzNmZzhupNA7G2CMeeoCwZOJeZkyd3zvtAnRZkf7FrQBO123On30pJt7TA==",
+      "version": "2.27.2",
+      "resolved": "https://registry.npmjs.org/@tiptap/extension-color/-/extension-color-2.27.2.tgz",
+      "integrity": "sha512-sOKCP8/2V3sRM3FdWgMe1lFE5ewsWNCRafiVoujS1+TTHGCj4jw6W+LiumBUk7cRI8kXW/rqGWVC4RVdknYUCA==",
       "license": "MIT",
       "funding": {
         "type": "github",
         "url": "https://github.com/sponsors/ueberdosis"
       },
       "peerDependencies": {
-        "@tiptap/extension-text-style": "^3.17.1"
+        "@tiptap/core": "^2.7.0",
+        "@tiptap/extension-text-style": "^2.7.0"
       }
     },
     "node_modules/@tiptap/extension-document": {
@@ -2748,16 +2749,16 @@
       }
     },
     "node_modules/@tiptap/extension-highlight": {
-      "version": "3.17.1",
-      "resolved": "https://registry.npmjs.org/@tiptap/extension-highlight/-/extension-highlight-3.17.1.tgz",
-      "integrity": "sha512-I4EdBhPVzJd4ECMI9kP0NE4aG4Numd46jy/AqeZyf3dqVgCxRyAbSyU7oy4aXUnsojYODrKKG6+djm07KgOGoQ==",
+      "version": "2.27.2",
+      "resolved": "https://registry.npmjs.org/@tiptap/extension-highlight/-/extension-highlight-2.27.2.tgz",
+      "integrity": "sha512-ZjlktDdMjruMJFAVz0TbQf0v92Jqkc7Ri1iZJqBXuLid+r+GxUzl2CVAV7qq5yagkGQgvAG+WGsMk880HgR3MA==",
       "license": "MIT",
       "funding": {
         "type": "github",
         "url": "https://github.com/sponsors/ueberdosis"
       },
       "peerDependencies": {
-        "@tiptap/core": "^3.17.1"
+        "@tiptap/core": "^2.7.0"
       }
     },
     "node_modules/@tiptap/extension-history": {
@@ -2841,16 +2842,17 @@
       }
     },
     "node_modules/@tiptap/extension-placeholder": {
-      "version": "3.17.1",
-      "resolved": "https://registry.npmjs.org/@tiptap/extension-placeholder/-/extension-placeholder-3.17.1.tgz",
-      "integrity": "sha512-cE8Rij5/1t4KnWE7GaDewhBek9DKNB+97yrxyggMegILg6v195hOmOkRZkyfnFMYZoBDlrfSAtX9wBvbZBqIsg==",
+      "version": "2.27.2",
+      "resolved": "https://registry.npmjs.org/@tiptap/extension-placeholder/-/extension-placeholder-2.27.2.tgz",
+      "integrity": "sha512-IjsgSVYJRjpAKmIoapU0E2R4E2FPY3kpvU7/1i7PUYisylqejSJxmtJPGYw0FOMQY9oxnEEvfZHMBA610tqKpg==",
       "license": "MIT",
       "funding": {
         "type": "github",
         "url": "https://github.com/sponsors/ueberdosis"
       },
       "peerDependencies": {
-        "@tiptap/extensions": "^3.17.1"
+        "@tiptap/core": "^2.7.0",
+        "@tiptap/pm": "^2.7.0"
       }
     },
     "node_modules/@tiptap/extension-strike": {
@@ -2880,42 +2882,42 @@
       }
     },
     "node_modules/@tiptap/extension-text-align": {
-      "version": "3.17.1",
-      "resolved": "https://registry.npmjs.org/@tiptap/extension-text-align/-/extension-text-align-3.17.1.tgz",
-      "integrity": "sha512-CyJbZf823dqPZ/1zwRsza5pk/NQwFZwILdFYLVkV88I4+Ua9YVztI9kmwTB6dJyuKT4kTc7nhQHdaa957alGZQ==",
+      "version": "2.27.2",
+      "resolved": "https://registry.npmjs.org/@tiptap/extension-text-align/-/extension-text-align-2.27.2.tgz",
+      "integrity": "sha512-0Pyks6Hu+Q/+9+5/osoSv0SP6jIerdWMYbi13aaZLsJoj3lBj5WNaE11JtAwSFN5sx0IbqhDSlp1zkvRnzgZ8g==",
       "license": "MIT",
       "funding": {
         "type": "github",
         "url": "https://github.com/sponsors/ueberdosis"
       },
       "peerDependencies": {
-        "@tiptap/core": "^3.17.1"
+        "@tiptap/core": "^2.7.0"
       }
     },
     "node_modules/@tiptap/extension-text-style": {
-      "version": "3.17.1",
-      "resolved": "https://registry.npmjs.org/@tiptap/extension-text-style/-/extension-text-style-3.17.1.tgz",
-      "integrity": "sha512-TCMsEU92r/TfZkN8AKo/WIcJ1uNq/5NiZxloq5drF1HXxDDjwliurgwBw3OTGUlKQmer0N9hV0AAePY/G+5Akw==",
+      "version": "2.27.2",
+      "resolved": "https://registry.npmjs.org/@tiptap/extension-text-style/-/extension-text-style-2.27.2.tgz",
+      "integrity": "sha512-Omk+uxjJLyEY69KStpCw5fA9asvV+MGcAX2HOxyISDFoLaL49TMrNjhGAuz09P1L1b0KGXo4ml7Q3v/Lfy4WPA==",
       "license": "MIT",
       "funding": {
         "type": "github",
         "url": "https://github.com/sponsors/ueberdosis"
       },
       "peerDependencies": {
-        "@tiptap/core": "^3.17.1"
+        "@tiptap/core": "^2.7.0"
       }
     },
     "node_modules/@tiptap/extension-underline": {
-      "version": "3.17.1",
-      "resolved": "https://registry.npmjs.org/@tiptap/extension-underline/-/extension-underline-3.17.1.tgz",
-      "integrity": "sha512-6RdBzmkg6DYs0EqPyoqLGkISXzCnPqM/q3A6nh3EmFmORcIDfuNmcidvA6EImebK8KQGmtZKsRhQSnK4CNQ39g==",
+      "version": "2.27.2",
+      "resolved": "https://registry.npmjs.org/@tiptap/extension-underline/-/extension-underline-2.27.2.tgz",
+      "integrity": "sha512-gPOsbAcw1S07ezpAISwoO8f0RxpjcSH7VsHEFDVuXm4ODE32nhvSinvHQjv2icRLOXev+bnA7oIBu7Oy859gWQ==",
       "license": "MIT",
       "funding": {
         "type": "github",
         "url": "https://github.com/sponsors/ueberdosis"
       },
       "peerDependencies": {
-        "@tiptap/core": "^3.17.1"
+        "@tiptap/core": "^2.7.0"
       }
     },
     "node_modules/@tiptap/pm": {
@@ -3004,19 +3006,6 @@
         "url": "https://github.com/sponsors/ueberdosis"
       }
     },
-    "node_modules/@tiptap/starter-kit/node_modules/@tiptap/extension-text-style": {
-      "version": "2.27.2",
-      "resolved": "https://registry.npmjs.org/@tiptap/extension-text-style/-/extension-text-style-2.27.2.tgz",
-      "integrity": "sha512-Omk+uxjJLyEY69KStpCw5fA9asvV+MGcAX2HOxyISDFoLaL49TMrNjhGAuz09P1L1b0KGXo4ml7Q3v/Lfy4WPA==",
-      "license": "MIT",
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/ueberdosis"
-      },
-      "peerDependencies": {
-        "@tiptap/core": "^2.7.0"
-      }
-    },
     "node_modules/@types/aws-lambda": {
       "version": "8.10.160",
       "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.160.tgz",
@@ -3144,7 +3133,7 @@
       "version": "19.2.9",
       "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.9.tgz",
       "integrity": "sha512-Lpo8kgb/igvMIPeNV2rsYKTgaORYdO1XGVZ4Qz3akwOj0ySGYMPlQWa8BaLn0G63D1aSaAQ5ldR06wCpChQCjA==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT",
       "dependencies": {
         "csstype": "^3.2.2"
@@ -3590,7 +3579,7 @@
       "version": "3.2.3",
       "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz",
       "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT"
     },
     "node_modules/debug": {

package.json

@@ -14,12 +14,12 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@tiptap/extension-color": "^3.17.1",
-    "@tiptap/extension-highlight": "^3.17.1",
-    "@tiptap/extension-placeholder": "^3.17.1",
-    "@tiptap/extension-text-align": "^3.17.1",
-    "@tiptap/extension-text-style": "^3.17.1",
-    "@tiptap/extension-underline": "^3.17.1",
+    "@tiptap/extension-color": "^2.10.2",
+    "@tiptap/extension-highlight": "^2.10.2",
+    "@tiptap/extension-placeholder": "^2.10.2",
+    "@tiptap/extension-text-align": "^2.10.2",
+    "@tiptap/extension-text-style": "^2.10.2",
+    "@tiptap/extension-underline": "^2.10.2",
     "@tiptap/pm": "^2.10.2",
     "@tiptap/react": "^2.10.2",
     "@tiptap/starter-kit": "^2.10.2",

services/api.ts

@@ -10,37 +10,54 @@ const getApiBaseUrl = () => {
 
 export const API_BASE_URL = getApiBaseUrl();
 
-// Generic fetch wrapper with error handling
+// Default request timeout (30 seconds)
+const DEFAULT_TIMEOUT_MS = 30_000;
+
+// Generic fetch wrapper with error handling and timeout
 async function fetchAPI<T>(
   endpoint: string,
-  options: RequestInit = {},
+  options: RequestInit & { timeoutMs?: number } = {},
 ): Promise<T> {
   const url = `${API_BASE_URL}${endpoint}`;
+  const { timeoutMs = DEFAULT_TIMEOUT_MS, ...fetchOptions } = options;
 
   const defaultHeaders: HeadersInit = {
     "Content-Type": "application/json",
   };
 
-  const response = await fetch(url, {
-    ...options,
-    headers: {
-      ...defaultHeaders,
-      ...options.headers,
-    },
-  });
+  // AbortController for request timeout
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
 
-  if (!response.ok) {
-    const error = await response
-      .json()
-      .catch(() => ({ detail: "Unknown error" }));
-    // Handle Pydantic validation errors (array of objects) or string errors
-    const errorMessage = Array.isArray(error.detail)
-      ? error.detail.map((e: any) => e.msg || e.message || JSON.stringify(e)).join(', ')
-      : (error.detail || `HTTP ${response.status}`);
-    throw new Error(errorMessage);
-  }
+  try {
+    const response = await fetch(url, {
+      ...fetchOptions,
+      headers: {
+        ...defaultHeaders,
+        ...fetchOptions.headers,
+      },
+      signal: controller.signal,
+    });
 
-  return response.json();
+    if (!response.ok) {
+      const error = await response
+        .json()
+        .catch(() => ({ detail: "Unknown error" }));
+      const errorMessage = Array.isArray(error.detail)
+        ? error.detail.map((e: any) => e.msg || e.message || JSON.stringify(e)).join(', ')
+        : (error.detail || `HTTP ${response.status}`);
+      throw new Error(errorMessage);
+    }
+
+    return response.json();
+  } catch (err: any) {
+    if (err.name === 'AbortError') {
+      throw new Error(`Request timed out after ${Math.round(timeoutMs / 1000)}s — the server may be starting up. Please try again.`);
+    }
+    throw err;
+  } finally {
+    clearTimeout(timeoutId);
+  }
 }
 
 // File upload helper
@@ -1223,7 +1240,8 @@ export const canadianRFPAPI = {
     fetchAPI<CanadianSearchResponse>("/opportunities/canadian/search", {
       method: "POST",
       body: JSON.stringify(query),
-    }),
+      timeoutMs: 120_000, // 2 min — scraping multiple sources takes time
+    } as any),
 
   /** Scrape a single Canadian tender source */
   scrapeSource: (source: string, maxResults: number = 20) =>

views/Dashboard.tsx

@@ -2,7 +2,7 @@
 
 import React, { useState, useEffect, useMemo } from 'react';
 import { useRouter } from 'next/navigation';
-import api from '../services/api';
+import api, { API_BASE_URL } from '../services/api';
 
 interface DashboardStats {
   total_rfps: number;
@@ -29,8 +29,6 @@ interface OnboardingTask {
   action?: string;
 }
 
-const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000/api/v1';
-
 const Dashboard = () => {
   const router = useRouter();
   const [stats, setStats] = useState<DashboardStats>({
@@ -84,15 +82,19 @@ const Dashboard = () => {
   const fetchDashboardData = async () => {
     setLoading(true);
     setBackendError(null);
+
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 15_000);
+
     try {
       // Fetch stats
-      const statsRes = await fetch(`${API_BASE_URL}/stats`);
+      const statsRes = await fetch(`${API_BASE_URL}/stats`, { signal: controller.signal });
       if (statsRes.ok) {
         setStats(await statsRes.json());
       }
 
       // Fetch projects
-      const projectsRes = await fetch(`${API_BASE_URL}/projects`);
+      const projectsRes = await fetch(`${API_BASE_URL}/projects`, { signal: controller.signal });
       if (projectsRes.ok) {
         const data = await projectsRes.json();
         const mappedProjects = data.map((p: any) => ({
@@ -105,17 +107,19 @@ const Dashboard = () => {
           win_probability: 70,
           file_count: p.file_count || 0
         }));
-        console.log('Fetched and mapped projects:', mappedProjects);
         setProjects(mappedProjects);
       }
     } catch (err: any) {
       console.error('Failed to fetch dashboard data:', err);
-      if (err.message?.includes('Failed to fetch') || err.name === 'TypeError') {
+      if (err.name === 'AbortError') {
+        setBackendError('Backend request timed out. Please ensure the backend is running.');
+      } else if (err.message?.includes('Failed to fetch') || err.name === 'TypeError') {
         setBackendError('Cannot connect to backend server. Please start the backend.');
       } else {
         setBackendError(err.message || 'Failed to load dashboard data');
       }
     } finally {
+      clearTimeout(timeoutId);
       setLoading(false);
     }
   };

views/Opportunities.tsx

@@ -76,7 +76,7 @@ const Opportunities = () => {
     setCurrentPage(1);
     try {
       const result = await api.canadianRFP.search({
-        max_per_source: 250,
+        max_per_source: 20,
         min_relevance: 0,
         level: levelFilter !== 'all' ? levelFilter : undefined,
         province: provinceFilter !== 'all' ? provinceFilter : undefined,