Add FastMCP tool for web scrapping.

Add logging info for tools calling

README.md

@@ -22,9 +22,10 @@ Disable Smart App Control (SAC) on Windows
 3.Click the Smart App Control settings link.
 4.Select Off to disable the feature
 
-6) Setup OPENAI_API_KEY from terminal
+6) Setup OPENAI_API_KEY, LANGCHAIN_API_KEY, and SERPAPI_API_KEY from terminal
 set LANGCHAIN_API_KEY=<your-langsmith-key>
 set OPENAI_API_KEY=<your-api-key>
+sett SERPAPI_API_KEY=<your-serpapi-key>
 
 7) Running the Streamlit Application
 python -m streamlit run app.py

agent_workflows/crew_setup.py

@@ -1,13 +1,20 @@
+import json
 import os
+from langchain_community.utilities import SerpAPIWrapper
 from pydantic import BaseModel, Field as PydanticField  # Alias it to avoid conflicts
 from crewai.tools import tool
 from typing import List, Dict, Any
-from shared_config import verbose, logger
+from shared_config import verbose, logger, useSerpAPI
 from crewai import Agent, Task, Crew, Process, LLM, TaskOutput, TaskOutput
 from crewai.tools import tool
-from langchain_community.tools import DuckDuckGoSearchRun
+from langchain_community.utilities import DuckDuckGoSearchAPIWrapper
 from datetime import datetime
-import os
+import requests
+
+import asyncio
+from fastmcp.client import Client
+from fastmcp.client.transports import StreamableHttpTransport
+
 
 ###########################################################################################
 #Define a callback function to log CrewAI step task
@@ -34,25 +41,166 @@ def create_crew(config):
     # 1. DEFINE CUSTOM TOOLS & OUTPUT SCHEMAS
     # ==========================================
 
-    # Simple search tool for the Research Agent
-    @tool("web_search")
-    def web_search_tool(query: str) -> str:
-        """Search the web for current events, news, or factual information."""
-        ddg = DuckDuckGoSearchRun()
-        return ddg.invoke(query)
-
-  
-
     class ResearchEvidence(BaseModel):
         source_url: str = PydanticField(..., description="The source URL of the evidence.")
         claim: str = PydanticField(..., description="The factual finding or data point.")
         context: str = PydanticField(..., description="The context or time frame of this data point.")
-
+    
     class ResearchOutputSchema(BaseModel):
-        competitors: List[Dict[str, Any]] = PydanticField(..., description="List of competitors with pricing, features, and target audience.")
-        market_trends: List[str] = PydanticField(..., description="Key macro trends observed.")
+        competitors: List[Dict[str, Any]] = PydanticField(..., description="List of competitors with products pricing range, features, and target audience.")
+        market_trends: List[str] = PydanticField(..., description="Market Analysis & Growth Trends")
         evidence_citations: List[ResearchEvidence] = PydanticField(..., description="Valid source citations for facts collected.")
 
+    if useSerpAPI:
+        #Define SerpAPI Search Tool
+        @tool("Web Search Tool")
+        def web_search_tool(search_query: str) -> str:
+            """
+            Search Google for real-time information, news, and facts using SerpAPI.
+            Input should be a simple text search query string.
+            """
+            # Strip potential quotes added by the agent to clean up the query
+            search_query = search_query.strip("'\"")
+            if not search_query:
+                logger.info(f"Web Search tool used. Error: Empty query")
+                return "Error: Received an empty search query."
+
+            url = "https://serpapi.com/search"
+            payload = {
+                "q": search_query,
+                "api_key": os.getenv("SERPAPI_API_KEY"),
+                "engine": "google",
+                "num": 10
+            }
+
+            try:
+                response = requests.get(url, params=payload, timeout=10)
+                
+                # 1. Check HTTP Status (e.g., 403 Forbidden, 400 Bad Request)
+                if response.status_code != 200:
+                    logger.info(f"Web Search tool used. API Error (Status {response.status_code}): {response.text}")
+                    return f"API Error (Status {response.status_code}): {response.text}"
+                    
+                # 2. Safely parse JSON
+                try:
+                    data = response.json()
+                except ValueError:
+                    logger.info(f"Web Search tool used. Error: Server returned non-JSON data. Raw Response: {response.text[:200]}")
+                    return f"Error: Server returned non-JSON data. Raw Response: {response.text[:200]}"
+                
+                # 3. Handle backend errors sent inside valid JSON
+                if "error" in data:
+                    logger.info(f"Web Search tool used. SerpAPI Error: {data['error']}")
+                    return f"SerpAPI Error: {data['error']}"
+                    
+                organic_results = data.get("organic_results", [])
+                if not organic_results:
+                    logger.info(f"Web Search tool used. No organic results found for query: '{search_query}'")
+                    return f"No organic results found for query: '{search_query}'"
+                    
+                results_summary = []
+                for result in organic_results:
+                    title = result.get("title")
+                    link = result.get("link")
+                    snippet = result.get("snippet")
+                    results_summary.append(f"Title: {title}\nLink: {link}\nSnippet: {snippet}\n---")
+                    
+                strResult = "\n".join(results_summary)
+                logger.info(f"""Web Search tool used.\nQuery: {search_query}\nResult:\n{strResult}""")
+                return strResult
+                
+            except requests.exceptions.RequestException as e:
+                logger.info(f"Web Search tool used. Network connection error occurred: {str(e)}")
+                return f"Network connection error occurred: {str(e)}"
+    else:   
+        #Define DuckDuckGo Search Tool
+        @tool("Web Search Tool")
+        def web_search_tool(query: str) -> str:
+            """
+            Search the web for real-time information, news, and facts using DuckDuckGo and return the structured Title, Link, and Snippet for each result.
+                
+            Args:
+                query (str): The search query text.
+                    
+            Returns:
+                str: Text containing Title, Link, and Snippet fields for each search result.
+            """
+            # Max results set to 5 for optimal context length
+            api_wrapper = DuckDuckGoSearchAPIWrapper()
+                
+            try:
+                # Fetch structured list of dictionaries
+                results = api_wrapper.results(query, 15)
+                if not results:
+                    logger.info(f"""Web Search tool used. No search results found for""")
+                    return f"No search results found for: '{query}'"
+                        
+                output = []
+                for i, res in enumerate(results, start=1):
+                    item_str = (
+                        f"Result #{i}\n"
+                        f"Title: {res.get('title', 'N/A')}\n"
+                        f"Link: {res.get('link', 'N/A')}\n"
+                        f"Snippet: {res.get('snippet', 'N/A')}\n"
+                        f"{'-'*40}"
+                    )
+                    output.append(item_str)
+        
+                logger.info(f"""Web Search tool used.\nQuery: {query}\nResult:\n{"\n".join(output)}""")   
+                return "\n".join(output)
+        
+            except Exception as e:
+                logger.info(f"""Web Search tool used. Error executing search: {str(e)}""")   
+                return f"Error executing search: {str(e)}"
+
+    @tool("FastMCP Batch Web Content extraction Tool")
+    def fastmcp_batch_web_content_extraction_tool(urls: list[str]) -> str:
+        """
+        Connects directly to the FastMCP HTTP server to scrape and extract text from 
+        a list of multiple URLs simultaneously in parallel.
+        """
+        async def call_fast_mcp():
+            # Initialize client using FastMCP's precise matching transport protocol
+            transport = StreamableHttpTransport("http://localhost:8000/mcp/")
+            
+            async with Client(transport) as client:
+                # call_tool abstracts away deep JSON-RPC structures perfectly
+                result = await client.call_tool("batch_web_content_extraction", {"urls": urls})
+                return result
+
+        try:
+            # Spin up clean execution loop for this worker thread
+            mcp_response = asyncio.run(call_fast_mcp())
+            
+            # FastMCP Client response objects expose string contents naturally via .content
+            # Ensure we safely extract the raw text string from the MCP content blocks
+            if hasattr(mcp_response, "content") and mcp_response.content:
+                # The text is inside the first content block object
+                raw_content = mcp_response.content[0].text
+            else:
+                # Fallback to string casting if it's already a plain string
+                raw_content = str(mcp_response)
+
+            # Parse JSON and pretty-format the string back to the CrewAI Agent
+            try:
+                parsed_data = json.loads(raw_content)
+                formatted_output = []
+                for item in parsed_data:
+                    formatted_output.append(f"=== SOURCE URL: {item['url']} ===")
+                    formatted_output.append(f"STATUS: {item['status']}")
+                    formatted_output.append(f"CONTENT:\n{item['content']}\n")
+                    formatted_output.append("=" * 40 + "\n")
+
+                logger.info(f"""FastMCP Batch Web Content extraction tool used.\nURLs:\n{"\n".join(urls)}\nResult:\n{"\n".join(formatted_output)}""")    
+                return "\n".join(formatted_output)
+            except json.JSONDecodeError:
+                logger.info(f"""FastMCP Batch Web Content extraction tool used. Error in JSON formatting so raw data returned:\nURLs:\n{"\n".join(urls)}\nResult:\n{raw_content}""")    
+                return raw_content
+                
+        except Exception as e:
+            logger.info(f"""FastMCP Batch Web Content extraction tool used. Error executing the tool and no data returned.""") 
+            return f"Error executing Parallel FastMCP tool over Streamable HTTP: {str(e)}"
+
 
     # ---------------------------------------------------------
     #Create llm instance
@@ -93,13 +241,14 @@ def create_crew(config):
     # 2. researcher
     researcher_agent = Agent(
         role="Market Research Specialist",
-        goal="Gather raw competitive intelligence, target audience signals, and pricing tiers with structural citations.",
-        backstory="An elite OSINT researcher who tracks digital signals. You ignore marketing fluff and extract verifiable numbers, product capabilities, and structural URLs.",
-        #role='Research Agent',
-        #goal='Gather comprehensive market data, identify industry trends, and map out the competitive landscape.',
-        #backstory='You are an elite Market Research Analyst. You are an expert at mining deep web data, extracting customer pain points, and finding precise statistical facts from modern industry reports.',
+        #goal="Gather raw competitive intelligence, target audience signals, and pricing tiers with structural citations.",
+        #backstory="An elite OSINT researcher who tracks digital signals. You ignore marketing fluff and extract verifiable numbers, product capabilities, and structural URLs.",
+        goal ="Gather search market data and extract multiple deep resources simultaneously to speed up workflows. Identify industry trends, pricing tiers, core features, and target customer profiles, while ensuring all data is verifiable with source URLs.",
+        #goal='Gather search market data and extract multiple deep resources simultaneously to speed up workflows. Identify industry trends, and map out the competitive landscape.',
+        backstory="""You are an elite Market Research Specialist. You are an expert at mining deep web data and finding precise statistical facts from modern industry reports.
+        You use your Web Search tool to find matching links, and then pass all relevant URLs at the exact same time into your batch web content extraction tool for immediate, parallel analysis.""",
         verbose=verbose,
-        tools=[web_search_tool],
+        tools=[web_search_tool, fastmcp_batch_web_content_extraction_tool],
         use_system_prompt=False, 
         llm=llm
     )
@@ -136,14 +285,24 @@ def create_crew(config):
 
     # Task 1: Research Agent gathers raw intelligence
     research_task = Task(
-        description=(
-            "Search the web for the top 3-5 players in the target market: '{query}'. "
-            "Extract pricing tiers, core feature modules, and target customer profiles. "
-            "Every single metric or claim MUST have a URL source tracked inside the JSON structure. "
-            "Do not fabricate any data and do not make assumptions. Only include verifiable facts using working URLs."
-            "If you cannot find a specific metric, leave it blank and do not fabricate data."
+        # description=(
+        #     "Search the web for the top 3-5 players in the target market: '{query}'. "
+        #     "Extract pricing tiers, core feature modules, and target customer profiles. "
+        #     "Every single metric or claim MUST have a URL source tracked inside the JSON structure. "
+        #     "Do not fabricate any data and do not make assumptions. Only include verifiable facts using working URLs."
+        #     "If you cannot find a specific metric, leave it blank and do not fabricate data."
+        description=("""
+            1. Using your Web Search Tool, search websites for the compagnies related the target market: '{query}'. When sending the search query, do not add any information about the year. It should be the most recent data.
+            2. Review the result snippets and keep URLs that match the target market. Make sure URLs are not borken or do not redirect to "Not Found" website
+            3. Collect those URLs into a list and feed them into the 'FastMCP Batch Web Content Extraction Tool' in a single call.
+            4. Review the returned parallel scrape data and extract following information:
+                a. List of compagnies with products pricing range, keys features, and target audience. Do not list more than one time a specific compagny. 
+                b. Market Analysis & Growth Trends
+                c. Evidence citations which include URL, the factual finding or data point and the context or time frame of this data point.
+            5. Every single metric or claim MUST have a URL source tracked inside the JSON structure.
+            """
     ),
-        expected_output="A structured JSON file matching the schema with validated competitive rows and strict citation tracking.",
+        expected_output="A structured JSON file matching the schema and strict citation tracking.",
         output_json=ResearchOutputSchema,
         agent=researcher_agent
     )
@@ -152,10 +311,11 @@ def create_crew(config):
     analysis_task = Task(
         description=(
             "Review the structured JSON output provided by the Research Agent. "
-            "Construct a detailed markdown competitive landscape table and a pricing matrix. "
+            "Construct a detailed markdown competitive landscape table and a pricing matrix. Include source URL in the table."
             "Perform a comprehensive SWOT, 4P, and 7P analysis based strictly on the evidence compiled."
         ),
         expected_output="A deep-dive analytical report featuring clean markdown tables, a SWOT quadrant, and an exhaustive 4P/7P matrix analysis.",
+        context=[research_task],
         agent=analyst_agent
     )
 
@@ -168,15 +328,32 @@ def create_crew(config):
             "Identify high-ROI acquisition channels and detail a 30-60-90 day milestone launch plan."
         ),
         expected_output="An actionable, comprehensive market entry strategy blueprint divided into logical execution phases.",
+        context=[analysis_task],
         agent=strategist_agent
     )
 
     # Task 4: Head Planner synthesizes, formats, and exports
     compilation_task = Task(
         description=(
-            "Synthesize the outputs from the Research, Analyst, and Strategy agents into a seamless, high-caliber Master GTM Report. "
-            "Include all research data tables, citations, SWOT/7P strategic matrices, and execution timelines. "
-            "Include a title that cleary identifies the target market and a subtitle that highlights the core value proposition. "
+            """Synthesize the outputs from the Research, Analyst, and Strategy agents into a seamless, high-caliber Master GTM Report.
+            Include all research data tables, citations, SWOT/7P strategic matrices, and execution timelines.
+            Include a title and small summary that cleary identifies the target market and highlights the core value proposition. Include the input query '{query}' for reference at the end of the summary.
+            Include a table of contents.
+            Include a conclusion and Evidence Citations section at the end.
+            Ensure the document is formatted for executive-level presentation.
+            [OUTPUT FORMAT SPECIFICATION]:
+            Format your entire document exclusively in clean, standard Markdown.
+
+            Strict Structural Formatting Rules:
+            1. DOCUMENT TITLE & SUMMARY: Start immediately with the document title as a level-1 heading (# Title) and your summary sections. Do NOT include numbers on these meta-sections.
+            2. MAIN SECTIONS & SUBSECTIONS: You must explicitly number all research content headings manually (e.g., "# 1. Competitive Landscape Analysis", "## 1.1 Competitive Landscape Table", "# 2. Go-To-Market Strategy"). 
+            3. TABLE OF CONTENTS: Directly after your summary block, include a manually written, clean Markdown Table of Contents listing your explicitly numbered headings. Example:
+            ### Table of Contents
+            - [1. Competitive Landscape Analysis](#1-competitive-landscape-analysis)
+                - [1.1 Competitive Landscape Table](#11-competitive-landscape-table)
+                - [1.2 Pricing Matrix](#12-pricing-matrix)
+            - [2. Go-To-Market (GTM) Strategy Blueprint](#2-go-to-market-gtm-strategy-blueprint)
+            4. NO MARKDOWN TOC MARKERS: Do NOT output the text string "[TOC]". Write out the table of contents yourself using bullet points and standard markdown links."""
         ),
         expected_output="Final master GTM document.",
         context=[research_task, analysis_task, strategy_task],

app.py

@@ -4,7 +4,7 @@ from shared_config import AgenticAIConfig, create_logger, logger
 import streamlit as st
 from dotenv import load_dotenv
 from agent_workflows.orchestration import runAgenticWorkflow
-from utilities.pdf_tools import export_to_pdf
+from utilities.pdf_tools_v2 import export_to_pdf
 from datetime import datetime
 
 
@@ -37,7 +37,7 @@ if "chat_history" not in st.session_state:
 #llm_model = "gpt-5.5"  #Output~ 30 US$ / 1M tokens
 MODEL_REGISTRY={
     "OpenAI":["gpt-4o-mini","gpt-5.4-nano","gpt-5-mini","gpt-4.1","gpt-4o","gpt-3.5-turbo"],
-    "Ollama":["llama3.1","gpt-oss:20b","gemma4:e4b","qwen2.5-coder:14b","mistral-small3.2"],
+    "Ollama":["gemma4:e4b","llama3.1","gpt-oss:20b","qwen2.5-coder:14b","mistral-small3.2"],
     }
 
 #7- Creating Sidebar
@@ -67,7 +67,7 @@ with st.sidebar:
     config.max_tokens=st.slider(
         "Max Tokens",
         min_value=100,
-        max_value=4000,
+        max_value=30000,
         value=config.max_tokens,
         step=100
     )
@@ -128,8 +128,8 @@ if st.button("Generate Response", type="primary"):
             formatted_DateTime = now.strftime("%Y-%m-%d_%H%M%S")
             pdfpath = f"""output/{pdf_fiename_prefix}_{formatted_DateTime}.pdf"""
             export_to_pdf(
-                inputText=config.response,
-                filename=pdfpath)
+                llm_response=config.response,
+                output_filename=pdfpath)
 
             logger.info("PDF Exported Successfully at: "+pdfpath)
 

requirements.txt

@@ -11,10 +11,11 @@ langsmith
 crewai
 opentelemetry-instrumentation-crewai
 opentelemetry-instrumentation-openai
-#fastmcp
+fastmcp
 
 #web search
 ddgs
+google-search-results
 
 # UI
 streamlit

shared_config.py

@@ -15,6 +15,10 @@ verbose = True  #Set to True to see CrewAI process information in the console
 printLogToConsole = False  #Set to True to save and print log information to console, else set to False to only save log information to file
 ###########################################################################################
 
+###########################################################################################
+useSerpAPI = True  #Set to True to use SerpAPI google engine for web search (requires an API Key). If set to False, DuckDuckGo will be used
+###########################################################################################
+
 ###########################################################################################
 #Define logger parameter
 ###########################################################################################
@@ -39,7 +43,7 @@ class AgenticAIConfig:
     provider:str = "OpenAI"
     model: str = "gpt-4o"
     temperature: float = 0.4
-    max_tokens:int = 3000
+    max_tokens:int = 5000
     
     # User Input
     user_query:str=""

utilities/logger.py

@@ -1,4 +1,5 @@
 import logging
+import sys
 from xmlrpc.client import boolean
 
 
@@ -12,13 +13,15 @@ def simple_logger(log_relative_path = str, printToConsole = boolean, logger_name
 
     formatter = logging.Formatter(fmt="%(asctime)s [%(levelname)s] %(message)s",
                                    datefmt="%Y-%m-%d %H:%M:%S")
-    file_handler = logging.FileHandler(log_relative_path)
+    file_handler = logging.FileHandler(log_relative_path, encoding="utf-8")
     file_handler.setFormatter(formatter)
     file_handler.setLevel(logging.INFO)
     logger.addHandler(file_handler)
 
     if printToConsole:
-        console_handler = logging.StreamHandler()
+        # 2. Force StreamHandler to use sys.stdout with utf-8 encoding
+        sys.stdout.reconfigure(encoding='utf-8')
+        console_handler = logging.StreamHandler(sys.stdout)
         console_handler.setFormatter(formatter)
         console_handler.setLevel(logging.INFO) 
         logger.addHandler(console_handler)

utilities/pdf_tools.py

@@ -1,40 +1,168 @@
+# html_template = """
+# <!DOCTYPE html>
+# <html>
+# <head>
+#     <meta charset="utf-8">
+#     <style>
+#         /* Existing styles... */
+
+#         /* Styles for Links */
+#         a {{
+#             color: #0000ff; /* Sets the link color to blue */
+#             text-decoration: underline; /* Ensures the underline is visible */
+#         }}
+#         a:visited {{
+#             color: #0000ff; /* Keeps the link blue even after it is clicked */
+#         }}
+
+#         /* Styles for clean Table Lines */
+#         table {{
+#             width: 100%;
+#             border-collapse: collapse; /* Merges adjacent cell borders into a single line */
+#             margin: 20px 0;
+#             font-size: 11pt;
+#         }}
+#         th, td {{
+#             border: 1px solid #bdc3c7; /* Draws the actual grid lines */
+#             padding: 10px;
+#             text-align: left;
+#         }}
+#         th {{
+#             background-color: #f2f4f4; /* Optional: adds a neat background header color */
+#             font-weight: bold;
+#             color: #2c3e50;
+#         }}
+#         tr:nth-child(even) {{
+#             background-color: #f9f9f9; /* Optional: adds zebra striping to rows */
+#         }}
+
+#         /* Table of Contents Styles */
+#         .toc-list {{
+#             list-style-type: decimal;
+#             padding-left: 20px;
+#             margin: 10px 0;
+#         }}
+#         .toc-list li {{
+#             margin-bottom: 6px;
+#             font-size: 11pt;
+#         }}
+#         /* Styles the nested sub-sections */
+#         .toc-list ul {{
+#             list-style-type: none; /* Removes numbers/bullets from sub-items */
+#             padding-left: 20px;    /* Creates the indent */
+#             margin: 4px 0;
+#         }}
+#         .toc-list ul li {{
+#             position: relative;
+#             font-size: 10.5pt;
+#             color: #555;
+#         }}
+#         /* Adds a clean dash prefix (-) to the sub-items */
+#         .toc-list ul li::before {{
+#             content: "- ";
+#             position: absolute;
+#             left: -12px;
+#         }}
+#     </style>
+# </head>
+# <body>
+#     {content}
+# </body>
+# </html>
+# """
+
 html_template = """
 <!DOCTYPE html>
 <html>
 <head>
     <meta charset="utf-8">
     <style>
-        /* Existing styles... */
-
-        /* Styles for Links */
-        a {{
-            color: #0000ff; /* Sets the link color to blue */
-            text-decoration: underline; /* Ensures the underline is visible */
-        }}
-        a:visited {{
-            color: #0000ff; /* Keeps the link blue even after it is clicked */
+        /* --- WeasyPrint Page Parameters --- */
+        @page {{
+            size: A4;
+            margin: 20mm;
+            @bottom-right {{
+                content: counter(page) " / " counter(pages);
+                font-family: Arial, sans-serif;
+                font-size: 9pt;
+                color: #7f8c8d;
+            }}
         }}
 
-        /* Styles for clean Table Lines */
-        table {{
-            width: 100%;
-            border-collapse: collapse; /* Merges adjacent cell borders into a single line */
-            margin: 20px 0;
+        body {{
+            font-family: Arial, sans-serif;
+            color: #2c3e50;
+            line-height: 1.6;
             font-size: 11pt;
         }}
-        th, td {{
-            border: 1px solid #bdc3c7; /* Draws the actual grid lines */
-            padding: 10px;
-            text-align: left;
+
+        /* --- Table of Contents Component --- */
+        .toc-box {{
+            margin: 25px 0;
+            padding: 20px;
+            background-color: #fcfcfc;
+            border: 1px solid #eaeaea;
+            border-radius: 4px;
         }}
-        th {{
-            background-color: #f2f4f4; /* Optional: adds a neat background header color */
+        
+        /* Direct Parents (1., 2., 3.) */
+        .toc-box > ol {{
+            list-style-type: decimal;
+            padding-left: 20px;
+            margin: 0;
+        }}
+        
+        .toc-box > ol > li {{
+            font-size: 12pt;
             font-weight: bold;
+            margin-top: 12px;
             color: #2c3e50;
         }}
-        tr:nth-child(even) {{
-            background-color: #f9f9f9; /* Optional: adds zebra striping to rows */
+
+        /* Targets sub-items that Python and Markdown nested inside the <li> tag */
+        .toc-box ol li ul {{
+            list-style-type: none; /* Destroys standard browser bullet points */
+            padding-left: 20px;    /* Indents sub-items perfectly to the right */
+            margin: 6px 0 0 0;
         }}
+
+        .toc-box ol li ul li {{
+            font-size: 11pt;
+            font-weight: normal;   /* Removes parent bold styles from sub-items */
+            color: #555555;
+            margin-bottom: 5px;
+            position: relative;
+        }}
+
+        /* Injects a clean layout hyphen prefix natively via CSS */
+        .toc-box ol li ul li::before {{
+            content: "-  ";
+            font-weight: bold;
+            color: #7f8c8d;
+        }}
+
+        /* --- Global Structural Typography --- */
+        h1, h2, h3, h4, h5, h6 {{
+            color: #2c3e50;
+            font-weight: bold;
+            page-break-after: avoid;
+            break-after: avoid;
+        }}
+
+        table {{
+            width: 100%;
+            border-collapse: collapse;
+            margin: 24px 0;
+            font-size: 10.5pt;
+            page-break-inside: auto;
+            break-inside: auto;
+        }}
+        tr {{ page-break-inside: avoid; break-inside: avoid; }}
+        th, td {{ border: 1px solid #bdc3c7; padding: 10px; text-align: left; }}
+        th {{ background-color: #f2f4f4; font-weight: bold; }}
+        
+        a {{ color: #2980b9; text-decoration: none; }}
+        .page-break {{ page-break-before: always; }}
     </style>
 </head>
 <body>
@@ -43,18 +171,104 @@ html_template = """
 </html>
 """
 
+import re
 import markdown
+from html.parser import HTMLParser
 from weasyprint import HTML
 
-def export_to_pdf(inputText, filename):
+class TOCHtmlRestructurer(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.output = []
+        self.level_stack = []      # Tracks nesting hierarchy
+        self.parent_count = 0
+        self.child_count = 0
+        self.capture_text = False
+        self.current_item_text = ""
+        self.current_href = None   # Tracks anchor references dynamically
 
-    # Step 1: Convert Markdown components to HTML chunks.
-    # The 'tables' and 'fenced_code' extensions keep standard LLM formatting neat.
+    def handle_starttag(self, tag, attrs):
+        if tag in ['ol', 'ul']:
+            if not self.level_stack:
+                self.level_stack.append('parent')
+            else:
+                self.level_stack.append('child')
+        elif tag == 'li':
+            self.capture_text = True
+            self.current_item_text = ""
+            self.current_href = None
+        elif tag == 'a' and self.capture_text:
+            # Capture the link anchor destination safely
+            attrs_dict = dict(attrs)
+            self.current_href = attrs_dict.get('href')
+
+    def handle_endtag(self, tag):
+        if tag in ['ol', 'ul']:
+            if self.level_stack:
+                self.level_stack.pop()
+        elif tag == 'li':
+            self.capture_text = False
+            self.process_accumulated_item()
+
+    def handle_data(self, data):
+        if self.capture_text:
+            self.current_item_text += data
+
+    def process_accumulated_item(self):
+        # 1. Clean out stray prefix digits and list flags safely
+        clean = self.current_item_text.strip()
+        clean = re.sub(r'^([\d\.\s\-•]*\d+\.\d+|[\d\.\s\-•]*\d+\.)\s*', '', clean)
+        clean = re.sub(r'^\d+\s+(?=[A-Za-z])', '', clean)
+        clean = re.sub(r'^[\s\-•]*', '', clean).strip()
+        
+        if not clean:
+            return
+
+        # 2. Check current depth level stack state
+        current_state = self.level_stack[-1] if self.level_stack else 'parent'
+        
+        # 3. Rebuild the text as a clickable link if an anchor was present
+        if self.current_href:
+            display_text = f'<a href="{self.current_href}">{clean}</a>'
+        else:
+            display_text = clean
+
+        # 4. Generate structured HTML outputs with calculated taxonomy counters
+        if current_state == 'parent':
+            self.parent_count += 1
+            self.child_count = 0
+            self.output.append(f'<div class="toc-parent">{self.parent_count}. {display_text}</div>')
+        else:
+            self.child_count += 1
+            self.output.append(f'<div class="toc-child">{self.parent_count}.{self.child_count} {display_text}</div>')
+
+def extract_and_fix_toc_blocks(html_content):
+    """
+    Finds the compiled HTML lists block, intercepts it, clears 
+    numbering artifacts structurally via HTML parsing tree mechanics.
+    """
+    match = re.search(r'(<(?:ol|ul)>[\s\S]*?</(?:ol|ul)>)', html_content)
+    if not match:
+        return html_content
+    
+    raw_toc_html = match.group(1)
+    
+    parser = TOCHtmlRestructurer()
+    parser.feed(raw_toc_html)
+    
+    new_toc_html = '<div class="toc-box">\n<h2>Table of Contents</h2>\n'
+    new_toc_html += '\n'.join(parser.output)
+    new_toc_html += '\n</div>'
+    
+    return html_content.replace(raw_toc_html, new_toc_html, 1)
+
+def export_to_pdf(inputText, filename):
+    """
+    Processes plain dynamic markdown, captures structural compilation output,
+    restructures TOC nodes downstream safely, and outputs a high-fidelity PDF.
+    """
     html_content = markdown.markdown(inputText, extensions=['tables', 'fenced_code'])
+    final_body_content = extract_and_fix_toc_blocks(html_content)
     
-    # Step 2: Inject the content into our CSS-styled HTML boilerplate template
-    final_html = html_template.format(content=html_content)
-    
-    # Step 3: Render directly to a high-fidelity PDF file
-    HTML(string=final_html).write_pdf(filename)
-    #print(f"Successfully generated styled PDF at: {output_pdf_path}")
+    final_html = html_template.format(content=final_body_content)
+    HTML(string=final_html).write_pdf(filename)

utilities/pdf_tools_v2.py

@@ -0,0 +1,133 @@
+import markdown
+from weasyprint import HTML
+
+# ==========================================
+# REUSABLE HTML & CSS LAYOUT TEMPLATE
+# ==========================================
+PDF_REPORT_TEMPLATE = """<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <style>
+        @page {{
+            size: A4;
+            margin: 20mm;
+            @bottom-right {{
+                content: counter(page);
+                font-family: Arial, sans-serif;
+                font-size: 9pt;
+            }}
+        }}
+        
+        body {{
+            font-family: Arial, sans-serif;
+            color: #333333;
+            line-height: 1.6;
+            font-size: 11pt;
+        }}
+
+        h1 {{ font-size: 18pt; margin-top: 24pt; border-bottom: 1px solid #ddd; padding-bottom: 6px; }}
+        h2 {{ font-size: 14pt; margin-top: 18pt; color: #2c3e50; }}
+        h3 {{ font-size: 12pt; margin-top: 14pt; color: #7f8c8d; }}
+
+        /* --- Layout Tables --- */
+        table {{
+            width: 100%;
+            border-collapse: collapse;
+            margin: 15pt 0;
+        }}
+        th, td {{
+            border: 1px solid #dddddd;
+            padding: 8px;
+            text-align: left;
+        }}
+        th {{ background-color: #f8f9fa; font-weight: bold; }}
+        tr:nth-child(even) {{ background-color: #fdfdfd; }}
+
+        /* --- Links --- */
+        a {{ color: #3498db; text-decoration: none; }}
+        a:hover {{ text-decoration: underline; }}
+
+        /* ==========================================
+           DYNAMIC PAGED MEDIA TOC LAYOUT
+        ========================================== */
+        .toc-wrapper {{
+            margin: 20pt 0;
+            padding: 15pt;
+            background: #fdfdfd;
+            border: 1px solid #eaeaea;
+            border-radius: 5px;
+            page-break-after: always;
+        }}
+        
+        /* Clear default markdown padding/bullets on root list element */
+        .toc-wrapper > ul {{
+            list-style-type: none;
+            padding-left: 0;
+        }}
+        
+        /* Main Level 1 entries (bolded for contrast) */
+        .toc-wrapper > ul > li {{
+            font-weight: bold;
+            margin-top: 12px;
+            margin-bottom: 6px;
+        }}
+
+        /* Indent Subsections (Level 2 nested lists) */
+        .toc-wrapper li ul {{
+            list-style-type: none;
+            padding-left: 25px;
+            margin-top: 4px;
+            margin-bottom: 4px;
+        }}
+        
+        /* Ensure subsection text lines are regular weight */
+        .toc-wrapper li ul li {{
+            font-weight: normal;
+            margin-bottom: 6px;
+        }}
+
+        .toc-wrapper li {{
+            position: relative;
+            display: block;
+        }}
+        
+        /* Pulls the target page number of the markdown heading reference link */
+        .toc-wrapper a::after {{
+            content: target-counter(attr(href), page);
+            float: right;
+            color: #7f8c8d;
+            font-weight: normal;
+        }}
+    </style>
+</head>
+<body>
+    <div class="content">
+        {html_body}
+    </div>
+</body>
+</html>
+"""
+
+def export_to_pdf(llm_response: str, output_filename: str = "output.pdf") -> None:
+    """Converts structured LLM response with a fully clickable, indented manual TOC into a clean PDF."""
+    
+    # 1. Standard markdown conversion with 'toc' extension enabled.
+    # This automatically adds unique ID anchors to your <h1> and <h2> elements 
+    # so that clicking the TOC links will correctly jump down to that section.
+    html_body = markdown.markdown(
+        llm_response, 
+        extensions=['fenced_code', 'tables', 'toc']
+    )
+    
+    # 2. Inject a styling hook class wrapper around your Table of Contents list block
+    html_body = html_body.replace('<h3>Table of Contents</h3>', '<h3>Table of Contents</h3><div class="toc-wrapper">')
+    
+    # Close the div wrapper cleanly right before the next main topic heading begins
+    html_body = html_body.replace('<h1>1.', '</div><h1>1.')
+    
+    # 3. Format the final output document string structure
+    final_html_content = PDF_REPORT_TEMPLATE.format(html_body=html_body)
+    
+    # 4. Generate the clickable PDF file
+    HTML(string=final_html_content).write_pdf(output_filename)