datarobot_genai.drtools.core.clients.dr_docs

dr_docs

DataRobot Agentic AI Documentation search client.

Searches the DataRobot agentic-ai documentation section without requiring any API keys. Parses the sitemap.xml to discover agentic-ai documentation page URLs, fetches their full HTML content concurrently at index-build time, and builds a TF-IDF index from real page titles and body text. The index is cached in memory for fast repeated queries.

Only pages under https://docs.datarobot.com/en/docs/agentic-ai/ are indexed, keeping the corpus small (~28 pages) so the full index can be built in a few seconds.

---

TODO: Pre-built index (push index build out of MCP): Build the TF-IDF index in a separate process (workload api), then store it (serialized file or blob). Sitemap fetch, 28 HTTP requests, HTML parsing, tokenization, and index construction run outside the MCP.

In the MCP: Load pre-built index from file/URL or bundled resource. search_docs = load index if needed + TF-IDF search. Same for fetch_page_content if cache page contents in the artifact. No sitemap or bulk HTML fetch, no parsing/tokenizing in the MCP. Only loads that artifact and runs search in memory.

DocPage

Represents a single documentation page.

Source code in datarobot_genai/drtools/core/clients/dr_docs.py

class DocPage:
    """Represents a single documentation page."""

    def __init__(self, url: str, title: str, text: str = "") -> None:
        self.url = url
        self.title = title
        self.text = text
        # Pre-compute tokens for searching (title weighted 3x over body text)
        self._title_tokens = _tokenize(title)
        self._text_tokens = _tokenize(text)
        self.tf = _compute_tf(self._title_tokens * 3 + self._text_tokens)

    def as_dict(self) -> dict[str, str]:
        """Return a dictionary representation of the page."""
        return {
            "url": self.url,
            "title": self.title,
            "description": self.text,
        }

as_dict

as_dict() -> dict[str, str]

Return a dictionary representation of the page.

Source code in datarobot_genai/drtools/core/clients/dr_docs.py

def as_dict(self) -> dict[str, str]:
    """Return a dictionary representation of the page."""
    return {
        "url": self.url,
        "title": self.title,
        "description": self.text,
    }

search_docs async

search_docs(query: str, max_results: int = MAX_RESULTS_DEFAULT) -> list[dict[str, Any]]

Search DataRobot agentic-ai documentation for pages relevant to a query.

Parameters:

Name	Type	Description	Default
query	str	The search query string.	required
max_results	int	Maximum number of results to return.	MAX_RESULTS_DEFAULT

Returns

List of dictionaries with 'url', 'title', and 'description' keys.

Source code in datarobot_genai/drtools/core/clients/dr_docs.py

async def search_docs(query: str, max_results: int = MAX_RESULTS_DEFAULT) -> list[dict[str, Any]]:
    """Search DataRobot agentic-ai documentation for pages relevant to a query.

    Args:
        query: The search query string.
        max_results: Maximum number of results to return.

    Returns
    -------
        List of dictionaries with 'url', 'title', and 'description' keys.
    """
    max_results = max(1, min(max_results, MAX_RESULTS))

    index = await _ensure_index()
    results = index.search(query, max_results=max_results)

    return [page.as_dict() for page in results]

fetch_page_content async

fetch_page_content(url: str) -> dict[str, Any]

Fetch and extract the text content of a specific documentation page.

Parameters:

Name	Type	Description	Default
url	str	The URL of the documentation page to fetch.	required

Returns

Dictionary with 'url', 'title', and 'content' keys.

Source code in datarobot_genai/drtools/core/clients/dr_docs.py

async def fetch_page_content(url: str) -> dict[str, Any]:
    """Fetch and extract the text content of a specific documentation page.

    Args:
        url: The URL of the documentation page to fetch.

    Returns
    -------
        Dictionary with 'url', 'title', and 'content' keys.
    """
    if "/en/docs/" not in url or not url.startswith(DOCS_BASE_URL):
        return {
            "url": url,
            "title": "Error",
            "content": "URL must be a DataRobot English documentation page "
            f"(must contain '/en/docs/' or start with '{DOCS_BASE_URL}').",
        }

    # Check if the page is already cached in the index to avoid a live HTTP request
    if not _index.is_stale:
        for page in _index.pages:
            if page.url.lower().rstrip("/") == url.lower().rstrip("/"):
                return {
                    "url": page.url,
                    "title": page.title,
                    "content": page.text,
                }

    async with aiohttp.ClientSession() as session:
        html = await _fetch_url_raw_text_content(session, url)

    if not html:
        return {
            "url": url,
            "title": "Error",
            "content": f"Failed to fetch content from {url}",
        }

    title, content = _extract_html_content(html)
    title = title or _title_from_url(url)

    return {
        "url": url,
        "title": title,
        "content": content,
    }