BGE Retriever

rankify.retrievers.bge_retriever

BaseRetriever

Bases: ABC

Abstract base class for all retrieval methods in the rankify framework.

This class defines the common interface that all retrievers must implement, ensuring consistency across different retrieval methods (BM25, DPR, etc.).

Source code in rankify/retrievers/base_retriever.py

class BaseRetriever(ABC):
    """
    Abstract base class for all retrieval methods in the rankify framework.

    This class defines the common interface that all retrievers must implement,
    ensuring consistency across different retrieval methods (BM25, DPR, etc.).
    """

    def __init__(self, n_docs: int = 10, batch_size: int = 36, threads: int = 30, **kwargs):
        """
        Initialize the base retriever.

        Args:
            n_docs (int): Number of documents to retrieve per query
            batch_size (int): Number of queries to process in a batch
            threads (int): Number of parallel threads for retrieval
            **kwargs: Additional parameters specific to each retriever
        """
        self.n_docs = n_docs
        self.batch_size = batch_size
        self.threads = threads
        self.config = kwargs

    @abstractmethod
    def _initialize_searcher(self) -> Any:
        """Initialize the specific searcher for this retriever type."""
        pass

    @abstractmethod
    def retrieve(self, documents: List[Document]) -> List[Document]:
        """
        Retrieve relevant contexts for the given documents.

        Args:
            documents (List[Document]): List of documents containing queries

        Returns:
            List[Document]: Documents updated with retrieved contexts
        """
        pass

    def _preprocess_query(self, query: str) -> str:
        """
        Preprocess query text. Can be overridden by specific retrievers.

        Args:
            query (str): Raw query text

        Returns:
            str: Preprocessed query text
        """
        return query.strip()

__init__(n_docs=10, batch_size=36, threads=30, **kwargs)

Initialize the base retriever.

Parameters:

Name	Type	Description	Default
n_docs	int	Number of documents to retrieve per query	10
batch_size	int	Number of queries to process in a batch	36
threads	int	Number of parallel threads for retrieval	30
**kwargs		Additional parameters specific to each retriever	{}

Source code in rankify/retrievers/base_retriever.py

def __init__(self, n_docs: int = 10, batch_size: int = 36, threads: int = 30, **kwargs):
    """
    Initialize the base retriever.

    Args:
        n_docs (int): Number of documents to retrieve per query
        batch_size (int): Number of queries to process in a batch
        threads (int): Number of parallel threads for retrieval
        **kwargs: Additional parameters specific to each retriever
    """
    self.n_docs = n_docs
    self.batch_size = batch_size
    self.threads = threads
    self.config = kwargs

retrieve(documents) abstractmethod

Retrieve relevant contexts for the given documents.

Parameters:

Name	Type	Description	Default
documents	List[Document]	List of documents containing queries	required

Returns:

Type	Description
List[Document]	List[Document]: Documents updated with retrieved contexts

Source code in rankify/retrievers/base_retriever.py

@abstractmethod
def retrieve(self, documents: List[Document]) -> List[Document]:
    """
    Retrieve relevant contexts for the given documents.

    Args:
        documents (List[Document]): List of documents containing queries

    Returns:
        List[Document]: Documents updated with retrieved contexts
    """
    pass

IndexManager

Manages downloading, caching, and loading of retrieval indexes.

Handles both prebuilt indexes (wiki, msmarco) and custom user indexes.

Source code in rankify/retrievers/index_manager.py

class IndexManager:
    """
    Manages downloading, caching, and loading of retrieval indexes.

    Handles both prebuilt indexes (wiki, msmarco) and custom user indexes.
    """

    def __init__(self, cache_dir: str = None):
        self.cache_dir = cache_dir or os.environ.get("RERANKING_CACHE_DIR", "./cache")
        self.index_configs = self._load_index_configs()

    def _load_index_configs(self) -> Dict:
        """Load index configuration mappings."""
        return {
            "bm25": {
                "wiki": {
                    "url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/bm25_wiki.zip",
                    "prebuilt": "wikipedia-dpr-100w"
                },
                "msmarco": {
                    "url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/bm25_index_msmarco.zip?download=true",
                    "prebuilt": None
                }
            },
            "dpr-multi": {
                "wiki": {
                    "prebuilt": "wikipedia-dpr-100w.dpr-multi",
                    "encoder": "facebook/dpr-question_encoder-multiset-base"
                },
                "msmarco": {
                    "url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/msmarco_passage_dpr_multi.zip",
                    "encoder": "facebook/dpr-question_encoder-multiset-base"
                }
            },
            "dpr-single": {
                "wiki": {
                    "prebuilt": "wikipedia-dpr-100w.dpr-single-nq",
                    "encoder": "facebook/dpr-question_encoder-single-nq-base"
                },
                "msmarco": {
                    "url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/msmarco_passage_dpr_single.zip",
                    "encoder": "facebook/dpr-question_encoder-single-nq-base"
                }
            },
            "ance-multi": {
                "wiki": {
                    "prebuilt": "wikipedia-dpr-100w.ance-multi",
                    "encoder": "castorini/ance-dpr-question-multi"
                },
                "msmarco": {
                    "prebuilt": "msmarco-v1-passage.ance", 
                    "encoder": "castorini/ance-msmarco-passage"
                }
            },

            "bpr-single": {
                "wiki": {
                    "prebuilt": "wikipedia-dpr-100w.bpr-single-nq",
                    "encoder": "castorini/bpr-nq-question-encoder"
                },
                "msmarco": {
                    "url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/msmarco_passage_bpr.zip",
                    "encoder": "castorini/bpr-nq-question-encoder"
                }
            },
            "bge": {
                "wiki": {
                    "urls": [
                        "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/bgb_index.tar.gz.part1?download=true",
                        "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/bgb_index.tar.gz.part2?download=true",
                    ],
                    "passages_url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/psgs_w100/psgs_w100.tsv?download=true",
                    "encoder": "BAAI/bge-large-en-v1.5"
                },
                "msmarco": {
                    "urls": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/msmarco_embeddings_bgb.zip?download=true",
                    "passages_url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/msmarco-passage-corpus/msmarco-passage-corpus.tsv?download=true",
                    "encoder": "BAAI/bge-large-en-v1.5"
                }
            },
            "colbert": {
                "wiki": {
                    "urls": [
                        "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/wikipedia_embeddings_colbert/wiki.zip.001?download=true",
                        "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/wikipedia_embeddings_colbert/wiki.zip.002?download=true",
                        "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/wikipedia_embeddings_colbert/wiki.zip.003?download=true",
                        "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/wikipedia_embeddings_colbert/wiki.zip.004?download=true",
                        "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/wikipedia_embeddings_colbert/wiki.zip.005?download=true"
                    ],
                    "passages_url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/psgs_w100/psgs_w100.tsv?download=true",
                    "model": "colbert-ir/colbertv2.0"
                },
                "msmarco": {
                    "urls": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/msmarco_embeddings_colbert.zip?download=true",
                    "passages_url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/msmarco-passage-corpus/msmarco-passage-corpus.tsv?download=true",
                    "model": "colbert-ir/colbertv2.0"
                }
            },
            "contriever": {
                "wiki": {
                    "url": "https://dl.fbaipublicfiles.com/contriever/embeddings/contriever-msmarco/wikipedia_embeddings.tar",
                    "passages_url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/psgs_w100/psgs_w100.tsv?download=true",
                    "model": "facebook/contriever-msmarco",
                    "vector_size": 768
                },
                "msmarco": {
                    "url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/msmarco_embeddings_contriever.zip?download=true",
                    "passages_url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/msmarco-passage-corpus/msmarco-passage-corpus.tsv?download=true",
                    "model": "facebook/contriever-msmarco",
                    "vector_size": 768
                }
            },
            "online": {
                "web": {
                    "search_provider": "web",
                    "chunk_size": 500,
                    "chunk_overlap": 50,
                    "requires_api_key": True
                }
            },
            "hyde": {
                "wiki": {
                    "base_retriever": "contriever",
                    "base_model": "facebook/contriever-msmarco",
                    "base_index_type": "wiki",
                    "task": "web search",
                    "llm_model": "gpt-3.5-turbo-0125",
                    "num_generated_docs": 1,
                    "max_token_generated_docs": 512,
                    "temperature": 0.7,
                    "requires_api_key": True
                },
                "msmarco": {
                    "base_retriever": "contriever", 
                    "base_model": "facebook/contriever-msmarco",
                    "base_index_type": "msmarco",
                    "task": "web search",
                    "llm_model": "gpt-3.5-turbo-0125",
                    "num_generated_docs": 1,
                    "max_token_generated_docs": 512,
                    "temperature": 0.7,
                    "requires_api_key": True
                }
            }
        }

    def get_index_path(self, method: str, index_type: str, custom_path: str = None) -> str:
        """
        Get the path to the index for a given method and index type.

        Args:
            method (str): Retrieval method (e.g., 'bm25', 'dpr-multi', 'ance')
            index_type (str): Index type ('wiki', 'msmarco', or 'custom')
            custom_path (str): Path to custom index if index_type is 'custom'

        Returns:
            str: Path to the index
        """
        if custom_path:
            return custom_path

        if method not in self.index_configs:
            raise ValueError(f"Unsupported method: {method}")

        if index_type not in self.index_configs[method]:
            raise ValueError(f"Unsupported index type '{index_type}' for method '{method}'")

        config = self.index_configs[method][index_type]

        # If it's a prebuilt index, return the identifier
        if "prebuilt" in config and config["prebuilt"]:
            return config["prebuilt"]

        # Otherwise, download and return local path
        return self._ensure_index_downloaded(method, index_type)

    def _ensure_index_downloaded(self, method: str, index_type: str) -> str:
        """Download and extract index if not already available."""
        config = self.index_configs[method][index_type]
        url = config["url"]

        # Create local directory path
        index_name = f"{method}_{index_type}"
        local_dir = os.path.join(self.cache_dir, "index", index_name)

        if not os.path.exists(local_dir):
            print(f"Downloading {method} index for {index_type}...")
            self._download_and_extract(url, local_dir)

        return local_dir

    def _download_and_extract(self, url: str, destination: str):
        """Download and extract a ZIP file."""
        os.makedirs(destination, exist_ok=True)

        zip_name = os.path.basename(url).split("?")[0]
        zip_path = os.path.join(self.cache_dir, "temp", zip_name)

        os.makedirs(os.path.dirname(zip_path), exist_ok=True)

        if not os.path.exists(zip_path):
            response = requests.get(url, stream=True)
            response.raise_for_status()

            with open(zip_path, "wb") as f:
                for chunk in tqdm(response.iter_content(chunk_size=1024), 
                                desc=f"Downloading {zip_name}"):
                    f.write(chunk)

        print(f"Extracting {zip_name}...")
        with zipfile.ZipFile(zip_path, "r") as zip_ref:
            zip_ref.extractall(destination)

        # Clean up ZIP file
        os.remove(zip_path)
        print("Extraction complete.")

    def load_id_mapping(self, index_path: str) -> Optional[Dict]:
        """Load ID mapping if available."""
        mapping_path = os.path.join(index_path, "id_mapping.json")
        if os.path.exists(mapping_path):
            with open(mapping_path, "r", encoding="utf-8") as f:
                mapping = json.load(f)
                return {v: k for k, v in mapping.items()}  # Reverse mapping
        return None

    # def load_corpus(self, index_path: str) -> Dict:
    #     """Load corpus data from index path."""
    #     corpus_file = os.path.join(index_path, "corpus.jsonl")
    #     if not os.path.exists(corpus_file):
    #         return {}

    #     corpus = {}
    #     with open(corpus_file, "r", encoding="utf-8") as f:
    #         for line in f:
    #             doc = json.loads(line.strip())
    #             doc_id = doc.get("docid") or doc.get("id")
    #             contents = doc.get("contents", "")
    #             title = doc.get("title", contents[:100] if contents else "No Title")
    #             corpus[str(doc_id)] = {"contents": contents, "title": title}

    #     return corpus
    def load_corpus(self, index_path: str) -> Dict:
        """Load corpus data from index path (supports multiple formats)."""
        # Try different corpus file formats in order of preference
        corpus_files = [
            "corpus_metadata.json",  # Your custom format
            "corpus.jsonl",          # Standard JSONL format
            "corpus.json"            # Alternative JSON format
        ]

        for corpus_filename in corpus_files:
            corpus_file = os.path.join(index_path, corpus_filename)
            if os.path.exists(corpus_file):
                print(f"📚 Loading corpus from {corpus_filename}")

                if corpus_filename.endswith('.jsonl'):
                    # JSONL format (one JSON object per line)
                    return self._load_corpus_jsonl(corpus_file)
                else:
                    # JSON format (single JSON object)
                    return self._load_corpus_json(corpus_file)

        print(f"❌ No corpus file found in {index_path}")
        return {}

    def _load_corpus_jsonl(self, corpus_file: str) -> Dict:
        """Load corpus from JSONL format (one JSON object per line)."""
        corpus = {}
        with open(corpus_file, "r", encoding="utf-8") as f:
            for line in f:
                doc = json.loads(line.strip())
                doc_id = doc.get("docid") or doc.get("id")
                contents = doc.get("contents", "")
                title = doc.get("title", contents[:100] if contents else "No Title")
                corpus[str(doc_id)] = {"contents": contents, "title": title}
        return corpus

    def _load_corpus_json(self, corpus_file: str) -> Dict:
        """Load corpus from JSON format (single JSON object)."""
        corpus = {}
        with open(corpus_file, "r", encoding="utf-8") as f:
            data = json.load(f)

        # Handle different JSON structures
        if isinstance(data, dict):
            # If it's a dict, iterate through it
            for doc_id, doc_data in data.items():
                if isinstance(doc_data, dict):
                    contents = doc_data.get("contents") or doc_data.get("text") or ""
                    title = doc_data.get("title", contents[:100] if contents else "No Title")
                else:
                    # If doc_data is a string, use it as contents
                    contents = str(doc_data)
                    title = contents[:100] if contents else "No Title"
                corpus[str(doc_id)] = {"contents": contents, "title": title}

        elif isinstance(data, list):
            # If it's a list, iterate through documents
            for doc in data:
                doc_id = doc.get("docid") or doc.get("id")
                contents = doc.get("contents", "")
                title = doc.get("title", contents[:100] if contents else "No Title")
                corpus[str(doc_id)] = {"contents": contents, "title": title}

        return corpus
    def download_and_extract_index(self, url: str) -> str:
        """Download and extract index from URL, return local path."""
        # This is used by DenseRetriever pattern
        import tempfile
        import zipfile

        # Create temporary file
        with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as tmp_file:
            tmp_path = tmp_file.name

        try:
            # Download
            print(f"Downloading from {url}...")
            response = requests.get(url, stream=True)
            response.raise_for_status()

            with open(tmp_path, 'wb') as f:
                for chunk in tqdm(response.iter_content(chunk_size=1024)):
                    f.write(chunk)

            # Extract to cache directory
            extract_dir = os.path.join(self.cache_dir, "downloaded_index")
            os.makedirs(extract_dir, exist_ok=True)

            with zipfile.ZipFile(tmp_path, 'r') as zip_ref:
                zip_ref.extractall(extract_dir)

            return extract_dir

        finally:
            # Clean up temporary file
            if os.path.exists(tmp_path):
                os.remove(tmp_path)

get_index_path(method, index_type, custom_path=None)

Get the path to the index for a given method and index type.

Parameters:

Name	Type	Description	Default
method	str	Retrieval method (e.g., 'bm25', 'dpr-multi', 'ance')	required
index_type	str	Index type ('wiki', 'msmarco', or 'custom')	required
custom_path	str	Path to custom index if index_type is 'custom'	None

Returns:

Name	Type	Description
str	str	Path to the index

Source code in rankify/retrievers/index_manager.py

def get_index_path(self, method: str, index_type: str, custom_path: str = None) -> str:
    """
    Get the path to the index for a given method and index type.

    Args:
        method (str): Retrieval method (e.g., 'bm25', 'dpr-multi', 'ance')
        index_type (str): Index type ('wiki', 'msmarco', or 'custom')
        custom_path (str): Path to custom index if index_type is 'custom'

    Returns:
        str: Path to the index
    """
    if custom_path:
        return custom_path

    if method not in self.index_configs:
        raise ValueError(f"Unsupported method: {method}")

    if index_type not in self.index_configs[method]:
        raise ValueError(f"Unsupported index type '{index_type}' for method '{method}'")

    config = self.index_configs[method][index_type]

    # If it's a prebuilt index, return the identifier
    if "prebuilt" in config and config["prebuilt"]:
        return config["prebuilt"]

    # Otherwise, download and return local path
    return self._ensure_index_downloaded(method, index_type)

load_id_mapping(index_path)

Load ID mapping if available.

Source code in rankify/retrievers/index_manager.py

def load_id_mapping(self, index_path: str) -> Optional[Dict]:
    """Load ID mapping if available."""
    mapping_path = os.path.join(index_path, "id_mapping.json")
    if os.path.exists(mapping_path):
        with open(mapping_path, "r", encoding="utf-8") as f:
            mapping = json.load(f)
            return {v: k for k, v in mapping.items()}  # Reverse mapping
    return None

load_corpus(index_path)

Load corpus data from index path (supports multiple formats).

Source code in rankify/retrievers/index_manager.py

def load_corpus(self, index_path: str) -> Dict:
    """Load corpus data from index path (supports multiple formats)."""
    # Try different corpus file formats in order of preference
    corpus_files = [
        "corpus_metadata.json",  # Your custom format
        "corpus.jsonl",          # Standard JSONL format
        "corpus.json"            # Alternative JSON format
    ]

    for corpus_filename in corpus_files:
        corpus_file = os.path.join(index_path, corpus_filename)
        if os.path.exists(corpus_file):
            print(f"📚 Loading corpus from {corpus_filename}")

            if corpus_filename.endswith('.jsonl'):
                # JSONL format (one JSON object per line)
                return self._load_corpus_jsonl(corpus_file)
            else:
                # JSON format (single JSON object)
                return self._load_corpus_json(corpus_file)

    print(f"❌ No corpus file found in {index_path}")
    return {}

download_and_extract_index(url)

Download and extract index from URL, return local path.

Source code in rankify/retrievers/index_manager.py

def download_and_extract_index(self, url: str) -> str:
    """Download and extract index from URL, return local path."""
    # This is used by DenseRetriever pattern
    import tempfile
    import zipfile

    # Create temporary file
    with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as tmp_file:
        tmp_path = tmp_file.name

    try:
        # Download
        print(f"Downloading from {url}...")
        response = requests.get(url, stream=True)
        response.raise_for_status()

        with open(tmp_path, 'wb') as f:
            for chunk in tqdm(response.iter_content(chunk_size=1024)):
                f.write(chunk)

        # Extract to cache directory
        extract_dir = os.path.join(self.cache_dir, "downloaded_index")
        os.makedirs(extract_dir, exist_ok=True)

        with zipfile.ZipFile(tmp_path, 'r') as zip_ref:
            zip_ref.extractall(extract_dir)

        return extract_dir

    finally:
        # Clean up temporary file
        if os.path.exists(tmp_path):
            os.remove(tmp_path)

Document

Represents a document consisting of a question, answers, and contexts.

Attributes:

Name	Type	Description
question	Question	The question associated with the document.
answers	Answer	The answers to the question.
contexts	list[Context]	A list of related contexts.
reorder_contexts	list[Context] or None	A reordered list of contexts based on relevance.

Source code in rankify/dataset/dataset.py

class Document:
    """
    Represents a document consisting of a question, answers, and contexts.

    Attributes:
        question (Question): The question associated with the document.
        answers (Answer): The answers to the question.
        contexts (list[Context]): A list of related contexts.
        reorder_contexts (list[Context] or None): A reordered list of contexts based on relevance.
    """
    def __init__(self, question: Question, answers: Answer, contexts: list = None , id: int = None) -> None:
        """
        Initializes a Document instance.

        Args:
            question (Question): The question associated with the document.
            answers (Answer): The answers to the question.
            contexts (list[Context], optional): A list of contexts related to the question.

        Example:
            ```python
            q = Question("What is the capital of France?")
            a = Answer(["Paris"])
            c1 = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
            c2 = Context(score=0.5, has_answer=False, id=2, title="Berlin", text="Berlin is the capital of Germany.")
            d = Document(question=q, answers=a, contexts=[c1, c2])
            print(d)
            ```
        """
        self.question: Question = question
        self.answers: Answer = answers
        self.contexts: List[Context] = contexts
        self.reorder_contexts: List[Context] = None
        self.id = str(id) 

    @classmethod
    def from_dict(cls, data: dict,n_docs:int=100) -> 'Document':
        """
        Creates a Document instance from a dictionary.

        Args:
            data (dict): A dictionary containing the question, answers, and contexts.
            n_docs (int, optional): The number of contexts to include. Defaults to 100.

        Returns:
            Document: A new Document instance.

        Example:
            ```python
            data = {
                "question": "What is the capital of France?",
                "answers": ["Paris"],
                "ctxs": [
                    {"score": 0.9, "has_answer": True, "id": 1, "title": "Paris", "text": "The capital of France is Paris."},
                    {"score": 0.5, "has_answer": False, "id": 2, "title": "Berlin", "text": "Berlin is the capital of Germany."}
                ]
            }
            d = Document.from_dict(data)
            print(d.question)
            ```
        """
        question = Question(data["question"])
        if "answers" in data:
            answers = Answer(data["answers"])
        else:
            answers =Answer('')

        if "query_id" in data:
            id = data["query_id"]
        else:
            id = None
        contexts = [Context(**ctx) for ctx in data["ctxs"][:n_docs]]
        return cls(question, answers, contexts, id=id)

    def to_dict(self) -> Dict[str, Optional[object]]:
        """
        Converts the document into a dictionary representation.

        Returns:
            dict: A dictionary containing the question, answers, and contexts.
        """
        return {
            "question": self.question.question,
            "answers": self.answers.answers,
            "contexts": [ctx.to_dict() for ctx in self.contexts]
        }
    def to_dict_reoreder(self) -> Dict[str,Optional[object]]:
        return {
            "question" : self.question.question,
            "answers" : self.answers.answers,
            "contexts" : [ctx.to_dict() for ctx in self.reorder_contexts]
        }
    def __str__(self) -> str:
        """
        Returns a string representation of the Document instance.

        Returns:
            str: The formatted document information.

        Example:
            ```python
            d = Document(Question("What is the capital of France?"), Answer(["Paris"]))
            print(d)
            ```
        """
        contexts_str = "\n\n".join([str(ctx) for ctx in self.contexts])
        reorder_contexts_str= ''
        if self.reorder_contexts is not None:
            reorder_contexts_str = "\n\n".join([str(ctx) for ctx in self.reorder_contexts])
        return f"{self.question}\n\n{self.answers}\n\nContext: \n\n{contexts_str}\nReorder contexts: \n\n{reorder_contexts_str}"

__init__(question, answers, contexts=None, id=None)

Initializes a Document instance.

Parameters:

Name	Type	Description	Default
question	Question	The question associated with the document.	required
answers	Answer	The answers to the question.	required
contexts	list[Context]	A list of contexts related to the question.	None

Example

q = Question("What is the capital of France?")
a = Answer(["Paris"])
c1 = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
c2 = Context(score=0.5, has_answer=False, id=2, title="Berlin", text="Berlin is the capital of Germany.")
d = Document(question=q, answers=a, contexts=[c1, c2])
print(d)

Source code in rankify/dataset/dataset.py

def __init__(self, question: Question, answers: Answer, contexts: list = None , id: int = None) -> None:
    """
    Initializes a Document instance.

    Args:
        question (Question): The question associated with the document.
        answers (Answer): The answers to the question.
        contexts (list[Context], optional): A list of contexts related to the question.

    Example:
        ```python
        q = Question("What is the capital of France?")
        a = Answer(["Paris"])
        c1 = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
        c2 = Context(score=0.5, has_answer=False, id=2, title="Berlin", text="Berlin is the capital of Germany.")
        d = Document(question=q, answers=a, contexts=[c1, c2])
        print(d)
        ```
    """
    self.question: Question = question
    self.answers: Answer = answers
    self.contexts: List[Context] = contexts
    self.reorder_contexts: List[Context] = None
    self.id = str(id) 

from_dict(data, n_docs=100) classmethod

Creates a Document instance from a dictionary.

Parameters:

Name	Type	Description	Default
data	dict	A dictionary containing the question, answers, and contexts.	required
n_docs	int	The number of contexts to include. Defaults to 100.	100

Returns:

Name	Type	Description
Document	Document	A new Document instance.

Example

data = {
    "question": "What is the capital of France?",
    "answers": ["Paris"],
    "ctxs": [
        {"score": 0.9, "has_answer": True, "id": 1, "title": "Paris", "text": "The capital of France is Paris."},
        {"score": 0.5, "has_answer": False, "id": 2, "title": "Berlin", "text": "Berlin is the capital of Germany."}
    ]
}
d = Document.from_dict(data)
print(d.question)

Source code in rankify/dataset/dataset.py

@classmethod
def from_dict(cls, data: dict,n_docs:int=100) -> 'Document':
    """
    Creates a Document instance from a dictionary.

    Args:
        data (dict): A dictionary containing the question, answers, and contexts.
        n_docs (int, optional): The number of contexts to include. Defaults to 100.

    Returns:
        Document: A new Document instance.

    Example:
        ```python
        data = {
            "question": "What is the capital of France?",
            "answers": ["Paris"],
            "ctxs": [
                {"score": 0.9, "has_answer": True, "id": 1, "title": "Paris", "text": "The capital of France is Paris."},
                {"score": 0.5, "has_answer": False, "id": 2, "title": "Berlin", "text": "Berlin is the capital of Germany."}
            ]
        }
        d = Document.from_dict(data)
        print(d.question)
        ```
    """
    question = Question(data["question"])
    if "answers" in data:
        answers = Answer(data["answers"])
    else:
        answers =Answer('')

    if "query_id" in data:
        id = data["query_id"]
    else:
        id = None
    contexts = [Context(**ctx) for ctx in data["ctxs"][:n_docs]]
    return cls(question, answers, contexts, id=id)

to_dict()

Converts the document into a dictionary representation.

Returns:

Name	Type	Description
dict	Dict[str, Optional[object]]	A dictionary containing the question, answers, and contexts.

Source code in rankify/dataset/dataset.py

def to_dict(self) -> Dict[str, Optional[object]]:
    """
    Converts the document into a dictionary representation.

    Returns:
        dict: A dictionary containing the question, answers, and contexts.
    """
    return {
        "question": self.question.question,
        "answers": self.answers.answers,
        "contexts": [ctx.to_dict() for ctx in self.contexts]
    }

__str__()

Returns a string representation of the Document instance.

Returns:

Name	Type	Description
str	str	The formatted document information.

Example

d = Document(Question("What is the capital of France?"), Answer(["Paris"]))
print(d)

Source code in rankify/dataset/dataset.py

def __str__(self) -> str:
    """
    Returns a string representation of the Document instance.

    Returns:
        str: The formatted document information.

    Example:
        ```python
        d = Document(Question("What is the capital of France?"), Answer(["Paris"]))
        print(d)
        ```
    """
    contexts_str = "\n\n".join([str(ctx) for ctx in self.contexts])
    reorder_contexts_str= ''
    if self.reorder_contexts is not None:
        reorder_contexts_str = "\n\n".join([str(ctx) for ctx in self.reorder_contexts])
    return f"{self.question}\n\n{self.answers}\n\nContext: \n\n{contexts_str}\nReorder contexts: \n\n{reorder_contexts_str}"

Context

Represents a context with metadata such as score and title.

Attributes:

Name	Type	Description
score	float	The relevance score of the context.
has_answer	bool	Whether the context contains an answer.
id	int	The identifier of the context.
title	str	The title of the context.
text	str	The text of the context.

Source code in rankify/dataset/dataset.py

class Context:
    """
    Represents a context with metadata such as score and title.

    Attributes:
        score (float, optional): The relevance score of the context.
        has_answer (bool, optional): Whether the context contains an answer.
        id (int, optional): The identifier of the context.
        title (str, optional): The title of the context.
        text (str, optional): The text of the context.
    """
    def __init__(self, score: float=None, has_answer: bool=None, id: str=None, title: str=None, text: str=None)-> None:
        """
        Initializes a Context instance.

        Args:
            score (float, optional): The relevance score.
            has_answer (bool, optional): Whether the context contains an answer.
            id (int, optional): The identifier of the context.
            title (str, optional): The title of the context.
            text (str, optional): The text of the context.

        Example:
            ```python
            c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
            print(c)
            ```
        """
        self.score: Optional[float] = score
        self.has_answer: Optional[bool] = has_answer
        self.id: Optional[str] = id
        self.title: Optional[str] = title
        self.text: Optional[str] = text

    def to_dict(self, save_text: bool=False) -> Dict[str, Optional[object]]:

        """
        Converts the Context instance to a dictionary.

        Args:
            save_text (bool): Whether to include text in the output dictionary.

        Returns:
            dict: The context data.

        Example:
            ```python
            c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
            print(c.to_dict())
            ```
        """
        context_dict = {
            "score": float(self.score) if self.score is not None else None,
            "has_answer": self.has_answer,
            "id": self.id,
            }

        # Include 'text' only if save_text is True
        if save_text:
            context_dict["text"] = self.text
            context_dict["title"] =  self.title

        return context_dict
    def __str__(self) -> str:
        """
        Returns a string representation of the Context instance.

        Returns:
            str: The formatted context.

        Example:
            ```python
            c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
            print(str(c))
            ```
        """
        return f"ID: {self.id}\nHas Answer: {self.has_answer}\nTitle: {self.title}\nText: {self.text}\nScore: {self.score}"

__init__(score=None, has_answer=None, id=None, title=None, text=None)

Initializes a Context instance.

Parameters:

Name	Type	Description	Default
score	float	The relevance score.	None
has_answer	bool	Whether the context contains an answer.	None
id	int	The identifier of the context.	None
title	str	The title of the context.	None
text	str	The text of the context.	None

Example

c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
print(c)

Source code in rankify/dataset/dataset.py

def __init__(self, score: float=None, has_answer: bool=None, id: str=None, title: str=None, text: str=None)-> None:
    """
    Initializes a Context instance.

    Args:
        score (float, optional): The relevance score.
        has_answer (bool, optional): Whether the context contains an answer.
        id (int, optional): The identifier of the context.
        title (str, optional): The title of the context.
        text (str, optional): The text of the context.

    Example:
        ```python
        c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
        print(c)
        ```
    """
    self.score: Optional[float] = score
    self.has_answer: Optional[bool] = has_answer
    self.id: Optional[str] = id
    self.title: Optional[str] = title
    self.text: Optional[str] = text

to_dict(save_text=False)

Converts the Context instance to a dictionary.

Parameters:

Name	Type	Description	Default
save_text	bool	Whether to include text in the output dictionary.	False

Returns:

Name	Type	Description
dict	Dict[str, Optional[object]]	The context data.

Example

c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
print(c.to_dict())

Source code in rankify/dataset/dataset.py

def to_dict(self, save_text: bool=False) -> Dict[str, Optional[object]]:

    """
    Converts the Context instance to a dictionary.

    Args:
        save_text (bool): Whether to include text in the output dictionary.

    Returns:
        dict: The context data.

    Example:
        ```python
        c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
        print(c.to_dict())
        ```
    """
    context_dict = {
        "score": float(self.score) if self.score is not None else None,
        "has_answer": self.has_answer,
        "id": self.id,
        }

    # Include 'text' only if save_text is True
    if save_text:
        context_dict["text"] = self.text
        context_dict["title"] =  self.title

    return context_dict

__str__()

Returns a string representation of the Context instance.

Returns:

Name	Type	Description
str	str	The formatted context.

Example

c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
print(str(c))

Source code in rankify/dataset/dataset.py

def __str__(self) -> str:
    """
    Returns a string representation of the Context instance.

    Returns:
        str: The formatted context.

    Example:
        ```python
        c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
        print(str(c))
        ```
    """
    return f"ID: {self.id}\nHas Answer: {self.has_answer}\nTitle: {self.title}\nText: {self.text}\nScore: {self.score}"

BGERetriever

Bases: BaseRetriever

BGE retriever implementation using precomputed embeddings and FAISS indexing.

Implements BGE (Beijing Academy of Artificial Intelligence General Embedding) model for dense passage retrieval with efficient FAISS-based search.

Source code in rankify/retrievers/bge_retriever.py

class BGERetriever(BaseRetriever):
    """
    BGE retriever implementation using precomputed embeddings and FAISS indexing.

    Implements BGE (Beijing Academy of Artificial Intelligence General Embedding) model
    for dense passage retrieval with efficient FAISS-based search.
    """

    def __init__(self, model: str = "BAAI/bge-large-en-v1.5", index_type: str = "wiki", 
                 index_folder: str = None, device: str = "cuda", **kwargs):
        super().__init__(**kwargs)
        self.model_name = model
        self.index_type = index_type
        self.index_folder = index_folder
        self.device = device
        self.tokenizer_simple = SimpleTokenizer()

        # Initialize index manager
        self.index_manager = IndexManager()

        # Setup paths and download if needed
        if index_folder:
            self.index_path = index_folder
            self.passage_path = os.path.join(self.index_folder, 'passages.tsv')
        else:
            self.index_path = os.path.join(self.index_manager.cache_dir, "index", f"bge_index_{index_type}")
            self._ensure_index_and_passages_downloaded()
            self.passage_path = self._get_passage_path()

        # Load components
        self.doc_ids = self._load_document_ids()
        self.doc_texts = self._load_tsv()
        self.index = self._initialize_searcher()

        # Load model and tokenizer
        self.model_hf = AutoModel.from_pretrained(self.model_name).to(self.device).eval()
        self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)

    def _initialize_searcher(self):
        """Initialize FAISS index for BGE retrieval."""
        return self._build_faiss_index()

    def _ensure_index_and_passages_downloaded(self):
        """Download and extract BGE index and passages if needed."""
        os.makedirs(self.index_path, exist_ok=True)

        if self.index_type not in self.index_manager.index_configs.get("bge", {}):
            raise ValueError(f"Unsupported BGE index type: {self.index_type}")

        config = self.index_manager.index_configs["bge"][self.index_type]

        # Check if required files exist
        required_files = [
            os.path.join(self.index_path, "bge_doc_ids.pkl"),
            os.path.join(self.index_path, "bge_embeddings.h5")
        ]

        if not all(os.path.exists(f) for f in required_files):
            urls = config.get("urls")
            if isinstance(urls, list):
                # Multi-part download
                for url in urls:
                    filename = self._extract_filename_from_url(url)
                    local_path = os.path.join(self.index_path, filename)
                    if not os.path.exists(local_path):
                        self._download_file(url, local_path)
                self._extract_multi_part_archive()
            else:
                # Single file download
                zip_path = os.path.join(self.index_path, "index.zip")
                if not os.path.exists(zip_path):
                    self._download_file(urls, zip_path)
                self._extract_zip_files()

        # Download passages if needed
        passages_url = config.get("passages_url")
        if passages_url:
            passage_filename = self._extract_filename_from_url(passages_url)
            passage_path = os.path.join(self.index_manager.cache_dir, passage_filename)
            if not os.path.exists(passage_path):
                self._download_file(passages_url, passage_path)

    def _get_passage_path(self):
        """Get path to passages file."""
        if self.index_type not in self.index_manager.index_configs.get("bge", {}):
            raise ValueError(f"Unsupported BGE index type: {self.index_type}")

        config = self.index_manager.index_configs["bge"][self.index_type]
        passages_url = config.get("passages_url")
        if passages_url:
            passage_filename = self._extract_filename_from_url(passages_url)
            return os.path.join(self.index_manager.cache_dir, passage_filename)
        return None

    def _extract_filename_from_url(self, url):
        """Extract filename from URL."""
        parsed_url = urlparse(url)
        return os.path.basename(parsed_url.path).split('?')[0]

    def _download_file(self, url: str, save_path: str):
        """Download file from URL with progress bar."""
        os.makedirs(os.path.dirname(save_path), exist_ok=True)
        response = requests.get(url, stream=True)
        response.raise_for_status()

        with open(save_path, "wb") as f:
            for chunk in tqdm(response.iter_content(chunk_size=1024), 
                            desc=f"Downloading {os.path.basename(save_path)}"):
                f.write(chunk)

    def _extract_zip_files(self):
        """Extract ZIP files in the index folder."""
        zip_files = [f for f in os.listdir(self.index_path) if f.endswith(".zip")]

        for zip_file in zip_files:
            zip_path = os.path.join(self.index_path, zip_file)

            with zipfile.ZipFile(zip_path, "r") as zip_ref:
                for member in zip_ref.namelist():
                    filename = os.path.basename(member)
                    if not filename:
                        continue

                    extracted_path = os.path.join(self.index_path, filename)
                    with zip_ref.open(member) as source, open(extracted_path, "wb") as target:
                        shutil.copyfileobj(source, target)

            print(f"Extracted {zip_file}")
            os.remove(zip_path)

    def _extract_multi_part_archive(self):
        """Extract multi-part tar.gz archives."""
        parts = sorted([f for f in os.listdir(self.index_path) if f.startswith("bgb_index.tar.")],
                      key=lambda x: x.split('.')[-1])

        if not parts:
            return

        combined_path = os.path.join(self.index_path, "combined.tar.gz")

        # Combine parts
        with open(combined_path, "wb") as combined:
            for part in parts:
                with open(os.path.join(self.index_path, part), "rb") as part_file:
                    shutil.copyfileobj(part_file, combined)

        try:
            # Decompress and extract
            tar_file = combined_path.replace(".gz", "")
            with gzip.open(combined_path, "rb") as f_in, open(tar_file, "wb") as f_out:
                shutil.copyfileobj(f_in, f_out)

            shutil.unpack_archive(tar_file, self.index_path)
            os.remove(tar_file)

        except Exception as e:
            raise RuntimeError(f"Error extracting multi-part archive: {e}")
        finally:
            # Cleanup
            if os.path.exists(combined_path):
                os.remove(combined_path)
            for part in parts:
                part_path = os.path.join(self.index_path, part)
                if os.path.exists(part_path):
                    os.remove(part_path)

    def _build_faiss_index(self):
        """Build or load FAISS index."""
        print("Handling FAISS index...")

        index_path = os.path.join(self.index_path, "faiss_index.bin")
        embeddings_path = os.path.join(self.index_path, "bge_embeddings.h5")

        if os.path.exists(index_path):
            print(f"Loading existing FAISS index from {index_path}...")
            index = faiss.read_index(index_path)
        else:
            print(f"Building FAISS index from embeddings at {embeddings_path}...")
            with h5py.File(embeddings_path, "r") as f:
                embeddings = f["embeddings"][:].astype(np.float32)

            embedding_dim = embeddings.shape[1]
            index = faiss.IndexFlatIP(embedding_dim)

            # Add embeddings in chunks
            chunk_size = 50000
            for start in tqdm(range(0, embeddings.shape[0], chunk_size), 
                            desc="Adding embeddings to FAISS index"):
                end = min(start + chunk_size, embeddings.shape[0])
                chunk = embeddings[start:end]
                index.add(chunk)

            print(f"Saving FAISS index to {index_path}...")
            faiss.write_index(index, index_path)

        print(f"FAISS index loaded with {index.ntotal} embeddings.")
        return index

    def _load_document_ids(self):
        """Load document IDs from pickle file."""
        doc_ids_path = os.path.join(self.index_path, "bge_doc_ids.pkl")
        print(f"Loading document IDs from {doc_ids_path}...")

        doc_ids = []
        with open(doc_ids_path, "rb") as f:
            while True:
                try:
                    doc_ids.extend(pickle.load(f))
                except EOFError:
                    break

        print(f"Loaded {len(doc_ids)} document IDs.")
        return doc_ids

    def _load_tsv(self):
        """Load document texts from TSV file."""
        if not self.passage_path or not os.path.exists(self.passage_path):
            print("Warning: Passage file not found, using empty corpus")
            return {}

        doc_texts = {}
        with open(self.passage_path, "r", encoding="utf-8") as f:
            next(f)  # Skip header
            for line in f:
                try:
                    doc_id, passage, title = line.strip().split("\t")
                    doc_texts[doc_id] = {"text": passage, "title": title}
                except ValueError:
                    continue  # Skip malformed lines

        print(f"Loaded {len(doc_texts)} passages.")
        return doc_texts

    def _encode_queries(self, queries: List[str]):
        """Encode queries into dense embeddings."""
        all_embeddings = []
        with torch.no_grad():
            for i in tqdm(range(0, len(queries), self.batch_size), desc="Encoding queries"):
                batch = queries[i:i + self.batch_size]
                tokenized = self.tokenizer(batch, padding=True, truncation=True, 
                                         return_tensors="pt").to(self.device)
                model_output = self.model_hf(**tokenized)
                embeddings = model_output.last_hidden_state[:, 0, :]  # CLS token
                embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=1)
                all_embeddings.append(embeddings.cpu().numpy())

        return np.vstack(all_embeddings)

    def retrieve(self, documents: List[Document]) -> List[Document]:
        """Retrieve relevant contexts using BGE."""
        queries = [doc.question.question for doc in documents]
        print(f"Retrieving {len(documents)} documents with BGE...")

        # Encode queries
        query_embeddings = self._encode_queries(queries)

        # Check dimension compatibility
        if query_embeddings.shape[1] != self.index.d:
            raise ValueError(f"Dimension mismatch: queries={query_embeddings.shape[1]}, "
                           f"index={self.index.d}")

        # Batch FAISS search
        all_distances = []
        all_indices = []

        for start_idx in tqdm(range(0, len(query_embeddings), self.batch_size), 
                            desc="FAISS search"):
            end_idx = min(start_idx + self.batch_size, len(query_embeddings))
            batch_embeddings = query_embeddings[start_idx:end_idx]

            distances, indices = self.index.search(batch_embeddings, self.n_docs)
            all_distances.append(distances)
            all_indices.append(indices)

        # Combine results
        all_distances = np.vstack(all_distances)
        all_indices = np.vstack(all_indices)

        # Process results
        for i, document in enumerate(tqdm(documents, desc="Processing documents")):
            contexts = []
            for dist, idx in zip(all_distances[i], all_indices[i]):
                try:
                    context = self._create_context_from_result(idx, dist, document)
                    if context:
                        contexts.append(context)
                except Exception as e:
                    print(f"Error processing result {idx}: {e}")

            document.contexts = contexts

        return documents

    def _create_context_from_result(self, idx: int, score: float, document: Document) -> Context:
        """Create Context object from search result."""
        doc_id = self.doc_ids[idx]
        doc_data = self.doc_texts.get(str(doc_id), {
            "text": "Text not found", 
            "title": "No Title"
        })

        return Context(
            id=doc_id,
            title=doc_data["title"],
            text=doc_data["text"],
            score=float(score),
            has_answer=has_answers(doc_data["text"], document.answers.answers, self.tokenizer_simple)
        )

retrieve(documents)

Retrieve relevant contexts using BGE.

Source code in rankify/retrievers/bge_retriever.py

def retrieve(self, documents: List[Document]) -> List[Document]:
    """Retrieve relevant contexts using BGE."""
    queries = [doc.question.question for doc in documents]
    print(f"Retrieving {len(documents)} documents with BGE...")

    # Encode queries
    query_embeddings = self._encode_queries(queries)

    # Check dimension compatibility
    if query_embeddings.shape[1] != self.index.d:
        raise ValueError(f"Dimension mismatch: queries={query_embeddings.shape[1]}, "
                       f"index={self.index.d}")

    # Batch FAISS search
    all_distances = []
    all_indices = []

    for start_idx in tqdm(range(0, len(query_embeddings), self.batch_size), 
                        desc="FAISS search"):
        end_idx = min(start_idx + self.batch_size, len(query_embeddings))
        batch_embeddings = query_embeddings[start_idx:end_idx]

        distances, indices = self.index.search(batch_embeddings, self.n_docs)
        all_distances.append(distances)
        all_indices.append(indices)

    # Combine results
    all_distances = np.vstack(all_distances)
    all_indices = np.vstack(all_indices)

    # Process results
    for i, document in enumerate(tqdm(documents, desc="Processing documents")):
        contexts = []
        for dist, idx in zip(all_distances[i], all_indices[i]):
            try:
                context = self._create_context_from_result(idx, dist, document)
                if context:
                    contexts.append(context)
            except Exception as e:
                print(f"Error processing result {idx}: {e}")

        document.contexts = contexts

    return documents