Self Consistency RAG

rankify.generator.rag_methods.self_consistency_rag

BaseRAGModel

Bases: ABC

Base RAG Model for Retrieval-Augmented Generation (RAG).

This is an abstract base class for implementing LLM endpoints in rankify. It defines the interface for generating responses and optional embedding generation.

Methods:

Name	Description
generate	str, **kwargs) -> str: Abstract method to generate a response based on the given prompt.
embed	str, **kwargs) -> List[float]: Optional method to generate embeddings for the given text.

Notes

• This class serves as a blueprint for RAG models like OpenAIModel and HuggingFaceModel.
• The embed method is optional and can be implemented if needed.
• This class needs to be extended to include new LLM endpoints in Rankify.

Source code in rankify/generator/models/base_rag_model.py

class BaseRAGModel(ABC):
    """
    **Base RAG Model** for Retrieval-Augmented Generation (RAG).

    This is an abstract base class for implementing LLM endpoints in rankify. 
    It defines the interface for generating responses and optional embedding generation.

    Methods:
        generate(prompt: str, **kwargs) -> str:
            Abstract method to generate a response based on the given prompt.
        embed(text: str, **kwargs) -> List[float]:
            Optional method to generate embeddings for the given text.

    Notes:
        - This class serves as a blueprint for RAG models like `OpenAIModel` and `HuggingFaceModel`.
        - The `embed` method is optional and can be implemented if needed.
        - This class needs to be extended to include new LLM endpoints in Rankify.
    """

    @abstractmethod
    def generate(self, prompt: str, **kwargs) -> str:
        """Generate a response based on the given prompt."""
        pass

    def embed(self, text: str, **kwargs) -> List[float]:
        """Optional: Generate embeddings for the given text."""
        raise NotImplementedError("Embedding is not required for this implementation.")

generate(prompt, **kwargs) abstractmethod

Generate a response based on the given prompt.

Source code in rankify/generator/models/base_rag_model.py

@abstractmethod
def generate(self, prompt: str, **kwargs) -> str:
    """Generate a response based on the given prompt."""
    pass

embed(text, **kwargs)

Optional: Generate embeddings for the given text.

Source code in rankify/generator/models/base_rag_model.py

def embed(self, text: str, **kwargs) -> List[float]:
    """Optional: Generate embeddings for the given text."""
    raise NotImplementedError("Embedding is not required for this implementation.")

Document

Represents a document consisting of a question, answers, and contexts.

Attributes:

Name	Type	Description
question	Question	The question associated with the document.
answers	Answer	The answers to the question.
contexts	list[Context]	A list of related contexts.
reorder_contexts	list[Context] or None	A reordered list of contexts based on relevance.

Source code in rankify/dataset/dataset.py

class Document:
    """
    Represents a document consisting of a question, answers, and contexts.

    Attributes:
        question (Question): The question associated with the document.
        answers (Answer): The answers to the question.
        contexts (list[Context]): A list of related contexts.
        reorder_contexts (list[Context] or None): A reordered list of contexts based on relevance.
    """
    def __init__(self, question: Question, answers: Answer, contexts: list = None , id: int = None) -> None:
        """
        Initializes a Document instance.

        Args:
            question (Question): The question associated with the document.
            answers (Answer): The answers to the question.
            contexts (list[Context], optional): A list of contexts related to the question.

        Example:
            ```python
            q = Question("What is the capital of France?")
            a = Answer(["Paris"])
            c1 = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
            c2 = Context(score=0.5, has_answer=False, id=2, title="Berlin", text="Berlin is the capital of Germany.")
            d = Document(question=q, answers=a, contexts=[c1, c2])
            print(d)
            ```
        """
        self.question: Question = question
        self.answers: Answer = answers
        self.contexts: List[Context] = contexts
        self.reorder_contexts: List[Context] = None
        self.id = str(id) 

    @classmethod
    def from_dict(cls, data: dict,n_docs:int=100) -> 'Document':
        """
        Creates a Document instance from a dictionary.

        Args:
            data (dict): A dictionary containing the question, answers, and contexts.
            n_docs (int, optional): The number of contexts to include. Defaults to 100.

        Returns:
            Document: A new Document instance.

        Example:
            ```python
            data = {
                "question": "What is the capital of France?",
                "answers": ["Paris"],
                "ctxs": [
                    {"score": 0.9, "has_answer": True, "id": 1, "title": "Paris", "text": "The capital of France is Paris."},
                    {"score": 0.5, "has_answer": False, "id": 2, "title": "Berlin", "text": "Berlin is the capital of Germany."}
                ]
            }
            d = Document.from_dict(data)
            print(d.question)
            ```
        """
        question = Question(data["question"])
        if "answers" in data:
            answers = Answer(data["answers"])
        else:
            answers =Answer('')

        if "query_id" in data:
            id = data["query_id"]
        else:
            id = None
        contexts = [Context(**ctx) for ctx in data["ctxs"][:n_docs]]
        return cls(question, answers, contexts, id=id)

    def to_dict(self) -> Dict[str, Optional[object]]:
        """
        Converts the document into a dictionary representation.

        Returns:
            dict: A dictionary containing the question, answers, and contexts.
        """
        return {
            "question": self.question.question,
            "answers": self.answers.answers,
            "contexts": [ctx.to_dict() for ctx in self.contexts]
        }
    def to_dict_reoreder(self) -> Dict[str,Optional[object]]:
        return {
            "question" : self.question.question,
            "answers" : self.answers.answers,
            "contexts" : [ctx.to_dict() for ctx in self.reorder_contexts]
        }
    def __str__(self) -> str:
        """
        Returns a string representation of the Document instance.

        Returns:
            str: The formatted document information.

        Example:
            ```python
            d = Document(Question("What is the capital of France?"), Answer(["Paris"]))
            print(d)
            ```
        """
        contexts_str = "\n\n".join([str(ctx) for ctx in self.contexts])
        reorder_contexts_str= ''
        if self.reorder_contexts is not None:
            reorder_contexts_str = "\n\n".join([str(ctx) for ctx in self.reorder_contexts])
        return f"{self.question}\n\n{self.answers}\n\nContext: \n\n{contexts_str}\nReorder contexts: \n\n{reorder_contexts_str}"

__init__(question, answers, contexts=None, id=None)

Initializes a Document instance.

Parameters:

Name	Type	Description	Default
question	Question	The question associated with the document.	required
answers	Answer	The answers to the question.	required
contexts	list[Context]	A list of contexts related to the question.	None

Example

q = Question("What is the capital of France?")
a = Answer(["Paris"])
c1 = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
c2 = Context(score=0.5, has_answer=False, id=2, title="Berlin", text="Berlin is the capital of Germany.")
d = Document(question=q, answers=a, contexts=[c1, c2])
print(d)

Source code in rankify/dataset/dataset.py

def __init__(self, question: Question, answers: Answer, contexts: list = None , id: int = None) -> None:
    """
    Initializes a Document instance.

    Args:
        question (Question): The question associated with the document.
        answers (Answer): The answers to the question.
        contexts (list[Context], optional): A list of contexts related to the question.

    Example:
        ```python
        q = Question("What is the capital of France?")
        a = Answer(["Paris"])
        c1 = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
        c2 = Context(score=0.5, has_answer=False, id=2, title="Berlin", text="Berlin is the capital of Germany.")
        d = Document(question=q, answers=a, contexts=[c1, c2])
        print(d)
        ```
    """
    self.question: Question = question
    self.answers: Answer = answers
    self.contexts: List[Context] = contexts
    self.reorder_contexts: List[Context] = None
    self.id = str(id) 

from_dict(data, n_docs=100) classmethod

Creates a Document instance from a dictionary.

Parameters:

Name	Type	Description	Default
data	dict	A dictionary containing the question, answers, and contexts.	required
n_docs	int	The number of contexts to include. Defaults to 100.	100

Returns:

Name	Type	Description
Document	Document	A new Document instance.

Example

data = {
    "question": "What is the capital of France?",
    "answers": ["Paris"],
    "ctxs": [
        {"score": 0.9, "has_answer": True, "id": 1, "title": "Paris", "text": "The capital of France is Paris."},
        {"score": 0.5, "has_answer": False, "id": 2, "title": "Berlin", "text": "Berlin is the capital of Germany."}
    ]
}
d = Document.from_dict(data)
print(d.question)

Source code in rankify/dataset/dataset.py

@classmethod
def from_dict(cls, data: dict,n_docs:int=100) -> 'Document':
    """
    Creates a Document instance from a dictionary.

    Args:
        data (dict): A dictionary containing the question, answers, and contexts.
        n_docs (int, optional): The number of contexts to include. Defaults to 100.

    Returns:
        Document: A new Document instance.

    Example:
        ```python
        data = {
            "question": "What is the capital of France?",
            "answers": ["Paris"],
            "ctxs": [
                {"score": 0.9, "has_answer": True, "id": 1, "title": "Paris", "text": "The capital of France is Paris."},
                {"score": 0.5, "has_answer": False, "id": 2, "title": "Berlin", "text": "Berlin is the capital of Germany."}
            ]
        }
        d = Document.from_dict(data)
        print(d.question)
        ```
    """
    question = Question(data["question"])
    if "answers" in data:
        answers = Answer(data["answers"])
    else:
        answers =Answer('')

    if "query_id" in data:
        id = data["query_id"]
    else:
        id = None
    contexts = [Context(**ctx) for ctx in data["ctxs"][:n_docs]]
    return cls(question, answers, contexts, id=id)

to_dict()

Converts the document into a dictionary representation.

Returns:

Name	Type	Description
dict	Dict[str, Optional[object]]	A dictionary containing the question, answers, and contexts.

Source code in rankify/dataset/dataset.py

def to_dict(self) -> Dict[str, Optional[object]]:
    """
    Converts the document into a dictionary representation.

    Returns:
        dict: A dictionary containing the question, answers, and contexts.
    """
    return {
        "question": self.question.question,
        "answers": self.answers.answers,
        "contexts": [ctx.to_dict() for ctx in self.contexts]
    }

__str__()

Returns a string representation of the Document instance.

Returns:

Name	Type	Description
str	str	The formatted document information.

Example

d = Document(Question("What is the capital of France?"), Answer(["Paris"]))
print(d)

Source code in rankify/dataset/dataset.py

def __str__(self) -> str:
    """
    Returns a string representation of the Document instance.

    Returns:
        str: The formatted document information.

    Example:
        ```python
        d = Document(Question("What is the capital of France?"), Answer(["Paris"]))
        print(d)
        ```
    """
    contexts_str = "\n\n".join([str(ctx) for ctx in self.contexts])
    reorder_contexts_str= ''
    if self.reorder_contexts is not None:
        reorder_contexts_str = "\n\n".join([str(ctx) for ctx in self.reorder_contexts])
    return f"{self.question}\n\n{self.answers}\n\nContext: \n\n{contexts_str}\nReorder contexts: \n\n{reorder_contexts_str}"

Answer

Represents answers to a question.

Attributes:

Name	Type	Description
answers	list[str]	A list of possible answers.

Source code in rankify/dataset/dataset.py

class Answer:
    """
    Represents answers to a question.

    Attributes:
        answers (list[str]): A list of possible answers.
    """
    def __init__(self, answers:list=None) -> None:
        """
        Initializes an Answer instance.

        Args:
            answers (list[str] or str, optional): A list of possible answers. Defaults to None.

        Example:
            ```python
            a = Answer(["Paris", "Lyon"])
            print(a)  
            # Output:
            # Answer:
            # - Paris
            # - Lyon
            ```
        """
        if isinstance(answers, str):  # If it's a string, convert it to a list
            self.answers = [answers]
        if isinstance(answers, int):
             self.answers = [str(answers)]
        elif isinstance(answers, list):  # If it's a list, ensure all elements are strings
            self.answers = [str(answer) for answer in answers]
        else:  # If it's neither, initialize with an empty list
            self.answers = []

    def __str__(self) -> str:
        """
        Returns a string representation of the Answer instance.

        Returns:
            str: The formatted answers.

        Example:
            ```python
            a = Answer(["Paris", "Lyon"])
            str(a)  
            # Output: 
            # Answer: 
            # - Paris
            # - Lyon
            ```
        """
        return f"Answer: \n- "+ "\n- ".join(self.answers)

__init__(answers=None)

Initializes an Answer instance.

Parameters:

Name	Type	Description	Default
answers	list[str] or str	A list of possible answers. Defaults to None.	None

Example

a = Answer(["Paris", "Lyon"])
print(a)  
# Output:
# Answer:
# - Paris
# - Lyon

Source code in rankify/dataset/dataset.py

def __init__(self, answers:list=None) -> None:
    """
    Initializes an Answer instance.

    Args:
        answers (list[str] or str, optional): A list of possible answers. Defaults to None.

    Example:
        ```python
        a = Answer(["Paris", "Lyon"])
        print(a)  
        # Output:
        # Answer:
        # - Paris
        # - Lyon
        ```
    """
    if isinstance(answers, str):  # If it's a string, convert it to a list
        self.answers = [answers]
    if isinstance(answers, int):
         self.answers = [str(answers)]
    elif isinstance(answers, list):  # If it's a list, ensure all elements are strings
        self.answers = [str(answer) for answer in answers]
    else:  # If it's neither, initialize with an empty list
        self.answers = []

__str__()

Returns a string representation of the Answer instance.

Returns:

Name	Type	Description
str	str	The formatted answers.

Example

a = Answer(["Paris", "Lyon"])
str(a)  
# Output: 
# Answer: 
# - Paris
# - Lyon

Source code in rankify/dataset/dataset.py

def __str__(self) -> str:
    """
    Returns a string representation of the Answer instance.

    Returns:
        str: The formatted answers.

    Example:
        ```python
        a = Answer(["Paris", "Lyon"])
        str(a)  
        # Output: 
        # Answer: 
        # - Paris
        # - Lyon
        ```
    """
    return f"Answer: \n- "+ "\n- ".join(self.answers)

BaseRAGMethod

Bases: ABC

Base RAG Method for Retrieval-Augmented Generation (RAG) techniques.

This abstract base class defines the blueprint for implementing RAG methods in Rankify. Each RAG method (e.g., zero-shot, chain-of-thought, Fusion-in-Decoder) should inherit from this class and implement the logic for answering questions using a provided RAG model.

Attributes:

Name	Type	Description
model	BaseRAGModel	The RAG model instance used for generation.

Methods:

Name	Description
answer_questions	List[Document], custom_prompt=None, **kwargs) -> List[str]: Abstract method to answer questions based on a list of documents and optional custom prompt.

Notes

• Extend this class to implement new RAG techniques or strategies.
• The answer_questions method must be implemented by all subclasses.
• This class enables modularity and extensibility for different retrieval-augmented generation approaches.

Source code in rankify/generator/rag_methods/base_rag_method.py

class BaseRAGMethod(ABC):
    """
    **Base RAG Method** for Retrieval-Augmented Generation (RAG) techniques.

    This abstract base class defines the blueprint for implementing RAG methods in Rankify.
    Each RAG method (e.g., zero-shot, chain-of-thought, Fusion-in-Decoder) should inherit from this class
    and implement the logic for answering questions using a provided RAG model.

    Attributes:
        model (BaseRAGModel): The RAG model instance used for generation.

    Methods:
        answer_questions(documents: List[Document], custom_prompt=None, **kwargs) -> List[str]:
            Abstract method to answer questions based on a list of documents and optional custom prompt.

    Notes:
        - Extend this class to implement new RAG techniques or strategies.
        - The `answer_questions` method must be implemented by all subclasses.
        - This class enables modularity and extensibility for different retrieval-augmented generation approaches.
    """
    def __init__(self, model: BaseRAGModel, **kwargs):
        """
        Initialize the BaseRAGMethod.

        Args:
            model (BaseRAGModel): The RAG model instance used for generation.
            **kwargs: Additional configuration parameters for the RAG method.
        """
        self.model = model

    @abstractmethod
    def answer_questions(self, documents: List[Document], custom_prompt=None, **kwargs) -> List[str]:
        """
        Abstract method to answer questions based on a list of documents.

        Args:
            documents (List[Document]): List of Document objects containing questions and contexts.
            custom_prompt (str, optional): Custom prompt to override default prompt generation.
            **kwargs: Additional parameters for the answering logic.

        Returns:
            List[str]: List of generated answers, one per document.

        Notes:
            - Must be implemented by subclasses to define the RAG technique's answering logic.
            - Enables flexible integration of different prompting or generation strategies.
        """
        pass

__init__(model, **kwargs)

Initialize the BaseRAGMethod.

Parameters:

Name	Type	Description	Default
model	BaseRAGModel	The RAG model instance used for generation.	required
**kwargs		Additional configuration parameters for the RAG method.	{}

Source code in rankify/generator/rag_methods/base_rag_method.py

def __init__(self, model: BaseRAGModel, **kwargs):
    """
    Initialize the BaseRAGMethod.

    Args:
        model (BaseRAGModel): The RAG model instance used for generation.
        **kwargs: Additional configuration parameters for the RAG method.
    """
    self.model = model

answer_questions(documents, custom_prompt=None, **kwargs) abstractmethod

Abstract method to answer questions based on a list of documents.

Parameters:

Name	Type	Description	Default
documents	List[Document]	List of Document objects containing questions and contexts.	required
custom_prompt	str	Custom prompt to override default prompt generation.	None
**kwargs		Additional parameters for the answering logic.	{}

Returns:

Type	Description
List[str]	List[str]: List of generated answers, one per document.

Notes

• Must be implemented by subclasses to define the RAG technique's answering logic.
• Enables flexible integration of different prompting or generation strategies.

Source code in rankify/generator/rag_methods/base_rag_method.py

@abstractmethod
def answer_questions(self, documents: List[Document], custom_prompt=None, **kwargs) -> List[str]:
    """
    Abstract method to answer questions based on a list of documents.

    Args:
        documents (List[Document]): List of Document objects containing questions and contexts.
        custom_prompt (str, optional): Custom prompt to override default prompt generation.
        **kwargs: Additional parameters for the answering logic.

    Returns:
        List[str]: List of generated answers, one per document.

    Notes:
        - Must be implemented by subclasses to define the RAG technique's answering logic.
        - Enables flexible integration of different prompting or generation strategies.
    """
    pass

SelfConsistencyRAG

Bases: BaseRAGMethod

SelfConsistencyRAG Method for Retrieval-Augmented Generation.

Implements the self-consistency technique for multi-step question answering. For each question, the model generates multiple answers with a Chain-of-Thought style prompt and aggregates them using majority vote or an optional reranker, improving reliability and robustness of the final answer.

Attributes:

Name	Type	Description
model	BaseRAGModel	The RAG model instance used for generation.
num_samples	int	Number of answer samples to generate per question (default: 5).
reranker		Optional reranker instance for ranking generated answers.

Methods:

Name	Description
__init__	Initializes the SelfConsistencyRAG method with the provided model, sample count, and optional reranker.
answer_questions	List[Document], custom_prompt=None, **kwargs) -> List[str]: Generates multiple answers for each document and aggregates by majority vote or reranker.

References

• Wang et al. Self-consistency improves chain of thought reasoning in language models
  Paper

Example

from rankify.dataset.dataset import Document, Question, Answer, Context
from rankify.generator.generator import Generator
import torch

from rankify.models.sentence_transformer_reranker import SentenceTransformerReranker

# Define question and answer placeholder
question = Question("What is the largest Portuguese-speaking city in the world?")
answers = Answer("")

# Define contexts
contexts = [
    Context(id=1, title="Portuguese Language Distribution", text=(
        "Portuguese is spoken in many countries, including Portugal, Brazil, Angola, Mozambique, and others. "
        "Among these, Brazil has the largest population of Portuguese speakers."), score=0.9),

    Context(id=2, title="Population Facts", text=(
        "Brazil has the most populous city in these countries."), score=0.8),

    Context(id=3, title="São Paulo vs Lisbon", text=(
        "São Paulo is the most populous city in Brazil. It is widely recognized as the largest Portuguese-speaking city in the world. "
        "In comparison, Lisbon, the capital of Portugal, has a much smaller population than São Paulo."), score=0.95),
]

# Construct document
doc = Document(question=question, answers=answers, contexts=contexts)

generation_kwargs = dict(
    temperature=0.8,      # or even 1.0 for more diversity
    top_p=0.95,           # allow some randomness
    max_new_tokens=16,    # enough for a short answer
)

#Use a reranker if preferred over majority voting (optional)
#reranker = SentenceTransformerReranker(method="sentence_transformer_reranker", model_name="all-MiniLM-L6-v2")

generator = Generator(
    method="self-consistency-rag",
    model_name='meta-llama/Meta-Llama-3.1-8B-Instruct',
    backend="huggingface",
    torch_dtype=torch.float16,
    num_samples=10,  # Number of samples for self-consistency
    #reranker=reranker,  # Optional reranker for better answer selection
)

generated_answers = generator.generate([doc], **generation_kwargs)
print(generated_answers) 

Notes

• Generates multiple answers per question using the underlying model's sampling capabilities.
• If a reranker is provided, uses it to select the best answer; otherwise, applies majority voting.
• Suitable for reducing randomness and improving answer reliability in generative QA.

Source code in rankify/generator/rag_methods/self_consistency_rag.py

class SelfConsistencyRAG(BaseRAGMethod):
    """
    **SelfConsistencyRAG Method** for Retrieval-Augmented Generation.

    Implements the self-consistency technique for multi-step question answering.
    For each question, the model generates multiple answers with a Chain-of-Thought style prompt
    and aggregates them using majority vote or an optional reranker, improving reliability
    and robustness of the final answer.

    Attributes:
        model (BaseRAGModel): The RAG model instance used for generation.
        num_samples (int): Number of answer samples to generate per question (default: 5).
        reranker: Optional reranker instance for ranking generated answers.

    Methods:
        __init__(model, num_samples=5, reranker=None, **kwargs):
            Initializes the SelfConsistencyRAG method with the provided model, sample count, and optional reranker.
        answer_questions(documents: List[Document], custom_prompt=None, **kwargs) -> List[str]:
            Generates multiple answers for each document and aggregates by majority vote or reranker.

    References:
        - Wang et al. *Self-consistency improves chain of thought reasoning in language models*  
            [Paper](https://arxiv.org/abs/2203.11171)

    Example:
        ```python
        from rankify.dataset.dataset import Document, Question, Answer, Context
        from rankify.generator.generator import Generator
        import torch

        from rankify.models.sentence_transformer_reranker import SentenceTransformerReranker

        # Define question and answer placeholder
        question = Question("What is the largest Portuguese-speaking city in the world?")
        answers = Answer("")

        # Define contexts
        contexts = [
            Context(id=1, title="Portuguese Language Distribution", text=(
                "Portuguese is spoken in many countries, including Portugal, Brazil, Angola, Mozambique, and others. "
                "Among these, Brazil has the largest population of Portuguese speakers."), score=0.9),

            Context(id=2, title="Population Facts", text=(
                "Brazil has the most populous city in these countries."), score=0.8),

            Context(id=3, title="São Paulo vs Lisbon", text=(
                "São Paulo is the most populous city in Brazil. It is widely recognized as the largest Portuguese-speaking city in the world. "
                "In comparison, Lisbon, the capital of Portugal, has a much smaller population than São Paulo."), score=0.95),
        ]

        # Construct document
        doc = Document(question=question, answers=answers, contexts=contexts)

        generation_kwargs = dict(
            temperature=0.8,      # or even 1.0 for more diversity
            top_p=0.95,           # allow some randomness
            max_new_tokens=16,    # enough for a short answer
        )

        #Use a reranker if preferred over majority voting (optional)
        #reranker = SentenceTransformerReranker(method="sentence_transformer_reranker", model_name="all-MiniLM-L6-v2")

        generator = Generator(
            method="self-consistency-rag",
            model_name='meta-llama/Meta-Llama-3.1-8B-Instruct',
            backend="huggingface",
            torch_dtype=torch.float16,
            num_samples=10,  # Number of samples for self-consistency
            #reranker=reranker,  # Optional reranker for better answer selection
        )

        generated_answers = generator.generate([doc], **generation_kwargs)
        print(generated_answers) 
        ```

    Notes:
        - Generates multiple answers per question using the underlying model's sampling capabilities.
        - If a reranker is provided, uses it to select the best answer; otherwise, applies majority voting.
        - Suitable for reducing randomness and improving answer reliability in generative QA.
    """
    def __init__(self, model: BaseRAGModel, num_samples: int = 5, reranker=None, **kwargs):
        """
        Initialize the SelfConsistencyRAG method.

        Args:
            model (BaseRAGModel): The RAG model instance used for generation.
            num_samples (int, optional): Number of answer samples to generate per question (default: 5).
            reranker (optional): Reranker instance for ranking generated answers.
            **kwargs: Additional configuration parameters (unused).
        """
        super().__init__(model=model)
        self.num_samples = num_samples
        self.reranker = reranker  # Optional: pass a reranker instance

    def answer_questions(self, documents: List[Document], custom_prompt: Optional[str] = None, **kwargs) -> List[str]:
        """
        For each document, generate multiple answers and aggregate by majority vote or reranker.

        Args:
            documents (List[Document]): A list of Document objects containing questions and contexts.
            custom_prompt (str, optional): Custom prompt to override default prompt generation.
            **kwargs: Additional parameters for the model's generate method.

        Returns:
            List[str]: Aggregated answers for each document.

        Notes:
            - Uses the model's prompt generator to construct Chain-of-Thought style prompt.
            - Generates multiple answers per question using sampling and aggregates results.
            - If a reranker is provided, selects the best answer using reranking; otherwise, applies majority voting.
        """
        answers = []
        for document in tqdm(documents, desc="Answering questions", unit="q"):
            question = document.question.question
            contexts = [context.text for context in document.contexts]
            prompt = self.model.prompt_generator.generate_user_prompt(question, contexts, custom_prompt)

            # Generate multiple answers in one call
            sample_answers = self.model.generate(
                prompt=prompt,
                #do_sample=True,
                num_return_sequences=self.num_samples,
                **kwargs
            )

            # Ensure sample_answers is a list
            if isinstance(sample_answers, str):
                sample_answers = [sample_answers]

            print(f"Generated {len(sample_answers)} samples: {sample_answers}")

            # Use reranker if provided, otherwise majority vote
            if self.reranker:
                rerank_docs = []
                for ans in sample_answers:
                    doc = Document(question=document.question, answers=Answer(answers=[ans]), contexts=document.contexts)
                    rerank_docs.append(doc)
                reranked = self.reranker.rank(rerank_docs)
                best_answer = reranked[0].answers.answers
                if isinstance(best_answer, list):
                    best_answer = best_answer[0]
                answers.append(best_answer)
            else:
                # Majority vote (with normalization)
                def normalize(text):
                    return text.strip()
                normalized = [normalize(ans) for ans in sample_answers]
                most_common, _ = Counter(normalized).most_common(1)[0]
                answers.append(most_common)
        return answers

__init__(model, num_samples=5, reranker=None, **kwargs)

Initialize the SelfConsistencyRAG method.

Parameters:

Name	Type	Description	Default
model	BaseRAGModel	The RAG model instance used for generation.	required
num_samples	int	Number of answer samples to generate per question (default: 5).	5
reranker	optional	Reranker instance for ranking generated answers.	None
**kwargs		Additional configuration parameters (unused).	{}

Source code in rankify/generator/rag_methods/self_consistency_rag.py

def __init__(self, model: BaseRAGModel, num_samples: int = 5, reranker=None, **kwargs):
    """
    Initialize the SelfConsistencyRAG method.

    Args:
        model (BaseRAGModel): The RAG model instance used for generation.
        num_samples (int, optional): Number of answer samples to generate per question (default: 5).
        reranker (optional): Reranker instance for ranking generated answers.
        **kwargs: Additional configuration parameters (unused).
    """
    super().__init__(model=model)
    self.num_samples = num_samples
    self.reranker = reranker  # Optional: pass a reranker instance

answer_questions(documents, custom_prompt=None, **kwargs)

For each document, generate multiple answers and aggregate by majority vote or reranker.

Parameters:

Name	Type	Description	Default
documents	List[Document]	A list of Document objects containing questions and contexts.	required
custom_prompt	str	Custom prompt to override default prompt generation.	None
**kwargs		Additional parameters for the model's generate method.	{}

Returns:

Type	Description
List[str]	List[str]: Aggregated answers for each document.

Notes

• Uses the model's prompt generator to construct Chain-of-Thought style prompt.
• Generates multiple answers per question using sampling and aggregates results.
• If a reranker is provided, selects the best answer using reranking; otherwise, applies majority voting.

Source code in rankify/generator/rag_methods/self_consistency_rag.py

def answer_questions(self, documents: List[Document], custom_prompt: Optional[str] = None, **kwargs) -> List[str]:
    """
    For each document, generate multiple answers and aggregate by majority vote or reranker.

    Args:
        documents (List[Document]): A list of Document objects containing questions and contexts.
        custom_prompt (str, optional): Custom prompt to override default prompt generation.
        **kwargs: Additional parameters for the model's generate method.

    Returns:
        List[str]: Aggregated answers for each document.

    Notes:
        - Uses the model's prompt generator to construct Chain-of-Thought style prompt.
        - Generates multiple answers per question using sampling and aggregates results.
        - If a reranker is provided, selects the best answer using reranking; otherwise, applies majority voting.
    """
    answers = []
    for document in tqdm(documents, desc="Answering questions", unit="q"):
        question = document.question.question
        contexts = [context.text for context in document.contexts]
        prompt = self.model.prompt_generator.generate_user_prompt(question, contexts, custom_prompt)

        # Generate multiple answers in one call
        sample_answers = self.model.generate(
            prompt=prompt,
            #do_sample=True,
            num_return_sequences=self.num_samples,
            **kwargs
        )

        # Ensure sample_answers is a list
        if isinstance(sample_answers, str):
            sample_answers = [sample_answers]

        print(f"Generated {len(sample_answers)} samples: {sample_answers}")

        # Use reranker if provided, otherwise majority vote
        if self.reranker:
            rerank_docs = []
            for ans in sample_answers:
                doc = Document(question=document.question, answers=Answer(answers=[ans]), contexts=document.contexts)
                rerank_docs.append(doc)
            reranked = self.reranker.rank(rerank_docs)
            best_answer = reranked[0].answers.answers
            if isinstance(best_answer, list):
                best_answer = best_answer[0]
            answers.append(best_answer)
        else:
            # Majority vote (with normalization)
            def normalize(text):
                return text.strip()
            normalized = [normalize(ans) for ans in sample_answers]
            most_common, _ = Counter(normalized).most_common(1)[0]
            answers.append(most_common)
    return answers