Skip to content

RankGPT

rankify.models.rankgpt

HF_PRE_DEFIND_MODELS = {'upr': {'t5-small': 'google/t5-small-lm-adapt', 't5-base': 'google/t5-base-lm-adapt', 't5-large': 'google/t5-large-lm-adapt', 't0-3b': 'bigscience/T0_3B', 't0-11b': 'bigscience/T0', 'gpt-neo-2.7b': 'EleutherAI/gpt-neo-2.7B', 'gpt-j-6b': 'EleutherAI/gpt-j-6b', 'gpt2': 'openai-community/gpt2', 'gpt2-medium': 'openai-community/gpt2-medium', 'gpt2-large': 'openai-community/gpt2-large', 'gpt2-xl': 'openai-community/gpt2-xl', 'flan-t5-xl': 'google/flan-t5-xl'}, 'rankgpt-api': {'gpt-3.5': 'gpt-3.5', 'gpt-4': 'gpt-4o', 'gpt-4-mini': 'gpt-4o-mini', 'llamav3.1-8b': 'llamav3.1-8b', 'llamav3.1-70b': 'llamav3.1-70b', 'claude-3-5': 'claude-3-5'}, 'rankgpt': {'llamav3.1-8b': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'llamav3.1-70b': 'meta-llama/Llama-3.1-70B-Instruct', 'Llama-3.2-1B': 'meta-llama/Llama-3.2-1B-Instruct', 'Llama-3.2-3B': 'meta-llama/Llama-3.2-3B-Instruct', 'Qwen2.5-7B': 'Qwen/Qwen2.5-7B', 'Mistral-7B-Instruct-v0.2': 'mistralai/Mistral-7B-Instruct-v0.2', 'Mistral-7B-Instruct-v0.3': 'mistralai/Mistral-7B-Instruct-v0.3'}, 'flashrank': {'ms-marco-TinyBERT-L-2-v2': 'ms-marco-TinyBERT-L-2-v2', 'ms-marco-MiniLM-L-12-v2': 'ms-marco-MiniLM-L-12-v2', 'ms-marco-MultiBERT-L-12': 'ms-marco-MultiBERT-L-12', 'rank-T5-flan': 'rank-T5-flan', 'ce-esci-MiniLM-L12-v2': 'ce-esci-MiniLM-L12-v2', 'rank_zephyr_7b_v1_full': 'rank_zephyr_7b_v1_full', 'miniReranker_arabic_v1': 'miniReranker_arabic_v1'}, 'flashrank-model-file': {'ms-marco-TinyBERT-L-2-v2': 'flashrank-TinyBERT-L-2-v2.onnx', 'ms-marco-MiniLM-L-12-v2': 'flashrank-MiniLM-L-12-v2_Q.onnx', 'ms-marco-MultiBERT-L-12': 'flashrank-MultiBERT-L12_Q.onnx', 'rank-T5-flan': 'flashrank-rankt5_Q.onnx', 'ce-esci-MiniLM-L12-v2': 'flashrank-ce-esci-MiniLM-L12-v2_Q.onnx', 'rank_zephyr_7b_v1_full': 'rank_zephyr_7b_v1_full.Q4_K_M.gguf', 'miniReranker_arabic_v1': 'miniReranker_arabic_v1.onnx'}, 'monot5': {'monot5-base-msmarco': 'castorini/monot5-base-msmarco', 'monot5-base-msmarco-10k': 'castorini/monot5-base-msmarco-10k', 'monot5-large-msmarco': 'castorini/monot5-large-msmarco', 'monot5-large-msmarco-10k': 'castorini/monot5-large-msmarco-10k', 'monot5-base-med-msmarco': 'castorini/monot5-base-med-msmarco', 'monot5-3b-med-msmarco': 'castorini/monot5-3b-med-msmarco', 'monot5-3b-msmarco-10k': 'castorini/monot5-3b-msmarco-10k', 'mt5-base-en-msmarco': 'unicamp-dl/mt5-base-en-msmarco', 'ptt5-base-pt-msmarco-10k-v2': 'unicamp-dl/ptt5-base-pt-msmarco-10k-v2', 'ptt5-base-pt-msmarco-100k-v2': 'unicamp-dl/ptt5-base-pt-msmarco-100k-v2', 'ptt5-base-en-pt-msmarco-100k-v2': 'unicamp-dl/ptt5-base-en-pt-msmarco-100k-v2', 'mt5-base-en-pt-msmarco-v2': 'unicamp-dl/mt5-base-en-pt-msmarco-v2', 'mt5-base-mmarco-v2': 'unicamp-dl/mt5-base-mmarco-v2', 'mt5-base-en-pt-msmarco-v1': 'unicamp-dl/mt5-base-en-pt-msmarco-v1', 'mt5-base-mmarco-v1': 'unicamp-dl/mt5-base-mmarco-v1', 'ptt5-base-pt-msmarco-10k-v1': 'unicamp-dl/ptt5-base-pt-msmarco-10k-v1', 'ptt5-base-pt-msmarco-100k-v1': 'unicamp-dl/ptt5-base-pt-msmarco-100k-v1', 'ptt5-base-en-pt-msmarco-10k-v1': 'unicamp-dl/ptt5-base-en-pt-msmarco-10k-v1', 'mt5-3B-mmarco-en-pt': 'unicamp-dl/mt5-3B-mmarco-en-pt', 'mt5-13b-mmarco-100k': 'unicamp-dl/mt5-13b-mmarco-100k', 'monoptt5-small': 'unicamp-dl/monoptt5-small', 'monoptt5-base': 'unicamp-dl/monoptt5-base', 'monoptt5-large': 'unicamp-dl/monoptt5-large', 'monoptt5-3b': 'unicamp-dl/monoptt5-3b'}, 'rankt5': {'rankt5-base': 'Soyoung97/RankT5-base', 'rankt5-large': 'Soyoung97/RankT5-large', 'rankt5-3b': 'Soyoung97/RankT5-3b'}, 'listt5': {'listt5-base': 'Soyoung97/ListT5-base', 'listt5-3b': 'Soyoung97/ListT5-3b'}, 'inranker': {'inranker-small': 'unicamp-dl/InRanker-small', 'inranker-base': 'unicamp-dl/InRanker-base', 'inranker-3b': 'unicamp-dl/InRanker-3B'}, 'apiranker': {'cohere': 'cohere', 'jina': 'jina', 'voyage': 'voyage', 'mixedbread.ai': 'mixedbread.ai'}, 'transformer_ranker': {'mxbai-rerank-xsmall': 'mixedbread-ai/mxbai-rerank-xsmall-v1', 'mxbai-rerank-base': 'mixedbread-ai/mxbai-rerank-base-v1', 'mxbai-rerank-large': 'mixedbread-ai/mxbai-rerank-large-v1', 'bge-reranker-base': 'BAAI/bge-reranker-base', 'bge-reranker-large': 'BAAI/bge-reranker-large', 'bge-reranker-v2-m3': 'BAAI/bge-reranker-v2-m3', 'bce-reranker-base': 'maidalun1020/bce-reranker-base_v1', 'jina-reranker-tiny': 'jinaai/jina-reranker-v1-tiny-en', 'jina-reranker-turbo': 'jinaai/jina-reranker-v1-turbo-en', 'jina-reranker-base-multilingual': 'jinaai/jina-reranker-v2-base-multilingual', 'gte-multilingual-reranker-base': 'Alibaba-NLP/gte-multilingual-reranker-base', 'camembert-base-mmarcoFR': 'antoinelouis/crossencoder-camembert-base-mmarcoFR', 'camembert-large-mmarcoFR': 'antoinelouis/crossencoder-camembert-large-mmarcoFR', 'camemberta-base-mmarcoFR': 'antoinelouis/crossencoder-camemberta-base-mmarcoFR', 'distilcamembert-mmarcoFR': 'antoinelouis/crossencoder-distilcamembert-mmarcoFR', 'cross-encoder-mmarco-mMiniLMv2-L12-H384-v1': 'corrius/cross-encoder-mmarco-mMiniLMv2-L12-H384-v1', 'nli-deberta-v3-large': 'cross-encoder/nli-deberta-v3-large', 'ms-marco-MiniLM-L-12-v2': 'cross-encoder/ms-marco-MiniLM-L-12-v2', 'ms-marco-MiniLM-L-6-v2': 'cross-encoder/ms-marco-MiniLM-L-6-v2', 'ms-marco-MiniLM-L-4-v2': 'cross-encoder/ms-marco-MiniLM-L-4-v2', 'ms-marco-MiniLM-L-2-v2': 'cross-encoder/ms-marco-MiniLM-L-2-v2', 'ms-marco-TinyBERT-L-2-v2': 'cross-encoder/ms-marco-TinyBERT-L-2-v2', 'ms-marco-electra-base': 'cross-encoder/ms-marco-electra-base', 'ms-marco-TinyBERT-L-6': 'cross-encoder/ms-marco-TinyBERT-L-6', 'ms-marco-TinyBERT-L-4': 'cross-encoder/ms-marco-TinyBERT-L-4', 'ms-marco-TinyBERT-L-2': 'cross-encoder/ms-marco-TinyBERT-L-2', 'msmarco-MiniLM-L12-en-de-v1': 'cross-encoder/msmarco-MiniLM-L12-en-de-v1', 'msmarco-MiniLM-L6-en-de-v1': 'cross-encoder/msmarco-MiniLM-L6-en-de-v1'}, 'llm_layerwise_ranker': {'bge-multilingual-gemma2': 'BAAI/bge-multilingual-gemma2', 'bge-reranker-v2-gemma': 'BAAI/bge-reranker-v2-gemma', 'bge-reranker-v2-minicpm-layerwise': 'BAAI/bge-reranker-v2-minicpm-layerwise', 'bge-reranker-v2.5-gemma2-lightweight': 'BAAI/bge-reranker-v2.5-gemma2-lightweight'}, 'first_ranker': {'First-Model': 'rryisthebest/First_Model', 'Llama-3-8B': 'meta-llama/Meta-Llama-3-8B-Instruct'}, 'lit5dist': {'LiT5-Distill-base': 'castorini/LiT5-Distill-base', 'LiT5-Distill-large': 'castorini/LiT5-Distill-large', 'LiT5-Distill-xl': 'castorini/LiT5-Distill-xl', 'LiT5-Distill-base-v2': 'castorini/LiT5-Distill-base-v2', 'LiT5-Distill-large-v2': 'castorini/LiT5-Distill-large-v2', 'LiT5-Distill-xl-v2': 'castorini/LiT5-Distill-xl-v2'}, 'lit5score': {'LiT5-Score-base': 'castorini/LiT5-Score-base', 'LiT5-Score-large': 'castorini/LiT5-Score-large', 'LiT5-Score-xl': 'castorini/LiT5-Score-xl'}, 'vicuna_reranker': {'rank_vicuna_7b_v1': 'castorini/rank_vicuna_7b_v1', 'rank_vicuna_7b_v1_noda': 'castorini/rank_vicuna_7b_v1_noda', 'rank_vicuna_7b_v1_fp16': 'castorini/rank_vicuna_7b_v1_fp16', 'rank_vicuna_7b_v1_noda_fp16': 'castorini/rank_vicuna_7b_v1_noda_fp16'}, 'zephyr_reranker': {'rank_zephyr_7b_v1_full': 'castorini/rank_zephyr_7b_v1_full'}, 'blender_reranker': {'PairRM': 'llm-blender/PairRM'}, 'splade_reranker': {'splade-cocondenser': 'naver/splade-cocondenser-ensembledistil'}, 'sentence_transformer_reranker': {'all-MiniLM-L6-v2': 'all-MiniLM-L6-v2', 'gtr-t5-base': 'sentence-transformers/gtr-t5-base', 'gtr-t5-large': 'sentence-transformers/gtr-t5-large', 'gtr-t5-xl': 'sentence-transformers/gtr-t5-xl', 'gtr-t5-xxl': 'sentence-transformers/gtr-t5-xxl', 'sentence-t5-base': 'sentence-transformers/sentence-t5-base', 'sentence-t5-xl': 'sentence-transformers/sentence-t5-xl', 'sentence-t5-xxl': 'sentence-transformers/sentence-t5-xxl', 'sentence-t5-large': 'sentence-transformers/sentence-t5-large', 'distilbert-multilingual-nli-stsb-quora-ranking': 'sentence-transformers/distilbert-multilingual-nli-stsb-quora-ranking', 'msmarco-bert-co-condensor': 'sentence-transformers/msmarco-bert-co-condensor', 'msmarco-roberta-base-v2': 'sentence-transformers/msmarco-roberta-base-v2'}, 'colbert_ranker': {'colbertv2.0': 'colbert-ir/colbertv2.0', 'FranchColBERT': 'bclavie/FraColBERTv2', 'JapanColBERT': 'bclavie/JaColBERTv2', 'SpanishColBERT': 'AdrienB134/ColBERTv2.0-spanish-mmarcoES', 'jina-colbert-v1-en': 'jinaai/jina-colbert-v1-en', 'ArabicColBERT-250k': 'akhooli/arabic-colbertv2-250k-norm', 'ArabicColBERT-711k': 'akhooli/arabic-colbertv2-711k-norm', 'BengaliColBERT': 'turjo4nis/colbertv2.0-bn', 'mxbai-colbert-large-v1': 'mixedbread-ai/mxbai-colbert-large-v1'}, 'monobert': {'monobert-large': 'castorini/monobert-large-msmarco'}, 'llm2vec': {'Meta-Llama-31-8B': 'McGill-NLP/LLM2Vec-Meta-Llama-31-8B-Instruct-mntp', 'Meta-Llama-3-8B': 'McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp', 'Mistral-7B': 'McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp', 'Llama-2-7B': 'McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp', 'Sheared-LLaMA': 'McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp'}, 'twolar': {'twolar-xl': 'Dundalia/TWOLAR-xl', 'twolar-large': 'Dundalia/TWOLAR-large'}, 'duot5': {'duot5-base-msmarco': 'castorini/duot5-base-msmarco', 'duot5-3b-msmarco': 'castorini/duot5-3b-msmarco', 'duot5-large-msmarco': 'castorini/duot5-large-msmarco-10k'}, 'rankllama': {'rankllama-v1-7b-lora-passage': 'castorini/rankllama-v1-7b-lora-passage'}, 'dear_reranker': {'dear-3b-reranker-ce-v1': 'abdoelsayed/dear-3b-reranker-ce-v1', 'dear-3b-reranker-ranknet-v1': 'abdoelsayed/dear-3b-reranker-ranknet-v1', 'dear-3b-reranker-ce-lora-v1': 'abdoelsayed/dear-3b-reranker-ce-lora-v1', 'dear-8b-reranker-ce-v1': 'abdoelsayed/dear-8b-reranker-ce-v1'}, 'echorank': {'flan-t5-large': 'google/flan-t5-large', 'flan-t5-xl': 'google/flan-t5-xl'}, 'incontext_reranker': {'llamav3.1-8b': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'llamav3.1-70b': 'meta-llama/Llama-3.1-70B-Instruct', 'Mistral-7B-Instruct-v0.2': 'mistralai/Mistral-7B-Instruct-v0.2'}, 'tart': {'tart-full-flan-t5-xl': 'facebook/tart-full-flan-t5-xl', 'tart-dual-flan-t5-xl': 'facebook/tart-dual-flan-t5-xl'}, 'prp': {'llamav3.1-8b': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'llamav3.1-70b': 'meta-llama/Llama-3.1-70B-Instruct', 'Llama-3.2-1B': 'meta-llama/Llama-3.2-1B-Instruct', 'Llama-3.2-3B': 'meta-llama/Llama-3.2-3B-Instruct', 'Mistral-7B-Instruct-v0.3': 'mistralai/Mistral-7B-Instruct-v0.3'}, 'prp-api': {'gpt-3.5': 'gpt-3.5-turbo', 'gpt-4': 'gpt-4o', 'gpt-4-mini': 'gpt-4o-mini', 'llamav3.1-8b': 'llamav3.1-8b', 'llamav3.1-70b': 'llamav3.1-70b'}, 'rankgemma': {'gemma-2-2b': 'google/gemma-2-2b-it', 'gemma-2-9b': 'google/gemma-2-9b-it', 'gemma-2-27b': 'google/gemma-2-27b-it'}, 'rankmistral': {'mistral-7b': 'mistralai/Mistral-7B-Instruct-v0.3', 'mistral-7b-v0.2': 'mistralai/Mistral-7B-Instruct-v0.2', 'mixtral-8x7b': 'mistralai/Mixtral-8x7B-Instruct-v0.1'}} module-attribute

URL = {'default': {'url': 'https://api.openai.com/v1', 'model_name': 'gpt-3.5-turbo-0125', 'class': OpenaiClient}, 'gpt-3.5': {'url': 'https://api.openai.com/v1', 'model_name': 'gpt-3.5-turbo-0125', 'class': OpenaiClient}, 'gpt-4': {'url': 'https://api.openai.com/v1', 'model_name': 'gpt-4o', 'class': OpenaiClient}, 'gpt-4-mini': {'url': 'https://api.openai.com/v1', 'model_name': 'gpt-4o-mini', 'class': OpenaiClient}, 'llamav3.1-8b': {'url': 'https://api.together.xyz/v1', 'model_name': 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo', 'class': OpenaiClient}, 'llamav3.1-70b': {'url': 'https://api.together.xyz/v1', 'model_name': 'meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo', 'class': OpenaiClient}, 'claude-3-5': {'url': 'https://api.anthropic.com', 'model_name': 'claude-3-5-sonnet-20241022', 'class': ClaudeClient}, 'cohere': {'url': 'https://api.cohere.ai/v2/rerank', 'model_name': 'rerank-english-v3.0'}, 'jina': {'url': 'https://api.jina.ai/v1/rerank', 'model_name': 'jina-reranker-v1-base-en'}, 'voyage': {'url': 'https://api.voyageai.com/v1/rerank', 'model_name': 'rerank-lite-1'}, 'mixedbread.ai': {'url': 'https://api.mixedbread.ai/v1/reranking', 'model_name': 'mixedbread-ai/mxbai-rerank-large-v1'}} module-attribute

BaseRanking

Bases: ABC

An abstract base class for implementing different ranking models.

This class defines the interface for all ranking models, ensuring that all subclasses implement the required methods.

Attributes:

Name Type Description
method str

The name of the ranking method.
model_name str

The name of the model being used for ranking.
api_key str

An optional API key for accessing remote models or services.
Source code in rankify/models/base.py 
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
class BaseRanking(ABC):
    """
    An abstract base class for implementing different ranking models.

    This class defines the interface for all ranking models, ensuring that all subclasses implement the required methods.

    Attributes:
        method (str): The name of the ranking method.
        model_name (str): The name of the model being used for ranking.
        api_key (str, optional): An optional API key for accessing remote models or services.
    """

    @abstractmethod
    def __init__(self, method: str= None, model_name: str= None, api_key: str= None, **kwargs) ->None:
        """
        Initializes the base ranking model.

        Args:
            method (str, optional): The name of the ranking method. Defaults to None.
            model_name (str, optional): The name of the model being used for ranking. Defaults to None.
            api_key (str, optional): An optional API key for accessing remote models or services. Defaults to None.

        Example:
            ```python
            class MyRanking(BaseRanking):
                def __init__(self, method, model_name):
                    super().__init__(method, model_name)
            ```
        """
        pass

    @abstractmethod
    def rank(self, documents: list[Document] ):
        """
        Abstract method to rank a list of documents.

        Args:
            documents (list[Document]): A list of Document instances that need to be ranked.

        Raises:
            NotImplementedError: This method must be implemented by subclasses.

        Example:
            ```python
            class MyRanking(BaseRanking):
                def __init__(self, method, model_name):
                    super().__init__(method, model_name)

                def rank(self, documents):
                    # Ranking implementation here
                    pass
            ```
        """
        pass

__init__(method=None, model_name=None, api_key=None, **kwargs) abstractmethod

Initializes the base ranking model.

Parameters:

Name Type Description Default
method str

The name of the ranking method. Defaults to None.

 None
model_name str

The name of the model being used for ranking. Defaults to None.

 None
api_key str

An optional API key for accessing remote models or services. Defaults to None.

 None
Example 
class MyRanking(BaseRanking):
    def __init__(self, method, model_name):
        super().__init__(method, model_name)
Source code in rankify/models/base.py 
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
@abstractmethod
def __init__(self, method: str= None, model_name: str= None, api_key: str= None, **kwargs) ->None:
    """
    Initializes the base ranking model.

    Args:
        method (str, optional): The name of the ranking method. Defaults to None.
        model_name (str, optional): The name of the model being used for ranking. Defaults to None.
        api_key (str, optional): An optional API key for accessing remote models or services. Defaults to None.

    Example:
        ```python
        class MyRanking(BaseRanking):
            def __init__(self, method, model_name):
                super().__init__(method, model_name)
        ```
    """
    pass

rank(documents) abstractmethod

Abstract method to rank a list of documents.

Parameters:

Name Type Description Default
documents list[Document]

A list of Document instances that need to be ranked.

 required

Raises:

Type Description
NotImplementedError

This method must be implemented by subclasses.
Example 
class MyRanking(BaseRanking):
    def __init__(self, method, model_name):
        super().__init__(method, model_name)

    def rank(self, documents):
        # Ranking implementation here
        pass
Source code in rankify/models/base.py 
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
@abstractmethod
def rank(self, documents: list[Document] ):
    """
    Abstract method to rank a list of documents.

    Args:
        documents (list[Document]): A list of Document instances that need to be ranked.

    Raises:
        NotImplementedError: This method must be implemented by subclasses.

    Example:
        ```python
        class MyRanking(BaseRanking):
            def __init__(self, method, model_name):
                super().__init__(method, model_name)

            def rank(self, documents):
                # Ranking implementation here
                pass
        ```
    """
    pass

Document

Represents a document consisting of a question, answers, and contexts.

Attributes:

Name Type Description
question Question

The question associated with the document.
answers Answer

The answers to the question.
contexts list[Context]

A list of related contexts.
reorder_contexts list[Context] or None

A reordered list of contexts based on relevance.
Source code in rankify/dataset/dataset.py 
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
class Document:
    """
    Represents a document consisting of a question, answers, and contexts.

    Attributes:
        question (Question): The question associated with the document.
        answers (Answer): The answers to the question.
        contexts (list[Context]): A list of related contexts.
        reorder_contexts (list[Context] or None): A reordered list of contexts based on relevance.
    """
    def __init__(self, question: Question, answers: Answer, contexts: list = None , id: int = None) -> None:
        """
        Initializes a Document instance.

        Args:
            question (Question): The question associated with the document.
            answers (Answer): The answers to the question.
            contexts (list[Context], optional): A list of contexts related to the question.

        Example:
            ```python
            q = Question("What is the capital of France?")
            a = Answer(["Paris"])
            c1 = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
            c2 = Context(score=0.5, has_answer=False, id=2, title="Berlin", text="Berlin is the capital of Germany.")
            d = Document(question=q, answers=a, contexts=[c1, c2])
            print(d)
            ```
        """
        self.question: Question = question
        self.answers: Answer = answers
        self.contexts: List[Context] = contexts
        self.reorder_contexts: List[Context] = None
        self.id = str(id) 

    @classmethod
    def from_dict(cls, data: dict,n_docs:int=100) -> 'Document':
        """
        Creates a Document instance from a dictionary.

        Args:
            data (dict): A dictionary containing the question, answers, and contexts.
            n_docs (int, optional): The number of contexts to include. Defaults to 100.

        Returns:
            Document: A new Document instance.

        Example:
            ```python
            data = {
                "question": "What is the capital of France?",
                "answers": ["Paris"],
                "ctxs": [
                    {"score": 0.9, "has_answer": True, "id": 1, "title": "Paris", "text": "The capital of France is Paris."},
                    {"score": 0.5, "has_answer": False, "id": 2, "title": "Berlin", "text": "Berlin is the capital of Germany."}
                ]
            }
            d = Document.from_dict(data)
            print(d.question)
            ```
        """
        question = Question(data["question"])
        if "answers" in data:
            answers = Answer(data["answers"])
        else:
            answers =Answer('')

        if "query_id" in data:
            id = data["query_id"]
        else:
            id = None
        contexts = [Context(**ctx) for ctx in data["ctxs"][:n_docs]]
        return cls(question, answers, contexts, id=id)

    def to_dict(self) -> Dict[str, Optional[object]]:
        """
        Converts the document into a dictionary representation.

        Returns:
            dict: A dictionary containing the question, answers, and contexts.
        """
        return {
            "question": self.question.question,
            "answers": self.answers.answers,
            "contexts": [ctx.to_dict() for ctx in self.contexts]
        }
    def to_dict_reoreder(self) -> Dict[str,Optional[object]]:
        return {
            "question" : self.question.question,
            "answers" : self.answers.answers,
            "contexts" : [ctx.to_dict() for ctx in self.reorder_contexts]
        }
    def __str__(self) -> str:
        """
        Returns a string representation of the Document instance.

        Returns:
            str: The formatted document information.

        Example:
            ```python
            d = Document(Question("What is the capital of France?"), Answer(["Paris"]))
            print(d)
            ```
        """
        contexts_str = "\n\n".join([str(ctx) for ctx in self.contexts])
        reorder_contexts_str= ''
        if self.reorder_contexts is not None:
            reorder_contexts_str = "\n\n".join([str(ctx) for ctx in self.reorder_contexts])
        return f"{self.question}\n\n{self.answers}\n\nContext: \n\n{contexts_str}\nReorder contexts: \n\n{reorder_contexts_str}"

__init__(question, answers, contexts=None, id=None)

Initializes a Document instance.

Parameters:

Name Type Description Default
question Question

The question associated with the document.

 required
answers Answer

The answers to the question.

 required
contexts list[Context]

A list of contexts related to the question.

 None
Example 
q = Question("What is the capital of France?")
a = Answer(["Paris"])
c1 = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
c2 = Context(score=0.5, has_answer=False, id=2, title="Berlin", text="Berlin is the capital of Germany.")
d = Document(question=q, answers=a, contexts=[c1, c2])
print(d)
Source code in rankify/dataset/dataset.py 
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
def __init__(self, question: Question, answers: Answer, contexts: list = None , id: int = None) -> None:
    """
    Initializes a Document instance.

    Args:
        question (Question): The question associated with the document.
        answers (Answer): The answers to the question.
        contexts (list[Context], optional): A list of contexts related to the question.

    Example:
        ```python
        q = Question("What is the capital of France?")
        a = Answer(["Paris"])
        c1 = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
        c2 = Context(score=0.5, has_answer=False, id=2, title="Berlin", text="Berlin is the capital of Germany.")
        d = Document(question=q, answers=a, contexts=[c1, c2])
        print(d)
        ```
    """
    self.question: Question = question
    self.answers: Answer = answers
    self.contexts: List[Context] = contexts
    self.reorder_contexts: List[Context] = None
    self.id = str(id)

from_dict(data, n_docs=100) classmethod

Creates a Document instance from a dictionary.

Parameters:

Name Type Description Default
data dict

A dictionary containing the question, answers, and contexts.

 required
n_docs int

The number of contexts to include. Defaults to 100.

 100

Returns:

Name Type Description
Document Document

A new Document instance.
Example 
data = {
    "question": "What is the capital of France?",
    "answers": ["Paris"],
    "ctxs": [
        {"score": 0.9, "has_answer": True, "id": 1, "title": "Paris", "text": "The capital of France is Paris."},
        {"score": 0.5, "has_answer": False, "id": 2, "title": "Berlin", "text": "Berlin is the capital of Germany."}
    ]
}
d = Document.from_dict(data)
print(d.question)
Source code in rankify/dataset/dataset.py 
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
@classmethod
def from_dict(cls, data: dict,n_docs:int=100) -> 'Document':
    """
    Creates a Document instance from a dictionary.

    Args:
        data (dict): A dictionary containing the question, answers, and contexts.
        n_docs (int, optional): The number of contexts to include. Defaults to 100.

    Returns:
        Document: A new Document instance.

    Example:
        ```python
        data = {
            "question": "What is the capital of France?",
            "answers": ["Paris"],
            "ctxs": [
                {"score": 0.9, "has_answer": True, "id": 1, "title": "Paris", "text": "The capital of France is Paris."},
                {"score": 0.5, "has_answer": False, "id": 2, "title": "Berlin", "text": "Berlin is the capital of Germany."}
            ]
        }
        d = Document.from_dict(data)
        print(d.question)
        ```
    """
    question = Question(data["question"])
    if "answers" in data:
        answers = Answer(data["answers"])
    else:
        answers =Answer('')

    if "query_id" in data:
        id = data["query_id"]
    else:
        id = None
    contexts = [Context(**ctx) for ctx in data["ctxs"][:n_docs]]
    return cls(question, answers, contexts, id=id)

to_dict()

Converts the document into a dictionary representation.

Returns:

Name Type Description
dict Dict[str, Optional[object]]

A dictionary containing the question, answers, and contexts.
Source code in rankify/dataset/dataset.py 
269
270
271
272
273
274
275
276
277
278
279
280
def to_dict(self) -> Dict[str, Optional[object]]:
    """
    Converts the document into a dictionary representation.

    Returns:
        dict: A dictionary containing the question, answers, and contexts.
    """
    return {
        "question": self.question.question,
        "answers": self.answers.answers,
        "contexts": [ctx.to_dict() for ctx in self.contexts]
    }

__str__()

Returns a string representation of the Document instance.

Returns:

Name Type Description
str str

The formatted document information.
Example 
d = Document(Question("What is the capital of France?"), Answer(["Paris"]))
print(d)
Source code in rankify/dataset/dataset.py 
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
def __str__(self) -> str:
    """
    Returns a string representation of the Document instance.

    Returns:
        str: The formatted document information.

    Example:
        ```python
        d = Document(Question("What is the capital of France?"), Answer(["Paris"]))
        print(d)
        ```
    """
    contexts_str = "\n\n".join([str(ctx) for ctx in self.contexts])
    reorder_contexts_str= ''
    if self.reorder_contexts is not None:
        reorder_contexts_str = "\n\n".join([str(ctx) for ctx in self.reorder_contexts])
    return f"{self.question}\n\n{self.answers}\n\nContext: \n\n{contexts_str}\nReorder contexts: \n\n{reorder_contexts_str}"

RankGPT

Bases: BaseRanking

Implements RankGPT with detailed GPU memory and performance logging.

Source code in rankify/models/rankgpt.py 
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
class RankGPT(BaseRanking):
    """
    Implements **RankGPT** with detailed GPU memory and performance logging.
    """

    def __init__(self, method: str = None, model_name: str = None, api_key: str = None , **kwargs):
        """
        Initializes a RankGPT instance.
        """
        self.method = method
        self.window_size = kwargs.get("window_size", 20) 
        self.step = kwargs.get("step", 10)
        self.endpoint = kwargs.get("endpoint", "https://api.openai.com/v1")
        self.api_key = api_key
        self.model = None
        self.tokenizer = None
        self.use_gpu = True
        self.use_bf16 = True

        # Performance tracking
        self.inference_times = []
        self.memory_snapshots = []

        if self.method == 'rankgpt-api':
            if model_name in URL:
                self.model_name = URL[model_name]['model_name']
                self.url = URL[model_name]['url']
            else:
                self.model_name = model_name
                self.url = self.endpoint
        else:
            self.model_name = model_name

        self._load(model_name)

    def _load(self, model_name: str = None) -> None:
        """
        Loads the GPT model and tokenizer with GPU optimizations.
        """
        if self.method == 'rankgpt-api':
            if model_name in URL:
                self.model = URL[model_name]['class'](self.api_key, self.url)
            else:
                self.model = URL['default']['class'](self.api_key, self.url)
        else:
            print(f"🚀 Loading {self.model_name} with GPU optimizations...")

            # Clear GPU memory first
            if torch.cuda.is_available():
                torch.cuda.empty_cache()
                self._log_gpu_memory("Before Loading")

            # OPTIMIZED MODEL LOADING
            self.model = AutoModelForCausalLM.from_pretrained(
                self.model_name,
                torch_dtype=torch.float16,  # Force float16
                device_map="auto",          # Auto GPU placement
                low_cpu_mem_usage=True,
                trust_remote_code=True
            )

            self.tokenizer = AutoTokenizer.from_pretrained(
                self.model_name,
                use_fast=True,              # Fast tokenizer
                trust_remote_code=True
            )

            if self.tokenizer.pad_token is None:
                self.tokenizer.pad_token = self.tokenizer.eos_token

            # VERIFY GPU PLACEMENT
            model_device = next(self.model.parameters()).device
            print(f"📍 Model loaded on: {model_device}")

            if 'cuda' not in str(model_device) and torch.cuda.is_available():
                #print("🚨 Forcing GPU placement...")
                self.model = self.model.cuda()
                #print(f"📍 Model now on: {next(self.model.parameters()).device}")

            # Show GPU memory usage after loading
            if torch.cuda.is_available():
                self._log_gpu_memory("After Loading")

    def _log_gpu_memory(self, stage="", detailed=False):
        """Log detailed GPU memory usage."""
        if not torch.cuda.is_available():
            print(f"   💻 {stage}: CPU mode")
            return

        # Force GPU synchronization for accurate measurements
        torch.cuda.synchronize()

        allocated = torch.cuda.memory_allocated() / 1024**3
        reserved = torch.cuda.memory_reserved() / 1024**3
        max_allocated = torch.cuda.max_memory_allocated() / 1024**3

        # if detailed:
        #     total_memory = torch.cuda.get_device_properties(0).total_memory / 1024**3
        #     free_memory = total_memory - reserved
        #     utilization = (allocated / total_memory) * 100

        #     print(f"   🎮 GPU Memory {stage}:")
        #     print(f"      Allocated: {allocated:.3f}GB")
        #     print(f"      Reserved:  {reserved:.3f}GB") 
        #     print(f"      Max Used:  {max_allocated:.3f}GB")
        #     print(f"      Free:      {free_memory:.3f}GB")
        #     print(f"      Usage:     {utilization:.1f}%")
        # else:
        #     print(f"   🎮 GPU Memory {stage}: {allocated:.3f}GB allocated, {reserved:.3f}GB reserved")

        return allocated, reserved

    def rank(self, documents: List[Document]) -> List[Document]:
        """
        Ranks documents with comprehensive logging.
        """
        print(f"\n🎯 Starting ranking for {len(documents)} documents")
        self._log_gpu_memory("Before Ranking", detailed=True)

        start_total = time.time()

        for doc_idx, document in enumerate(tqdm(documents, desc="Reranking Documents")):
            #print(f"\n📄 Document {doc_idx+1}/{len(documents)} - {len(document.contexts)} contexts")
            doc_start = time.time()

            reorder_contexts = self.sliding_windows(
                document, 
                rank_start=0, 
                rank_end=len(document.contexts), 
                window_size=self.window_size, 
                step=self.step
            )
            document.reorder_contexts = reorder_contexts

            doc_time = time.time() - doc_start
            #print(f"   ⏱️  Document {doc_idx+1} completed in {doc_time:.2f}s")

            # Memory cleanup and logging every document
            if torch.cuda.is_available():
                torch.cuda.empty_cache()
                gc.collect()
                self._log_gpu_memory(f"After Doc {doc_idx+1}")
            #break
        total_time = time.time() - start_total
        print(f"\n✅ All documents ranked in {total_time:.2f}s")
        self._print_performance_summary()

        return documents

    def sliding_windows(self, item=None, rank_start=0, rank_end=100, window_size=20, step=10):
        """
        Applies sliding window ranking with detailed logging.
        """
        #print(f"   🪟 Starting sliding windows: {rank_start}-{rank_end}, window={window_size}, step={step}")

        item.reorder_contexts = item.contexts
        item = copy.copy(item)
        item.reorder_contexts = item.contexts.copy()

        # Calculate total windows
        total_windows = 0
        temp_end = rank_end
        temp_start = rank_end - window_size
        while temp_start >= rank_start:
            temp_start = max(temp_start, rank_start)
            total_windows += 1
            temp_end = temp_end - step
            temp_start = temp_start - step

        #print(f"   📊 Will process {total_windows} windows")

        # Process windows with detailed logging
        window_num = 0
        end_pos = rank_end
        start_pos = rank_end - window_size

        while start_pos >= rank_start:
            start_pos = max(start_pos, rank_start)
            window_num += 1

            #print(f"\n     🔹 Window {window_num}/{total_windows}: [{start_pos}:{end_pos}] ({end_pos-start_pos} items)")

            # Log memory before window processing
            mem_before_window = self._log_gpu_memory(f"Before Window {window_num}")
            window_start_time = time.time()

            item = self.permutation_pipeline(
                item=item, 
                rank_start=start_pos, 
                rank_end=end_pos,
                window_info=f"{window_num}/{total_windows}"
            )

            window_time = time.time() - window_start_time
            mem_after_window = self._log_gpu_memory(f"After Window {window_num}")

            # Calculate memory delta for this window
            # if torch.cuda.is_available() and mem_before_window and mem_after_window:
            #     mem_delta = mem_after_window[0] - mem_before_window[0]
            #     print(f"     ⏱️  Window {window_num} completed in {window_time:.3f}s (Δ{mem_delta:+.3f}GB)")
            # else:
            #     print(f"     ⏱️  Window {window_num} completed in {window_time:.3f}s")

            end_pos = end_pos - step
            start_pos = start_pos - step

        #print(f"   ✅ All {total_windows} windows completed")
        return item.reorder_contexts

    def permutation_pipeline(self, item=None, rank_start=0, rank_end=100, window_info=""):
        """
        Pipeline with detailed timing and memory logging.
        """
        # Create instruction phase
        inst_start = time.time()
        messages = self.create_permutation_instruction(item=item, rank_start=rank_start, rank_end=rank_end)
        inst_time = time.time() - inst_start

        # LLM inference phase with detailed logging
        llm_start = time.time()
        permutation = self.run_llm_logged(messages, window_info)
        llm_time = time.time() - llm_start

        # Process result phase  
        proc_start = time.time()
        item = self.receive_permutation(item=item, permutation=permutation, rank_start=rank_start, rank_end=rank_end)
        proc_time = time.time() - proc_start

        total_time = inst_time + llm_time + proc_time
        #print(f"       📊 Pipeline: Inst:{inst_time:.3f}s + LLM:{llm_time:.3f}s + Proc:{proc_time:.3f}s = {total_time:.3f}s")

        # Store performance data
        self.inference_times.append(llm_time)

        return item

    def run_llm_logged(self, messages, window_info=""):
        """
        LLM inference with detailed GPU memory and timing logging.
        """
        if self.method == 'rankgpt-api':
            return self.model.chat(model=self.model_name, messages=messages, temperature=0, return_text=True)

        # Pre-inference logging
        if torch.cuda.is_available():
            torch.cuda.synchronize()
            gpu_before = torch.cuda.memory_allocated() / 1024**3

        with torch.no_grad():
            # Tokenization phase
            tokenize_start = time.time()
            try:
                input_ids = self.tokenizer.apply_chat_template(
                    messages,
                    add_generation_prompt=True,
                    return_tensors="pt",
                    max_length=1024,        
                    truncation=True         
                )

                # Log tokenization results
                seq_len = input_ids.shape[1]
                tokenize_time = time.time() - tokenize_start

                # Device transfer phase
                transfer_start = time.time()
                model_device = next(self.model.parameters()).device
                input_ids = input_ids.to(model_device, non_blocking=True)
                transfer_time = time.time() - transfer_start

                if torch.cuda.is_available():
                    torch.cuda.synchronize()
                    gpu_after_tokenize = torch.cuda.memory_allocated() / 1024**3
                    tokenize_mem_delta = gpu_after_tokenize - gpu_before
                else:
                    gpu_after_tokenize = 0
                    tokenize_mem_delta = 0

            except Exception as e:
                print(f"         ❌ Tokenization failed: {e}")
                return ""

            # Generation phase with detailed monitoring
            generate_start = time.time()
            try:
                terminators = [self.tokenizer.eos_token_id]
                eot_token = self.tokenizer.convert_tokens_to_ids("<|eot_id|>")
                if eot_token is not None and eot_token != self.tokenizer.unk_token_id:
                    terminators.append(eot_token)

                # Memory snapshot before generation
                if torch.cuda.is_available():
                    torch.cuda.synchronize()
                    gpu_before_gen = torch.cuda.memory_allocated() / 1024**3

                # GPU generation with mixed precision
                with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16):
                    responses = self.model.generate(
                        input_ids,
                        max_new_tokens=32,      
                        eos_token_id=terminators,
                        do_sample=False,        
                        temperature=1.0,
                        top_p=1.0,
                        use_cache=True,         
                        pad_token_id=self.tokenizer.pad_token_id,
                        num_beams=1,           
                        early_stopping=True,
                        output_attentions=False,
                        output_hidden_states=False
                    )

                generate_time = time.time() - generate_start

                # Memory snapshot after generation
                if torch.cuda.is_available():
                    torch.cuda.synchronize()
                    gpu_after_gen = torch.cuda.memory_allocated() / 1024**3
                    gen_mem_delta = gpu_after_gen - gpu_before_gen
                    gen_mem_peak = torch.cuda.max_memory_allocated() / 1024**3
                else:
                    gpu_after_gen = 0
                    gen_mem_delta = 0
                    gen_mem_peak = 0

            except Exception as e:
                print(f"         ❌ Generation failed: {e}")
                return ""

            # Decoding phase
            decode_start = time.time()
            try:
                generated_tokens = responses[0][input_ids.shape[-1]:]
                response = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)
                decode_time = time.time() - decode_start

                # Final memory check
                if torch.cuda.is_available():
                    torch.cuda.synchronize()
                    gpu_final = torch.cuda.memory_allocated() / 1024**3
                    total_mem_delta = gpu_final - gpu_before
                else:
                    gpu_final = 0
                    total_mem_delta = 0

            except Exception as e:
                print(f"         ❌ Decoding failed: {e}")
                return ""

        # Comprehensive logging
        total_time = tokenize_time + transfer_time + generate_time + decode_time
        tokens_per_sec = len(generated_tokens) / generate_time if generate_time > 0 else 0

        # print(f"         🔧 Detailed Breakdown {window_info}:")
        # print(f"            Input tokens: {seq_len}")
        # print(f"            Output tokens: {len(generated_tokens)}")
        # print(f"            Tokenize: {tokenize_time:.3f}s (Δ{tokenize_mem_delta:+.3f}GB)")
        # print(f"            Transfer: {transfer_time:.3f}s")
        # print(f"            Generate: {generate_time:.3f}s (Δ{gen_mem_delta:+.3f}GB, Peak:{gen_mem_peak:.3f}GB)")
        # print(f"            Decode: {decode_time:.3f}s")
        # print(f"            Total: {total_time:.3f}s (Δ{total_mem_delta:+.3f}GB)")
        # print(f"            Speed: {tokens_per_sec:.1f} tokens/sec")

        # Performance assessment
        # if generate_time < 0.5:
        #     perf_status = "🚀 EXCELLENT"
        # elif generate_time < 1.0:
        #     perf_status = "✅ GOOD"
        # elif generate_time < 2.0:
        #     perf_status = "⚠️ SLOW"
        # else:
        #     perf_status = "🐌 VERY SLOW"

        # print(f"            Performance: {perf_status}")

        return response

    def _print_performance_summary(self):
        """Print comprehensive performance summary."""
        if not self.inference_times:
            return

        times = self.inference_times
        avg_time = sum(times) / len(times)
        min_time = min(times)
        max_time = max(times)

        # print(f"\n📈 PERFORMANCE SUMMARY:")
        # print(f"   🧠 LLM Inference Statistics:")
        # print(f"      Total inferences: {len(times)}")
        # print(f"      Average time: {avg_time:.3f}s")
        # print(f"      Fastest: {min_time:.3f}s")
        # print(f"      Slowest: {max_time:.3f}s")
        # print(f"      Std deviation: {(sum((t - avg_time)**2 for t in times) / len(times))**0.5:.3f}s")

        # Throughput calculation
        total_inference_time = sum(times)
        throughput = len(times) / total_inference_time if total_inference_time > 0 else 0
        # print(f"      Throughput: {throughput:.2f} inferences/second")

        # Performance categorization
        # if avg_time > 3.0:
        #     print(f"   🚨 CRITICAL: Very slow performance - check GPU usage!")
        # elif avg_time > 1.5:
        #     print(f"   ⚠️ WARNING: Slow performance - optimization needed")
        # elif avg_time > 0.8:
        #     print(f"   ✅ ACCEPTABLE: Moderate performance")
        # else:
        #     print(f"   🚀 EXCELLENT: Fast performance!")

        # GPU memory summary
        # if torch.cuda.is_available():
        #     current_allocated = torch.cuda.memory_allocated() / 1024**3
        #     max_allocated = torch.cuda.max_memory_allocated() / 1024**3
        #     total_memory = torch.cuda.get_device_properties(0).total_memory / 1024**3

        #     print(f"   🎮 GPU Memory Summary:")
        #     print(f"      Current usage: {current_allocated:.2f}GB")
        #     print(f"      Peak usage: {max_allocated:.2f}GB")
        #     print(f"      Total available: {total_memory:.2f}GB")
        #     print(f"      Efficiency: {(max_allocated/total_memory)*100:.1f}% peak utilization")

    # Keep all existing methods unchanged
    def create_permutation_instruction(self, item=None, rank_start=0, rank_end=100):
        query = item.question.question
        num = len(item.contexts[rank_start: rank_end])
        max_length = 300
        messages = self.get_prefix_prompt(query, num)
        rank = 0
        for hit in item.contexts[rank_start: rank_end]:
            rank += 1
            content = hit.text
            content = content.replace("Title: Content: ", "")
            content = content.strip()
            content = " ".join(content.split()[: int(max_length)])
            messages.append({"role": "user", "content": f"[{rank}] {content}"})
            messages.append({"role": "assistant", "content": f"Received passage [{rank}]."})
        messages.append({"role": "user", "content": self.get_post_prompt(query, num)})
        return messages

    def run_llm(self, messages):
        """Wrapper for backward compatibility."""
        return self.run_llm_logged(messages)

    def receive_permutation(self, item, permutation, rank_start=0, rank_end=100):
        response = self.clean_response(permutation)
        if not response.strip():
            return item
        try:
            response = [int(x) - 1 for x in response.split()]
        except ValueError:
            return item
        response = self.remove_duplicate(response)
        cut_range = item.contexts[rank_start:rank_end]
        original_rank = [tt for tt in range(len(cut_range))]
        response = [ss for ss in response if ss in original_rank]
        response = response + [tt for tt in original_rank if tt not in response]
        for j, x in enumerate(response):
            item.reorder_contexts[j + rank_start] = copy.copy(cut_range[x])
            if hasattr(item.reorder_contexts[j + rank_start], 'rank'):
                item.reorder_contexts[j + rank_start].rank = cut_range[j].rank
            if hasattr(item.reorder_contexts[j + rank_start], 'score'):
                item.reorder_contexts[j + rank_start].score = cut_range[j].score
        return item

    def get_prefix_prompt(self, query, num):
        return [{'role': 'system',
                 'content': "You are RankGPT, an intelligent assistant that can rank passages based on their relevancy to the query."},
                {'role': 'user',
                 'content': f"I will provide you with {num} passages, each indicated by number identifier []. \nRank the passages based on their relevance to query: {query}."},
                {'role': 'assistant', 'content': 'Okay, please provide the passages.'}]

    def clean_response(self, response: str):
        new_response = ''
        for c in response:
            if not c.isdigit():
                new_response += ' '
            else:
                new_response += c
        new_response = new_response.strip()
        return new_response

    def remove_duplicate(self, response):
        new_response = []
        for c in response:
            if c not in new_response:
                new_response.append(c)
        return new_response

    def get_post_prompt(self, query, num):
        return f"Search Query: {query}. \nRank the {num} passages above based on their relevance to search query. The passages should be listed in descending order using identifiers. The most relevant passages should be listed first. The output format should be [] > [], e.g., [1] > [2]. Only response the ranking results, do not say any word or explain."

__init__(method=None, model_name=None, api_key=None, **kwargs)

Initializes a RankGPT instance.

Source code in rankify/models/rankgpt.py 
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
def __init__(self, method: str = None, model_name: str = None, api_key: str = None , **kwargs):
    """
    Initializes a RankGPT instance.
    """
    self.method = method
    self.window_size = kwargs.get("window_size", 20) 
    self.step = kwargs.get("step", 10)
    self.endpoint = kwargs.get("endpoint", "https://api.openai.com/v1")
    self.api_key = api_key
    self.model = None
    self.tokenizer = None
    self.use_gpu = True
    self.use_bf16 = True

    # Performance tracking
    self.inference_times = []
    self.memory_snapshots = []

    if self.method == 'rankgpt-api':
        if model_name in URL:
            self.model_name = URL[model_name]['model_name']
            self.url = URL[model_name]['url']
        else:
            self.model_name = model_name
            self.url = self.endpoint
    else:
        self.model_name = model_name

    self._load(model_name)

rank(documents)

Ranks documents with comprehensive logging.

Source code in rankify/models/rankgpt.py 
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
def rank(self, documents: List[Document]) -> List[Document]:
    """
    Ranks documents with comprehensive logging.
    """
    print(f"\n🎯 Starting ranking for {len(documents)} documents")
    self._log_gpu_memory("Before Ranking", detailed=True)

    start_total = time.time()

    for doc_idx, document in enumerate(tqdm(documents, desc="Reranking Documents")):
        #print(f"\n📄 Document {doc_idx+1}/{len(documents)} - {len(document.contexts)} contexts")
        doc_start = time.time()

        reorder_contexts = self.sliding_windows(
            document, 
            rank_start=0, 
            rank_end=len(document.contexts), 
            window_size=self.window_size, 
            step=self.step
        )
        document.reorder_contexts = reorder_contexts

        doc_time = time.time() - doc_start
        #print(f"   ⏱️  Document {doc_idx+1} completed in {doc_time:.2f}s")

        # Memory cleanup and logging every document
        if torch.cuda.is_available():
            torch.cuda.empty_cache()
            gc.collect()
            self._log_gpu_memory(f"After Doc {doc_idx+1}")
        #break
    total_time = time.time() - start_total
    print(f"\n✅ All documents ranked in {total_time:.2f}s")
    self._print_performance_summary()

    return documents

sliding_windows(item=None, rank_start=0, rank_end=100, window_size=20, step=10)

Applies sliding window ranking with detailed logging.

Source code in rankify/models/rankgpt.py 
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
def sliding_windows(self, item=None, rank_start=0, rank_end=100, window_size=20, step=10):
    """
    Applies sliding window ranking with detailed logging.
    """
    #print(f"   🪟 Starting sliding windows: {rank_start}-{rank_end}, window={window_size}, step={step}")

    item.reorder_contexts = item.contexts
    item = copy.copy(item)
    item.reorder_contexts = item.contexts.copy()

    # Calculate total windows
    total_windows = 0
    temp_end = rank_end
    temp_start = rank_end - window_size
    while temp_start >= rank_start:
        temp_start = max(temp_start, rank_start)
        total_windows += 1
        temp_end = temp_end - step
        temp_start = temp_start - step

    #print(f"   📊 Will process {total_windows} windows")

    # Process windows with detailed logging
    window_num = 0
    end_pos = rank_end
    start_pos = rank_end - window_size

    while start_pos >= rank_start:
        start_pos = max(start_pos, rank_start)
        window_num += 1

        #print(f"\n     🔹 Window {window_num}/{total_windows}: [{start_pos}:{end_pos}] ({end_pos-start_pos} items)")

        # Log memory before window processing
        mem_before_window = self._log_gpu_memory(f"Before Window {window_num}")
        window_start_time = time.time()

        item = self.permutation_pipeline(
            item=item, 
            rank_start=start_pos, 
            rank_end=end_pos,
            window_info=f"{window_num}/{total_windows}"
        )

        window_time = time.time() - window_start_time
        mem_after_window = self._log_gpu_memory(f"After Window {window_num}")

        # Calculate memory delta for this window
        # if torch.cuda.is_available() and mem_before_window and mem_after_window:
        #     mem_delta = mem_after_window[0] - mem_before_window[0]
        #     print(f"     ⏱️  Window {window_num} completed in {window_time:.3f}s (Δ{mem_delta:+.3f}GB)")
        # else:
        #     print(f"     ⏱️  Window {window_num} completed in {window_time:.3f}s")

        end_pos = end_pos - step
        start_pos = start_pos - step

    #print(f"   ✅ All {total_windows} windows completed")
    return item.reorder_contexts

permutation_pipeline(item=None, rank_start=0, rank_end=100, window_info='')

Pipeline with detailed timing and memory logging.

Source code in rankify/models/rankgpt.py 
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
def permutation_pipeline(self, item=None, rank_start=0, rank_end=100, window_info=""):
    """
    Pipeline with detailed timing and memory logging.
    """
    # Create instruction phase
    inst_start = time.time()
    messages = self.create_permutation_instruction(item=item, rank_start=rank_start, rank_end=rank_end)
    inst_time = time.time() - inst_start

    # LLM inference phase with detailed logging
    llm_start = time.time()
    permutation = self.run_llm_logged(messages, window_info)
    llm_time = time.time() - llm_start

    # Process result phase  
    proc_start = time.time()
    item = self.receive_permutation(item=item, permutation=permutation, rank_start=rank_start, rank_end=rank_end)
    proc_time = time.time() - proc_start

    total_time = inst_time + llm_time + proc_time
    #print(f"       📊 Pipeline: Inst:{inst_time:.3f}s + LLM:{llm_time:.3f}s + Proc:{proc_time:.3f}s = {total_time:.3f}s")

    # Store performance data
    self.inference_times.append(llm_time)

    return item

run_llm_logged(messages, window_info='')

LLM inference with detailed GPU memory and timing logging.

Source code in rankify/models/rankgpt.py 
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
def run_llm_logged(self, messages, window_info=""):
    """
    LLM inference with detailed GPU memory and timing logging.
    """
    if self.method == 'rankgpt-api':
        return self.model.chat(model=self.model_name, messages=messages, temperature=0, return_text=True)

    # Pre-inference logging
    if torch.cuda.is_available():
        torch.cuda.synchronize()
        gpu_before = torch.cuda.memory_allocated() / 1024**3

    with torch.no_grad():
        # Tokenization phase
        tokenize_start = time.time()
        try:
            input_ids = self.tokenizer.apply_chat_template(
                messages,
                add_generation_prompt=True,
                return_tensors="pt",
                max_length=1024,        
                truncation=True         
            )

            # Log tokenization results
            seq_len = input_ids.shape[1]
            tokenize_time = time.time() - tokenize_start

            # Device transfer phase
            transfer_start = time.time()
            model_device = next(self.model.parameters()).device
            input_ids = input_ids.to(model_device, non_blocking=True)
            transfer_time = time.time() - transfer_start

            if torch.cuda.is_available():
                torch.cuda.synchronize()
                gpu_after_tokenize = torch.cuda.memory_allocated() / 1024**3
                tokenize_mem_delta = gpu_after_tokenize - gpu_before
            else:
                gpu_after_tokenize = 0
                tokenize_mem_delta = 0

        except Exception as e:
            print(f"         ❌ Tokenization failed: {e}")
            return ""

        # Generation phase with detailed monitoring
        generate_start = time.time()
        try:
            terminators = [self.tokenizer.eos_token_id]
            eot_token = self.tokenizer.convert_tokens_to_ids("<|eot_id|>")
            if eot_token is not None and eot_token != self.tokenizer.unk_token_id:
                terminators.append(eot_token)

            # Memory snapshot before generation
            if torch.cuda.is_available():
                torch.cuda.synchronize()
                gpu_before_gen = torch.cuda.memory_allocated() / 1024**3

            # GPU generation with mixed precision
            with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16):
                responses = self.model.generate(
                    input_ids,
                    max_new_tokens=32,      
                    eos_token_id=terminators,
                    do_sample=False,        
                    temperature=1.0,
                    top_p=1.0,
                    use_cache=True,         
                    pad_token_id=self.tokenizer.pad_token_id,
                    num_beams=1,           
                    early_stopping=True,
                    output_attentions=False,
                    output_hidden_states=False
                )

            generate_time = time.time() - generate_start

            # Memory snapshot after generation
            if torch.cuda.is_available():
                torch.cuda.synchronize()
                gpu_after_gen = torch.cuda.memory_allocated() / 1024**3
                gen_mem_delta = gpu_after_gen - gpu_before_gen
                gen_mem_peak = torch.cuda.max_memory_allocated() / 1024**3
            else:
                gpu_after_gen = 0
                gen_mem_delta = 0
                gen_mem_peak = 0

        except Exception as e:
            print(f"         ❌ Generation failed: {e}")
            return ""

        # Decoding phase
        decode_start = time.time()
        try:
            generated_tokens = responses[0][input_ids.shape[-1]:]
            response = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)
            decode_time = time.time() - decode_start

            # Final memory check
            if torch.cuda.is_available():
                torch.cuda.synchronize()
                gpu_final = torch.cuda.memory_allocated() / 1024**3
                total_mem_delta = gpu_final - gpu_before
            else:
                gpu_final = 0
                total_mem_delta = 0

        except Exception as e:
            print(f"         ❌ Decoding failed: {e}")
            return ""

    # Comprehensive logging
    total_time = tokenize_time + transfer_time + generate_time + decode_time
    tokens_per_sec = len(generated_tokens) / generate_time if generate_time > 0 else 0

    # print(f"         🔧 Detailed Breakdown {window_info}:")
    # print(f"            Input tokens: {seq_len}")
    # print(f"            Output tokens: {len(generated_tokens)}")
    # print(f"            Tokenize: {tokenize_time:.3f}s (Δ{tokenize_mem_delta:+.3f}GB)")
    # print(f"            Transfer: {transfer_time:.3f}s")
    # print(f"            Generate: {generate_time:.3f}s (Δ{gen_mem_delta:+.3f}GB, Peak:{gen_mem_peak:.3f}GB)")
    # print(f"            Decode: {decode_time:.3f}s")
    # print(f"            Total: {total_time:.3f}s (Δ{total_mem_delta:+.3f}GB)")
    # print(f"            Speed: {tokens_per_sec:.1f} tokens/sec")

    # Performance assessment
    # if generate_time < 0.5:
    #     perf_status = "🚀 EXCELLENT"
    # elif generate_time < 1.0:
    #     perf_status = "✅ GOOD"
    # elif generate_time < 2.0:
    #     perf_status = "⚠️ SLOW"
    # else:
    #     perf_status = "🐌 VERY SLOW"

    # print(f"            Performance: {perf_status}")

    return response

run_llm(messages)

Wrapper for backward compatibility.

Source code in rankify/models/rankgpt.py 
453
454
455
def run_llm(self, messages):
    """Wrapper for backward compatibility."""
    return self.run_llm_logged(messages)