Skip to content

Dense Retriever

rankify.retrievers.dense_retriever

BaseRetriever

Bases: ABC

Abstract base class for all retrieval methods in the rankify framework.

This class defines the common interface that all retrievers must implement, ensuring consistency across different retrieval methods (BM25, DPR, etc.).

Source code in rankify/retrievers/base_retriever.py 
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
class BaseRetriever(ABC):
    """
    Abstract base class for all retrieval methods in the rankify framework.

    This class defines the common interface that all retrievers must implement,
    ensuring consistency across different retrieval methods (BM25, DPR, etc.).
    """

    def __init__(self, n_docs: int = 10, batch_size: int = 36, threads: int = 30, **kwargs):
        """
        Initialize the base retriever.

        Args:
            n_docs (int): Number of documents to retrieve per query
            batch_size (int): Number of queries to process in a batch
            threads (int): Number of parallel threads for retrieval
            **kwargs: Additional parameters specific to each retriever
        """
        self.n_docs = n_docs
        self.batch_size = batch_size
        self.threads = threads
        self.config = kwargs

    @abstractmethod
    def _initialize_searcher(self) -> Any:
        """Initialize the specific searcher for this retriever type."""
        pass

    @abstractmethod
    def retrieve(self, documents: List[Document]) -> List[Document]:
        """
        Retrieve relevant contexts for the given documents.

        Args:
            documents (List[Document]): List of documents containing queries

        Returns:
            List[Document]: Documents updated with retrieved contexts
        """
        pass

    def _preprocess_query(self, query: str) -> str:
        """
        Preprocess query text. Can be overridden by specific retrievers.

        Args:
            query (str): Raw query text

        Returns:
            str: Preprocessed query text
        """
        return query.strip()

__init__(n_docs=10, batch_size=36, threads=30, **kwargs)

Initialize the base retriever.

Parameters:

Name Type Description Default
n_docs int

Number of documents to retrieve per query

 10
batch_size int

Number of queries to process in a batch

 36
threads int

Number of parallel threads for retrieval

 30
**kwargs

Additional parameters specific to each retriever

 {}
Source code in rankify/retrievers/base_retriever.py 
14
15
16
17
18
19
20
21
22
23
24
25
26
27
def __init__(self, n_docs: int = 10, batch_size: int = 36, threads: int = 30, **kwargs):
    """
    Initialize the base retriever.

    Args:
        n_docs (int): Number of documents to retrieve per query
        batch_size (int): Number of queries to process in a batch
        threads (int): Number of parallel threads for retrieval
        **kwargs: Additional parameters specific to each retriever
    """
    self.n_docs = n_docs
    self.batch_size = batch_size
    self.threads = threads
    self.config = kwargs

retrieve(documents) abstractmethod

Retrieve relevant contexts for the given documents.

Parameters:

Name Type Description Default
documents List[Document]

List of documents containing queries

 required

Returns:

Type Description
List[Document]

List[Document]: Documents updated with retrieved contexts
Source code in rankify/retrievers/base_retriever.py 
34
35
36
37
38
39
40
41
42
43
44
45
@abstractmethod
def retrieve(self, documents: List[Document]) -> List[Document]:
    """
    Retrieve relevant contexts for the given documents.

    Args:
        documents (List[Document]): List of documents containing queries

    Returns:
        List[Document]: Documents updated with retrieved contexts
    """
    pass

IndexManager

Manages downloading, caching, and loading of retrieval indexes.

Handles both prebuilt indexes (wiki, msmarco) and custom user indexes.

Source code in rankify/retrievers/index_manager.py 
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
class IndexManager:
    """
    Manages downloading, caching, and loading of retrieval indexes.

    Handles both prebuilt indexes (wiki, msmarco) and custom user indexes.
    """

    def __init__(self, cache_dir: str = None):
        self.cache_dir = cache_dir or os.environ.get("RERANKING_CACHE_DIR", "./cache")
        self.index_configs = self._load_index_configs()

    def _load_index_configs(self) -> Dict:
        """Load index configuration mappings."""
        return {
            "bm25": {
                "wiki": {
                    "url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/bm25_wiki.zip",
                    "prebuilt": "wikipedia-dpr-100w"
                },
                "msmarco": {
                    "url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/bm25_index_msmarco.zip?download=true",
                    "prebuilt": None
                }
            },
            "dpr-multi": {
                "wiki": {
                    "prebuilt": "wikipedia-dpr-100w.dpr-multi",
                    "encoder": "facebook/dpr-question_encoder-multiset-base"
                },
                "msmarco": {
                    "url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/msmarco_passage_dpr_multi.zip",
                    "encoder": "facebook/dpr-question_encoder-multiset-base"
                }
            },
            "dpr-single": {
                "wiki": {
                    "prebuilt": "wikipedia-dpr-100w.dpr-single-nq",
                    "encoder": "facebook/dpr-question_encoder-single-nq-base"
                },
                "msmarco": {
                    "url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/msmarco_passage_dpr_single.zip",
                    "encoder": "facebook/dpr-question_encoder-single-nq-base"
                }
            },
            "ance-multi": {
                "wiki": {
                    "prebuilt": "wikipedia-dpr-100w.ance-multi",
                    "encoder": "castorini/ance-dpr-question-multi"
                },
                "msmarco": {
                    "prebuilt": "msmarco-v1-passage.ance", 
                    "encoder": "castorini/ance-msmarco-passage"
                }
            },

            "bpr-single": {
                "wiki": {
                    "prebuilt": "wikipedia-dpr-100w.bpr-single-nq",
                    "encoder": "castorini/bpr-nq-question-encoder"
                },
                "msmarco": {
                    "url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/msmarco_passage_bpr.zip",
                    "encoder": "castorini/bpr-nq-question-encoder"
                }
            },
            "bge": {
                "wiki": {
                    "urls": [
                        "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/bgb_index.tar.gz.part1?download=true",
                        "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/bgb_index.tar.gz.part2?download=true",
                    ],
                    "passages_url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/psgs_w100/psgs_w100.tsv?download=true",
                    "encoder": "BAAI/bge-large-en-v1.5"
                },
                "msmarco": {
                    "urls": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/msmarco_embeddings_bgb.zip?download=true",
                    "passages_url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/msmarco-passage-corpus/msmarco-passage-corpus.tsv?download=true",
                    "encoder": "BAAI/bge-large-en-v1.5"
                }
            },
            "colbert": {
                "wiki": {
                    "urls": [
                        "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/wikipedia_embeddings_colbert/wiki.zip.001?download=true",
                        "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/wikipedia_embeddings_colbert/wiki.zip.002?download=true",
                        "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/wikipedia_embeddings_colbert/wiki.zip.003?download=true",
                        "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/wikipedia_embeddings_colbert/wiki.zip.004?download=true",
                        "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/wikipedia_embeddings_colbert/wiki.zip.005?download=true"
                    ],
                    "passages_url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/psgs_w100/psgs_w100.tsv?download=true",
                    "model": "colbert-ir/colbertv2.0"
                },
                "msmarco": {
                    "urls": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/msmarco_embeddings_colbert.zip?download=true",
                    "passages_url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/msmarco-passage-corpus/msmarco-passage-corpus.tsv?download=true",
                    "model": "colbert-ir/colbertv2.0"
                }
            },
            "contriever": {
                "wiki": {
                    "url": "https://dl.fbaipublicfiles.com/contriever/embeddings/contriever-msmarco/wikipedia_embeddings.tar",
                    "passages_url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/psgs_w100/psgs_w100.tsv?download=true",
                    "model": "facebook/contriever-msmarco",
                    "vector_size": 768
                },
                "msmarco": {
                    "url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/index/msmarco_embeddings_contriever.zip?download=true",
                    "passages_url": "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/msmarco-passage-corpus/msmarco-passage-corpus.tsv?download=true",
                    "model": "facebook/contriever-msmarco",
                    "vector_size": 768
                }
            },
            "online": {
                "web": {
                    "search_provider": "web",
                    "chunk_size": 500,
                    "chunk_overlap": 50,
                    "requires_api_key": True
                }
            },
            "hyde": {
                "wiki": {
                    "base_retriever": "contriever",
                    "base_model": "facebook/contriever-msmarco",
                    "base_index_type": "wiki",
                    "task": "web search",
                    "llm_model": "gpt-3.5-turbo-0125",
                    "num_generated_docs": 1,
                    "max_token_generated_docs": 512,
                    "temperature": 0.7,
                    "requires_api_key": True
                },
                "msmarco": {
                    "base_retriever": "contriever", 
                    "base_model": "facebook/contriever-msmarco",
                    "base_index_type": "msmarco",
                    "task": "web search",
                    "llm_model": "gpt-3.5-turbo-0125",
                    "num_generated_docs": 1,
                    "max_token_generated_docs": 512,
                    "temperature": 0.7,
                    "requires_api_key": True
                }
            }
        }

    def get_index_path(self, method: str, index_type: str, custom_path: str = None) -> str:
        """
        Get the path to the index for a given method and index type.

        Args:
            method (str): Retrieval method (e.g., 'bm25', 'dpr-multi', 'ance')
            index_type (str): Index type ('wiki', 'msmarco', or 'custom')
            custom_path (str): Path to custom index if index_type is 'custom'

        Returns:
            str: Path to the index
        """
        if custom_path:
            return custom_path

        if method not in self.index_configs:
            raise ValueError(f"Unsupported method: {method}")

        if index_type not in self.index_configs[method]:
            raise ValueError(f"Unsupported index type '{index_type}' for method '{method}'")

        config = self.index_configs[method][index_type]

        # If it's a prebuilt index, return the identifier
        if "prebuilt" in config and config["prebuilt"]:
            return config["prebuilt"]

        # Otherwise, download and return local path
        return self._ensure_index_downloaded(method, index_type)

    def _ensure_index_downloaded(self, method: str, index_type: str) -> str:
        """Download and extract index if not already available."""
        config = self.index_configs[method][index_type]
        url = config["url"]

        # Create local directory path
        index_name = f"{method}_{index_type}"
        local_dir = os.path.join(self.cache_dir, "index", index_name)

        if not os.path.exists(local_dir):
            print(f"Downloading {method} index for {index_type}...")
            self._download_and_extract(url, local_dir)

        return local_dir

    def _download_and_extract(self, url: str, destination: str):
        """Download and extract a ZIP file."""
        os.makedirs(destination, exist_ok=True)

        zip_name = os.path.basename(url).split("?")[0]
        zip_path = os.path.join(self.cache_dir, "temp", zip_name)

        os.makedirs(os.path.dirname(zip_path), exist_ok=True)

        if not os.path.exists(zip_path):
            response = requests.get(url, stream=True)
            response.raise_for_status()

            with open(zip_path, "wb") as f:
                for chunk in tqdm(response.iter_content(chunk_size=1024), 
                                desc=f"Downloading {zip_name}"):
                    f.write(chunk)

        print(f"Extracting {zip_name}...")
        with zipfile.ZipFile(zip_path, "r") as zip_ref:
            zip_ref.extractall(destination)

        # Clean up ZIP file
        os.remove(zip_path)
        print("Extraction complete.")

    def load_id_mapping(self, index_path: str) -> Optional[Dict]:
        """Load ID mapping if available."""
        mapping_path = os.path.join(index_path, "id_mapping.json")
        if os.path.exists(mapping_path):
            with open(mapping_path, "r", encoding="utf-8") as f:
                mapping = json.load(f)
                return {v: k for k, v in mapping.items()}  # Reverse mapping
        return None

    # def load_corpus(self, index_path: str) -> Dict:
    #     """Load corpus data from index path."""
    #     corpus_file = os.path.join(index_path, "corpus.jsonl")
    #     if not os.path.exists(corpus_file):
    #         return {}

    #     corpus = {}
    #     with open(corpus_file, "r", encoding="utf-8") as f:
    #         for line in f:
    #             doc = json.loads(line.strip())
    #             doc_id = doc.get("docid") or doc.get("id")
    #             contents = doc.get("contents", "")
    #             title = doc.get("title", contents[:100] if contents else "No Title")
    #             corpus[str(doc_id)] = {"contents": contents, "title": title}

    #     return corpus
    def load_corpus(self, index_path: str) -> Dict:
        """Load corpus data from index path (supports multiple formats)."""
        # Try different corpus file formats in order of preference
        corpus_files = [
            "corpus_metadata.json",  # Your custom format
            "corpus.jsonl",          # Standard JSONL format
            "corpus.json"            # Alternative JSON format
        ]

        for corpus_filename in corpus_files:
            corpus_file = os.path.join(index_path, corpus_filename)
            if os.path.exists(corpus_file):
                print(f"📚 Loading corpus from {corpus_filename}")

                if corpus_filename.endswith('.jsonl'):
                    # JSONL format (one JSON object per line)
                    return self._load_corpus_jsonl(corpus_file)
                else:
                    # JSON format (single JSON object)
                    return self._load_corpus_json(corpus_file)

        print(f"❌ No corpus file found in {index_path}")
        return {}

    def _load_corpus_jsonl(self, corpus_file: str) -> Dict:
        """Load corpus from JSONL format (one JSON object per line)."""
        corpus = {}
        with open(corpus_file, "r", encoding="utf-8") as f:
            for line in f:
                doc = json.loads(line.strip())
                doc_id = doc.get("docid") or doc.get("id")
                contents = doc.get("contents", "")
                title = doc.get("title", contents[:100] if contents else "No Title")
                corpus[str(doc_id)] = {"contents": contents, "title": title}
        return corpus

    def _load_corpus_json(self, corpus_file: str) -> Dict:
        """Load corpus from JSON format (single JSON object)."""
        corpus = {}
        with open(corpus_file, "r", encoding="utf-8") as f:
            data = json.load(f)

        # Handle different JSON structures
        if isinstance(data, dict):
            # If it's a dict, iterate through it
            for doc_id, doc_data in data.items():
                if isinstance(doc_data, dict):
                    contents = doc_data.get("contents") or doc_data.get("text") or ""
                    title = doc_data.get("title", contents[:100] if contents else "No Title")
                else:
                    # If doc_data is a string, use it as contents
                    contents = str(doc_data)
                    title = contents[:100] if contents else "No Title"
                corpus[str(doc_id)] = {"contents": contents, "title": title}

        elif isinstance(data, list):
            # If it's a list, iterate through documents
            for doc in data:
                doc_id = doc.get("docid") or doc.get("id")
                contents = doc.get("contents", "")
                title = doc.get("title", contents[:100] if contents else "No Title")
                corpus[str(doc_id)] = {"contents": contents, "title": title}

        return corpus
    def download_and_extract_index(self, url: str) -> str:
        """Download and extract index from URL, return local path."""
        # This is used by DenseRetriever pattern
        import tempfile
        import zipfile

        # Create temporary file
        with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as tmp_file:
            tmp_path = tmp_file.name

        try:
            # Download
            print(f"Downloading from {url}...")
            response = requests.get(url, stream=True)
            response.raise_for_status()

            with open(tmp_path, 'wb') as f:
                for chunk in tqdm(response.iter_content(chunk_size=1024)):
                    f.write(chunk)

            # Extract to cache directory
            extract_dir = os.path.join(self.cache_dir, "downloaded_index")
            os.makedirs(extract_dir, exist_ok=True)

            with zipfile.ZipFile(tmp_path, 'r') as zip_ref:
                zip_ref.extractall(extract_dir)

            return extract_dir

        finally:
            # Clean up temporary file
            if os.path.exists(tmp_path):
                os.remove(tmp_path)

get_index_path(method, index_type, custom_path=None)

Get the path to the index for a given method and index type.

Parameters:

Name Type Description Default
method str

Retrieval method (e.g., 'bm25', 'dpr-multi', 'ance')

 required
index_type str

Index type ('wiki', 'msmarco', or 'custom')

 required
custom_path str

Path to custom index if index_type is 'custom'

 None

Returns:

Name Type Description
str str

Path to the index
Source code in rankify/retrievers/index_manager.py 
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
def get_index_path(self, method: str, index_type: str, custom_path: str = None) -> str:
    """
    Get the path to the index for a given method and index type.

    Args:
        method (str): Retrieval method (e.g., 'bm25', 'dpr-multi', 'ance')
        index_type (str): Index type ('wiki', 'msmarco', or 'custom')
        custom_path (str): Path to custom index if index_type is 'custom'

    Returns:
        str: Path to the index
    """
    if custom_path:
        return custom_path

    if method not in self.index_configs:
        raise ValueError(f"Unsupported method: {method}")

    if index_type not in self.index_configs[method]:
        raise ValueError(f"Unsupported index type '{index_type}' for method '{method}'")

    config = self.index_configs[method][index_type]

    # If it's a prebuilt index, return the identifier
    if "prebuilt" in config and config["prebuilt"]:
        return config["prebuilt"]

    # Otherwise, download and return local path
    return self._ensure_index_downloaded(method, index_type)

load_id_mapping(index_path)

Load ID mapping if available.

Source code in rankify/retrievers/index_manager.py 
227
228
229
230
231
232
233
234
def load_id_mapping(self, index_path: str) -> Optional[Dict]:
    """Load ID mapping if available."""
    mapping_path = os.path.join(index_path, "id_mapping.json")
    if os.path.exists(mapping_path):
        with open(mapping_path, "r", encoding="utf-8") as f:
            mapping = json.load(f)
            return {v: k for k, v in mapping.items()}  # Reverse mapping
    return None

load_corpus(index_path)

Load corpus data from index path (supports multiple formats).

Source code in rankify/retrievers/index_manager.py 
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
def load_corpus(self, index_path: str) -> Dict:
    """Load corpus data from index path (supports multiple formats)."""
    # Try different corpus file formats in order of preference
    corpus_files = [
        "corpus_metadata.json",  # Your custom format
        "corpus.jsonl",          # Standard JSONL format
        "corpus.json"            # Alternative JSON format
    ]

    for corpus_filename in corpus_files:
        corpus_file = os.path.join(index_path, corpus_filename)
        if os.path.exists(corpus_file):
            print(f"📚 Loading corpus from {corpus_filename}")

            if corpus_filename.endswith('.jsonl'):
                # JSONL format (one JSON object per line)
                return self._load_corpus_jsonl(corpus_file)
            else:
                # JSON format (single JSON object)
                return self._load_corpus_json(corpus_file)

    print(f"❌ No corpus file found in {index_path}")
    return {}

download_and_extract_index(url)

Download and extract index from URL, return local path.

Source code in rankify/retrievers/index_manager.py 
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
def download_and_extract_index(self, url: str) -> str:
    """Download and extract index from URL, return local path."""
    # This is used by DenseRetriever pattern
    import tempfile
    import zipfile

    # Create temporary file
    with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as tmp_file:
        tmp_path = tmp_file.name

    try:
        # Download
        print(f"Downloading from {url}...")
        response = requests.get(url, stream=True)
        response.raise_for_status()

        with open(tmp_path, 'wb') as f:
            for chunk in tqdm(response.iter_content(chunk_size=1024)):
                f.write(chunk)

        # Extract to cache directory
        extract_dir = os.path.join(self.cache_dir, "downloaded_index")
        os.makedirs(extract_dir, exist_ok=True)

        with zipfile.ZipFile(tmp_path, 'r') as zip_ref:
            zip_ref.extractall(extract_dir)

        return extract_dir

    finally:
        # Clean up temporary file
        if os.path.exists(tmp_path):
            os.remove(tmp_path)

Document

Represents a document consisting of a question, answers, and contexts.

Attributes:

Name Type Description
question Question

The question associated with the document.
answers Answer

The answers to the question.
contexts list[Context]

A list of related contexts.
reorder_contexts list[Context] or None

A reordered list of contexts based on relevance.
Source code in rankify/dataset/dataset.py 
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
class Document:
    """
    Represents a document consisting of a question, answers, and contexts.

    Attributes:
        question (Question): The question associated with the document.
        answers (Answer): The answers to the question.
        contexts (list[Context]): A list of related contexts.
        reorder_contexts (list[Context] or None): A reordered list of contexts based on relevance.
    """
    def __init__(self, question: Question, answers: Answer, contexts: list = None , id: int = None) -> None:
        """
        Initializes a Document instance.

        Args:
            question (Question): The question associated with the document.
            answers (Answer): The answers to the question.
            contexts (list[Context], optional): A list of contexts related to the question.

        Example:
            ```python
            q = Question("What is the capital of France?")
            a = Answer(["Paris"])
            c1 = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
            c2 = Context(score=0.5, has_answer=False, id=2, title="Berlin", text="Berlin is the capital of Germany.")
            d = Document(question=q, answers=a, contexts=[c1, c2])
            print(d)
            ```
        """
        self.question: Question = question
        self.answers: Answer = answers
        self.contexts: List[Context] = contexts
        self.reorder_contexts: List[Context] = None
        self.id = str(id) 

    @classmethod
    def from_dict(cls, data: dict,n_docs:int=100) -> 'Document':
        """
        Creates a Document instance from a dictionary.

        Args:
            data (dict): A dictionary containing the question, answers, and contexts.
            n_docs (int, optional): The number of contexts to include. Defaults to 100.

        Returns:
            Document: A new Document instance.

        Example:
            ```python
            data = {
                "question": "What is the capital of France?",
                "answers": ["Paris"],
                "ctxs": [
                    {"score": 0.9, "has_answer": True, "id": 1, "title": "Paris", "text": "The capital of France is Paris."},
                    {"score": 0.5, "has_answer": False, "id": 2, "title": "Berlin", "text": "Berlin is the capital of Germany."}
                ]
            }
            d = Document.from_dict(data)
            print(d.question)
            ```
        """
        question = Question(data["question"])
        if "answers" in data:
            answers = Answer(data["answers"])
        else:
            answers =Answer('')

        if "query_id" in data:
            id = data["query_id"]
        else:
            id = None
        contexts = [Context(**ctx) for ctx in data["ctxs"][:n_docs]]
        return cls(question, answers, contexts, id=id)

    def to_dict(self) -> Dict[str, Optional[object]]:
        """
        Converts the document into a dictionary representation.

        Returns:
            dict: A dictionary containing the question, answers, and contexts.
        """
        return {
            "question": self.question.question,
            "answers": self.answers.answers,
            "contexts": [ctx.to_dict() for ctx in self.contexts]
        }
    def to_dict_reoreder(self) -> Dict[str,Optional[object]]:
        return {
            "question" : self.question.question,
            "answers" : self.answers.answers,
            "contexts" : [ctx.to_dict() for ctx in self.reorder_contexts]
        }
    def __str__(self) -> str:
        """
        Returns a string representation of the Document instance.

        Returns:
            str: The formatted document information.

        Example:
            ```python
            d = Document(Question("What is the capital of France?"), Answer(["Paris"]))
            print(d)
            ```
        """
        contexts_str = "\n\n".join([str(ctx) for ctx in self.contexts])
        reorder_contexts_str= ''
        if self.reorder_contexts is not None:
            reorder_contexts_str = "\n\n".join([str(ctx) for ctx in self.reorder_contexts])
        return f"{self.question}\n\n{self.answers}\n\nContext: \n\n{contexts_str}\nReorder contexts: \n\n{reorder_contexts_str}"

__init__(question, answers, contexts=None, id=None)

Initializes a Document instance.

Parameters:

Name Type Description Default
question Question

The question associated with the document.

 required
answers Answer

The answers to the question.

 required
contexts list[Context]

A list of contexts related to the question.

 None
Example 
q = Question("What is the capital of France?")
a = Answer(["Paris"])
c1 = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
c2 = Context(score=0.5, has_answer=False, id=2, title="Berlin", text="Berlin is the capital of Germany.")
d = Document(question=q, answers=a, contexts=[c1, c2])
print(d)
Source code in rankify/dataset/dataset.py 
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
def __init__(self, question: Question, answers: Answer, contexts: list = None , id: int = None) -> None:
    """
    Initializes a Document instance.

    Args:
        question (Question): The question associated with the document.
        answers (Answer): The answers to the question.
        contexts (list[Context], optional): A list of contexts related to the question.

    Example:
        ```python
        q = Question("What is the capital of France?")
        a = Answer(["Paris"])
        c1 = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
        c2 = Context(score=0.5, has_answer=False, id=2, title="Berlin", text="Berlin is the capital of Germany.")
        d = Document(question=q, answers=a, contexts=[c1, c2])
        print(d)
        ```
    """
    self.question: Question = question
    self.answers: Answer = answers
    self.contexts: List[Context] = contexts
    self.reorder_contexts: List[Context] = None
    self.id = str(id)

from_dict(data, n_docs=100) classmethod

Creates a Document instance from a dictionary.

Parameters:

Name Type Description Default
data dict

A dictionary containing the question, answers, and contexts.

 required
n_docs int

The number of contexts to include. Defaults to 100.

 100

Returns:

Name Type Description
Document Document

A new Document instance.
Example 
data = {
    "question": "What is the capital of France?",
    "answers": ["Paris"],
    "ctxs": [
        {"score": 0.9, "has_answer": True, "id": 1, "title": "Paris", "text": "The capital of France is Paris."},
        {"score": 0.5, "has_answer": False, "id": 2, "title": "Berlin", "text": "Berlin is the capital of Germany."}
    ]
}
d = Document.from_dict(data)
print(d.question)
Source code in rankify/dataset/dataset.py 
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
@classmethod
def from_dict(cls, data: dict,n_docs:int=100) -> 'Document':
    """
    Creates a Document instance from a dictionary.

    Args:
        data (dict): A dictionary containing the question, answers, and contexts.
        n_docs (int, optional): The number of contexts to include. Defaults to 100.

    Returns:
        Document: A new Document instance.

    Example:
        ```python
        data = {
            "question": "What is the capital of France?",
            "answers": ["Paris"],
            "ctxs": [
                {"score": 0.9, "has_answer": True, "id": 1, "title": "Paris", "text": "The capital of France is Paris."},
                {"score": 0.5, "has_answer": False, "id": 2, "title": "Berlin", "text": "Berlin is the capital of Germany."}
            ]
        }
        d = Document.from_dict(data)
        print(d.question)
        ```
    """
    question = Question(data["question"])
    if "answers" in data:
        answers = Answer(data["answers"])
    else:
        answers =Answer('')

    if "query_id" in data:
        id = data["query_id"]
    else:
        id = None
    contexts = [Context(**ctx) for ctx in data["ctxs"][:n_docs]]
    return cls(question, answers, contexts, id=id)

to_dict()

Converts the document into a dictionary representation.

Returns:

Name Type Description
dict Dict[str, Optional[object]]

A dictionary containing the question, answers, and contexts.
Source code in rankify/dataset/dataset.py 
269
270
271
272
273
274
275
276
277
278
279
280
def to_dict(self) -> Dict[str, Optional[object]]:
    """
    Converts the document into a dictionary representation.

    Returns:
        dict: A dictionary containing the question, answers, and contexts.
    """
    return {
        "question": self.question.question,
        "answers": self.answers.answers,
        "contexts": [ctx.to_dict() for ctx in self.contexts]
    }

__str__()

Returns a string representation of the Document instance.

Returns:

Name Type Description
str str

The formatted document information.
Example 
d = Document(Question("What is the capital of France?"), Answer(["Paris"]))
print(d)
Source code in rankify/dataset/dataset.py 
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
def __str__(self) -> str:
    """
    Returns a string representation of the Document instance.

    Returns:
        str: The formatted document information.

    Example:
        ```python
        d = Document(Question("What is the capital of France?"), Answer(["Paris"]))
        print(d)
        ```
    """
    contexts_str = "\n\n".join([str(ctx) for ctx in self.contexts])
    reorder_contexts_str= ''
    if self.reorder_contexts is not None:
        reorder_contexts_str = "\n\n".join([str(ctx) for ctx in self.reorder_contexts])
    return f"{self.question}\n\n{self.answers}\n\nContext: \n\n{contexts_str}\nReorder contexts: \n\n{reorder_contexts_str}"

Context

Represents a context with metadata such as score and title.

Attributes:

Name Type Description
score float

The relevance score of the context.
has_answer bool

Whether the context contains an answer.
id int

The identifier of the context.
title str

The title of the context.
text str

The text of the context.
Source code in rankify/dataset/dataset.py 
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
class Context:
    """
    Represents a context with metadata such as score and title.

    Attributes:
        score (float, optional): The relevance score of the context.
        has_answer (bool, optional): Whether the context contains an answer.
        id (int, optional): The identifier of the context.
        title (str, optional): The title of the context.
        text (str, optional): The text of the context.
    """
    def __init__(self, score: float=None, has_answer: bool=None, id: str=None, title: str=None, text: str=None)-> None:
        """
        Initializes a Context instance.

        Args:
            score (float, optional): The relevance score.
            has_answer (bool, optional): Whether the context contains an answer.
            id (int, optional): The identifier of the context.
            title (str, optional): The title of the context.
            text (str, optional): The text of the context.

        Example:
            ```python
            c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
            print(c)
            ```
        """
        self.score: Optional[float] = score
        self.has_answer: Optional[bool] = has_answer
        self.id: Optional[str] = id
        self.title: Optional[str] = title
        self.text: Optional[str] = text

    def to_dict(self, save_text: bool=False) -> Dict[str, Optional[object]]:

        """
        Converts the Context instance to a dictionary.

        Args:
            save_text (bool): Whether to include text in the output dictionary.

        Returns:
            dict: The context data.

        Example:
            ```python
            c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
            print(c.to_dict())
            ```
        """
        context_dict = {
            "score": float(self.score) if self.score is not None else None,
            "has_answer": self.has_answer,
            "id": self.id,
            }

        # Include 'text' only if save_text is True
        if save_text:
            context_dict["text"] = self.text
            context_dict["title"] =  self.title

        return context_dict
    def __str__(self) -> str:
        """
        Returns a string representation of the Context instance.

        Returns:
            str: The formatted context.

        Example:
            ```python
            c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
            print(str(c))
            ```
        """
        return f"ID: {self.id}\nHas Answer: {self.has_answer}\nTitle: {self.title}\nText: {self.text}\nScore: {self.score}"

__init__(score=None, has_answer=None, id=None, title=None, text=None)

Initializes a Context instance.

Parameters:

Name Type Description Default
score float

The relevance score.

 None
has_answer bool

Whether the context contains an answer.

 None
id int

The identifier of the context.

 None
title str

The title of the context.

 None
text str

The text of the context.

 None
Example 
c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
print(c)
Source code in rankify/dataset/dataset.py 
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
def __init__(self, score: float=None, has_answer: bool=None, id: str=None, title: str=None, text: str=None)-> None:
    """
    Initializes a Context instance.

    Args:
        score (float, optional): The relevance score.
        has_answer (bool, optional): Whether the context contains an answer.
        id (int, optional): The identifier of the context.
        title (str, optional): The title of the context.
        text (str, optional): The text of the context.

    Example:
        ```python
        c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
        print(c)
        ```
    """
    self.score: Optional[float] = score
    self.has_answer: Optional[bool] = has_answer
    self.id: Optional[str] = id
    self.title: Optional[str] = title
    self.text: Optional[str] = text

to_dict(save_text=False)

Converts the Context instance to a dictionary.

Parameters:

Name Type Description Default
save_text bool

Whether to include text in the output dictionary.

 False

Returns:

Name Type Description
dict Dict[str, Optional[object]]

The context data.
Example 
c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
print(c.to_dict())
Source code in rankify/dataset/dataset.py 
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
def to_dict(self, save_text: bool=False) -> Dict[str, Optional[object]]:

    """
    Converts the Context instance to a dictionary.

    Args:
        save_text (bool): Whether to include text in the output dictionary.

    Returns:
        dict: The context data.

    Example:
        ```python
        c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
        print(c.to_dict())
        ```
    """
    context_dict = {
        "score": float(self.score) if self.score is not None else None,
        "has_answer": self.has_answer,
        "id": self.id,
        }

    # Include 'text' only if save_text is True
    if save_text:
        context_dict["text"] = self.text
        context_dict["title"] =  self.title

    return context_dict

__str__()

Returns a string representation of the Context instance.

Returns:

Name Type Description
str str

The formatted context.
Example 
c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
print(str(c))
Source code in rankify/dataset/dataset.py 
180
181
182
183
184
185
186
187
188
189
190
191
192
193
def __str__(self) -> str:
    """
    Returns a string representation of the Context instance.

    Returns:
        str: The formatted context.

    Example:
        ```python
        c = Context(score=0.9, has_answer=True, id=1, title="Paris", text="The capital of France is Paris.")
        print(str(c))
        ```
    """
    return f"ID: {self.id}\nHas Answer: {self.has_answer}\nTitle: {self.title}\nText: {self.text}\nScore: {self.score}"

DenseRetriever

Bases: BaseRetriever

Dense retriever implementation supporting DPR, ANCE, and BPR models.

Uses FAISS indexes for efficient dense vector retrieval.

Source code in rankify/retrievers/dense_retriever.py 
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
class DenseRetriever(BaseRetriever):
    """
    Dense retriever implementation supporting DPR, ANCE, and BPR models.

    Uses FAISS indexes for efficient dense vector retrieval.
    """

    MSMARCO_CORPUS_URL = "https://huggingface.co/datasets/abdoelsayed/reranking-datasets/resolve/main/msmarco-passage-corpus/msmarco-passage-corpus.tsv?download=true"

    def __init__(self, method: str = "dpr-multi", index_type: str = "wiki", 
                 index_folder: str = None, **kwargs):
        super().__init__(**kwargs)
        self.method = method
        self.index_type = index_type
        self.index_folder = index_folder
        self.tokenizer = SimpleTokenizer()

        # Initialize index manager
        self.index_manager = IndexManager()

        # Handle custom index folder case
        if index_folder:
            # For custom indexes, get encoder from method and default index_type
            if method in self.index_manager.index_configs:
                # Use wiki as default if the provided index_type is a path
                default_index_type = "wiki" if (index_type.startswith("./") or 
                                               index_type.startswith("/") or 
                                               os.path.exists(index_type)) else index_type
                if default_index_type in self.index_manager.index_configs[method]:
                    self.query_encoder = self.index_manager.index_configs[method][default_index_type]["encoder"]
                else:
                    # Fallback to first available encoder for this method
                    available_types = list(self.index_manager.index_configs[method].keys())
                    self.query_encoder = self.index_manager.index_configs[method][available_types[0]]["encoder"]
                    print(f"Warning: Using default encoder {self.query_encoder} for custom index")
            else:
                raise ValueError(f"Unsupported method: {method}")
        else:
            # Get query encoder for standard indexes
            if method not in self.index_manager.index_configs:
                raise ValueError(f"Unsupported method: {method}")
            if index_type not in self.index_manager.index_configs[method]:
                raise ValueError(f"Unsupported index_type '{index_type}' for method '{method}'")
            self.query_encoder = self.index_manager.index_configs[method][index_type]["encoder"]

        self.question_tokenizer = AutoTokenizer.from_pretrained(self.query_encoder)

        # Initialize variables
        self.id_mapping = None
        self.corpus = {}

        # Handle custom vs prebuilt indexes
        if index_folder:
            self.index_path = index_folder
            self.id_mapping = self.index_manager.load_id_mapping(index_folder)
            self.corpus = self.index_manager.load_corpus(index_folder)
            print(f"Loaded corpus with {len(self.corpus)} documents")
            if self.id_mapping:
                print(f"Loaded ID mapping with {len(self.id_mapping)} entries")
        else:
            self.index_path = self.index_manager.get_index_path(method, index_type)

        # Initialize searcher
        self.searcher = self._initialize_searcher()
        if self.searcher is None:
            raise RuntimeError("Failed to initialize searcher")

        # Load corpus for MSMARCO if needed
        if index_type == "msmarco" and not index_folder:
            self._load_msmarco_corpus()

    def _initialize_searcher(self):
        """
        Initialize FAISS searcher for all cases.
        This is PURE DPR - no LuceneSearcher anywhere!
        """
        try:
            # Case 1: Custom index folder provided
            if self.index_folder:
                print(f"Initializing FaissSearcher with custom index folder: {self.index_folder}")
                if not os.path.exists(self.index_folder):
                    raise ValueError(f"Custom index folder '{self.index_folder}' does not exist.")
                return FaissSearcher(self.index_folder, self.query_encoder)

            # Case 2: Standard indexes (prebuilt or downloaded)
            print(f"Index path: {self.index_path}")

            # Case 2a: URL - need to download first
            if self.index_path.startswith("http"):
                print(f"Downloading index from URL: {self.index_path}")
                local_dir = self.index_manager.download_and_extract_index(self.index_path)
                print(f"Downloaded index to: {local_dir}")
                return FaissSearcher(local_dir, self.query_encoder)

            # Case 2b: Local file path (already downloaded)
            elif (os.path.exists(self.index_path) or 
                  self.index_path.startswith('/') or 
                  self.index_path.startswith('./')):
                print(f"Using local index path: {self.index_path}")
                if not os.path.exists(self.index_path):
                    raise ValueError(f"Local index path '{self.index_path}' does not exist.")
                return FaissSearcher(self.index_path, self.query_encoder)

            # Case 2c: Prebuilt index name (like "wikipedia-dpr-100w.dpr-multi")
            else:
                print(f"Using prebuilt index: {self.index_path}")
                return FaissSearcher.from_prebuilt_index(self.index_path, self.query_encoder)

        except Exception as e:
            print(f"ERROR initializing searcher:")
            print(f"  - Method: {self.method}")
            print(f"  - Index type: {self.index_type}")
            print(f"  - Index folder: {self.index_folder}")
            print(f"  - Index path: {self.index_path}")
            print(f"  - Query encoder: {self.query_encoder}")
            print(f"  - Error: {e}")
            import traceback
            traceback.print_exc()
            return None

    def _load_msmarco_corpus(self):
        """Load MSMARCO corpus into memory."""
        corpus_file = os.path.join(self.index_manager.cache_dir, "msmarco-passage-corpus.tsv")

        if not os.path.exists(corpus_file):
            print("Downloading MSMARCO corpus...")
            self._download_file(self.MSMARCO_CORPUS_URL, corpus_file)

        print("Loading MSMARCO corpus...")
        self.corpus = {}
        with open(corpus_file, "r", encoding="utf-8") as f:
            next(f)  # Skip header
            for line in f:
                parts = line.strip().split("\t")
                if len(parts) >= 3:
                    doc_id, text, title = parts[0], parts[1], parts[2]
                    self.corpus[doc_id] = {"text": text, "title": title}

    def _download_file(self, url: str, save_path: str):
        """Download a file from URL."""
        os.makedirs(os.path.dirname(save_path), exist_ok=True)
        print(f"Downloading {os.path.basename(save_path)}...")
        response = requests.get(url, stream=True)
        response.raise_for_status()

        with open(save_path, "wb") as f:
            for chunk in tqdm(response.iter_content(chunk_size=1024), 
                            desc=f"Downloading {os.path.basename(save_path)}"):
                f.write(chunk)

    def retrieve(self, documents: List[Document]) -> List[Document]:
        """Retrieve relevant contexts using dense retrieval."""
        queries = [doc.question.question for doc in documents]
        qids = [str(i) for i in range(len(queries))]

        print(f"Retrieving {len(documents)} documents with {self.method}...")

        # Perform batch search
        batch_results = self._batch_search(queries, qids)

        # Process results
        for i, document in enumerate(tqdm(documents, desc="Processing documents")):
            contexts = []
            hits = batch_results.get(str(i), [])

            for hit in hits:
                try:
                    context = self._create_context_from_hit(hit, document)
                    if context:
                        contexts.append(context)
                except Exception as e:
                    print(f"Error processing document ID {hit.docid}: {e}")

            document.contexts = contexts
            #break

        return documents

    def _create_context_from_hit(self, hit, document) -> Context:
        """Create Context object from search hit."""
        doc_id = hit.docid
        text = ""
        title = ""

        try:
            # Strategy 1: Use loaded corpus (MSMARCO or custom)
            if self.corpus:
                if self.index_type == "msmarco":
                    # MSMARCO corpus format
                    doc_data = self.corpus.get(str(hit.docid), {})
                    if doc_data:
                        text = doc_data.get("text", "")
                        title = doc_data.get("title", "")
                    else:
                        text = f"Document {hit.docid} not found in MSMARCO corpus"
                        title = f"Document {hit.docid}"
                else:
                    # Custom corpus format
                    doc_data = self.corpus.get(str(hit.docid), {})
                    if doc_data:
                        text = doc_data.get("contents") or doc_data.get("text", "") or ""
                        title = doc_data.get("title", "")
                        # Use ID mapping if available
                        if self.id_mapping:
                            doc_id = self.id_mapping.get(int(hit.docid), hit.docid)
                    else:
                        text = f"Document {hit.docid} not found in custom corpus"
                        title = f"Document {hit.docid}"

            # Strategy 2: Try to get document from searcher
            elif hasattr(self.searcher, 'doc'):
                try:
                    doc = self.searcher.doc(hit.docid)
                    raw_content = json.loads(doc.raw())
                    content = raw_content.get("contents", "")

                    if '\n' in content:
                        lines = content.split('\n', 1)
                        title = lines[0].strip()
                        text = lines[1].strip() if len(lines) > 1 else ""
                    else:
                        title = "No Title"
                        text = content

                except Exception as e:
                    print(f"Error retrieving document {hit.docid} from searcher: {e}")
                    text = f"Error retrieving document {hit.docid}"
                    title = f"Document {hit.docid}"

            # Strategy 3: Fallback
            else:
                print(f"Warning: No way to retrieve content for document {hit.docid}")
                text = f"Document {hit.docid}"
                title = f"Document {hit.docid}"

        except Exception as e:
            print(f"Error processing hit {hit.docid}: {e}")
            text = f"Error: {str(e)}"
            title = f"Document {hit.docid}"

        return Context(
            id=doc_id,
            title=title,
            text=text,
            score=hit.score,
            has_answer=has_answers(text, document.answers.answers, self.tokenizer)
        )

    def _batch_search(self, queries: List[str], qids: List[str]) -> dict:
        """Perform batch search using FAISS searcher."""
        if self.searcher is None:
            raise RuntimeError("Searcher not initialized properly")

        if not hasattr(self.searcher, 'batch_search'):
            raise AttributeError(f"Searcher {type(self.searcher)} does not have batch_search method")

        batch_results = {}
        batch_qids, batch_queries = [], []

        for idx, (qid, query) in enumerate(tqdm(zip(qids, queries), desc="Batch search", total=len(qids))):
            query = self._preprocess_query(query)
            batch_qids.append(qid)
            batch_queries.append(query)

            if (idx + 1) % self.batch_size == 0 or idx == len(qids) - 1:
                try:
                    results = self.searcher.batch_search(batch_queries, batch_qids, 
                                                       self.n_docs, self.threads)
                    batch_results.update(results)
                except Exception as e:
                    print(f"Batch search failed:")
                    print(f"  - Searcher type: {type(self.searcher)}")
                    print(f"  - Queries: {batch_queries}")
                    print(f"  - Error: {e}")
                    raise e

                batch_qids.clear()
                batch_queries.clear()

        return batch_results

    def _preprocess_query(self, query: str) -> str:
        """Preprocess query to handle token length limits."""
        try:
            tokenized = self.question_tokenizer(query, add_special_tokens=True)

            if len(tokenized["input_ids"]) > 511:
                truncated = self.question_tokenizer.decode(
                    tokenized["input_ids"][:511], skip_special_tokens=True
                )
                return truncated
            return query
        except Exception as e:
            print(f"Error preprocessing query '{query}': {e}")
            return query

retrieve(documents)

Retrieve relevant contexts using dense retrieval.

Source code in rankify/retrievers/dense_retriever.py 
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
def retrieve(self, documents: List[Document]) -> List[Document]:
    """Retrieve relevant contexts using dense retrieval."""
    queries = [doc.question.question for doc in documents]
    qids = [str(i) for i in range(len(queries))]

    print(f"Retrieving {len(documents)} documents with {self.method}...")

    # Perform batch search
    batch_results = self._batch_search(queries, qids)

    # Process results
    for i, document in enumerate(tqdm(documents, desc="Processing documents")):
        contexts = []
        hits = batch_results.get(str(i), [])

        for hit in hits:
            try:
                context = self._create_context_from_hit(hit, document)
                if context:
                    contexts.append(context)
            except Exception as e:
                print(f"Error processing document ID {hit.docid}: {e}")

        document.contexts = contexts
        #break

    return documents