From 3d45c01522aa9208f55b5961313ae196e08bde97 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Mon, 24 Jan 2022 08:38:36 -0300
Subject: [PATCH 01/27] include pyvespa dependency

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index 29de4fdc..c57ca226 100644
--- a/setup.py
+++ b/setup.py
@@ -21,6 +21,7 @@
         'pytrec_eval',
         'faiss_cpu',
         'elasticsearch',
+        'pyvespa',
         'tensorflow>=2.2.0',
         'tensorflow-text',
         'tensorflow-hub'

From 27b3cfd6712be47e797facfbba08031be4401297 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Mon, 24 Jan 2022 08:38:56 -0300
Subject: [PATCH 02/27] include pycharm config files on gitignore

---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index df2b2b25..47bd55d5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -134,3 +134,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# PyCharm
+.idea
\ No newline at end of file

From 567403463f41d20e2b26d3a3705ca64d10e40e02 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Mon, 24 Jan 2022 08:40:22 -0300
Subject: [PATCH 03/27] create a class for vespa lexical search

---
 beir/retrieval/search/lexical/vespa_search.py | 189 ++++++++++++++++++
 1 file changed, 189 insertions(+)
 create mode 100644 beir/retrieval/search/lexical/vespa_search.py

diff --git a/beir/retrieval/search/lexical/vespa_search.py b/beir/retrieval/search/lexical/vespa_search.py
new file mode 100644
index 00000000..7d2358a1
--- /dev/null
+++ b/beir/retrieval/search/lexical/vespa_search.py
@@ -0,0 +1,189 @@
+from typing import Dict, Optional
+from vespa.application import Vespa
+from vespa.package import ApplicationPackage, Field, FieldSet, RankProfile, QueryField
+from vespa.query import QueryModel, OR, RankProfile as Ranking, WeakAnd
+from vespa.deployment import VespaDocker, VespaCloud
+
+
+class VespaLexicalSearch:
+    def __init__(
+        self,
+        application_name: str,
+        match_phase: str = "or",
+        rank_phase: str = "bm25",
+        deployment_type: str = "docker",
+        deployment_parameters: Optional[Dict] = None,
+        initialize: bool = True,
+    ):
+        self.results = {}
+        self.application_name = application_name
+        assert match_phase in [
+            "or",
+            "weak_and",
+        ], "'match_phase' should be either 'or' or 'weak_and'"
+        self.match_phase = match_phase
+        assert rank_phase in [
+            "bm25",
+            "native_rank",
+        ], "'rank_phase' should be either 'bm25' or 'native_rank'"
+        self.rank_phase = rank_phase
+        assert deployment_type in [
+            "docker",
+            "cloud",
+        ], "deployment_type should be either 'docker' or 'cloud'"
+        self.deployment_type = deployment_type
+        self.deployment_parameters = deployment_parameters
+        self.initialize = initialize
+        if self.initialize:
+            self.app = self.initialise()
+            assert (
+                self.app.get_application_status().status_code == 200
+            ), "Application status different than 200."
+        else:
+            assert self.deployment_parameters is not None, (
+                "if 'initialize' is set to false, 'deployment_parameters' should contain Vespa "
+                "connection parameters such as 'url' and 'port'"
+            )
+            self.app = Vespa(**self.deployment_parameters)
+            assert (
+                self.app.get_application_status().status_code == 200
+            ), "Application status different than 200."
+
+    def initialise(self):
+        #
+        # Create Vespa application package
+        #
+        app_package = ApplicationPackage(name=self.application_name)
+        app_package.schema.add_fields(
+            Field(name="id", type="string", indexing=["attribute", "summary"]),
+            Field(
+                name="title",
+                type="string",
+                indexing=["index", "summary"],
+                index="enable-bm25",
+            ),
+            Field(
+                name="body",
+                type="string",
+                indexing=["index", "summary"],
+                index="enable-bm25",
+            ),
+        )
+        app_package.schema.add_field_set(
+            FieldSet(name="default", fields=["title", "body"])
+        )
+        app_package.schema.add_rank_profile(
+            rank_profile=RankProfile(
+                name="bm25", first_phase="bm25(title) + bm25(body)"
+            )
+        )
+        app_package.schema.add_rank_profile(
+            rank_profile=RankProfile(
+                name="native_rank", first_phase="nativeRank(title,body)"
+            )
+        )
+        app_package.query_profile.add_fields(QueryField(name="maxHits", value=10000))
+        #
+        # Deploy application
+        #
+        if self.deployment_type == "docker":
+            if not self.deployment_parameters:
+                self.deployment_parameters = {"port": 8089}
+            vespa_docker = VespaDocker(**self.deployment_parameters)
+            app = vespa_docker.deploy(application_package=app_package)
+            app.delete_all_docs(
+                content_cluster_name=self.application_name + "_content",
+                schema=self.application_name,
+            )
+            return app
+        elif self.deployment_type == "cloud":
+            assert self.deployment_parameters is not None and all(
+                [
+                    x
+                    in [
+                        "tenant_name",
+                        "application_name",
+                        "user_key_path",
+                        "instance_name",
+                    ]
+                    for x in self.deployment_parameters.keys()
+                ]
+            ), (
+                "'deployment_parameters' should be a dict containing the following keys: "
+                "['tenant_name', 'application_name', 'user_key_path', 'instance_name']"
+            )
+            # Cloud
+            vespa_cloud = VespaCloud(
+                tenant=self.deployment_parameters["tenant_name"],
+                application=self.deployment_parameters["application_name"],
+                application_package=app_package,
+                key_location=self.deployment_parameters["user_key_path"],
+            )
+            app = vespa_cloud.deploy(
+                instance=self.deployment_parameters["instance_name"]
+            )
+            app.delete_all_docs(
+                content_cluster_name=self.application_name + "_content",
+                schema=self.application_name,
+            )
+            return app
+        else:
+            ValueError("deployment_type should be either 'docker' or 'cloud'")
+
+    def search(
+        self,
+        corpus: Dict[str, Dict[str, str]],
+        queries: Dict[str, str],
+        top_k: int,
+        *args,
+        **kwargs
+    ) -> Dict[str, Dict[str, float]]:
+
+        if self.initialize:
+            _ = self.index(corpus)
+
+        # retrieve results from BM25
+        query_ids = list(queries.keys())
+        queries = [queries[qid] for qid in query_ids]
+
+        if self.match_phase == "or":
+            match_phase = OR()
+        elif self.match_phase == "weak_and":
+            match_phase = WeakAnd(hits=top_k)
+        else:
+            ValueError("'match_phase' should be either 'or' or 'weak_and'")
+
+        if self.rank_phase not in ["bm25", "native_rank"]:
+            ValueError("'rank_phase' should be either 'bm25' or 'native_rank'")
+
+        query_model = QueryModel(
+            name=self.match_phase + "_bm25",
+            match_phase=match_phase,
+            rank_profile=Ranking(name=self.rank_phase, list_features=False),
+        )
+
+        for query_id, query in zip(query_ids, queries):
+            scores = {}
+            query_result = self.app.query(
+                query=query,
+                query_model=query_model,
+                hits=top_k,
+            )
+            for hit in query_result.hits:
+                scores[hit["fields"]["id"]] = hit["relevance"]
+            self.results[query_id] = scores
+        return self.results
+
+    def index(self, corpus: Dict[str, Dict[str, str]]):
+        batch_feed = [
+            {
+                "id": idx,
+                "fields": {
+                    "id": idx,
+                    "title": corpus[idx].get("title", None),
+                    "body": corpus[idx].get("text", None),
+                },
+            }
+            for idx in list(corpus.keys())
+        ]
+        return self.app.feed_batch(batch=batch_feed)

From 064c34e81f09716ca4c36046e5ef448f0f78995d Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Mon, 24 Jan 2022 08:40:40 -0300
Subject: [PATCH 04/27] basic tests for vespa lexical search

---
 beir/test_retrieval_lexical_vespa.py | 69 ++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 beir/test_retrieval_lexical_vespa.py

diff --git a/beir/test_retrieval_lexical_vespa.py b/beir/test_retrieval_lexical_vespa.py
new file mode 100644
index 00000000..eb1349f3
--- /dev/null
+++ b/beir/test_retrieval_lexical_vespa.py
@@ -0,0 +1,69 @@
+import unittest
+import shutil
+import docker
+from beir.retrieval.search.lexical.vespa_search import VespaLexicalSearch
+from beir.retrieval.evaluation import EvaluateRetrieval
+
+
+class TestVespaSearch(unittest.TestCase):
+    def setUp(self) -> None:
+        self.application_name = "vespa_test"
+        self.corpus = {
+            "1": {"title": "this is a title 1", "text": "this is text 1"},
+            "2": {"title": "this is a title 2", "text": "this is text 2"},
+            "3": {"title": "this is a title 3", "text": "this is text 3"},
+        }
+        self.queries = {"1": "this is query 1", "2": "this is query 2"}
+
+    def test_or_bm25(self):
+        model = VespaLexicalSearch(application_name=self.application_name, initialize=True)
+        retriever = EvaluateRetrieval(model)
+        results = retriever.retrieve(corpus=self.corpus, queries=self.queries)
+        self.assertEqual({"1", "2"}, set(results.keys()))
+        for query_id in results.keys():
+            self.assertEqual({"1", "2", "3"}, set(results[query_id].keys()))
+
+    def test_or_native_rank(self):
+        model = VespaLexicalSearch(
+            application_name=self.application_name,
+            initialize=True,
+            match_phase="or",
+            rank_phase="native_rank",
+        )
+        retriever = EvaluateRetrieval(model)
+        results = retriever.retrieve(corpus=self.corpus, queries=self.queries)
+        self.assertEqual({"1", "2"}, set(results.keys()))
+        for query_id in results.keys():
+            self.assertEqual({"1", "2", "3"}, set(results[query_id].keys()))
+
+    def test_weakand_bm25(self):
+        model = VespaLexicalSearch(
+            application_name=self.application_name,
+            initialize=True,
+            match_phase="weak_and",
+        )
+        retriever = EvaluateRetrieval(model)
+        results = retriever.retrieve(corpus=self.corpus, queries=self.queries)
+        self.assertEqual({"1", "2"}, set(results.keys()))
+        for query_id in results.keys():
+            self.assertEqual({"1", "2", "3"}, set(results[query_id].keys()))
+
+    def test_weakand_native_rank(self):
+        model = VespaLexicalSearch(
+            application_name=self.application_name,
+            initialize=True,
+            match_phase="weak_and",
+            rank_phase="native_rank"
+        )
+        retriever = EvaluateRetrieval(model)
+        results = retriever.retrieve(corpus=self.corpus, queries=self.queries)
+        self.assertEqual({"1", "2"}, set(results.keys()))
+        for query_id in results.keys():
+            self.assertEqual({"1", "2", "3"}, set(results[query_id].keys()))
+
+    def tearDown(self) -> None:
+        shutil.rmtree(self.application_name, ignore_errors=True)
+        client = docker.from_env()
+        container = client.containers.get(self.application_name)
+        container.stop()
+        container.remove()

From d764954f2c1caa100927490dc5bcd479b0439d6e Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Mon, 24 Jan 2022 08:41:18 -0300
Subject: [PATCH 05/27] include vespa lexical search as a possivle argument in
 the evaluator

---
 beir/retrieval/evaluation.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/beir/retrieval/evaluation.py b/beir/retrieval/evaluation.py
index 918929a1..e61bde7c 100644
--- a/beir/retrieval/evaluation.py
+++ b/beir/retrieval/evaluation.py
@@ -4,6 +4,7 @@
 from .search.dense import DenseRetrievalExactSearch as DRES
 from .search.dense import DenseRetrievalFaissSearch as DRFS
 from .search.lexical import BM25Search as BM25
+from .search.lexical.vespa_search import VespaLexicalSearch
 from .search.sparse import SparseSearch as SS
 from .custom_metrics import mrr, recall_cap, hole, top_k_accuracy
 
@@ -11,7 +12,7 @@
 
 class EvaluateRetrieval:
     
-    def __init__(self, retriever: Union[Type[DRES], Type[DRFS], Type[BM25], Type[SS]] = None, k_values: List[int] = [1,3,5,10,100,1000], score_function: str = "cos_sim"):
+    def __init__(self, retriever: Union[Type[DRES], Type[DRFS], Type[BM25], Type[SS], VespaLexicalSearch] = None, k_values: List[int] = [1, 3, 5, 10, 100, 1000], score_function: str = "cos_sim"):
         self.k_values = k_values
         self.top_k = max(k_values)
         self.retriever = retriever

From eac2910ce04c7249477e0af777c6ebe7e85b2ef9 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Mon, 24 Jan 2022 10:34:57 -0300
Subject: [PATCH 06/27] add vespa benchmark script

---
 .../benchmarking/benchmark_lexical_vespa.py   | 87 +++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100644 examples/benchmarking/benchmark_lexical_vespa.py

diff --git a/examples/benchmarking/benchmark_lexical_vespa.py b/examples/benchmarking/benchmark_lexical_vespa.py
new file mode 100644
index 00000000..793972c2
--- /dev/null
+++ b/examples/benchmarking/benchmark_lexical_vespa.py
@@ -0,0 +1,87 @@
+#! /usr/bin/env python3
+
+import os
+from beir import util
+from beir.datasets.data_loader import GenericDataLoader
+from beir.retrieval.search.lexical.vespa_search import VespaLexicalSearch
+from beir.retrieval.evaluation import EvaluateRetrieval
+from pandas import DataFrame
+
+
+def download_and_unzip_dataset(data_dir, dataset_name):
+    url = "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/{}.zip".format(
+        dataset_name
+    )
+    data_path = util.download_and_unzip(url, data_dir)
+    print("Dataset downloaded here: {}".format(data_path))
+    return data_path
+
+
+def prepare_data(data_path):
+    corpus, queries, qrels = GenericDataLoader(data_path).load(
+        split="test"
+    )  # or split = "train" or "dev"
+    return corpus, queries, qrels
+
+
+def get_search_results(dataset_name, corpus, queries, qrels):
+    initialize = True
+    metrics = []
+    for match_phase in ["or", "weak_and"]:
+        for rank_phase in ["bm25", "native_rank"]:
+            model = VespaLexicalSearch(
+                application_name=dataset_name,
+                match_phase=match_phase,
+                rank_phase=rank_phase,
+                initialize=initialize,
+            )
+            initialize = False
+            retriever = EvaluateRetrieval(model)
+            results = retriever.retrieve(corpus, queries)
+            ndcg, _map, recall, precision = retriever.evaluate(
+                qrels, results, retriever.k_values
+            )
+            metrics.append(
+                {
+                    "dataset_name": dataset_name,
+                    "match_phase": match_phase,
+                    "rank_phase": rank_phase,
+                    "ndcg": ndcg,
+                    "map": _map,
+                    "recall": recall,
+                    "precision": precision,
+                }
+            )
+    return metrics
+
+
+def benchmark_vespa_lexical(data_dir, dataset_names):
+    result = []
+    for dataset_name in dataset_names:
+        data_path = download_and_unzip_dataset(
+            data_dir=data_dir, dataset_name=dataset_name
+        )
+        corpus, queries, qrels = prepare_data(data_path=data_path)
+        metrics = get_search_results(
+            dataset_name=dataset_name, corpus=corpus, queries=queries, qrels=qrels
+        )
+        output_file = os.path.join(data_dir, "metrics.csv")
+        if os.path.isfile(output_file):
+            DataFrame.from_records(metrics).to_csv(
+                output_file, mode="a", header=False, index=False
+            )
+        else:
+            DataFrame.from_records(metrics).to_csv(
+                output_file, mode="w", header=True, index=False
+            )
+        print(metrics)
+        result.extend(metrics)
+    return result
+
+
+if __name__ == "__main__":
+
+    data_dir = "/Users/tmartins/projects/sw/beir/data"
+    dataset_names = ["scifact"]
+
+    result = benchmark_vespa_lexical(data_dir=data_dir, dataset_names=dataset_names)

From 0c54ada28570c773ece283a00a95b09c34dd0f1b Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Mon, 24 Jan 2022 10:49:07 -0300
Subject: [PATCH 07/27] increase timeout

---
 beir/retrieval/search/lexical/vespa_search.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/beir/retrieval/search/lexical/vespa_search.py b/beir/retrieval/search/lexical/vespa_search.py
index 7d2358a1..2b5039d1 100644
--- a/beir/retrieval/search/lexical/vespa_search.py
+++ b/beir/retrieval/search/lexical/vespa_search.py
@@ -168,6 +168,7 @@ def search(
                 query=query,
                 query_model=query_model,
                 hits=top_k,
+                timeout="10 s"
             )
             for hit in query_result.hits:
                 scores[hit["fields"]["id"]] = hit["relevance"]

From 8197a736b8ea60dfd4816ac1129b4d871187a9ac Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Mon, 24 Jan 2022 14:06:56 -0300
Subject: [PATCH 08/27] include deployment parameters. Fix result object.

---
 .../benchmarking/benchmark_lexical_vespa.py   | 27 ++++++++++---------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/examples/benchmarking/benchmark_lexical_vespa.py b/examples/benchmarking/benchmark_lexical_vespa.py
index 793972c2..68184cf6 100644
--- a/examples/benchmarking/benchmark_lexical_vespa.py
+++ b/examples/benchmarking/benchmark_lexical_vespa.py
@@ -26,6 +26,7 @@ def prepare_data(data_path):
 
 def get_search_results(dataset_name, corpus, queries, qrels):
     initialize = True
+    deployment_parameters = None
     metrics = []
     for match_phase in ["or", "weak_and"]:
         for rank_phase in ["bm25", "native_rank"]:
@@ -34,30 +35,32 @@ def get_search_results(dataset_name, corpus, queries, qrels):
                 match_phase=match_phase,
                 rank_phase=rank_phase,
                 initialize=initialize,
+                deployment_parameters=deployment_parameters,
             )
             initialize = False
+            deployment_parameters = {"url": "http://localhost", "port": 8089}
             retriever = EvaluateRetrieval(model)
             results = retriever.retrieve(corpus, queries)
             ndcg, _map, recall, precision = retriever.evaluate(
                 qrels, results, retriever.k_values
             )
-            metrics.append(
-                {
-                    "dataset_name": dataset_name,
-                    "match_phase": match_phase,
-                    "rank_phase": rank_phase,
-                    "ndcg": ndcg,
-                    "map": _map,
-                    "recall": recall,
-                    "precision": precision,
-                }
-            )
+            metric = {
+                "dataset_name": dataset_name,
+                "match_phase": match_phase,
+                "rank_phase": rank_phase,
+            }
+            metric.update(ndcg)
+            metric.update(_map)
+            metric.update(recall)
+            metric.update(precision)
+            metrics.append(metric)
     return metrics
 
 
 def benchmark_vespa_lexical(data_dir, dataset_names):
     result = []
     for dataset_name in dataset_names:
+        print("Dataset: {}".format(dataset_name))
         data_path = download_and_unzip_dataset(
             data_dir=data_dir, dataset_name=dataset_name
         )
@@ -82,6 +85,6 @@ def benchmark_vespa_lexical(data_dir, dataset_names):
 if __name__ == "__main__":
 
     data_dir = "/Users/tmartins/projects/sw/beir/data"
-    dataset_names = ["scifact"]
+    dataset_names = ["scifact", "nfcorpus"]
 
     result = benchmark_vespa_lexical(data_dir=data_dir, dataset_names=dataset_names)

From 14ab0c560bbca454fb3e7025a17d4dc848bafdf7 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Mon, 24 Jan 2022 14:30:33 -0300
Subject: [PATCH 09/27] include remove app method

---
 beir/retrieval/search/lexical/vespa_search.py | 75 ++++++-------------
 1 file changed, 21 insertions(+), 54 deletions(-)

diff --git a/beir/retrieval/search/lexical/vespa_search.py b/beir/retrieval/search/lexical/vespa_search.py
index 2b5039d1..b2fd3cc9 100644
--- a/beir/retrieval/search/lexical/vespa_search.py
+++ b/beir/retrieval/search/lexical/vespa_search.py
@@ -1,8 +1,9 @@
+import shutil
 from typing import Dict, Optional
 from vespa.application import Vespa
 from vespa.package import ApplicationPackage, Field, FieldSet, RankProfile, QueryField
 from vespa.query import QueryModel, OR, RankProfile as Ranking, WeakAnd
-from vespa.deployment import VespaDocker, VespaCloud
+from vespa.deployment import VespaDocker
 
 
 class VespaLexicalSearch:
@@ -11,7 +12,6 @@ def __init__(
         application_name: str,
         match_phase: str = "or",
         rank_phase: str = "bm25",
-        deployment_type: str = "docker",
         deployment_parameters: Optional[Dict] = None,
         initialize: bool = True,
     ):
@@ -27,13 +27,9 @@ def __init__(
             "native_rank",
         ], "'rank_phase' should be either 'bm25' or 'native_rank'"
         self.rank_phase = rank_phase
-        assert deployment_type in [
-            "docker",
-            "cloud",
-        ], "deployment_type should be either 'docker' or 'cloud'"
-        self.deployment_type = deployment_type
         self.deployment_parameters = deployment_parameters
         self.initialize = initialize
+        self.vespa_docker = None
         if self.initialize:
             self.app = self.initialise()
             assert (
@@ -86,49 +82,23 @@ def initialise(self):
         #
         # Deploy application
         #
-        if self.deployment_type == "docker":
-            if not self.deployment_parameters:
-                self.deployment_parameters = {"port": 8089}
-            vespa_docker = VespaDocker(**self.deployment_parameters)
-            app = vespa_docker.deploy(application_package=app_package)
-            app.delete_all_docs(
-                content_cluster_name=self.application_name + "_content",
-                schema=self.application_name,
-            )
-            return app
-        elif self.deployment_type == "cloud":
-            assert self.deployment_parameters is not None and all(
-                [
-                    x
-                    in [
-                        "tenant_name",
-                        "application_name",
-                        "user_key_path",
-                        "instance_name",
-                    ]
-                    for x in self.deployment_parameters.keys()
-                ]
-            ), (
-                "'deployment_parameters' should be a dict containing the following keys: "
-                "['tenant_name', 'application_name', 'user_key_path', 'instance_name']"
-            )
-            # Cloud
-            vespa_cloud = VespaCloud(
-                tenant=self.deployment_parameters["tenant_name"],
-                application=self.deployment_parameters["application_name"],
-                application_package=app_package,
-                key_location=self.deployment_parameters["user_key_path"],
-            )
-            app = vespa_cloud.deploy(
-                instance=self.deployment_parameters["instance_name"]
-            )
-            app.delete_all_docs(
-                content_cluster_name=self.application_name + "_content",
-                schema=self.application_name,
-            )
-            return app
-        else:
-            ValueError("deployment_type should be either 'docker' or 'cloud'")
+        if not self.deployment_parameters:
+            self.deployment_parameters = {"port": 8089}
+        self.vespa_docker = VespaDocker(**self.deployment_parameters)
+        app = self.vespa_docker.deploy(application_package=app_package)
+        app.delete_all_docs(
+            content_cluster_name=self.application_name + "_content",
+            schema=self.application_name,
+        )
+        return app
+
+    def remove_app(self):
+        if self.vespa_docker:
+            shutil.rmtree(
+                self.application_name, ignore_errors=True
+            )  # remove application package folder
+            self.vespa_docker.container.stop()  # stop docker container
+            self.vespa_docker.container.remove()  # stop docker container
 
     def search(
         self,
@@ -165,10 +135,7 @@ def search(
         for query_id, query in zip(query_ids, queries):
             scores = {}
             query_result = self.app.query(
-                query=query,
-                query_model=query_model,
-                hits=top_k,
-                timeout="10 s"
+                query=query, query_model=query_model, hits=top_k, timeout="10 s"
             )
             for hit in query_result.hits:
                 scores[hit["fields"]["id"]] = hit["relevance"]

From caf5131c7b73ab20b809d4f0ec8ce6c507dfa4aa Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Mon, 24 Jan 2022 14:31:05 -0300
Subject: [PATCH 10/27] use remove app method on the unit tests

---
 beir/test_retrieval_lexical_vespa.py | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/beir/test_retrieval_lexical_vespa.py b/beir/test_retrieval_lexical_vespa.py
index eb1349f3..21af5b73 100644
--- a/beir/test_retrieval_lexical_vespa.py
+++ b/beir/test_retrieval_lexical_vespa.py
@@ -1,6 +1,4 @@
 import unittest
-import shutil
-import docker
 from beir.retrieval.search.lexical.vespa_search import VespaLexicalSearch
 from beir.retrieval.evaluation import EvaluateRetrieval
 
@@ -16,54 +14,52 @@ def setUp(self) -> None:
         self.queries = {"1": "this is query 1", "2": "this is query 2"}
 
     def test_or_bm25(self):
-        model = VespaLexicalSearch(application_name=self.application_name, initialize=True)
-        retriever = EvaluateRetrieval(model)
+        self.model = VespaLexicalSearch(
+            application_name=self.application_name, initialize=True
+        )
+        retriever = EvaluateRetrieval(self.model)
         results = retriever.retrieve(corpus=self.corpus, queries=self.queries)
         self.assertEqual({"1", "2"}, set(results.keys()))
         for query_id in results.keys():
             self.assertEqual({"1", "2", "3"}, set(results[query_id].keys()))
 
     def test_or_native_rank(self):
-        model = VespaLexicalSearch(
+        self.model = VespaLexicalSearch(
             application_name=self.application_name,
             initialize=True,
             match_phase="or",
             rank_phase="native_rank",
         )
-        retriever = EvaluateRetrieval(model)
+        retriever = EvaluateRetrieval(self.model)
         results = retriever.retrieve(corpus=self.corpus, queries=self.queries)
         self.assertEqual({"1", "2"}, set(results.keys()))
         for query_id in results.keys():
             self.assertEqual({"1", "2", "3"}, set(results[query_id].keys()))
 
     def test_weakand_bm25(self):
-        model = VespaLexicalSearch(
+        self.model = VespaLexicalSearch(
             application_name=self.application_name,
             initialize=True,
             match_phase="weak_and",
         )
-        retriever = EvaluateRetrieval(model)
+        retriever = EvaluateRetrieval(self.model)
         results = retriever.retrieve(corpus=self.corpus, queries=self.queries)
         self.assertEqual({"1", "2"}, set(results.keys()))
         for query_id in results.keys():
             self.assertEqual({"1", "2", "3"}, set(results[query_id].keys()))
 
     def test_weakand_native_rank(self):
-        model = VespaLexicalSearch(
+        self.model = VespaLexicalSearch(
             application_name=self.application_name,
             initialize=True,
             match_phase="weak_and",
-            rank_phase="native_rank"
+            rank_phase="native_rank",
         )
-        retriever = EvaluateRetrieval(model)
+        retriever = EvaluateRetrieval(self.model)
         results = retriever.retrieve(corpus=self.corpus, queries=self.queries)
         self.assertEqual({"1", "2"}, set(results.keys()))
         for query_id in results.keys():
             self.assertEqual({"1", "2", "3"}, set(results[query_id].keys()))
 
     def tearDown(self) -> None:
-        shutil.rmtree(self.application_name, ignore_errors=True)
-        client = docker.from_env()
-        container = client.containers.get(self.application_name)
-        container.stop()
-        container.remove()
+        self.model.remove_app()

From 2ba1211100fb7bcc454ca313fdcc76628c02e071 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Mon, 24 Jan 2022 14:41:21 -0300
Subject: [PATCH 11/27] use remove method

---
 examples/benchmarking/benchmark_lexical_vespa.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/benchmarking/benchmark_lexical_vespa.py b/examples/benchmarking/benchmark_lexical_vespa.py
index 68184cf6..3b594c9b 100644
--- a/examples/benchmarking/benchmark_lexical_vespa.py
+++ b/examples/benchmarking/benchmark_lexical_vespa.py
@@ -54,6 +54,7 @@ def get_search_results(dataset_name, corpus, queries, qrels):
             metric.update(recall)
             metric.update(precision)
             metrics.append(metric)
+    model.remove_app()
     return metrics
 
 

From 8d94af4e1c788cbd83db905818d1e7da4ff207f8 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Mon, 24 Jan 2022 14:52:52 -0300
Subject: [PATCH 12/27] get container by name

---
 beir/retrieval/search/lexical/vespa_search.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/beir/retrieval/search/lexical/vespa_search.py b/beir/retrieval/search/lexical/vespa_search.py
index b2fd3cc9..991aa75f 100644
--- a/beir/retrieval/search/lexical/vespa_search.py
+++ b/beir/retrieval/search/lexical/vespa_search.py
@@ -36,6 +36,7 @@ def __init__(
                 self.app.get_application_status().status_code == 200
             ), "Application status different than 200."
         else:
+            self.vespa_docker = VespaDocker.from_container_name_or_id(self.application_name)
             assert self.deployment_parameters is not None, (
                 "if 'initialize' is set to false, 'deployment_parameters' should contain Vespa "
                 "connection parameters such as 'url' and 'port'"

From ed06a1146b132623fb32e84f65707ab2c61f3934 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Mon, 24 Jan 2022 21:41:58 -0300
Subject: [PATCH 13/27] add progress info

---
 beir/retrieval/search/lexical/vespa_search.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/beir/retrieval/search/lexical/vespa_search.py b/beir/retrieval/search/lexical/vespa_search.py
index 991aa75f..c7d236e5 100644
--- a/beir/retrieval/search/lexical/vespa_search.py
+++ b/beir/retrieval/search/lexical/vespa_search.py
@@ -36,7 +36,9 @@ def __init__(
                 self.app.get_application_status().status_code == 200
             ), "Application status different than 200."
         else:
-            self.vespa_docker = VespaDocker.from_container_name_or_id(self.application_name)
+            self.vespa_docker = VespaDocker.from_container_name_or_id(
+                self.application_name
+            )
             assert self.deployment_parameters is not None, (
                 "if 'initialize' is set to false, 'deployment_parameters' should contain Vespa "
                 "connection parameters such as 'url' and 'port'"
@@ -84,7 +86,7 @@ def initialise(self):
         # Deploy application
         #
         if not self.deployment_parameters:
-            self.deployment_parameters = {"port": 8089}
+            self.deployment_parameters = {"port": 8089, "container_memory": "12G"}
         self.vespa_docker = VespaDocker(**self.deployment_parameters)
         app = self.vespa_docker.deploy(application_package=app_package)
         app.delete_all_docs(
@@ -133,7 +135,17 @@ def search(
             rank_profile=Ranking(name=self.rank_phase, list_features=False),
         )
 
-        for query_id, query in zip(query_ids, queries):
+        for idx, (query_id, query) in enumerate(zip(query_ids, queries)):
+            if (idx % 1000) == 0:
+                print(
+                    "{}, {}, {}: {}/{}".format(
+                        self.application_name,
+                        match_phase,
+                        self.rank_phase,
+                        idx,
+                        len(query_ids),
+                    )
+                )
             scores = {}
             query_result = self.app.query(
                 query=query, query_model=query_model, hits=top_k, timeout="10 s"

From 4faf3884d7010cfe5944489cce2de2b44b9b4e94 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Mon, 24 Jan 2022 21:42:21 -0300
Subject: [PATCH 14/27] include all datasets

---
 .../benchmarking/benchmark_lexical_vespa.py   | 27 ++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)
 mode change 100644 => 100755 examples/benchmarking/benchmark_lexical_vespa.py

diff --git a/examples/benchmarking/benchmark_lexical_vespa.py b/examples/benchmarking/benchmark_lexical_vespa.py
old mode 100644
new mode 100755
index 3b594c9b..7f1558ac
--- a/examples/benchmarking/benchmark_lexical_vespa.py
+++ b/examples/benchmarking/benchmark_lexical_vespa.py
@@ -1,6 +1,7 @@
 #! /usr/bin/env python3
 
 import os
+import shutil
 from beir import util
 from beir.datasets.data_loader import GenericDataLoader
 from beir.retrieval.search.lexical.vespa_search import VespaLexicalSearch
@@ -80,12 +81,32 @@ def benchmark_vespa_lexical(data_dir, dataset_names):
             )
         print(metrics)
         result.extend(metrics)
+        shutil.rmtree(os.path.join(data_dir, dataset_name))
+        os.remove(os.path.join(data_dir, dataset_name + ".zip"))
     return result
 
 
 if __name__ == "__main__":
 
-    data_dir = "/Users/tmartins/projects/sw/beir/data"
-    dataset_names = ["scifact", "nfcorpus"]
-
+    data_dir = "~/beir/data"
+    dataset_names = [
+        "msmarco",
+        "trec-covid",
+        "nfcorpus",
+        "bioasq",
+        "nq",
+        "hotpotqa",
+        "figa",
+        "signallm",
+        "trec-news",
+        "arguana",
+        "webis-touche2020",
+        "cqadupstack",
+        "quora",
+        "dbpedia-entity",
+        "scidocs",
+        "fever",
+        "climate-fever",
+        "scifact",
+    ]
     result = benchmark_vespa_lexical(data_dir=data_dir, dataset_names=dataset_names)

From 7ae1a2981649a9dbb11f51254fc2cce98cdec0f0 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Sun, 30 Jan 2022 20:30:19 -0300
Subject: [PATCH 15/27] Include tenacity

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index c57ca226..652ebc76 100644
--- a/setup.py
+++ b/setup.py
@@ -22,6 +22,7 @@
         'faiss_cpu',
         'elasticsearch',
         'pyvespa',
+        'tenacity',
         'tensorflow>=2.2.0',
         'tensorflow-text',
         'tensorflow-hub'

From 5be07eab78366cc549728c74c5a5937ded7afb75 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Sun, 30 Jan 2022 20:41:14 -0300
Subject: [PATCH 16/27] add retry strategy

---
 beir/retrieval/search/lexical/vespa_search.py | 35 ++++++++++++-------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/beir/retrieval/search/lexical/vespa_search.py b/beir/retrieval/search/lexical/vespa_search.py
index c7d236e5..0ca83c13 100644
--- a/beir/retrieval/search/lexical/vespa_search.py
+++ b/beir/retrieval/search/lexical/vespa_search.py
@@ -4,6 +4,7 @@
 from vespa.package import ApplicationPackage, Field, FieldSet, RankProfile, QueryField
 from vespa.query import QueryModel, OR, RankProfile as Ranking, WeakAnd
 from vespa.deployment import VespaDocker
+from tenacity import retry, wait_exponential, stop_after_attempt, RetryError
 
 
 class VespaLexicalSearch:
@@ -16,7 +17,7 @@ def __init__(
         initialize: bool = True,
     ):
         self.results = {}
-        self.application_name = application_name
+        self.application_name = application_name.replace("-", "")
         assert match_phase in [
             "or",
             "weak_and",
@@ -58,13 +59,13 @@ def initialise(self):
             Field(
                 name="title",
                 type="string",
-                indexing=["index", "summary"],
+                indexing=["index"],
                 index="enable-bm25",
             ),
             Field(
                 name="body",
                 type="string",
-                indexing=["index", "summary"],
+                indexing=["index"],
                 index="enable-bm25",
             ),
         )
@@ -100,8 +101,18 @@ def remove_app(self):
             shutil.rmtree(
                 self.application_name, ignore_errors=True
             )  # remove application package folder
-            self.vespa_docker.container.stop()  # stop docker container
-            self.vespa_docker.container.remove()  # stop docker container
+            self.vespa_docker.container.stop(timeout=600)  # stop docker container
+            self.vespa_docker.container.remove()  # rm docker container
+
+    @retry(wait=wait_exponential(multiplier=1), stop=stop_after_attempt(3))
+    def send_query(self, query, query_model, hits, timeout="10 s"):
+        scores = {}
+        query_result = self.app.query(
+            query=query, query_model=query_model, hits=hits, timeout=timeout
+        )
+        for hit in query_result.hits:
+            scores[hit["fields"]["id"]] = hit["relevance"]
+        return scores
 
     def search(
         self,
@@ -140,18 +151,18 @@ def search(
                 print(
                     "{}, {}, {}: {}/{}".format(
                         self.application_name,
-                        match_phase,
+                        self.match_phase,
                         self.rank_phase,
                         idx,
                         len(query_ids),
                     )
                 )
-            scores = {}
-            query_result = self.app.query(
-                query=query, query_model=query_model, hits=top_k, timeout="10 s"
-            )
-            for hit in query_result.hits:
-                scores[hit["fields"]["id"]] = hit["relevance"]
+            try:
+                scores = self.send_query(
+                    query=query, query_model=query_model, hits=top_k, timeout="10 s"
+                )
+            except RetryError:
+                continue
             self.results[query_id] = scores
         return self.results
 

From 34cd6c539c2fceeffeae4df210add6f061ab63d6 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Sun, 30 Jan 2022 22:08:36 -0300
Subject: [PATCH 17/27] update script

---
 .../benchmarking/benchmark_lexical_vespa.py   | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/examples/benchmarking/benchmark_lexical_vespa.py b/examples/benchmarking/benchmark_lexical_vespa.py
index 7f1558ac..2eb9c337 100755
--- a/examples/benchmarking/benchmark_lexical_vespa.py
+++ b/examples/benchmarking/benchmark_lexical_vespa.py
@@ -55,7 +55,10 @@ def get_search_results(dataset_name, corpus, queries, qrels):
             metric.update(recall)
             metric.update(precision)
             metrics.append(metric)
-    model.remove_app()
+    try:
+        model.remove_app()
+    except: # todo: could not find how to increase container.remove() timeout
+        pass
     return metrics
 
 
@@ -88,25 +91,22 @@ def benchmark_vespa_lexical(data_dir, dataset_names):
 
 if __name__ == "__main__":
 
-    data_dir = "~/beir/data"
+    data_dir = os.environ["DATA_DIR"]
     dataset_names = [
-        "msmarco",
+        "scifact",
         "trec-covid",
         "nfcorpus",
-        "bioasq",
         "nq",
-        "hotpotqa",
-        "figa",
-        "signallm",
-        "trec-news",
+        "fiqa",
         "arguana",
         "webis-touche2020",
-        "cqadupstack",
+        "cqadupstack", 
         "quora",
         "dbpedia-entity",
         "scidocs",
         "fever",
         "climate-fever",
-        "scifact",
+        "msmarco",
+        "hotpotqa",
     ]
     result = benchmark_vespa_lexical(data_dir=data_dir, dataset_names=dataset_names)

From 066ef2d26114cc34ed3a21763c34f1251a4a2b99 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Tue, 15 Feb 2022 15:03:51 -0300
Subject: [PATCH 18/27] add issue link

---
 examples/benchmarking/benchmark_lexical_vespa.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/benchmarking/benchmark_lexical_vespa.py b/examples/benchmarking/benchmark_lexical_vespa.py
index 2eb9c337..f4923013 100755
--- a/examples/benchmarking/benchmark_lexical_vespa.py
+++ b/examples/benchmarking/benchmark_lexical_vespa.py
@@ -57,7 +57,7 @@ def get_search_results(dataset_name, corpus, queries, qrels):
             metrics.append(metric)
     try:
         model.remove_app()
-    except: # todo: could not find how to increase container.remove() timeout
+    except: # todo: could not find how to increase container.remove() timeout (https://github.com/docker/docker-py/issues/2951)
         pass
     return metrics
 

From e5b24b603ebccbe67b7abf867d336508dacf4de8 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Wed, 16 Feb 2022 09:54:57 -0300
Subject: [PATCH 19/27] process queries in parallel

---
 beir/retrieval/search/lexical/vespa_search.py | 83 +++++++++++++------
 1 file changed, 57 insertions(+), 26 deletions(-)

diff --git a/beir/retrieval/search/lexical/vespa_search.py b/beir/retrieval/search/lexical/vespa_search.py
index 0ca83c13..79492e1b 100644
--- a/beir/retrieval/search/lexical/vespa_search.py
+++ b/beir/retrieval/search/lexical/vespa_search.py
@@ -104,15 +104,56 @@ def remove_app(self):
             self.vespa_docker.container.stop(timeout=600)  # stop docker container
             self.vespa_docker.container.remove()  # rm docker container
 
-    @retry(wait=wait_exponential(multiplier=1), stop=stop_after_attempt(3))
-    def send_query(self, query, query_model, hits, timeout="10 s"):
-        scores = {}
-        query_result = self.app.query(
-            query=query, query_model=query_model, hits=hits, timeout=timeout
+    @retry(wait=wait_exponential(multiplier=1), stop=stop_after_attempt(10))
+    def send_query_batch(self, query_batch, query_model, hits, timeout="100 s"):
+        query_results = self.app.query_batch(
+            query_batch=query_batch,
+            query_model=query_model,
+            hits=hits,
+            **{"timeout": timeout, "ranking.softtimeout.enable": "false"}
         )
-        for hit in query_result.hits:
-            scores[hit["fields"]["id"]] = hit["relevance"]
-        return scores
+        return query_results
+
+    def process_queries(
+        self, query_ids, queries, query_model, hits, batch_size, timeout="100 s"
+    ):
+        results = {}
+        assert len(query_ids) == len(
+            queries
+        ), "There must be one query_id for each query."
+        query_id_batches = [
+            query_ids[i : i + batch_size] for i in range(0, len(query_ids), batch_size)
+        ]
+        query_batches = [
+            queries[i : i + batch_size] for i in range(0, len(queries), batch_size)
+        ]
+        for idx, (query_id_batch, query_batch) in enumerate(
+            zip(query_id_batches, query_batches)
+        ):
+            print(
+                "{}, {}, {}: {}/{}".format(
+                    self.application_name,
+                    self.match_phase,
+                    self.rank_phase,
+                    idx,
+                    len(query_batches),
+                )
+            )
+            try:
+                query_results = self.send_query_batch(
+                    query_batch=query_batch,
+                    query_model=query_model,
+                    hits=hits,
+                    timeout="100 s",
+                )
+            except RetryError:
+                continue
+            for (query_id, query_result) in zip(query_id_batch, query_results):
+                scores = {}
+                for hit in query_result.hits:
+                    scores[hit["fields"]["id"]] = hit["relevance"]
+                results[query_id] = scores
+        return results
 
     def search(
         self,
@@ -146,24 +187,14 @@ def search(
             rank_profile=Ranking(name=self.rank_phase, list_features=False),
         )
 
-        for idx, (query_id, query) in enumerate(zip(query_ids, queries)):
-            if (idx % 1000) == 0:
-                print(
-                    "{}, {}, {}: {}/{}".format(
-                        self.application_name,
-                        self.match_phase,
-                        self.rank_phase,
-                        idx,
-                        len(query_ids),
-                    )
-                )
-            try:
-                scores = self.send_query(
-                    query=query, query_model=query_model, hits=top_k, timeout="10 s"
-                )
-            except RetryError:
-                continue
-            self.results[query_id] = scores
+        self.results = self.process_queries(
+            query_ids=query_ids,
+            queries=queries,
+            query_model=query_model,
+            hits=top_k,
+            batch_size=1000,
+            timeout="100 s",
+        )
         return self.results
 
     def index(self, corpus: Dict[str, Dict[str, str]]):

From 4c5ac5c764b33d2cbcc68ece266968eb95acfe59 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Fri, 18 Feb 2022 07:05:49 -0300
Subject: [PATCH 20/27] pre-process queries

---
 beir/retrieval/search/lexical/vespa_search.py | 52 ++++++++++++++++++-
 1 file changed, 51 insertions(+), 1 deletion(-)

diff --git a/beir/retrieval/search/lexical/vespa_search.py b/beir/retrieval/search/lexical/vespa_search.py
index 79492e1b..8bc45894 100644
--- a/beir/retrieval/search/lexical/vespa_search.py
+++ b/beir/retrieval/search/lexical/vespa_search.py
@@ -1,4 +1,6 @@
 import shutil
+import re
+from collections import Counter
 from typing import Dict, Optional
 from vespa.application import Vespa
 from vespa.package import ApplicationPackage, Field, FieldSet, RankProfile, QueryField
@@ -6,6 +8,44 @@
 from vespa.deployment import VespaDocker
 from tenacity import retry, wait_exponential, stop_after_attempt, RetryError
 
+QUOTES = [
+    "\u0022",  # quotation mark (")
+    "\u0027",  # apostrophe (')
+    "\u00ab",  # left-pointing double-angle quotation mark
+    "\u00bb",  # right-pointing double-angle quotation mark
+    "\u2018",  # left single quotation mark
+    "\u2019",  # right single quotation mark
+    "\u201a",  # single low-9 quotation mark
+    "\u201b",  # single high-reversed-9 quotation mark
+    "\u201c",  # left double quotation mark
+    "\u201d",  # right double quotation mark
+    "\u201e",  # double low-9 quotation mark
+    "\u201f",  # double high-reversed-9 quotation mark
+    "\u2039",  # single left-pointing angle quotation mark
+    "\u203a",  # single right-pointing angle quotation mark
+    "\u300c",  # left corner bracket
+    "\u300d",  # right corner bracket
+    "\u300e",  # left white corner bracket
+    "\u300f",  # right white corner bracket
+    "\u301d",  # reversed double prime quotation mark
+    "\u301e",  # double prime quotation mark
+    "\u301f",  # low double prime quotation mark
+    "\ufe41",  # presentation form for vertical left corner bracket
+    "\ufe42",  # presentation form for vertical right corner bracket
+    "\ufe43",  # presentation form for vertical left corner white bracket
+    "\ufe44",  # presentation form for vertical right corner white bracket
+    "\uff02",  # fullwidth quotation mark
+    "\uff07",  # fullwidth apostrophe
+    "\uff62",  # halfwidth left corner bracket
+    "\uff63",  # halfwidth right corner bracket
+]
+
+
+def replace_quotes(x):
+    for symbol in QUOTES:
+        x = x.replace(symbol, "")
+    return x
+
 
 class VespaLexicalSearch:
     def __init__(
@@ -146,6 +186,12 @@ def process_queries(
                     hits=hits,
                     timeout="100 s",
                 )
+                status_code_summary = Counter([x.status_code for x in query_results])
+                print(
+                    "Sucessfull queries: {}/{}".format(
+                        status_code_summary[200], len(query_batch)
+                    )
+                )
             except RetryError:
                 continue
             for (query_id, query_result) in zip(query_id_batch, query_results):
@@ -171,6 +217,10 @@ def search(
         query_ids = list(queries.keys())
         queries = [queries[qid] for qid in query_ids]
 
+        queries = [
+            re.sub(" +", " ", replace_quotes(x)).strip() for x in queries
+        ]  # remove quotes and double spaces from queries
+
         if self.match_phase == "or":
             match_phase = OR()
         elif self.match_phase == "weak_and":
@@ -182,7 +232,7 @@ def search(
             ValueError("'rank_phase' should be either 'bm25' or 'native_rank'")
 
         query_model = QueryModel(
-            name=self.match_phase + "_bm25",
+            name=self.match_phase + "_" + self.rank_phase,
             match_phase=match_phase,
             rank_profile=Ranking(name=self.rank_phase, list_features=False),
         )

From 48a7c14272707af725cf1f3eae0899db9823bb8c Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Fri, 18 Feb 2022 16:39:36 -0300
Subject: [PATCH 21/27] improve feeding and output information

---
 beir/retrieval/search/lexical/vespa_search.py | 42 +++++++++++++++----
 1 file changed, 35 insertions(+), 7 deletions(-)

diff --git a/beir/retrieval/search/lexical/vespa_search.py b/beir/retrieval/search/lexical/vespa_search.py
index 8bc45894..e1ab8637 100644
--- a/beir/retrieval/search/lexical/vespa_search.py
+++ b/beir/retrieval/search/lexical/vespa_search.py
@@ -1,5 +1,6 @@
 import shutil
 import re
+from statistics import mean, median
 from collections import Counter
 from typing import Dict, Optional
 from vespa.application import Vespa
@@ -8,6 +9,7 @@
 from vespa.deployment import VespaDocker
 from tenacity import retry, wait_exponential, stop_after_attempt, RetryError
 
+REPLACE_SYMBOLS = ["(", ")", " -", " +"]
 QUOTES = [
     "\u0022",  # quotation mark (")
     "\u0027",  # apostrophe (')
@@ -39,10 +41,11 @@
     "\uff62",  # halfwidth left corner bracket
     "\uff63",  # halfwidth right corner bracket
 ]
+REPLACE_SYMBOLS.extend(QUOTES)
 
 
-def replace_quotes(x):
-    for symbol in QUOTES:
+def replace_symbols(x):
+    for symbol in REPLACE_SYMBOLS:
         x = x.replace(symbol, "")
     return x
 
@@ -186,10 +189,16 @@ def process_queries(
                     hits=hits,
                     timeout="100 s",
                 )
+                number_hits = [x.number_documents_retrieved for x in query_results]
                 status_code_summary = Counter([x.status_code for x in query_results])
                 print(
-                    "Sucessfull queries: {}/{}".format(
-                        status_code_summary[200], len(query_batch)
+                    "Sucessfull queries: {}/{}\nDocuments retrieved. Min: {}, Max: {}, Mean: {}, Median: {}.".format(
+                        status_code_summary[200],
+                        len(query_batch),
+                        min(number_hits),
+                        max(number_hits),
+                        round(mean(number_hits), 2),
+                        round(median(number_hits), 2),
                     )
                 )
             except RetryError:
@@ -218,7 +227,7 @@ def search(
         queries = [queries[qid] for qid in query_ids]
 
         queries = [
-            re.sub(" +", " ", replace_quotes(x)).strip() for x in queries
+            re.sub(" +", " ", replace_symbols(x)).strip() for x in queries
         ]  # remove quotes and double spaces from queries
 
         if self.match_phase == "or":
@@ -247,7 +256,14 @@ def search(
         )
         return self.results
 
-    def index(self, corpus: Dict[str, Dict[str, str]]):
+    @retry(wait=wait_exponential(multiplier=1), stop=stop_after_attempt(10))
+    def send_feed_batch(self, feed_batch, total_timeout=10000):
+        feed_results = self.app.feed_batch(
+            batch=feed_batch, total_timeout=total_timeout
+        )
+        return feed_results
+
+    def index(self, corpus: Dict[str, Dict[str, str]], batch_size=1000):
         batch_feed = [
             {
                 "id": idx,
@@ -259,4 +275,16 @@ def index(self, corpus: Dict[str, Dict[str, str]]):
             }
             for idx in list(corpus.keys())
         ]
-        return self.app.feed_batch(batch=batch_feed)
+        mini_batches = [
+            batch_feed[i : i + batch_size]
+            for i in range(0, len(batch_feed), batch_size)
+        ]
+        for idx, feed_batch in enumerate(mini_batches):
+            feed_results = self.send_feed_batch(feed_batch=feed_batch)
+            status_code_summary = Counter([x.status_code for x in feed_results])
+            print(
+                "Successful documents fed: {}/{}.\nBatch progress: {}/{}.".format(
+                    status_code_summary[200], len(feed_batch), idx, len(mini_batches)
+                )
+            )
+        return 0

From 106112291a9de9740c27dd094da8a7dd2e1aa5d4 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Thu, 24 Feb 2022 13:52:38 -0300
Subject: [PATCH 22/27] exclude cases where queries is returned as hits

---
 beir/retrieval/search/lexical/vespa_search.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/beir/retrieval/search/lexical/vespa_search.py b/beir/retrieval/search/lexical/vespa_search.py
index e1ab8637..7f5d59d0 100644
--- a/beir/retrieval/search/lexical/vespa_search.py
+++ b/beir/retrieval/search/lexical/vespa_search.py
@@ -206,7 +206,9 @@ def process_queries(
             for (query_id, query_result) in zip(query_id_batch, query_results):
                 scores = {}
                 for hit in query_result.hits:
-                    scores[hit["fields"]["id"]] = hit["relevance"]
+                    corpus_id = hit["fields"]["id"]
+                    if corpus_id != query_id:  # See https://github.com/UKPLab/beir/issues/72
+                        scores[corpus_id] = hit["relevance"]
                 results[query_id] = scores
         return results
 

From 80f4f3a14eb07faf6e2afd5298ff79359b711100 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Tue, 1 Mar 2022 08:06:41 -0300
Subject: [PATCH 23/27] add option to not exclude the dataset and not remove
 the app

---
 .../benchmarking/benchmark_lexical_vespa.py   | 28 ++++++++++++-------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/examples/benchmarking/benchmark_lexical_vespa.py b/examples/benchmarking/benchmark_lexical_vespa.py
index f4923013..1bf37f0c 100755
--- a/examples/benchmarking/benchmark_lexical_vespa.py
+++ b/examples/benchmarking/benchmark_lexical_vespa.py
@@ -25,7 +25,7 @@ def prepare_data(data_path):
     return corpus, queries, qrels
 
 
-def get_search_results(dataset_name, corpus, queries, qrels):
+def get_search_results(dataset_name, corpus, queries, qrels, remove_app=True):
     initialize = True
     deployment_parameters = None
     metrics = []
@@ -55,14 +55,17 @@ def get_search_results(dataset_name, corpus, queries, qrels):
             metric.update(recall)
             metric.update(precision)
             metrics.append(metric)
-    try:
-        model.remove_app()
-    except: # todo: could not find how to increase container.remove() timeout (https://github.com/docker/docker-py/issues/2951)
-        pass
+    if remove_app:
+        try:
+            model.remove_app()
+        except:  # todo: could not find how to increase container.remove() timeout (https://github.com/docker/docker-py/issues/2951)
+            pass
     return metrics
 
 
-def benchmark_vespa_lexical(data_dir, dataset_names):
+def benchmark_vespa_lexical(
+    data_dir, dataset_names, remove_dataset=True, remove_app=True
+):
     result = []
     for dataset_name in dataset_names:
         print("Dataset: {}".format(dataset_name))
@@ -71,7 +74,11 @@ def benchmark_vespa_lexical(data_dir, dataset_names):
         )
         corpus, queries, qrels = prepare_data(data_path=data_path)
         metrics = get_search_results(
-            dataset_name=dataset_name, corpus=corpus, queries=queries, qrels=qrels
+            dataset_name=dataset_name,
+            corpus=corpus,
+            queries=queries,
+            qrels=qrels,
+            remove_app=remove_app,
         )
         output_file = os.path.join(data_dir, "metrics.csv")
         if os.path.isfile(output_file):
@@ -84,8 +91,9 @@ def benchmark_vespa_lexical(data_dir, dataset_names):
             )
         print(metrics)
         result.extend(metrics)
-        shutil.rmtree(os.path.join(data_dir, dataset_name))
-        os.remove(os.path.join(data_dir, dataset_name + ".zip"))
+        if remove_dataset:
+            shutil.rmtree(os.path.join(data_dir, dataset_name))
+            os.remove(os.path.join(data_dir, dataset_name + ".zip"))
     return result
 
 
@@ -100,7 +108,7 @@ def benchmark_vespa_lexical(data_dir, dataset_names):
         "fiqa",
         "arguana",
         "webis-touche2020",
-        "cqadupstack", 
+        "cqadupstack",
         "quora",
         "dbpedia-entity",
         "scidocs",

From 7ee7962ecb55295e105abe7fa09e6984b3b69729 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Tue, 1 Mar 2022 16:03:56 -0300
Subject: [PATCH 24/27] document benchmark script

---
 .../benchmarking/benchmark_lexical_vespa.py   | 139 ++++++++++++++++--
 1 file changed, 127 insertions(+), 12 deletions(-)

diff --git a/examples/benchmarking/benchmark_lexical_vespa.py b/examples/benchmarking/benchmark_lexical_vespa.py
index 1bf37f0c..f22c210a 100755
--- a/examples/benchmarking/benchmark_lexical_vespa.py
+++ b/examples/benchmarking/benchmark_lexical_vespa.py
@@ -2,6 +2,7 @@
 
 import os
 import shutil
+from typing import Tuple, Optional, List, Dict
 from beir import util
 from beir.datasets.data_loader import GenericDataLoader
 from beir.retrieval.search.lexical.vespa_search import VespaLexicalSearch
@@ -9,7 +10,15 @@
 from pandas import DataFrame
 
 
-def download_and_unzip_dataset(data_dir, dataset_name):
+def download_and_unzip_dataset(data_dir: str, dataset_name: str) -> str:
+    """
+    Download and unzip dataset
+
+    :param data_dir: Folder path to hold the downloaded files
+    :param dataset_name: Name of the dataset according to BEIR benchmark
+
+    :return: Return the path of the folder containing the unzipped dataset files.
+    """
     url = "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/{}.zip".format(
         dataset_name
     )
@@ -18,19 +27,92 @@ def download_and_unzip_dataset(data_dir, dataset_name):
     return data_path
 
 
-def prepare_data(data_path):
+def prepare_data(data_path: str) -> Tuple:
+    """
+    Extract corpus, queries and qrels from the test dataset.
+
+    :param data_path: Folder path that contains the unzipped dataset files.
+
+    :return: a tuple containing 'corpus', 'queries' and 'qrels'.
+    """
     corpus, queries, qrels = GenericDataLoader(data_path).load(
         split="test"
     )  # or split = "train" or "dev"
     return corpus, queries, qrels
 
 
-def get_search_results(dataset_name, corpus, queries, qrels, remove_app=True):
-    initialize = True
-    deployment_parameters = None
+def parse_match_phase_argument(match_phase: Optional[List[str]] = None) -> List[str]:
+    """
+    Parse match phase argument.
+
+    :param match_phase: An optional list of match phase types to use in the experiments.
+        Currently supported types are 'weak_and' and 'or'. By default the experiments will use
+        'weak_and'.
+
+    :return: A list with all the match phase types to use in the experiments.
+    """
+    if not match_phase:
+        match_phase_list = ["weak_and"]
+    else:
+        assert all(
+            [x in ["or", "weak_and"] for x in match_phase]
+        ), "match_phase must be a list containing 'weak_and' and/or 'or'."
+        match_phase_list = match_phase
+    return match_phase_list
+
+
+def parse_rank_phase_argument(rank_phase: Optional[List[str]] = None) -> List[str]:
+    """
+    Parse rank phase argument.
+
+    :param rank_phase: An optional list of rank phase types to use in the experiments.
+        Currently supported types are 'bm25' and 'native_rank'. By default the experiments will use
+        'bm25'.
+
+    :return: A list with all the match phase types to use in the experiments.
+    """
+    if not rank_phase:
+        rank_phase_list = ["bm25"]
+    else:
+        assert all(
+            [x in ["native_rank", "bm25"] for x in rank_phase]
+        ), "rank_phase must be a list containing 'native_rank' and/or 'bm25'."
+        rank_phase_list = rank_phase
+    return rank_phase_list
+
+
+def get_search_results(
+    dataset_name: str,
+    corpus: Dict[str, Dict[str, str]],
+    queries: Dict[str, str],
+    qrels: Dict[str, Dict[str, int]],
+    match_phase: Optional[List[str]] = None,
+    rank_phase: Optional[List[str]] = None,
+    initialize: bool = True,
+    remove_app: bool = True,
+):
+    """
+    Deploy an Vespa app, feed, query and compute evaluation metrics
+
+    :param dataset_name: Name of the dataset according to BEIR benchmark
+    :param corpus: Corpus used to feed the app.
+    :param queries: Queries used to query the app.
+    :param qrels: Labeled data used to evaluate the query rresults.
+    :param match_phase: An optional list of match phase types to use in the experiments.
+        Currently supported types are 'weak_and' and 'or'. By default the experiments will use
+        'weak_and'.
+    :param rank_phase: An optional list of rank phase types to use in the experiments.
+        Currently supported types are 'bm25' and 'native_rank'. By default the experiments will use
+        'bm25'.
+    :param initialize: Deploy and feed the app on the first run of the experiments. Default to True.
+    :param remove_app: Stop and remove the app after the experiments are run. Default to True.
+    """
+    deployment_parameters = {"url": "http://localhost", "port": 8089}
+    match_phase_list = parse_match_phase_argument(match_phase=match_phase)
+    rank_phase_list = parse_rank_phase_argument(rank_phase=rank_phase)
     metrics = []
-    for match_phase in ["or", "weak_and"]:
-        for rank_phase in ["bm25", "native_rank"]:
+    for match_phase in match_phase_list:
+        for rank_phase in rank_phase_list:
             model = VespaLexicalSearch(
                 application_name=dataset_name,
                 match_phase=match_phase,
@@ -38,8 +120,7 @@ def get_search_results(dataset_name, corpus, queries, qrels, remove_app=True):
                 initialize=initialize,
                 deployment_parameters=deployment_parameters,
             )
-            initialize = False
-            deployment_parameters = {"url": "http://localhost", "port": 8089}
+            initialize = False  # only initialize the first run
             retriever = EvaluateRetrieval(model)
             results = retriever.retrieve(corpus, queries)
             ndcg, _map, recall, precision = retriever.evaluate(
@@ -64,8 +145,31 @@ def get_search_results(dataset_name, corpus, queries, qrels, remove_app=True):
 
 
 def benchmark_vespa_lexical(
-    data_dir, dataset_names, remove_dataset=True, remove_app=True
+    data_dir: str,
+    dataset_names: List[str],
+    match_phase: Optional[List[str]] = None,
+    rank_phase: Optional[List[str]] = None,
+    initialize: bool = True,
+    remove_dataset: bool = True,
+    remove_app: bool = True,
 ):
+    """
+    Benchmark Vespa lexical search app against a suite of BEIR datasets.
+
+    A metrics.csv file will be created at 'data_dir' containing the metrics computed in the experimets.
+
+    :param data_dir: Folder path to hold the downloaded files
+    :param dataset_names: A list of dataset names according to the BEIR benchmark.
+    :param match_phase: An optional list of match phase types to use in the experiments.
+        Currently supported types are 'weak_and' and 'or'. By default the experiments will use
+        'weak_and'.
+    :param rank_phase: An optional list of rank phase types to use in the experiments.
+        Currently supported types are 'bm25' and 'native_rank'. By default the experiments will use
+        'bm25'.
+    :param initialize: Deploy and feed the app on the first run of the experiments. Default to True.
+    :param remove_dataset: Remove dataset files after the experiments are run. Default to True.
+    :param remove_app: Stop and remove the app after the experiments are run. Default to True.
+    """
     result = []
     for dataset_name in dataset_names:
         print("Dataset: {}".format(dataset_name))
@@ -78,6 +182,9 @@ def benchmark_vespa_lexical(
             corpus=corpus,
             queries=queries,
             qrels=qrels,
+            match_phase=match_phase,
+            rank_phase=rank_phase,
+            initialize=initialize,
             remove_app=remove_app,
         )
         output_file = os.path.join(data_dir, "metrics.csv")
@@ -99,7 +206,7 @@ def benchmark_vespa_lexical(
 
 if __name__ == "__main__":
 
-    data_dir = os.environ["DATA_DIR"]
+    data_dir = os.environ.get("DATA_DIR", os.getcwd())
     dataset_names = [
         "scifact",
         "trec-covid",
@@ -117,4 +224,12 @@ def benchmark_vespa_lexical(
         "msmarco",
         "hotpotqa",
     ]
-    result = benchmark_vespa_lexical(data_dir=data_dir, dataset_names=dataset_names)
+    result = benchmark_vespa_lexical(
+        data_dir=data_dir,
+        dataset_names=dataset_names,
+        match_phase=["weak_and"],
+        rank_phase=["bm25"],
+        initialize=True,
+        remove_dataset=True,
+        remove_app=True,
+    )

From 713887ca3e79ba51c6f7bad9666ed492b7dc41fc Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Thu, 3 Mar 2022 10:24:08 -0300
Subject: [PATCH 25/27] add split_type and fix deployment parameters

---
 .../benchmarking/benchmark_lexical_vespa.py   | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/examples/benchmarking/benchmark_lexical_vespa.py b/examples/benchmarking/benchmark_lexical_vespa.py
index f22c210a..cb8f0714 100755
--- a/examples/benchmarking/benchmark_lexical_vespa.py
+++ b/examples/benchmarking/benchmark_lexical_vespa.py
@@ -27,16 +27,17 @@ def download_and_unzip_dataset(data_dir: str, dataset_name: str) -> str:
     return data_path
 
 
-def prepare_data(data_path: str) -> Tuple:
+def prepare_data(data_path: str, split_type: str = "test") -> Tuple:
     """
     Extract corpus, queries and qrels from the test dataset.
 
     :param data_path: Folder path that contains the unzipped dataset files.
+    :param split_type: One of 'train', 'dev' or 'test' set.
 
     :return: a tuple containing 'corpus', 'queries' and 'qrels'.
     """
     corpus, queries, qrels = GenericDataLoader(data_path).load(
-        split="test"
+        split=split_type
     )  # or split = "train" or "dev"
     return corpus, queries, qrels
 
@@ -97,7 +98,7 @@ def get_search_results(
     :param dataset_name: Name of the dataset according to BEIR benchmark
     :param corpus: Corpus used to feed the app.
     :param queries: Queries used to query the app.
-    :param qrels: Labeled data used to evaluate the query rresults.
+    :param qrels: Labeled data used to evaluate the query results.
     :param match_phase: An optional list of match phase types to use in the experiments.
         Currently supported types are 'weak_and' and 'or'. By default the experiments will use
         'weak_and'.
@@ -107,7 +108,10 @@ def get_search_results(
     :param initialize: Deploy and feed the app on the first run of the experiments. Default to True.
     :param remove_app: Stop and remove the app after the experiments are run. Default to True.
     """
-    deployment_parameters = {"url": "http://localhost", "port": 8089}
+    if initialize:
+        deployment_parameters = None
+    else:
+        deployment_parameters = {"url": "http://localhost", "port": 8089}
     match_phase_list = parse_match_phase_argument(match_phase=match_phase)
     rank_phase_list = parse_rank_phase_argument(rank_phase=rank_phase)
     metrics = []
@@ -121,6 +125,7 @@ def get_search_results(
                 deployment_parameters=deployment_parameters,
             )
             initialize = False  # only initialize the first run
+            deployment_parameters = {"url": "http://localhost", "port": 8089}
             retriever = EvaluateRetrieval(model)
             results = retriever.retrieve(corpus, queries)
             ndcg, _map, recall, precision = retriever.evaluate(
@@ -147,6 +152,7 @@ def get_search_results(
 def benchmark_vespa_lexical(
     data_dir: str,
     dataset_names: List[str],
+    split_type: str = "test",
     match_phase: Optional[List[str]] = None,
     rank_phase: Optional[List[str]] = None,
     initialize: bool = True,
@@ -156,10 +162,11 @@ def benchmark_vespa_lexical(
     """
     Benchmark Vespa lexical search app against a suite of BEIR datasets.
 
-    A metrics.csv file will be created at 'data_dir' containing the metrics computed in the experimets.
+    A metrics.csv file will be created at 'data_dir' containing the metrics computed in the experiments.
 
     :param data_dir: Folder path to hold the downloaded files
     :param dataset_names: A list of dataset names according to the BEIR benchmark.
+    :param split_type: One of 'train', 'dev' or 'test' set.
     :param match_phase: An optional list of match phase types to use in the experiments.
         Currently supported types are 'weak_and' and 'or'. By default the experiments will use
         'weak_and'.
@@ -176,7 +183,7 @@ def benchmark_vespa_lexical(
         data_path = download_and_unzip_dataset(
             data_dir=data_dir, dataset_name=dataset_name
         )
-        corpus, queries, qrels = prepare_data(data_path=data_path)
+        corpus, queries, qrels = prepare_data(data_path=data_path, split_type=split_type)
         metrics = get_search_results(
             dataset_name=dataset_name,
             corpus=corpus,

From b08a90e447f75973247b25b96b8241f7a62cf195 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Wed, 9 Mar 2022 09:43:12 -0300
Subject: [PATCH 26/27] expose timeout and async connections. Continue in case
 of empty hits.

---
 beir/retrieval/search/lexical/vespa_search.py | 27 +++++++++++++------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/beir/retrieval/search/lexical/vespa_search.py b/beir/retrieval/search/lexical/vespa_search.py
index 7f5d59d0..ebe2a143 100644
--- a/beir/retrieval/search/lexical/vespa_search.py
+++ b/beir/retrieval/search/lexical/vespa_search.py
@@ -148,17 +148,21 @@ def remove_app(self):
             self.vespa_docker.container.remove()  # rm docker container
 
     @retry(wait=wait_exponential(multiplier=1), stop=stop_after_attempt(10))
-    def send_query_batch(self, query_batch, query_model, hits, timeout="100 s"):
+    def send_query_batch(
+        self, query_batch, query_model, hits, timeout=100, async_connections=50
+    ):
         query_results = self.app.query_batch(
             query_batch=query_batch,
             query_model=query_model,
+            connections=async_connections,
+            total_timeout=timeout * len(query_batch),
             hits=hits,
-            **{"timeout": timeout, "ranking.softtimeout.enable": "false"}
+            **{"timeout": str(timeout) + " s", "ranking.softtimeout.enable": "false"}
         )
         return query_results
 
     def process_queries(
-        self, query_ids, queries, query_model, hits, batch_size, timeout="100 s"
+        self, query_ids, queries, query_model, hits, batch_size, timeout=100, async_connections=50
     ):
         results = {}
         assert len(query_ids) == len(
@@ -187,7 +191,8 @@ def process_queries(
                     query_batch=query_batch,
                     query_model=query_model,
                     hits=hits,
-                    timeout="100 s",
+                    timeout=timeout,
+                    async_connections=async_connections
                 )
                 number_hits = [x.number_documents_retrieved for x in query_results]
                 status_code_summary = Counter([x.status_code for x in query_results])
@@ -205,10 +210,16 @@ def process_queries(
                 continue
             for (query_id, query_result) in zip(query_id_batch, query_results):
                 scores = {}
-                for hit in query_result.hits:
-                    corpus_id = hit["fields"]["id"]
-                    if corpus_id != query_id:  # See https://github.com/UKPLab/beir/issues/72
-                        scores[corpus_id] = hit["relevance"]
+                try:
+                    if query_result.hits:
+                        for hit in query_result.hits:
+                            corpus_id = hit["fields"]["id"]
+                            if (
+                                corpus_id != query_id
+                            ):  # See https://github.com/UKPLab/beir/issues/72
+                                scores[corpus_id] = hit["relevance"]
+                except KeyError:
+                    continue
                 results[query_id] = scores
         return results
 

From cc51233a199cc7b7264781f562f052f91ba72cf2 Mon Sep 17 00:00:00 2001
From: tmartins <thigm85@gmail.com>
Date: Wed, 9 Mar 2022 09:47:50 -0300
Subject: [PATCH 27/27] msmarco uses dev set

---
 .../benchmarking/benchmark_lexical_vespa.py   | 21 ++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/examples/benchmarking/benchmark_lexical_vespa.py b/examples/benchmarking/benchmark_lexical_vespa.py
index cb8f0714..25c562d8 100755
--- a/examples/benchmarking/benchmark_lexical_vespa.py
+++ b/examples/benchmarking/benchmark_lexical_vespa.py
@@ -183,7 +183,9 @@ def benchmark_vespa_lexical(
         data_path = download_and_unzip_dataset(
             data_dir=data_dir, dataset_name=dataset_name
         )
-        corpus, queries, qrels = prepare_data(data_path=data_path, split_type=split_type)
+        corpus, queries, qrels = prepare_data(
+            data_path=data_path, split_type=split_type
+        )
         metrics = get_search_results(
             dataset_name=dataset_name,
             corpus=corpus,
@@ -228,12 +230,25 @@ def benchmark_vespa_lexical(
         "scidocs",
         "fever",
         "climate-fever",
-        "msmarco",
         "hotpotqa",
     ]
-    result = benchmark_vespa_lexical(
+    _ = benchmark_vespa_lexical(
         data_dir=data_dir,
         dataset_names=dataset_names,
+        split_type="test",
+        match_phase=["weak_and"],
+        rank_phase=["bm25"],
+        initialize=True,
+        remove_dataset=True,
+        remove_app=True,
+    )
+    #
+    # MS MARCO is the only dataset which uses the dev set to compute the metrics
+    #
+    _ = benchmark_vespa_lexical(
+        data_dir=data_dir,
+        dataset_names=["msmarco"],
+        split_type="dev",
         match_phase=["weak_and"],
         rank_phase=["bm25"],
         initialize=True,