File size: 2,866 Bytes
dbabef2
 
9281159
1b818bb
9281159
dbabef2
 
 
 
9281159
 
dbabef2
 
 
 
 
 
 
 
 
9281159
1b818bb
 
dbabef2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1b818bb
 
 
 
 
dbabef2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9281159
dbabef2
9281159
dbabef2
 
 
 
 
 
 
9281159
dbabef2
1b818bb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
from __future__ import annotations

import logging
from legora.performance import QueryCache, normalize_query

from sentence_transformers import (
    SentenceTransformer
)

logger = logging.getLogger(__name__)


class LegalEmbedder:

    def __init__(
        self,
        model_name: str =
        "BAAI/bge-large-en-v1.5"
    ):

        logger.info("Loading embedding model: %s", model_name)
        self.model_name = model_name
        self.query_cache = QueryCache()

        self.model = (
            SentenceTransformer(
                model_name
            )
        )

    # =====================================================
    # DOCUMENT EMBEDDINGS
    # =====================================================

    def embed(
        self,
        texts: list[str]
    ):

        return self.model.encode(
            texts,
            normalize_embeddings=True,
            convert_to_numpy=True,
            batch_size=16,
            show_progress_bar=True
        )

    # =====================================================
    # QUERY EMBEDDING
    # =====================================================

    def embed_query(
        self,
        query: str
    ):

        normalized = normalize_query(query) if self.query_cache.enabled else query
        return self.query_cache.get((self.model_name, normalized), lambda: self._embed_query(normalized))

    def _embed_query(self, query: str):
        """Encode one query with the unchanged BGE retrieval instruction."""
        query = (
            "Represent this sentence "
            "for searching relevant "
            f"passages: {query}"
        )

        return self.model.encode(
            query,
            normalize_embeddings=True,
            convert_to_numpy=True
        )

    # =====================================================
    # VECTOR SIZE
    # =====================================================

    def vector_size(
        self
    ) -> int:

        return (
            self.model
            .get_sentence_embedding_dimension()
        )


# =========================================================
# TEST
# =========================================================

if __name__ == "__main__":

    embedder = (
        LegalEmbedder()
    )

    texts = [
        "Section 52. Facts of which Court shall take judicial notice.",
        "Article 21. Protection of life and personal liberty."
    ]

    vectors = (
        embedder.embed(
            texts
        )
    )

    logger.debug('%s %s', "Vectors:", len(vectors))

    logger.debug('%s %s', "Dimension:", len(vectors[0]))

    query_vector = (
        embedder.embed_query(
            "facts judicially noticed by court"
        )
    )

    logger.debug('%s %s', "Query Dimension:", len(query_vector))

    logger.debug('%s %s', "Model Dimension:", embedder.vector_size())