Add files using upload-large-folder tool

.gitattributes

@@ -35,3 +35,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 tokenizer.json filter=lfs diff=lfs merge=lfs -text
 thumbnail.png filter=lfs diff=lfs merge=lfs -text
+.venv/lib/python3.11/site-packages/idna/__pycache__/idnadata.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
+.venv/lib/python3.11/site-packages/huggingface_hub/__pycache__/hf_api.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
+.venv/lib/python3.11/site-packages/prompt_toolkit/key_binding/bindings/__pycache__/vi.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
+.venv/lib/python3.11/site-packages/prompt_toolkit/layout/__pycache__/containers.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
+.venv/lib/python3.11/site-packages/yaml/_yaml.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+.venv/lib/python3.11/site-packages/hf_xet/hf_xet.abi3.so filter=lfs diff=lfs merge=lfs -text
+.venv/lib/python3.11/site-packages/charset_normalizer/md__mypyc.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text

.venv/lib/python3.11/site-packages/_virtualenv.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69ac3d8f27e679c81b94ab30b3b56e9cd138219b1ba94a1fa3606d5a76a1433d
+size 18

.venv/lib/python3.11/site-packages/charset_normalizer-3.4.3.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 TAHRI Ahmed R.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

.venv/lib/python3.11/site-packages/charset_normalizer/md__mypyc.cpython-311-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:631f96dfb24c0a4d8471724082545706b36c03079c36deca6abaf9a71dd17aa0
+size 281208

.venv/lib/python3.11/site-packages/fsspec-2025.9.0.dist-info/licenses/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2018, Martin Durant
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

.venv/lib/python3.11/site-packages/hf_xet/hf_xet.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5815fbc856f76eda6c435431e94fcf414dbd5121da2bc55beec4696245bd045b
+size 7942656

.venv/lib/python3.11/site-packages/huggingface_hub/__pycache__/hf_api.cpython-311.pyc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b32b254634467dd5d688832c7bdf422925ec2ce2729915d40fdf94a4e374a4cb
+size 506593

.venv/lib/python3.11/site-packages/idna/__pycache__/idnadata.cpython-311.pyc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2564ad128f80bff7f6dd8bbe7e5ab7c001ba11f9cf0d1f38a58ab5267b082d45
+size 101536

.venv/lib/python3.11/site-packages/packaging-25.0.dist-info/licenses/LICENSE

@@ -0,0 +1,3 @@
+This software is made available under the terms of *either* of the licenses
+found in LICENSE.APACHE or LICENSE.BSD. Contributions to this software is made
+under the terms of *both* these licenses.

.venv/lib/python3.11/site-packages/packaging-25.0.dist-info/licenses/LICENSE.APACHE

@@ -0,0 +1,177 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS

.venv/lib/python3.11/site-packages/packaging-25.0.dist-info/licenses/LICENSE.BSD

@@ -0,0 +1,23 @@
+Copyright (c) Donald Stufft and individual contributors.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+       this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

.venv/lib/python3.11/site-packages/prompt_toolkit-3.0.52.dist-info/licenses/AUTHORS.rst

@@ -0,0 +1,11 @@
+Authors
+=======
+
+Creator
+-------
+Jonathan Slenders <jonathan AT slenders.be>
+
+Contributors
+------------
+
+- Amjith Ramanujam <amjith.r AT gmail.com>

.venv/lib/python3.11/site-packages/prompt_toolkit-3.0.52.dist-info/licenses/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2014, Jonathan Slenders
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+* Neither the name of the {organization} nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

.venv/lib/python3.11/site-packages/prompt_toolkit/contrib/completers/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from .system import SystemCompleter
+
+__all__ = ["SystemCompleter"]

.venv/lib/python3.11/site-packages/prompt_toolkit/contrib/completers/system.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from prompt_toolkit.completion.filesystem import ExecutableCompleter, PathCompleter
+from prompt_toolkit.contrib.regular_languages.compiler import compile
+from prompt_toolkit.contrib.regular_languages.completion import GrammarCompleter
+
+__all__ = [
+    "SystemCompleter",
+]
+
+
+class SystemCompleter(GrammarCompleter):
+    """
+    Completer for system commands.
+    """
+
+    def __init__(self) -> None:
+        # Compile grammar.
+        g = compile(
+            r"""
+                # First we have an executable.
+                (?P<executable>[^\s]+)
+
+                # Ignore literals in between.
+                (
+                    \s+
+                    ("[^"]*" | '[^']*' | [^'"]+ )
+                )*
+
+                \s+
+
+                # Filename as parameters.
+                (
+                    (?P<filename>[^\s]+) |
+                    "(?P<double_quoted_filename>[^\s]+)" |
+                    '(?P<single_quoted_filename>[^\s]+)'
+                )
+            """,
+            escape_funcs={
+                "double_quoted_filename": (lambda string: string.replace('"', '\\"')),
+                "single_quoted_filename": (lambda string: string.replace("'", "\\'")),
+            },
+            unescape_funcs={
+                "double_quoted_filename": (
+                    lambda string: string.replace('\\"', '"')
+                ),  # XXX: not entirely correct.
+                "single_quoted_filename": (lambda string: string.replace("\\'", "'")),
+            },
+        )
+
+        # Create GrammarCompleter
+        super().__init__(
+            g,
+            {
+                "executable": ExecutableCompleter(),
+                "filename": PathCompleter(only_directories=False, expanduser=True),
+                "double_quoted_filename": PathCompleter(
+                    only_directories=False, expanduser=True
+                ),
+                "single_quoted_filename": PathCompleter(
+                    only_directories=False, expanduser=True
+                ),
+            },
+        )

.venv/lib/python3.11/site-packages/prompt_toolkit/contrib/regular_languages/__init__.py

@@ -0,0 +1,80 @@
+r"""
+Tool for expressing the grammar of an input as a regular language.
+==================================================================
+
+The grammar for the input of many simple command line interfaces can be
+expressed by a regular language. Examples are PDB (the Python debugger); a
+simple (bash-like) shell with "pwd", "cd", "cat" and "ls" commands; arguments
+that you can pass to an executable; etc. It is possible to use regular
+expressions for validation and parsing of such a grammar. (More about regular
+languages: http://en.wikipedia.org/wiki/Regular_language)
+
+Example
+-------
+
+Let's take the pwd/cd/cat/ls example. We want to have a shell that accepts
+these three commands. "cd" is followed by a quoted directory name and "cat" is
+followed by a quoted file name. (We allow quotes inside the filename when
+they're escaped with a backslash.) We could define the grammar using the
+following regular expression::
+
+    grammar = \s* (
+        pwd |
+        ls |
+        (cd  \s+ " ([^"]|\.)+ ") |
+        (cat \s+ " ([^"]|\.)+ ")
+    ) \s*
+
+
+What can we do with this grammar?
+---------------------------------
+
+- Syntax highlighting: We could use this for instance to give file names
+                       different color.
+- Parse the result: .. We can extract the file names and commands by using a
+                       regular expression with named groups.
+- Input validation: .. Don't accept anything that does not match this grammar.
+                       When combined with a parser, we can also recursively do
+                       filename validation (and accept only existing files.)
+- Autocompletion: .... Each part of the grammar can have its own autocompleter.
+                       "cat" has to be completed using file names, while "cd"
+                       has to be completed using directory names.
+
+How does it work?
+-----------------
+
+As a user of this library, you have to define the grammar of the input as a
+regular expression. The parts of this grammar where autocompletion, validation
+or any other processing is required need to be marked using a regex named
+group. Like ``(?P<varname>...)`` for instance.
+
+When the input is processed for validation (for instance), the regex will
+execute, the named group is captured, and the validator associated with this
+named group will test the captured string.
+
+There is one tricky bit:
+
+    Often we operate on incomplete input (this is by definition the case for
+    autocompletion) and we have to decide for the cursor position in which
+    possible state the grammar it could be and in which way variables could be
+    matched up to that point.
+
+To solve this problem, the compiler takes the original regular expression and
+translates it into a set of other regular expressions which each match certain
+prefixes of the original regular expression. We generate one prefix regular
+expression for every named variable (with this variable being the end of that
+expression).
+
+
+TODO: some examples of:
+    - How to create a highlighter from this grammar.
+    - How to create a validator from this grammar.
+    - How to create an autocompleter from this grammar.
+    - How to create a parser from this grammar.
+"""
+
+from __future__ import annotations
+
+from .compiler import compile
+
+__all__ = ["compile"]

.venv/lib/python3.11/site-packages/prompt_toolkit/contrib/regular_languages/compiler.py

@@ -0,0 +1,579 @@
+r"""
+Compiler for a regular grammar.
+
+Example usage::
+
+    # Create and compile grammar.
+    p = compile('add \s+ (?P<var1>[^\s]+)  \s+  (?P<var2>[^\s]+)')
+
+    # Match input string.
+    m = p.match('add 23 432')
+
+    # Get variables.
+    m.variables().get('var1')  # Returns "23"
+    m.variables().get('var2')  # Returns "432"
+
+
+Partial matches are possible::
+
+    # Create and compile grammar.
+    p = compile('''
+        # Operators with two arguments.
+        ((?P<operator1>[^\s]+)  \s+ (?P<var1>[^\s]+)  \s+  (?P<var2>[^\s]+)) |
+
+        # Operators with only one arguments.
+        ((?P<operator2>[^\s]+)  \s+ (?P<var1>[^\s]+))
+    ''')
+
+    # Match partial input string.
+    m = p.match_prefix('add 23')
+
+    # Get variables. (Notice that both operator1 and operator2 contain the
+    # value "add".) This is because our input is incomplete, and we don't know
+    # yet in which rule of the regex we we'll end up. It could also be that
+    # `operator1` and `operator2` have a different autocompleter and we want to
+    # call all possible autocompleters that would result in valid input.)
+    m.variables().get('var1')  # Returns "23"
+    m.variables().get('operator1')  # Returns "add"
+    m.variables().get('operator2')  # Returns "add"
+
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Callable, Dict, Iterable, Iterator, Pattern, TypeVar, overload
+from typing import Match as RegexMatch
+
+from .regex_parser import (
+    AnyNode,
+    Lookahead,
+    Node,
+    NodeSequence,
+    Regex,
+    Repeat,
+    Variable,
+    parse_regex,
+    tokenize_regex,
+)
+
+__all__ = ["compile", "Match", "Variables"]
+
+
+# Name of the named group in the regex, matching trailing input.
+# (Trailing input is when the input contains characters after the end of the
+# expression has been matched.)
+_INVALID_TRAILING_INPUT = "invalid_trailing"
+
+EscapeFuncDict = Dict[str, Callable[[str], str]]
+
+
+class _CompiledGrammar:
+    """
+    Compiles a grammar. This will take the parse tree of a regular expression
+    and compile the grammar.
+
+    :param root_node: :class~`.regex_parser.Node` instance.
+    :param escape_funcs: `dict` mapping variable names to escape callables.
+    :param unescape_funcs: `dict` mapping variable names to unescape callables.
+    """
+
+    def __init__(
+        self,
+        root_node: Node,
+        escape_funcs: EscapeFuncDict | None = None,
+        unescape_funcs: EscapeFuncDict | None = None,
+    ) -> None:
+        self.root_node = root_node
+        self.escape_funcs = escape_funcs or {}
+        self.unescape_funcs = unescape_funcs or {}
+
+        #: Dictionary that will map the regex names to Node instances.
+        self._group_names_to_nodes: dict[
+            str, str
+        ] = {}  # Maps regex group names to varnames.
+        counter = [0]
+
+        def create_group_func(node: Variable) -> str:
+            name = f"n{counter[0]}"
+            self._group_names_to_nodes[name] = node.varname
+            counter[0] += 1
+            return name
+
+        # Compile regex strings.
+        self._re_pattern = f"^{self._transform(root_node, create_group_func)}$"
+        self._re_prefix_patterns = list(
+            self._transform_prefix(root_node, create_group_func)
+        )
+
+        # Compile the regex itself.
+        flags = re.DOTALL  # Note that we don't need re.MULTILINE! (^ and $
+        # still represent the start and end of input text.)
+        self._re = re.compile(self._re_pattern, flags)
+        self._re_prefix = [re.compile(t, flags) for t in self._re_prefix_patterns]
+
+        # We compile one more set of regexes, similar to `_re_prefix`, but accept any trailing
+        # input. This will ensure that we can still highlight the input correctly, even when the
+        # input contains some additional characters at the end that don't match the grammar.)
+        self._re_prefix_with_trailing_input = [
+            re.compile(
+                r"(?:{})(?P<{}>.*?)$".format(t.rstrip("$"), _INVALID_TRAILING_INPUT),
+                flags,
+            )
+            for t in self._re_prefix_patterns
+        ]
+
+    def escape(self, varname: str, value: str) -> str:
+        """
+        Escape `value` to fit in the place of this variable into the grammar.
+        """
+        f = self.escape_funcs.get(varname)
+        return f(value) if f else value
+
+    def unescape(self, varname: str, value: str) -> str:
+        """
+        Unescape `value`.
+        """
+        f = self.unescape_funcs.get(varname)
+        return f(value) if f else value
+
+    @classmethod
+    def _transform(
+        cls, root_node: Node, create_group_func: Callable[[Variable], str]
+    ) -> str:
+        """
+        Turn a :class:`Node` object into a regular expression.
+
+        :param root_node: The :class:`Node` instance for which we generate the grammar.
+        :param create_group_func: A callable which takes a `Node` and returns the next
+            free name for this node.
+        """
+
+        def transform(node: Node) -> str:
+            # Turn `AnyNode` into an OR.
+            if isinstance(node, AnyNode):
+                return "(?:{})".format("|".join(transform(c) for c in node.children))
+
+            # Concatenate a `NodeSequence`
+            elif isinstance(node, NodeSequence):
+                return "".join(transform(c) for c in node.children)
+
+            # For Regex and Lookahead nodes, just insert them literally.
+            elif isinstance(node, Regex):
+                return node.regex
+
+            elif isinstance(node, Lookahead):
+                before = "(?!" if node.negative else "(="
+                return before + transform(node.childnode) + ")"
+
+            # A `Variable` wraps the children into a named group.
+            elif isinstance(node, Variable):
+                return f"(?P<{create_group_func(node)}>{transform(node.childnode)})"
+
+            # `Repeat`.
+            elif isinstance(node, Repeat):
+                if node.max_repeat is None:
+                    if node.min_repeat == 0:
+                        repeat_sign = "*"
+                    elif node.min_repeat == 1:
+                        repeat_sign = "+"
+                else:
+                    repeat_sign = "{%i,%s}" % (
+                        node.min_repeat,
+                        ("" if node.max_repeat is None else str(node.max_repeat)),
+                    )
+
+                return "(?:{}){}{}".format(
+                    transform(node.childnode),
+                    repeat_sign,
+                    ("" if node.greedy else "?"),
+                )
+            else:
+                raise TypeError(f"Got {node!r}")
+
+        return transform(root_node)
+
+    @classmethod
+    def _transform_prefix(
+        cls, root_node: Node, create_group_func: Callable[[Variable], str]
+    ) -> Iterable[str]:
+        """
+        Yield all the regular expressions matching a prefix of the grammar
+        defined by the `Node` instance.
+
+        For each `Variable`, one regex pattern will be generated, with this
+        named group at the end. This is required because a regex engine will
+        terminate once a match is found. For autocompletion however, we need
+        the matches for all possible paths, so that we can provide completions
+        for each `Variable`.
+
+        - So, in the case of an `Any` (`A|B|C)', we generate a pattern for each
+          clause. This is one for `A`, one for `B` and one for `C`. Unless some
+          groups don't contain a `Variable`, then these can be merged together.
+        - In the case of a `NodeSequence` (`ABC`), we generate a pattern for
+          each prefix that ends with a variable, and one pattern for the whole
+          sequence. So, that's one for `A`, one for `AB` and one for `ABC`.
+
+        :param root_node: The :class:`Node` instance for which we generate the grammar.
+        :param create_group_func: A callable which takes a `Node` and returns the next
+            free name for this node.
+        """
+
+        def contains_variable(node: Node) -> bool:
+            if isinstance(node, Regex):
+                return False
+            elif isinstance(node, Variable):
+                return True
+            elif isinstance(node, (Lookahead, Repeat)):
+                return contains_variable(node.childnode)
+            elif isinstance(node, (NodeSequence, AnyNode)):
+                return any(contains_variable(child) for child in node.children)
+
+            return False
+
+        def transform(node: Node) -> Iterable[str]:
+            # Generate separate pattern for all terms that contain variables
+            # within this OR. Terms that don't contain a variable can be merged
+            # together in one pattern.
+            if isinstance(node, AnyNode):
+                # If we have a definition like:
+                #           (?P<name> .*)  | (?P<city> .*)
+                # Then we want to be able to generate completions for both the
+                # name as well as the city. We do this by yielding two
+                # different regular expressions, because the engine won't
+                # follow multiple paths, if multiple are possible.
+                children_with_variable = []
+                children_without_variable = []
+                for c in node.children:
+                    if contains_variable(c):
+                        children_with_variable.append(c)
+                    else:
+                        children_without_variable.append(c)
+
+                for c in children_with_variable:
+                    yield from transform(c)
+
+                # Merge options without variable together.
+                if children_without_variable:
+                    yield "|".join(
+                        r for c in children_without_variable for r in transform(c)
+                    )
+
+            # For a sequence, generate a pattern for each prefix that ends with
+            # a variable + one pattern of the complete sequence.
+            # (This is because, for autocompletion, we match the text before
+            # the cursor, and completions are given for the variable that we
+            # match right before the cursor.)
+            elif isinstance(node, NodeSequence):
+                # For all components in the sequence, compute prefix patterns,
+                # as well as full patterns.
+                complete = [cls._transform(c, create_group_func) for c in node.children]
+                prefixes = [list(transform(c)) for c in node.children]
+                variable_nodes = [contains_variable(c) for c in node.children]
+
+                # If any child is contains a variable, we should yield a
+                # pattern up to that point, so that we are sure this will be
+                # matched.
+                for i in range(len(node.children)):
+                    if variable_nodes[i]:
+                        for c_str in prefixes[i]:
+                            yield "".join(complete[:i]) + c_str
+
+                # If there are non-variable nodes, merge all the prefixes into
+                # one pattern. If the input is: "[part1] [part2] [part3]", then
+                # this gets compiled into:
+                #  (complete1 + (complete2 + (complete3  | partial3) | partial2) | partial1 )
+                # For nodes that contain a variable, we skip the "|partial"
+                # part here, because thees are matched with the previous
+                # patterns.
+                if not all(variable_nodes):
+                    result = []
+
+                    # Start with complete patterns.
+                    for i in range(len(node.children)):
+                        result.append("(?:")
+                        result.append(complete[i])
+
+                    # Add prefix patterns.
+                    for i in range(len(node.children) - 1, -1, -1):
+                        if variable_nodes[i]:
+                            # No need to yield a prefix for this one, we did
+                            # the variable prefixes earlier.
+                            result.append(")")
+                        else:
+                            result.append("|(?:")
+                            # If this yields multiple, we should yield all combinations.
+                            assert len(prefixes[i]) == 1
+                            result.append(prefixes[i][0])
+                            result.append("))")
+
+                    yield "".join(result)
+
+            elif isinstance(node, Regex):
+                yield f"(?:{node.regex})?"
+
+            elif isinstance(node, Lookahead):
+                if node.negative:
+                    yield f"(?!{cls._transform(node.childnode, create_group_func)})"
+                else:
+                    # Not sure what the correct semantics are in this case.
+                    # (Probably it's not worth implementing this.)
+                    raise Exception("Positive lookahead not yet supported.")
+
+            elif isinstance(node, Variable):
+                # (Note that we should not append a '?' here. the 'transform'
+                # method will already recursively do that.)
+                for c_str in transform(node.childnode):
+                    yield f"(?P<{create_group_func(node)}>{c_str})"
+
+            elif isinstance(node, Repeat):
+                # If we have a repetition of 8 times. That would mean that the
+                # current input could have for instance 7 times a complete
+                # match, followed by a partial match.
+                prefix = cls._transform(node.childnode, create_group_func)
+
+                if node.max_repeat == 1:
+                    yield from transform(node.childnode)
+                else:
+                    for c_str in transform(node.childnode):
+                        if node.max_repeat:
+                            repeat_sign = "{,%i}" % (node.max_repeat - 1)
+                        else:
+                            repeat_sign = "*"
+                        yield "(?:{}){}{}{}".format(
+                            prefix,
+                            repeat_sign,
+                            ("" if node.greedy else "?"),
+                            c_str,
+                        )
+
+            else:
+                raise TypeError(f"Got {node!r}")
+
+        for r in transform(root_node):
+            yield f"^(?:{r})$"
+
+    def match(self, string: str) -> Match | None:
+        """
+        Match the string with the grammar.
+        Returns a :class:`Match` instance or `None` when the input doesn't match the grammar.
+
+        :param string: The input string.
+        """
+        m = self._re.match(string)
+
+        if m:
+            return Match(
+                string, [(self._re, m)], self._group_names_to_nodes, self.unescape_funcs
+            )
+        return None
+
+    def match_prefix(self, string: str) -> Match | None:
+        """
+        Do a partial match of the string with the grammar. The returned
+        :class:`Match` instance can contain multiple representations of the
+        match. This will never return `None`. If it doesn't match at all, the "trailing input"
+        part will capture all of the input.
+
+        :param string: The input string.
+        """
+        # First try to match using `_re_prefix`. If nothing is found, use the patterns that
+        # also accept trailing characters.
+        for patterns in [self._re_prefix, self._re_prefix_with_trailing_input]:
+            matches = [(r, r.match(string)) for r in patterns]
+            matches2 = [(r, m) for r, m in matches if m]
+
+            if matches2 != []:
+                return Match(
+                    string, matches2, self._group_names_to_nodes, self.unescape_funcs
+                )
+
+        return None
+
+
+class Match:
+    """
+    :param string: The input string.
+    :param re_matches: List of (compiled_re_pattern, re_match) tuples.
+    :param group_names_to_nodes: Dictionary mapping all the re group names to the matching Node instances.
+    """
+
+    def __init__(
+        self,
+        string: str,
+        re_matches: list[tuple[Pattern[str], RegexMatch[str]]],
+        group_names_to_nodes: dict[str, str],
+        unescape_funcs: dict[str, Callable[[str], str]],
+    ):
+        self.string = string
+        self._re_matches = re_matches
+        self._group_names_to_nodes = group_names_to_nodes
+        self._unescape_funcs = unescape_funcs
+
+    def _nodes_to_regs(self) -> list[tuple[str, tuple[int, int]]]:
+        """
+        Return a list of (varname, reg) tuples.
+        """
+
+        def get_tuples() -> Iterable[tuple[str, tuple[int, int]]]:
+            for r, re_match in self._re_matches:
+                for group_name, group_index in r.groupindex.items():
+                    if group_name != _INVALID_TRAILING_INPUT:
+                        regs = re_match.regs
+                        reg = regs[group_index]
+                        node = self._group_names_to_nodes[group_name]
+                        yield (node, reg)
+
+        return list(get_tuples())
+
+    def _nodes_to_values(self) -> list[tuple[str, str, tuple[int, int]]]:
+        """
+        Returns list of (Node, string_value) tuples.
+        """
+
+        def is_none(sl: tuple[int, int]) -> bool:
+            return sl[0] == -1 and sl[1] == -1
+
+        def get(sl: tuple[int, int]) -> str:
+            return self.string[sl[0] : sl[1]]
+
+        return [
+            (varname, get(slice), slice)
+            for varname, slice in self._nodes_to_regs()
+            if not is_none(slice)
+        ]
+
+    def _unescape(self, varname: str, value: str) -> str:
+        unwrapper = self._unescape_funcs.get(varname)
+        return unwrapper(value) if unwrapper else value
+
+    def variables(self) -> Variables:
+        """
+        Returns :class:`Variables` instance.
+        """
+        return Variables(
+            [(k, self._unescape(k, v), sl) for k, v, sl in self._nodes_to_values()]
+        )
+
+    def trailing_input(self) -> MatchVariable | None:
+        """
+        Get the `MatchVariable` instance, representing trailing input, if there is any.
+        "Trailing input" is input at the end that does not match the grammar anymore, but
+        when this is removed from the end of the input, the input would be a valid string.
+        """
+        slices: list[tuple[int, int]] = []
+
+        # Find all regex group for the name _INVALID_TRAILING_INPUT.
+        for r, re_match in self._re_matches:
+            for group_name, group_index in r.groupindex.items():
+                if group_name == _INVALID_TRAILING_INPUT:
+                    slices.append(re_match.regs[group_index])
+
+        # Take the smallest part. (Smaller trailing text means that a larger input has
+        # been matched, so that is better.)
+        if slices:
+            slice = (max(i[0] for i in slices), max(i[1] for i in slices))
+            value = self.string[slice[0] : slice[1]]
+            return MatchVariable("<trailing_input>", value, slice)
+        return None
+
+    def end_nodes(self) -> Iterable[MatchVariable]:
+        """
+        Yields `MatchVariable` instances for all the nodes having their end
+        position at the end of the input string.
+        """
+        for varname, reg in self._nodes_to_regs():
+            # If this part goes until the end of the input string.
+            if reg[1] == len(self.string):
+                value = self._unescape(varname, self.string[reg[0] : reg[1]])
+                yield MatchVariable(varname, value, (reg[0], reg[1]))
+
+
+_T = TypeVar("_T")
+
+
+class Variables:
+    def __init__(self, tuples: list[tuple[str, str, tuple[int, int]]]) -> None:
+        #: List of (varname, value, slice) tuples.
+        self._tuples = tuples
+
+    def __repr__(self) -> str:
+        return "{}({})".format(
+            self.__class__.__name__,
+            ", ".join(f"{k}={v!r}" for k, v, _ in self._tuples),
+        )
+
+    @overload
+    def get(self, key: str) -> str | None: ...
+
+    @overload
+    def get(self, key: str, default: str | _T) -> str | _T: ...
+
+    def get(self, key: str, default: str | _T | None = None) -> str | _T | None:
+        items = self.getall(key)
+        return items[0] if items else default
+
+    def getall(self, key: str) -> list[str]:
+        return [v for k, v, _ in self._tuples if k == key]
+
+    def __getitem__(self, key: str) -> str | None:
+        return self.get(key)
+
+    def __iter__(self) -> Iterator[MatchVariable]:
+        """
+        Yield `MatchVariable` instances.
+        """
+        for varname, value, slice in self._tuples:
+            yield MatchVariable(varname, value, slice)
+
+
+class MatchVariable:
+    """
+    Represents a match of a variable in the grammar.
+
+    :param varname: (string) Name of the variable.
+    :param value: (string) Value of this variable.
+    :param slice: (start, stop) tuple, indicating the position of this variable
+                  in the input string.
+    """
+
+    def __init__(self, varname: str, value: str, slice: tuple[int, int]) -> None:
+        self.varname = varname
+        self.value = value
+        self.slice = slice
+
+        self.start = self.slice[0]
+        self.stop = self.slice[1]
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.varname!r}, {self.value!r})"
+
+
+def compile(
+    expression: str,
+    escape_funcs: EscapeFuncDict | None = None,
+    unescape_funcs: EscapeFuncDict | None = None,
+) -> _CompiledGrammar:
+    """
+    Compile grammar (given as regex string), returning a `CompiledGrammar`
+    instance.
+    """
+    return _compile_from_parse_tree(
+        parse_regex(tokenize_regex(expression)),
+        escape_funcs=escape_funcs,
+        unescape_funcs=unescape_funcs,
+    )
+
+
+def _compile_from_parse_tree(
+    root_node: Node,
+    escape_funcs: EscapeFuncDict | None = None,
+    unescape_funcs: EscapeFuncDict | None = None,
+) -> _CompiledGrammar:
+    """
+    Compile grammar (given as parse tree), returning a `CompiledGrammar`
+    instance.
+    """
+    return _CompiledGrammar(
+        root_node, escape_funcs=escape_funcs, unescape_funcs=unescape_funcs
+    )

.venv/lib/python3.11/site-packages/prompt_toolkit/contrib/regular_languages/completion.py

@@ -0,0 +1,100 @@
+"""
+Completer for a regular grammar.
+"""
+
+from __future__ import annotations
+
+from typing import Iterable
+
+from prompt_toolkit.completion import CompleteEvent, Completer, Completion
+from prompt_toolkit.document import Document
+
+from .compiler import Match, _CompiledGrammar
+
+__all__ = [
+    "GrammarCompleter",
+]
+
+
+class GrammarCompleter(Completer):
+    """
+    Completer which can be used for autocompletion according to variables in
+    the grammar. Each variable can have a different autocompleter.
+
+    :param compiled_grammar: `GrammarCompleter` instance.
+    :param completers: `dict` mapping variable names of the grammar to the
+                       `Completer` instances to be used for each variable.
+    """
+
+    def __init__(
+        self, compiled_grammar: _CompiledGrammar, completers: dict[str, Completer]
+    ) -> None:
+        self.compiled_grammar = compiled_grammar
+        self.completers = completers
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        m = self.compiled_grammar.match_prefix(document.text_before_cursor)
+
+        if m:
+            yield from self._remove_duplicates(
+                self._get_completions_for_match(m, complete_event)
+            )
+
+    def _get_completions_for_match(
+        self, match: Match, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        """
+        Yield all the possible completions for this input string.
+        (The completer assumes that the cursor position was at the end of the
+        input string.)
+        """
+        for match_variable in match.end_nodes():
+            varname = match_variable.varname
+            start = match_variable.start
+
+            completer = self.completers.get(varname)
+
+            if completer:
+                text = match_variable.value
+
+                # Unwrap text.
+                unwrapped_text = self.compiled_grammar.unescape(varname, text)
+
+                # Create a document, for the completions API (text/cursor_position)
+                document = Document(unwrapped_text, len(unwrapped_text))
+
+                # Call completer
+                for completion in completer.get_completions(document, complete_event):
+                    new_text = (
+                        unwrapped_text[: len(text) + completion.start_position]
+                        + completion.text
+                    )
+
+                    # Wrap again.
+                    yield Completion(
+                        text=self.compiled_grammar.escape(varname, new_text),
+                        start_position=start - len(match.string),
+                        display=completion.display,
+                        display_meta=completion.display_meta,
+                    )
+
+    def _remove_duplicates(self, items: Iterable[Completion]) -> Iterable[Completion]:
+        """
+        Remove duplicates, while keeping the order.
+        (Sometimes we have duplicates, because the there several matches of the
+        same grammar, each yielding similar completions.)
+        """
+
+        def hash_completion(completion: Completion) -> tuple[str, int]:
+            return completion.text, completion.start_position
+
+        yielded_so_far: set[tuple[str, int]] = set()
+
+        for completion in items:
+            hash_value = hash_completion(completion)
+
+            if hash_value not in yielded_so_far:
+                yielded_so_far.add(hash_value)
+                yield completion

.venv/lib/python3.11/site-packages/prompt_toolkit/contrib/regular_languages/lexer.py

@@ -0,0 +1,94 @@
+"""
+`GrammarLexer` is compatible with other lexers and can be used to highlight
+the input using a regular grammar with annotations.
+"""
+
+from __future__ import annotations
+
+from typing import Callable
+
+from prompt_toolkit.document import Document
+from prompt_toolkit.formatted_text.base import StyleAndTextTuples
+from prompt_toolkit.formatted_text.utils import split_lines
+from prompt_toolkit.lexers import Lexer
+
+from .compiler import _CompiledGrammar
+
+__all__ = [
+    "GrammarLexer",
+]
+
+
+class GrammarLexer(Lexer):
+    """
+    Lexer which can be used for highlighting of fragments according to variables in the grammar.
+
+    (It does not actual lexing of the string, but it exposes an API, compatible
+    with the Pygments lexer class.)
+
+    :param compiled_grammar: Grammar as returned by the `compile()` function.
+    :param lexers: Dictionary mapping variable names of the regular grammar to
+                   the lexers that should be used for this part. (This can
+                   call other lexers recursively.) If you wish a part of the
+                   grammar to just get one fragment, use a
+                   `prompt_toolkit.lexers.SimpleLexer`.
+    """
+
+    def __init__(
+        self,
+        compiled_grammar: _CompiledGrammar,
+        default_style: str = "",
+        lexers: dict[str, Lexer] | None = None,
+    ) -> None:
+        self.compiled_grammar = compiled_grammar
+        self.default_style = default_style
+        self.lexers = lexers or {}
+
+    def _get_text_fragments(self, text: str) -> StyleAndTextTuples:
+        m = self.compiled_grammar.match_prefix(text)
+
+        if m:
+            characters: StyleAndTextTuples = [(self.default_style, c) for c in text]
+
+            for v in m.variables():
+                # If we have a `Lexer` instance for this part of the input.
+                # Tokenize recursively and apply tokens.
+                lexer = self.lexers.get(v.varname)
+
+                if lexer:
+                    document = Document(text[v.start : v.stop])
+                    lexer_tokens_for_line = lexer.lex_document(document)
+                    text_fragments: StyleAndTextTuples = []
+                    for i in range(len(document.lines)):
+                        text_fragments.extend(lexer_tokens_for_line(i))
+                        text_fragments.append(("", "\n"))
+                    if text_fragments:
+                        text_fragments.pop()
+
+                    i = v.start
+                    for t, s, *_ in text_fragments:
+                        for c in s:
+                            if characters[i][0] == self.default_style:
+                                characters[i] = (t, characters[i][1])
+                            i += 1
+
+            # Highlight trailing input.
+            trailing_input = m.trailing_input()
+            if trailing_input:
+                for i in range(trailing_input.start, trailing_input.stop):
+                    characters[i] = ("class:trailing-input", characters[i][1])
+
+            return characters
+        else:
+            return [("", text)]
+
+    def lex_document(self, document: Document) -> Callable[[int], StyleAndTextTuples]:
+        lines = list(split_lines(self._get_text_fragments(document.text)))
+
+        def get_line(lineno: int) -> StyleAndTextTuples:
+            try:
+                return lines[lineno]
+            except IndexError:
+                return []
+
+        return get_line