pylint for duplicated code for system prompts
Browse files
src/axolotl/datasets.py
CHANGED
|
@@ -126,6 +126,7 @@ class ConstantLengthDataset(IterableDataset):
|
|
| 126 |
buffer_len = 0
|
| 127 |
|
| 128 |
if example:
|
|
|
|
| 129 |
# just going to drop data points that are too long
|
| 130 |
if len(example["input_ids"]) <= self.seq_length:
|
| 131 |
input_ids = example["input_ids"]
|
|
|
|
| 126 |
buffer_len = 0
|
| 127 |
|
| 128 |
if example:
|
| 129 |
+
# FIXME
|
| 130 |
# just going to drop data points that are too long
|
| 131 |
if len(example["input_ids"]) <= self.seq_length:
|
| 132 |
input_ids = example["input_ids"]
|
src/axolotl/prompt_strategies/alpaca_w_system.py
CHANGED
|
@@ -21,6 +21,7 @@ class InstructionWSystemPromptTokenizingStrategy(PromptTokenizingStrategy):
|
|
| 21 |
)
|
| 22 |
|
| 23 |
def tokenize_prompt(self, prompt):
|
|
|
|
| 24 |
(
|
| 25 |
instruction,
|
| 26 |
input, # pylint: disable=redefined-builtin
|
|
|
|
| 21 |
)
|
| 22 |
|
| 23 |
def tokenize_prompt(self, prompt):
|
| 24 |
+
# pylint: disable=duplicate-code
|
| 25 |
(
|
| 26 |
instruction,
|
| 27 |
input, # pylint: disable=redefined-builtin
|