Fixes (#2569)

* fix rotary when very long first input
* fix valid transform at scoring and tokenize with onmt_tokenize when docify

Author: vince62s

onmt/modules/multi_headed_attn.py

@@ -466,7 +466,7 @@ def forward(
                     if self.max_relative_positions == -1:  # Rotary Embeddings
                         if seqlen + start_pos > self.rope.size(0):
                             # Resize rotary embeddings.
-                            self.rope, _, _ = rotaryembeddings(
+                            self.rope, self.cos, self.sin = rotaryembeddings(
                                 self.rotary_dim,
                                 maxseqlen=(seqlen + start_pos + 2048),
                                 base=self.rotary_theta,

onmt/tests/test_transform.py

@@ -327,12 +327,12 @@ def test_pyonmttok_bpe(self):
                 "struc￭",
                 "tion￭",
                 ":",
-                "\n￭",
+                "｟newline｠￭",
                 "in￭",
                 "struc￭",
                 "tion￭",
-                "\n￭",
-                "\n￭",
+                "｟newline｠￭",
+                "｟newline｠￭",
                 "#￭",
                 "#￭",
                 "#",
@@ -342,7 +342,7 @@ def test_pyonmttok_bpe(self):
                 "on￭",
                 "se",
                 ":",
-                "\n",
+                "｟newline｠",
                 "<blank>",
                 "respon￭",
                 "se",

onmt/trainer.py

@@ -297,7 +297,7 @@ def train(
             logger.info(
                 "Start training loop and validate every %d steps...", valid_steps
             )
-        logger.info("Scoring with: {}".format(self.scoring_preparator.transform))
+        logger.info("Scoring with: {}".format(self.scoring_preparator.transforms))
 
         total_stats = onmt.utils.Statistics()
         report_stats = onmt.utils.Statistics()

onmt/transforms/tokenize.py

@@ -157,7 +157,7 @@ def _tokenize(self, tokens, side="src", is_train=False):
         """Tokenize a list of words."""
         # This method embeds a custom logic to correctly handle certain placeholders
         # in case the tokenizer doesn't preserve them.
-        sentence = " ".join(tokens).replace(DefaultTokens.SEP, "\n")
+        sentence = " ".join(tokens)
         # Locate the end-of-sentence placeholders.
         sent_list = sentence.split(DefaultTokens.EOS)
         # Tokenize each sentence separately.
@@ -257,6 +257,7 @@ def tokenize_string(self, string, side="src", is_train=False):
         """Apply subword sampling or deterministic subwording"""
         sp_model = self.load_models[side]
         nbest_size = self.tgt_subword_nbest if side == "tgt" else self.src_subword_nbest
+        string = string.replace(DefaultTokens.SEP, "\n")
         if is_train is False or nbest_size in [0, 1]:
             # derterministic subwording
             tokens = sp_model.encode(string, out_type=str)
@@ -441,6 +442,9 @@ def _parse_opts(self):
         self.src_other_kwargs = self.opts.src_onmttok_kwargs
         self.tgt_other_kwargs = self.opts.tgt_onmttok_kwargs
         self.gpt2_pretok = self.opts.gpt2_pretok
+        self.preserve_placeholders = self.opts.tgt_onmttok_kwargs.get(
+            "preserve_placeholders", False
+        )
 
     @classmethod
     def get_specials(cls, opts):
@@ -558,6 +562,11 @@ def tokenize_string(self, sentence, side="src", is_train=False):
                     segmented.extend(["Ċ", "Ċ"])
                 else:
                     segmented.append(s)
+        elif (
+            self.src_subword_type == "sentencepiece" and not self.preserve_placeholders
+        ):
+            sentence = sentence.replace(DefaultTokens.SEP, "\n")
+            segmented = tokenizer(sentence)
         else:
             segmented = tokenizer(sentence)
         return segmented

onmt/transforms/transform.py

@@ -264,13 +264,16 @@ def _repr_args(self):
 def make_transforms(opts, transforms_cls, vocabs):
     """Build transforms in `transforms_cls` with vocab of `fields`."""
     transforms = {}
-    for name, transform_cls in transforms_cls.items():
-        if transform_cls.require_vocab() and vocabs is None:
-            logger.warning(f"{transform_cls.__name__} require vocab to apply, skip it.")
-            continue
-        transform_obj = transform_cls(opts)
-        transform_obj.warm_up(vocabs)
-        transforms[name] = transform_obj
+    if transforms_cls:
+        for name, transform_cls in transforms_cls.items():
+            if transform_cls.require_vocab() and vocabs is None:
+                logger.warning(
+                    f"{transform_cls.__name__} require vocab to apply, skip it."
+                )
+                continue
+            transform_obj = transform_cls(opts)
+            transform_obj.warm_up(vocabs)
+            transforms[name] = transform_obj
     return transforms
 
 

onmt/utils/scoring_utils.py

@@ -5,7 +5,7 @@
 from onmt.opts import translate_opts
 from onmt.constants import CorpusTask
 from onmt.inputters.dynamic_iterator import build_dynamic_dataset_iter
-from onmt.transforms import get_transforms_cls, make_transforms, TransformPipe
+from onmt.transforms import get_transforms_cls
 
 
 class ScoringPreparator:
@@ -19,16 +19,12 @@ def __init__(self, vocabs, opt):
         if self.opt.dump_preds is not None:
             if not os.path.exists(self.opt.dump_preds):
                 os.makedirs(self.opt.dump_preds)
-        self.transforms = opt.transforms
-        transforms_cls = get_transforms_cls(self.transforms)
-        transforms = make_transforms(self.opt, transforms_cls, self.vocabs)
-        self.transform = TransformPipe.build_from(transforms.values())
+        self.transforms = None
+        self.transforms_cls = None
 
     def warm_up(self, transforms):
         self.transforms = transforms
-        transforms_cls = get_transforms_cls(self.transforms)
-        transforms = make_transforms(self.opt, transforms_cls, self.vocabs)
-        self.transform = TransformPipe.build_from(transforms.values())
+        self.transforms_cls = get_transforms_cls(transforms)
 
     def translate(self, model, gpu_rank, step):
         """Compute and save the sentences predicted by the
@@ -84,7 +80,7 @@ def translate(self, model, gpu_rank, step):
 
         # Reinstantiate the validation iterator
 
-        transforms_cls = get_transforms_cls(model_opt._all_transform)
+        # transforms_cls = get_transforms_cls(model_opt._all_transform)
         model_opt.num_workers = 0
         model_opt.tgt = None
 
@@ -100,7 +96,7 @@ def translate(self, model, gpu_rank, step):
 
         valid_iter = build_dynamic_dataset_iter(
             model_opt,
-            transforms_cls,
+            self.transforms_cls,
             translator.vocabs,
             task=CorpusTask.VALID,
             tgt="",  # This force to clear the target side (needed when using tgt_file_prefix)
@@ -125,12 +121,11 @@ def translate(self, model, gpu_rank, step):
 
         # Flatten predictions
         preds = [x.lstrip() for sublist in preds for x in sublist]
-
         # Save results
         if len(preds) > 0 and self.opt.scoring_debug:
             path = os.path.join(self.opt.dump_preds, f"preds.valid_step_{step}.txt")
             with open(path, "a") as file:
-                for i in range(len(preds)):
+                for i in range(len(raw_srcs)):
                     file.write("SOURCE: {}\n".format(raw_srcs[i]))
                     file.write("REF: {}\n".format(raw_refs[i]))
                     file.write("PRED: {}\n\n".format(preds[i]))