Fix issue avoid-missing-comma found at https://codereview.doctor (#16768)

code-review-doctor · web-flow · commit 1b7de41a0742 · 2022-04-14T16:42:27.000-04:00
diff --git a/tests/bert_japanese/test_tokenization_bert_japanese.py b/tests/bert_japanese/test_tokenization_bert_japanese.py
@@ -173,7 +173,7 @@ def test_mecab_tokenizer_no_normalize(self):
         )
 
     def test_wordpiece_tokenizer(self):
-        vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは" "ばんは", "##こん", "##にちは", "##ばんは"]
+        vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]
 
         vocab = {}
         for (i, token) in enumerate(vocab_tokens):
@@ -246,7 +246,7 @@ def test_full_tokenizer(self):
         )
 
     def test_character_tokenizer(self):
-        vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界" "、", "。"]
+        vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]
 
         vocab = {}
         for (i, token) in enumerate(vocab_tokens):

Original file line number	Diff line number	Diff line change
`@@ -173,7 +173,7 @@ def test_mecab_tokenizer_no_normalize(self):`
`173`	`173`	`)`
`174`	`174`
`175`	`175`	`def test_wordpiece_tokenizer(self):`
`176`		`- vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは" "ばんは", "##こん", "##にちは", "##ばんは"]`
	`176`	`+ vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]`
`177`	`177`
`178`	`178`	`vocab = {}`
`179`	`179`	`for (i, token) in enumerate(vocab_tokens):`
`@@ -246,7 +246,7 @@ def test_full_tokenizer(self):`
`246`	`246`	`)`
`247`	`247`
`248`	`248`	`def test_character_tokenizer(self):`
`249`		`- vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界" "、", "。"]`
	`249`	`+ vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]`
`250`	`250`
`251`	`251`	`vocab = {}`
`252`	`252`	`for (i, token) in enumerate(vocab_tokens):`