read tokenizer path from config correctly (#230)

JustinAWei · web-flow · commit 005db47f0466 · 2023-01-26T20:43:50.000-05:00
diff --git a/examples/summarize_rlhf/trlx_gptj_text_summarization.py b/examples/summarize_rlhf/trlx_gptj_text_summarization.py
@@ -97,7 +97,7 @@ def reward_fn(samples: List[str], **kwargs):
     )
     config = TRLConfig.load_yaml(config_path)
 
-    tokenizer = AutoTokenizer.from_pretrained(config.model.tokenizer_path)
+    tokenizer = AutoTokenizer.from_pretrained(config.tokenizer.tokenizer_path)
     tokenizer.pad_token = tokenizer.eos_token
     tokenizer.padding_side = "left"
     max_length_input = (

Original file line number	Diff line number	Diff line change
`@@ -97,7 +97,7 @@ def reward_fn(samples: List[str], **kwargs):`
`97`	`97`	`)`
`98`	`98`	`config = TRLConfig.load_yaml(config_path)`
`99`	`99`
`100`		`- tokenizer = AutoTokenizer.from_pretrained(config.model.tokenizer_path)`
	`100`	`+ tokenizer = AutoTokenizer.from_pretrained(config.tokenizer.tokenizer_path)`
`101`	`101`	`tokenizer.pad_token = tokenizer.eos_token`
`102`	`102`	`tokenizer.padding_side = "left"`
`103`	`103`	`max_length_input = (`