File tree 1 file changed +5
-2
lines changed
1 file changed +5
-2
lines changed Original file line number Diff line number Diff line change 2702
2702
"inputs = tokenizer(\"Transformers are awesome!\", return_tensors=\"np\")\n",
2703
2703
"outputs = data_collator([{\"input_ids\": inputs[\"input_ids\"][0]}])\n",
2704
2704
"\n",
2705
+ "original_input_ids = inputs[\"input_ids\"][0]\n",
2706
+ "masked_input_ids = outputs[\"input_ids\"][0]\n",
2707
+ "\n",
2705
2708
"pd.DataFrame({\n",
2706
- " \"Original tokens\": tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"][0] ),\n",
2707
- " \"Masked tokens\": tokenizer.convert_ids_to_tokens(outputs[\"input_ids\"][0] ),\n",
2709
+ " \"Original tokens\": tokenizer.convert_ids_to_tokens(original_input_ids ),\n",
2710
+ " \"Masked tokens\": tokenizer.convert_ids_to_tokens(masked_input_ids ),\n",
2708
2711
" \"Original input_ids\": original_input_ids,\n",
2709
2712
" \"Masked input_ids\": masked_input_ids,\n",
2710
2713
" \"Labels\": outputs[\"labels\"][0]}).T"
You can’t perform that action at this time.
0 commit comments