We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 23f5ce4 commit 3c96c4fCopy full SHA for 3c96c4f
dataloading.py
@@ -14,7 +14,7 @@ def _load_data_shard(file: Path):
14
tokens = torch.empty(num_tokens, dtype=torch.uint8, pin_memory=True)
15
f.seek(256 * 4)
16
nbytes = f.readinto(tokens.numpy())
17
- assert nbytes == 2 * num_tokens, 'number of tokens read does not match header?'
+ assert nbytes == num_tokens, 'number of tokens read does not match header?'
18
return tokens
19
20
0 commit comments