mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-01 23:09:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix comment

Browse Source
This commit is contained in:
epwalsh
2019-12-30 14:18:08 -08:00
parent b28c3fd04c
commit fdb8ffca27
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tokenizers/src/models/bpe/model.rs
View File

@ -173,6 +173,7 @@ impl Model for BPE {
.map(|rank| { .map(|rank| {
if let Some(dropout) = self.dropout { if let Some(dropout) = self.dropout {
// With probability `dropout` we'll ignore // With probability `dropout` we'll ignore
// this merge.
if thread_rng().gen::<f32>() < dropout { if thread_rng().gen::<f32>() < dropout {
&(std::u32::MAX, std::u32::MAX) &(std::u32::MAX, std::u32::MAX)
} else { } else {