mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
fix comment
This commit is contained in:
@ -173,6 +173,7 @@ impl Model for BPE {
|
||||
.map(|rank| {
|
||||
if let Some(dropout) = self.dropout {
|
||||
// With probability `dropout` we'll ignore
|
||||
// this merge.
|
||||
if thread_rng().gen::<f32>() < dropout {
|
||||
&(std::u32::MAX, std::u32::MAX)
|
||||
} else {
|
||||
|
Reference in New Issue
Block a user