From f1c23b868006ee27acdd31796677f82fa10d6bd7 Mon Sep 17 00:00:00 2001
From: Steven Weiss <steventrouble@gmail.com>
Date: Wed, 3 Jan 2024 01:25:07 -0800
Subject: [PATCH] Add quick doc to byte_level.rs (#1420)

* Add quick doc to byte_level.rs

* Address PR comments
---
 tokenizers/src/pre_tokenizers/byte_level.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tokenizers/src/pre_tokenizers/byte_level.rs b/tokenizers/src/pre_tokenizers/byte_level.rs
index 03fb4b40..6343bbd0 100644
--- a/tokenizers/src/pre_tokenizers/byte_level.rs
+++ b/tokenizers/src/pre_tokenizers/byte_level.rs
@@ -9,6 +9,8 @@ use crate::tokenizer::{
 };
 use crate::utils::macro_rules_attribute;
 
+/// Converts bytes to unicode characters.
+/// See https://github.com/openai/gpt-2/blob/master/src/encoder.py#L9
 fn bytes_char() -> HashMap<u8, char> {
     let mut bs: Vec<u8> = vec![];
     bs.extend(b'!'..=b'~');
@@ -33,6 +35,8 @@ fn bytes_char() -> HashMap<u8, char> {
 }
 
 lazy_static! {
+    /// Regex that matches exactly one token.
+    /// See https://github.com/openai/gpt-2/blob/master/src/encoder.py#L98
     static ref RE: SysRegex = SysRegex::new(
         r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"
     )