mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
Workarounds publishing issues:
- Upgrade package-lock.json (cannot find VS code attempt) - Use published `macro_rules_attribute` so `cargo publish` works.
This commit is contained in:
Nicolas Patry
23
bindings/node/native/Cargo.lock
generated
23
bindings/node/native/Cargo.lock
generated
@ -815,6 +815,21 @@ dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "macro_rules_attribute"
|
||||
version = "0.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "641c64af6cd80b81cf9c2f2f6ee382b1050c71ce63e20800499971a4a4195005"
|
||||
dependencies = [
|
||||
"macro_rules_attribute-proc_macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "macro_rules_attribute-proc_macro"
|
||||
version = "0.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eb246ada5a8c47b8b6e90c9f9a0f84f294939cdf558f1bc8d17fbb30f9706598"
|
||||
|
||||
[[package]]
|
||||
name = "matches"
|
||||
version = "0.1.9"
|
||||
@ -1180,10 +1195,6 @@ dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc_macros"
|
||||
version = "0.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.10"
|
||||
@ -1678,7 +1689,7 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
|
||||
|
||||
[[package]]
|
||||
name = "tokenizers"
|
||||
version = "0.11.2"
|
||||
version = "0.11.3"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"cached-path",
|
||||
@ -1690,9 +1701,9 @@ dependencies = [
|
||||
"itertools 0.9.0",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"macro_rules_attribute",
|
||||
"onig",
|
||||
"paste",
|
||||
"proc_macros",
|
||||
"rand 0.7.3",
|
||||
"rayon",
|
||||
"rayon-cond",
|
||||
|
||||
8
bindings/node/package-lock.json
generated
8
bindings/node/package-lock.json
generated
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "tokenizers",
|
||||
"version": "0.8.0",
|
||||
"version": "0.8.3",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
@ -4840,9 +4840,9 @@
|
||||
"dev": true
|
||||
},
|
||||
"prettier": {
|
||||
"version": "2.3.2",
|
||||
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.3.2.tgz",
|
||||
"integrity": "sha512-lnJzDfJ66zkMy58OL5/NY5zp70S7Nz6KqcKkXYzn2tMVrNxvbqaBpg7H3qHaLxCJ5lNMsGuM8+ohS7cZrthdLQ==",
|
||||
"version": "2.5.1",
|
||||
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.5.1.tgz",
|
||||
"integrity": "sha512-vBZcPRUR5MZJwoyi3ZoyQlc1rXeEck8KgeC9AwwOn+exuxLxq5toTRDTSaVrXHxelDMHy9zlicw8u66yxoSUFg==",
|
||||
"dev": true
|
||||
},
|
||||
"prettier-linter-helpers": {
|
||||
|
||||
@ -30,7 +30,7 @@
|
||||
"eslint-plugin-simple-import-sort": "^5.0.3",
|
||||
"jest": "^26.6.3",
|
||||
"neon-cli": "^0.9.1",
|
||||
"prettier": "^2.3.2",
|
||||
"prettier": "^2.5.1",
|
||||
"shelljs": "^0.8.3",
|
||||
"ts-jest": "^26.5.6",
|
||||
"typescript": "^3.9.10"
|
||||
|
||||
23
bindings/python/Cargo.lock
generated
23
bindings/python/Cargo.lock
generated
@ -876,6 +876,21 @@ dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "macro_rules_attribute"
|
||||
version = "0.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "641c64af6cd80b81cf9c2f2f6ee382b1050c71ce63e20800499971a4a4195005"
|
||||
dependencies = [
|
||||
"macro_rules_attribute-proc_macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "macro_rules_attribute-proc_macro"
|
||||
version = "0.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eb246ada5a8c47b8b6e90c9f9a0f84f294939cdf558f1bc8d17fbb30f9706598"
|
||||
|
||||
[[package]]
|
||||
name = "matches"
|
||||
version = "0.1.9"
|
||||
@ -1218,10 +1233,6 @@ dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc_macros"
|
||||
version = "0.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "pyo3"
|
||||
version = "0.12.4"
|
||||
@ -1753,7 +1764,7 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
|
||||
|
||||
[[package]]
|
||||
name = "tokenizers"
|
||||
version = "0.11.2"
|
||||
version = "0.11.3"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"cached-path",
|
||||
@ -1765,9 +1776,9 @@ dependencies = [
|
||||
"itertools 0.9.0",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"macro_rules_attribute",
|
||||
"onig",
|
||||
"paste 1.0.6",
|
||||
"proc_macros",
|
||||
"rand 0.7.3",
|
||||
"rayon",
|
||||
"rayon-cond",
|
||||
|
||||
@ -58,7 +58,7 @@ reqwest = { version = "0.11", optional = true }
|
||||
cached-path = { version = "0.5", optional = true }
|
||||
aho-corasick = "0.7"
|
||||
paste = "1.0.6"
|
||||
proc_macros = { path = "./src/utils/proc_macros", version="0.1.0" }
|
||||
macro_rules_attribute = "0.0.2"
|
||||
|
||||
[features]
|
||||
default = ["progressbar", "http"]
|
||||
|
||||
@ -205,4 +205,4 @@ macro_rules! impl_serde_type{
|
||||
}
|
||||
|
||||
// Re-export macro_rules_attribute
|
||||
pub use proc_macros::macro_rules_attribute;
|
||||
pub use macro_rules_attribute::macro_rules_attribute;
|
||||
|
||||
@ -1,11 +0,0 @@
|
||||
[package]
|
||||
name = "proc_macros"
|
||||
version = "0.1.0"
|
||||
edition = "2018"
|
||||
|
||||
[lib]
|
||||
proc-macro = true
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
@ -1,4 +0,0 @@
|
||||
Crate that contains proc macro implementations.
|
||||
|
||||
According to the [book](https://doc.rust-lang.org/reference/procedural-macros.html),
|
||||
> procedural macros must be defined in a crate with the crate type of `proc-macro`.
|
||||
@ -1,180 +0,0 @@
|
||||
extern crate proc_macro;
|
||||
use ::proc_macro::*;
|
||||
|
||||
// Original: https://crates.io/crates/macro_rules_attribute
|
||||
/// Applies the given `macro_rules!` macro to the decorated item.
|
||||
///
|
||||
/// This, as with any `proc_macro_attribute`, **consumes** the item it
|
||||
/// decorates: it is the `macro_rules!` macro job to generate it (_it is thus
|
||||
/// able to modify it_!).
|
||||
///
|
||||
/// For a version with "read-only" access to the item it decorates, see
|
||||
/// [`macro_rules_derive`][`macro@macro_rules_derive`].
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// Deriving getters for a (non-generic) `struct`:
|
||||
///
|
||||
/// ```rust
|
||||
/// # macro_rules! ignore {($($tt:tt)*) => () }
|
||||
/// # ignore! {
|
||||
/// #[macro_use]
|
||||
/// extern crate macro_rules_attribute;
|
||||
/// # }
|
||||
///
|
||||
/// macro_rules! make_getters {(
|
||||
/// $(#[$struct_meta:meta])*
|
||||
/// $struct_vis:vis
|
||||
/// struct $StructName:ident {
|
||||
/// $(
|
||||
/// $(#[$field_meta:meta])*
|
||||
/// $field_vis:vis // this visibility will be applied to the getters instead
|
||||
/// $field_name:ident : $field_ty:ty
|
||||
/// ),* $(,)?
|
||||
/// }
|
||||
/// ) => (
|
||||
/// // First, generate the struct definition we have been given, but with
|
||||
/// // private fields instead.
|
||||
/// $(#[$struct_meta])*
|
||||
/// $struct_vis
|
||||
/// struct $StructName {
|
||||
/// $(
|
||||
/// $(#[$field_meta])*
|
||||
/// // notice the lack of visibility => private fields
|
||||
/// $field_name: $field_ty,
|
||||
/// )*
|
||||
/// }
|
||||
///
|
||||
/// // Then, implement the getters:
|
||||
/// impl $StructName {
|
||||
/// $(
|
||||
/// #[inline]
|
||||
/// $field_vis
|
||||
/// fn $field_name (self: &'_ Self)
|
||||
/// -> &'_ $field_ty
|
||||
/// {
|
||||
/// &self.$field_name
|
||||
/// }
|
||||
/// )*
|
||||
/// }
|
||||
/// )}
|
||||
///
|
||||
/// mod example {
|
||||
/// # use ::macro_rules_attribute_proc_macro::macro_rules_attribute;
|
||||
/// #[macro_rules_attribute(make_getters!)]
|
||||
/// /// The macro handles meta attributes such as docstrings
|
||||
/// pub
|
||||
/// struct Person {
|
||||
/// pub
|
||||
/// name: String,
|
||||
///
|
||||
/// pub
|
||||
/// age: u8,
|
||||
/// }
|
||||
/// }
|
||||
/// use example::Person;
|
||||
///
|
||||
/// fn is_new_born (person: &'_ Person)
|
||||
/// -> bool
|
||||
/// {
|
||||
/// // person.age == 0
|
||||
/// // ^ error[E0616]: field `age` of struct `example::Person` is private
|
||||
/// *person.age() == 0
|
||||
/// }
|
||||
/// ```
|
||||
#[proc_macro_attribute] pub
|
||||
fn macro_rules_attribute (
|
||||
attrs: TokenStream,
|
||||
input: TokenStream,
|
||||
) -> TokenStream
|
||||
{
|
||||
// check that `attrs` is indeed of the form `$macro_name:path !`
|
||||
{
|
||||
// FIXME: do this properly
|
||||
match attrs.clone().into_iter().last() {
|
||||
| Some(TokenTree::Punct(ref punct))
|
||||
if punct.as_char() == '!'
|
||||
=> {},
|
||||
|
||||
| _ => {
|
||||
panic!("Expected a parameter of the form `macro_name !`");
|
||||
},
|
||||
}
|
||||
}
|
||||
let mut ret = attrs;
|
||||
ret.extend(::std::iter::once(
|
||||
TokenTree::Group(Group::new(
|
||||
Delimiter::Brace,
|
||||
// FIXME: directly using `input` makes the token stream be seen
|
||||
// as a single token tree by the declarative macro !??
|
||||
input.into_iter().collect(),
|
||||
))
|
||||
));
|
||||
#[cfg(feature = "verbose-expansions")]
|
||||
eprintln!("{}", ret);
|
||||
ret
|
||||
}
|
||||
|
||||
// Original: https://crates.io/crates/macro_rules_attribute
|
||||
/// Applies the given `macro_rules!` macro to the decorated item.
|
||||
///
|
||||
/// This, as with any `#[derive(...)]`, **does not consume** the item it
|
||||
/// decorates: instead, it only generates code on top of it.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// Implementing `Into<Int>` for a given `#[repr(Int)]` `enum`:
|
||||
///
|
||||
/// ```rust
|
||||
/// # macro_rules! ignore {($($tt:tt)*) => () }
|
||||
/// # ignore! {
|
||||
/// #[macro_use]
|
||||
/// extern crate macro_rules_attribute;
|
||||
/// # }
|
||||
///
|
||||
/// macro_rules! ToInteger {(
|
||||
/// #[repr($Int:ident)]
|
||||
/// $(#[$enum_meta:meta])*
|
||||
/// $pub:vis
|
||||
/// enum $Enum:ident {
|
||||
/// $(
|
||||
/// $Variant:ident $(= $value:expr)?
|
||||
/// ),* $(,)?
|
||||
/// }
|
||||
/// ) => (
|
||||
/// impl ::core::convert::From<$Enum> for $Int {
|
||||
/// #[inline]
|
||||
/// fn from (x: $Enum)
|
||||
/// -> Self
|
||||
/// {
|
||||
/// x as _
|
||||
/// }
|
||||
/// }
|
||||
/// )}
|
||||
///
|
||||
/// # use ::macro_rules_attribute_proc_macro::macro_rules_derive;
|
||||
/// #[macro_rules_derive(ToInteger!)]
|
||||
/// #[repr(u32)]
|
||||
/// enum Bool {
|
||||
/// False,
|
||||
/// True,
|
||||
/// }
|
||||
///
|
||||
/// fn main ()
|
||||
/// {
|
||||
/// assert_eq!(u32::from(Bool::False), 0);
|
||||
/// assert_eq!(u32::from(Bool::True), 1);
|
||||
/// // assert_eq!(u8::from(Bool::False), 0);
|
||||
/// // ^ error[E0277]: the trait bound `u8: std::convert::From<main::Bool>` is not satisfied
|
||||
/// }
|
||||
/// ```
|
||||
#[proc_macro_attribute] pub
|
||||
fn macro_rules_derive (
|
||||
attrs: TokenStream,
|
||||
input: TokenStream,
|
||||
) -> TokenStream
|
||||
{
|
||||
let mut ret = input.clone();
|
||||
ret.extend(macro_rules_attribute(attrs, input));
|
||||
ret
|
||||
}
|
||||
Reference in New Issue
Block a user