caa3359c303338b794c91e54ed24184a *DESCRIPTION
dc86c093012c1c9bc880868b901be232 *LICENSE
58bc95c04d9a4a010790e81bdc31d872 *LICENSE.note
58d9e22276cf763469ce74123fff1916 *NAMESPACE
db565c09162f2acf2f7fb319e7893220 *NEWS.md
4a5b71cfd4fa08ca50304a7cd3da6cc3 *R/decoders.R
ae4057c6dcfef3aa329edf9f707e5902 *R/encoding.R
d86396f77f74e65df2444a8b5b2e3327 *R/extendr-wrappers.R
707f1fc798331be1ecf069398a3c0f52 *R/models.R
4c655db57131a1721363916808df14de *R/normalizers.R
581d066eda915aef3f4fe60a70f8ea86 *R/post_processors.R
d27088cbe26d4ad7aa76697d3d4f8574 *R/pre_tokenizers.R
78b306d392630e9bc9cc01f6ab22c85b *R/tokenizer.R
e06a7ea7000f6ad2ed0a5df60190bbf4 *R/trainers.R
5b09c83fbf9a930733e897c60652d6ee *README.md
6dc963801a6a45ae7fcb5e9d7f4237d4 *configure
1e6006521754e3a739cf9e515c11a90f *configure.win
ab6818dcb9d72a6e145d908496444c19 *inst/AUTHORS
7409903c965e154ff1a0e324f9ae3af2 *inst/po/fr/LC_MESSAGES/R-tok.mo
fc5c865d20fb1374514b7a7b2c79db72 *man/decoder_byte_level.Rd
f53249ee287578ae7f2f4ccf5e6b6e58 *man/encoding.Rd
db2a33f6da1c375a45c2fb460835cf10 *man/model_bpe.Rd
95a483956014ebaf7ba3376ca43439f0 *man/model_unigram.Rd
58ee54b034899dc580467f2656d10cdf *man/model_wordpiece.Rd
8ff8b5ad03a70de364e65beef6225f6a *man/normalizer_nfc.Rd
5eb6a22f8452904c5eaf483656268be8 *man/normalizer_nfkc.Rd
826ede43be3374bf68d5766b2dd62565 *man/pre_tokenizer.Rd
c9734b1d815ea73fbeefa7af0cc6c6ff *man/pre_tokenizer_byte_level.Rd
5bf53a1606c37f3e9525128961387aef *man/pre_tokenizer_whitespace.Rd
78b245cee42feeaa504876debb17dbfc *man/processor_byte_level.Rd
2e0ed460d6efe7d06e9276c1c21bce0a *man/tok_decoder.Rd
d0198a97f723ce4dfbe6ef4dbc53571e *man/tok_model.Rd
b2f8e5780581e6e05b0f7f5c546036b8 *man/tok_normalizer.Rd
6c968375a9926b5f42e499698db9aed4 *man/tok_processor.Rd
d1ccc7c4ae5804d9456eb2b656067e6c *man/tok_trainer.Rd
81e7f1e26f9d774a604aa4d190ac8358 *man/tokenizer.Rd
420bcd57270067bb8ca1b6eb11d3cbc8 *man/trainer_bpe.Rd
0c75314c63f5ee249aabf4f741bc573c *man/trainer_unigram.Rd
e0107b09c4dcd8ddb4596f7da4e6882b *man/trainer_wordpiece.Rd
c42835eacafd742f5c9e9d4b6647f6c5 *po/R-fr.po
5ca142c56b8e482a6215890a3c74c9e5 *po/R-tok.pot
2216c8af7c2a1039c3542a23fee724bf *src/Makevars.in
27957123fba16c05c53a66e68b0f3482 *src/Makevars.ucrt
cb1cf3fefa82428bb03afa7d81e6d2fe *src/Makevars.win.in
e475c45acff0aec8451fa9e0d2ade36c *src/entrypoint.c
41fcc569330976d1b5b12ef3e661f796 *src/rust/Cargo.lock
04fee7a7ebce5ddc73c27fb08a019ba8 *src/rust/Cargo.toml
943978c9bea0373e32250b1bb36dd9fc *src/rust/src/decoders.rs
0458be593b6569ecdd65dc527a5c4a2e *src/rust/src/lib.rs
896415978472025069aa475015cbf192 *src/rust/src/models.rs
4049fff976eb442349a8abcbf52d5dba *src/rust/src/normalizers.rs
45e17f8340a38f3a932085a03a18c3a8 *src/rust/src/post_processors.rs
066285c90b3515538fe16d1e0dbbf3e2 *src/rust/src/pre_tokenizers.rs
c9dcf4c12802b477c6002efa16da618f *src/rust/src/tokenizer.rs
f074818e63adef4a2d46e9d362f72457 *src/rust/src/trainers.rs
d4c7febd30951f7fbd584476dbb2dd70 *src/rust/vendor-config.toml
bf464ab248b11c7c13ffb8bfedd81b4c *src/rust/vendor.tar.xz
ba398b93085f305afd0f06437e58899f *src/tok-win.def
2fa9c1f6e421b6a79ceeee8fc410730b *tests/testthat.R
a379162c68dc3aac382a2d98facf4c44 *tests/testthat/assets/tokenizer.json
94ea82db26368d89a4fe5bc69f6e7de6 *tests/testthat/test-decoders.R
dcf23f7ae3377b349c5c54906d460a8b *tests/testthat/test-encoding.R
40f8effe797f06d3a9020d76d80c85c8 *tests/testthat/test-message-translations.R
e298050c9b2e5a47c1299ac438cde5a0 *tests/testthat/test-models.R
9bf4630914215ace43292d7e8f617c0a *tests/testthat/test-normalizers.R
d94a3ac8e8f1d828859ae0715b7b2fa0 *tests/testthat/test-post_processors.R
9cd4d8e57905d61a992b0d9945d48f90 *tests/testthat/test-pre_tokenizers.R
2847186a0d8b2fab5e187e5174fd6338 *tests/testthat/test-tokenizer.R
839814871c136e1452f80d76d68af077 *tests/testthat/test-trainers.R
c12f29979163c656f55be4c83943594c *tools/config.R
064e5e4e065fe2601c288550d112309d *tools/msrv.R
9da9f44f6b828277200c9971eebe15de *tools/patch.R
cd2990ec9b81ab0b80f56a753831fbac *tools/update_authors.R
caa74dadd0737705d92a72e8f37cb51e *tools/vendor.sh
