{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":219035799,"defaultBranch":"main","name":"tokenizers","ownerLogin":"huggingface","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2019-11-01T17:52:20.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/25720743?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1721038324.0","currentOid":""},"activityList":{"items":[{"before":"0dc66cc5d6475537802347b9cb5f2bdb483beab9","after":"1321f227b1bb6d4f8333a9c1635a38cd95669dd8","ref":"refs/heads/fix-pretokenizer","pushedAt":"2024-07-15T11:30:14.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"fix","shortMessageHtmlLink":"fix"}},{"before":"88abf90e06459b1840f3c295f0e3431ee478eabb","after":"0dc66cc5d6475537802347b9cb5f2bdb483beab9","ref":"refs/heads/fix-pretokenizer","pushedAt":"2024-07-15T11:15:11.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"clippy","shortMessageHtmlLink":"clippy"}},{"before":"90210c92366a083729dd27cd72d2c1d54044fcdc","after":"88abf90e06459b1840f3c295f0e3431ee478eabb","ref":"refs/heads/fix-pretokenizer","pushedAt":"2024-07-15T10:56:18.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"fix tests","shortMessageHtmlLink":"fix tests"}},{"before":"5d9ea9faa4fd90087ee83dd4c80ccd80e27385b4","after":"90210c92366a083729dd27cd72d2c1d54044fcdc","ref":"refs/heads/fix-pretokenizer","pushedAt":"2024-07-15T10:19:33.000Z","pushType":"push","commitsCount":4,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"Merge branch 'main' into fix-pretokenizer","shortMessageHtmlLink":"Merge branch 'main' into fix-pretokenizer"}},{"before":"ebf3a8e01d72f2e4eb0ee3b03967471f15d90c1e","after":null,"ref":"refs/heads/fix-decode","pushedAt":"2024-07-15T10:12:04.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"}},{"before":"f2a44dc5d1d77ef358820e2ccf822428efc67e30","after":"4ea2f235b0430f5db09f867b65306d6c0a5ec7ed","ref":"refs/heads/main","pushedAt":"2024-07-15T10:12:03.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"Add bytelevel normalizer to fix decode when adding tokens to BPE (#1555)\n\n* feature dependent test\r\n\r\n* nit about 嗎\r\n\r\n* update\r\n\r\n* actuallyfix it\r\n\r\n* update the test\r\n\r\nadd it\r\n\r\nfix\r\n\r\n* stub\r\n\r\n* Update tokenizers/src/pre_tokenizers/byte_level.rs\r\n\r\nCo-authored-by: Luc Georges <McPatate@users.noreply.github.com>\r\n\r\n* skip failing test\r\n\r\n* add normalizer to init\r\n\r\n---------\r\n\r\nCo-authored-by: Luc Georges <McPatate@users.noreply.github.com>","shortMessageHtmlLink":"Add bytelevel normalizer to fix decode when adding tokens to BPE (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2359074984\" data-permission-text=\"Title is private\" data-url=\"https://github.com/huggingface/tokenizers/issues/1555\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/huggingface/tokenizers/pull/1555/hovercard\" href=\"https://github.com/huggingface/tokenizers/pull/1555\">#1555</a>)"}},{"before":"7032172636956c90886fddc6390e7a04f1e0187a","after":"ebf3a8e01d72f2e4eb0ee3b03967471f15d90c1e","ref":"refs/heads/fix-decode","pushedAt":"2024-07-15T08:08:13.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"add normalizer to init","shortMessageHtmlLink":"add normalizer to init"}},{"before":"52a6f2e887c6923c893dfc2bea7df0d7865141bc","after":"5d9ea9faa4fd90087ee83dd4c80ccd80e27385b4","ref":"refs/heads/fix-pretokenizer","pushedAt":"2024-07-15T07:48:55.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"nit clippy","shortMessageHtmlLink":"nit clippy"}},{"before":"8213ad8a77b13aecca7251dee87f0fbb513665c9","after":"7032172636956c90886fddc6390e7a04f1e0187a","ref":"refs/heads/fix-decode","pushedAt":"2024-07-15T07:44:32.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"skip failing test","shortMessageHtmlLink":"skip failing test"}},{"before":"1e027dcc361c20e49115a1456ad221b5741eb7ec","after":"535814c71bc954e748d5e62f0f0f31fe1146566f","ref":"refs/heads/fast-encode","pushedAt":"2024-07-15T07:44:11.000Z","pushType":"push","commitsCount":3,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"update","shortMessageHtmlLink":"update"}},{"before":null,"after":"c071f57620876b663c22f9a64d336488e9523cc8","ref":"refs/heads/bench-python","pushedAt":"2024-07-13T10:29:44.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"initial commit","shortMessageHtmlLink":"initial commit"}},{"before":null,"after":"4c5a8e7f29cc3f9df58c40d2a667b0c80bfc4c5f","ref":"refs/heads/test-old-decode","pushedAt":"2024-07-12T14:39:05.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"things needed to run the benchmark","shortMessageHtmlLink":"things needed to run the benchmark"}},{"before":"7ff89358465ee303ea4dd45c641174ca785e3519","after":"0d4e8598e62cfda2edcf7a80a93450f05f069027","ref":"refs/heads/assign-token","pushedAt":"2024-07-12T08:47:41.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"remove print","shortMessageHtmlLink":"remove print"}},{"before":"8d4fae818032b3dd17ce5ecca22a4e185d30aeb1","after":"7ff89358465ee303ea4dd45c641174ca785e3519","ref":"refs/heads/assign-token","pushedAt":"2024-07-12T08:38:44.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"fix everything","shortMessageHtmlLink":"fix everything"}},{"before":"3d2705f20ebf968552c28dcbdf503d77795ad4ae","after":"8d4fae818032b3dd17ce5ecca22a4e185d30aeb1","ref":"refs/heads/assign-token","pushedAt":"2024-07-12T08:12:32.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"pass compilation","shortMessageHtmlLink":"pass compilation"}},{"before":"ae66fda15b80ecba252e6273948cb54000a8320e","after":"3d2705f20ebf968552c28dcbdf503d77795ad4ae","ref":"refs/heads/assign-token","pushedAt":"2024-07-12T08:07:29.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"fix","shortMessageHtmlLink":"fix"}},{"before":null,"after":"ae66fda15b80ecba252e6273948cb54000a8320e","ref":"refs/heads/assign-token","pushedAt":"2024-07-12T07:49:47.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"current update","shortMessageHtmlLink":"current update"}},{"before":"f0c40e5feaf2a9574bd12713b9b9e11691bb3648","after":"8213ad8a77b13aecca7251dee87f0fbb513665c9","ref":"refs/heads/fix-decode","pushedAt":"2024-07-12T05:39:31.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"Update tokenizers/src/pre_tokenizers/byte_level.rs\n\nCo-authored-by: Luc Georges <McPatate@users.noreply.github.com>","shortMessageHtmlLink":"Update tokenizers/src/pre_tokenizers/byte_level.rs"}},{"before":"bf8f6643054694d531624a319c13e66be0d917e7","after":"f0c40e5feaf2a9574bd12713b9b9e11691bb3648","ref":"refs/heads/fix-decode","pushedAt":"2024-07-12T05:34:32.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"Merge branch 'main' into fix-decode","shortMessageHtmlLink":"Merge branch 'main' into fix-decode"}},{"before":"1bd8d96255c85ecde1fe9b7abc59733488576883","after":"bf8f6643054694d531624a319c13e66be0d917e7","ref":"refs/heads/fix-decode","pushedAt":"2024-07-12T05:32:38.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"Merge branch 'main' into fix-decode","shortMessageHtmlLink":"Merge branch 'main' into fix-decode"}},{"before":"96ad3833ddff722ae763082e81423d7904204149","after":null,"ref":"refs/heads/revert-break","pushedAt":"2024-07-12T05:29:42.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"}},{"before":"fdd26ba9a3f0c133427aab0423888cbde91362d7","after":"f2a44dc5d1d77ef358820e2ccf822428efc67e30","ref":"refs/heads/main","pushedAt":"2024-07-12T05:29:40.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"Revert \"[BREAKING CHANGE] Ignore added_tokens (both special and not) … (#1569)\n\n* Revert \"[BREAKING CHANGE] Ignore added_tokens (both special and not) in the decoder (#1513)\"\r\n\r\nThis reverts commit 25aee8b88c8de3c5a52e2f9cb6281d6df00ad516.\r\n\r\n* don't remove audit\r\n\r\n* deprecate id_to_token\r\n\r\n* use simple id to token\r\n\r\n* don't break id_to_token since we are deprecating anyways?","shortMessageHtmlLink":"Revert \"[BREAKING CHANGE] Ignore added_tokens (both special and not) … ("}},{"before":"7761621afdfeb6df513ac0a3ac4d2cb324147ba0","after":"1bd8d96255c85ecde1fe9b7abc59733488576883","ref":"refs/heads/fix-decode","pushedAt":"2024-07-12T05:09:14.000Z","pushType":"push","commitsCount":4,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"Merge branch 'main' into fix-decode","shortMessageHtmlLink":"Merge branch 'main' into fix-decode"}},{"before":"0594b517c89f2c7ff55f86872d911086f14ff12e","after":"7761621afdfeb6df513ac0a3ac4d2cb324147ba0","ref":"refs/heads/fix-decode","pushedAt":"2024-07-12T05:09:03.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"revert unwanted pub for original","shortMessageHtmlLink":"revert unwanted pub for original"}},{"before":"d6123a65873e3885b076c8e3718a0d1a9ffd2fc7","after":"0594b517c89f2c7ff55f86872d911086f14ff12e","ref":"refs/heads/fix-decode","pushedAt":"2024-07-12T05:05:38.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"Update tokenizers/src/tokenizer/added_vocabulary.rs\n\nCo-authored-by: Luc Georges <McPatate@users.noreply.github.com>","shortMessageHtmlLink":"Update tokenizers/src/tokenizer/added_vocabulary.rs"}},{"before":"be0ae2e71754dc2a272d12da16ae9cc5a2bb4ca8","after":"d6123a65873e3885b076c8e3718a0d1a9ffd2fc7","ref":"refs/heads/fix-decode","pushedAt":"2024-07-12T05:05:22.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"Update tokenizers/src/pre_tokenizers/byte_level.rs\n\nCo-authored-by: Luc Georges <McPatate@users.noreply.github.com>","shortMessageHtmlLink":"Update tokenizers/src/pre_tokenizers/byte_level.rs"}},{"before":"51492f6affe426cbb198176f0a0188c740ebcb3b","after":"96ad3833ddff722ae763082e81423d7904204149","ref":"refs/heads/revert-break","pushedAt":"2024-07-12T05:04:05.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"don't break id_to_token since we are deprecating anyways?","shortMessageHtmlLink":"don't break id_to_token since we are deprecating anyways?"}},{"before":"a94432bc2ee66493e222f19db4fd49ee6875b855","after":"51492f6affe426cbb198176f0a0188c740ebcb3b","ref":"refs/heads/revert-break","pushedAt":"2024-07-12T04:59:45.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"use simple id to token","shortMessageHtmlLink":"use simple id to token"}},{"before":"3eed1347963206f24c30ac66d802370b24140951","after":"a94432bc2ee66493e222f19db4fd49ee6875b855","ref":"refs/heads/revert-break","pushedAt":"2024-07-12T04:54:22.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"don't remove audit","shortMessageHtmlLink":"don't remove audit"}},{"before":null,"after":"3eed1347963206f24c30ac66d802370b24140951","ref":"refs/heads/revert-break","pushedAt":"2024-07-12T04:48:01.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"Revert \"[BREAKING CHANGE] Ignore added_tokens (both special and not) in the decoder (#1513)\"\n\nThis reverts commit 25aee8b88c8de3c5a52e2f9cb6281d6df00ad516.","shortMessageHtmlLink":"Revert \"[BREAKING CHANGE] Ignore added_tokens (both special and not) …"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAEf59mDAA","startCursor":null,"endCursor":null}},"title":"Activity · huggingface/tokenizers"}