From 8a8628cbdfbc208413cf4fd02b08a39848f03d07 Mon Sep 17 00:00:00 2001 From: Steven Davidovitz Date: Tue, 2 Sep 2025 12:36:05 -0700 Subject: [PATCH] catch and return panics in encoders --- src/lib.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f4007d12..e1109484 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -433,15 +433,15 @@ pub extern "C" fn tokenizers_encode(ptr: *mut libc::c_void, message: *const libc let message_cow = String::from_utf8_lossy(message_bytes); let message = message_cow.as_ref(); - let encoding_details = match unified_tokenizer.encode_with_details(message, options.add_special_tokens) { - Ok(details) => details, - Err(_) => return tokenizers_buffer { - ids: ptr::null_mut(), - tokens: ptr::null_mut(), - len: 0, - type_ids: ptr::null_mut(), - special_tokens_mask: ptr::null_mut(), - attention_mask: ptr::null_mut(), + let encoding_details = match std::panic::catch_unwind(|| { unified_tokenizer.encode_with_details(message, options.add_special_tokens) }) { + Ok(Ok(details)) => details, + Ok(Err(_)) | Err(_) => return tokenizers_buffer { + ids: ptr::null_mut(), + tokens: ptr::null_mut(), + len: 0, + type_ids: ptr::null_mut(), + special_tokens_mask: ptr::null_mut(), + attention_mask: ptr::null_mut(), offsets: ptr::null_mut() } };