Rocketknight1 HF Staff commited on
Commit
6e8c319
·
verified ·
1 Parent(s): 5926caa

Add response_template to tokenizer_config.json

Browse files

Adds a `response_template` key to `tokenizer_config.json`.

Files changed (1) hide show
  1. tokenizer_config.json +1 -0
tokenizer_config.json CHANGED
@@ -63,6 +63,7 @@
63
  "type": "object",
64
  "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?P<content>(?:(?!\\<turn\\|\\>)(?!\\<\\|tool_response\\>).)+)?(?:\\<turn\\|\\>|\\<\\|tool_response\\>)?"
65
  },
 
66
  "soc_token": "<|channel>",
67
  "sot_token": "<|turn>",
68
  "stc_token": "<|tool_call>",
 
63
  "type": "object",
64
  "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?P<content>(?:(?!\\<turn\\|\\>)(?!\\<\\|tool_response\\>).)+)?(?:\\<turn\\|\\>|\\<\\|tool_response\\>)?"
65
  },
66
+ "response_template": {"defaults": {"role": "assistant"}, "start_anchor": ["<|turn>model\n", "<tool_response|>"], "fields": {"thinking": {"open": "<|channel>thought\n", "close": "<channel|>", "content": "text"}, "tool_calls": {"open_pattern": "<\\|tool_call>call:(?P<name>\\w+)", "close": "<tool_call|>", "repeats": true, "content": "json", "content_args": {"unquoted_keys": true, "string_delims": [["<|\"|>", "<|\"|>"]]}, "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}}, "content": {"close": ["<turn|>", "<|tool_response>", "<eos>"], "content": "text"}}},
67
  "soc_token": "<|channel>",
68
  "sot_token": "<|turn>",
69
  "stc_token": "<|tool_call>",