Home Explore Help
Register Sign In
ai-model
/
intent
6
0
Fork 0
Files Issues 0 Pull Requests 0 Wiki
Branch: master
Branches Tags
intent
/
m3e
/
tokenizer_config.json

tokenizer_config.json 1.2 KB
Permalink History Raw

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455 
  1. {
    2. 

  2.   "added_tokens_decoder": {
    3. 

  3.     "0": {
    4. 

  4.       "content": "[PAD]",
    5. 

  5.       "lstrip": false,
    6. 

  6.       "normalized": false,
    7. 

  7.       "rstrip": false,
    8. 

  8.       "single_word": false,
    9. 

  9.       "special": true
    10. 

  10.     },
    11. 

  11.     "100": {
    12. 

  12.       "content": "[UNK]",
    13. 

  13.       "lstrip": false,
    14. 

  14.       "normalized": false,
    15. 

  15.       "rstrip": false,
    16. 

  16.       "single_word": false,
    17. 

  17.       "special": true
    18. 

  18.     },
    19. 

  19.     "101": {
    20. 

  20.       "content": "[CLS]",
    21. 

  21.       "lstrip": false,
    22. 

  22.       "normalized": false,
    23. 

  23.       "rstrip": false,
    24. 

  24.       "single_word": false,
    25. 

  25.       "special": true
    26. 

  26.     },
    27. 

  27.     "102": {
    28. 

  28.       "content": "[SEP]",
    29. 

  29.       "lstrip": false,
    30. 

  30.       "normalized": false,
    31. 

  31.       "rstrip": false,
    32. 

  32.       "single_word": false,
    33. 

  33.       "special": true
    34. 

  34.     },
    35. 

  35.     "103": {
    36. 

  36.       "content": "[MASK]",
    37. 

  37.       "lstrip": false,
    38. 

  38.       "normalized": false,
    39. 

  39.       "rstrip": false,
    40. 

  40.       "single_word": false,
    41. 

  41.       "special": true
    42. 

  42.     }
    43. 

  43.   },
    44. 

  44.   "clean_up_tokenization_spaces": true,
    45. 

  45.   "cls_token": "[CLS]",
    46. 

  46.   "do_lower_case": true,
    47. 

  47.   "mask_token": "[MASK]",
    48. 

  48.   "model_max_length": 512,
    49. 

  49.   "pad_token": "[PAD]",
    50. 

  50.   "sep_token": "[SEP]",
    51. 

  51.   "strip_accents": null,
    52. 

  52.   "tokenize_chinese_chars": true,
    53. 

  53.   "tokenizer_class": "BertTokenizer",
    54. 

  54.   "unk_token": "[UNK]"
    55. 

  55. }
    56.