SFT-XLM-R / tokenizer_config.json
HannahGoossens's picture
Upload 8 files
5904735 verified
{
"add_prefix_space": true,
"added_tokens_decoder": {
"0": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<pad>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250001": {
"content": "<mask>",
"lstrip": true,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250002": {
"content": "[TakingUnderControl]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250003": {
"content": "[BeingDead]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250004": {
"content": "[Killing]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250005": {
"content": "[Arriving]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250006": {
"content": "[Translocation]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250007": {
"content": "[Communication]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250008": {
"content": "[Request]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250009": {
"content": "[Getting]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250010": {
"content": "[BeingInDebt]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250011": {
"content": "[Leaving]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250012": {
"content": "[ForceToAct]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250013": {
"content": "[Buying]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250014": {
"content": "[BeingAtAPlace]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250015": {
"content": "[FinancialTransaction]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250016": {
"content": "[Transportation]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250017": {
"content": "[HavingInPossession]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250018": {
"content": "[Enslaving]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250019": {
"content": "[Mismanagement]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250020": {
"content": "[BeingInARelationship]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250021": {
"content": "[RelationshipChange]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250022": {
"content": "[BeingAtPeace]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250023": {
"content": "[StartingConflict]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250024": {
"content": "[Collaboration]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250025": {
"content": "[Uprising]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250026": {
"content": "[BeingInConflict]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250027": {
"content": "[Giving]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250028": {
"content": "[Decreasing]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250029": {
"content": "[FallingIll]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250030": {
"content": "[ViolentContest]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250031": {
"content": "[Selling]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250032": {
"content": "[Voyage]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250033": {
"content": "[BeginningARelationship]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250034": {
"content": "[Attacking]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250035": {
"content": "[Trade]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250036": {
"content": "[HavingAMedicalCondition]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250037": {
"content": "[BeingEmployed]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250038": {
"content": "[Punishing]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250039": {
"content": "[BeingLeader]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250040": {
"content": "[Encounter]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250041": {
"content": "[Production]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250042": {
"content": "[Visit]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250043": {
"content": "[LosingPossession]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250044": {
"content": "[Dying]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250045": {
"content": "[AlteringARelationship]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250046": {
"content": "[ChangeOfPossession]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250047": {
"content": "[Increasing]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250048": {
"content": "[BeingDamaged]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250049": {
"content": "[None]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250050": {
"content": "[Destroying]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250051": {
"content": "[Damaging]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250052": {
"content": "[StartingAWar]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250053": {
"content": "[JoiningAnOrganization]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250054": {
"content": "[SocialInteraction]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250055": {
"content": "[LeavingAnOrganization]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250056": {
"content": "[InternalChange]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250057": {
"content": "[Replacing]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250058": {
"content": "[HavingInternalState-]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250059": {
"content": "[Occupation]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250060": {
"content": "[EndingARelationship]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250061": {
"content": "[IntentionalDamaging]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250062": {
"content": "[Mutiny]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250063": {
"content": "[EndingContractualAgreement]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250064": {
"content": "[SocialStatusChange]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250065": {
"content": "[BeingDestroyed]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250066": {
"content": "[Repairing]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250067": {
"content": "[HavingInternalState+]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250068": {
"content": "[BeginningContractualAgreement]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250069": {
"content": "[Sinking]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250070": {
"content": "[ExtendingContractualAgreement]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250071": {
"content": "[EndingConflict]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250072": {
"content": "[Besieging]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250073": {
"content": "[HavingContractualAgreement]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250074": {
"content": "[QuantityChange]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250075": {
"content": "[Invasion]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"additional_special_tokens": [
"[TakingUnderControl]",
"[BeingDead]",
"[Killing]",
"[Arriving]",
"[Translocation]",
"[Communication]",
"[Request]",
"[Getting]",
"[BeingInDebt]",
"[Leaving]",
"[ForceToAct]",
"[Buying]",
"[BeingAtAPlace]",
"[FinancialTransaction]",
"[Transportation]",
"[HavingInPossession]",
"[Enslaving]",
"[Mismanagement]",
"[BeingInARelationship]",
"[RelationshipChange]",
"[BeingAtPeace]",
"[StartingConflict]",
"[Collaboration]",
"[Uprising]",
"[BeingInConflict]",
"[Giving]",
"[Decreasing]",
"[FallingIll]",
"[ViolentContest]",
"[Selling]",
"[Voyage]",
"[BeginningARelationship]",
"[Attacking]",
"[Trade]",
"[HavingAMedicalCondition]",
"[BeingEmployed]",
"[Punishing]",
"[BeingLeader]",
"[Encounter]",
"[Production]",
"[Visit]",
"[LosingPossession]",
"[Dying]",
"[AlteringARelationship]",
"[ChangeOfPossession]",
"[Increasing]",
"[BeingDamaged]",
"[None]",
"[Destroying]",
"[Damaging]",
"[StartingAWar]",
"[JoiningAnOrganization]",
"[SocialInteraction]",
"[LeavingAnOrganization]",
"[InternalChange]",
"[Replacing]",
"[HavingInternalState-]",
"[Occupation]",
"[EndingARelationship]",
"[IntentionalDamaging]",
"[Mutiny]",
"[EndingContractualAgreement]",
"[SocialStatusChange]",
"[BeingDestroyed]",
"[Repairing]",
"[HavingInternalState+]",
"[BeginningContractualAgreement]",
"[Sinking]",
"[ExtendingContractualAgreement]",
"[EndingConflict]",
"[Besieging]",
"[HavingContractualAgreement]",
"[QuantityChange]",
"[Invasion]"
],
"bos_token": "<s>",
"clean_up_tokenization_spaces": false,
"cls_token": "<s>",
"eos_token": "</s>",
"extra_special_tokens": {},
"mask_token": "<mask>",
"model_max_length": 512,
"pad_token": "<pad>",
"sep_token": "</s>",
"tokenizer_class": "XLMRobertaTokenizer",
"unk_token": "<unk>"
}