from tokenizers import Tokenizer from tokenizers.models import BPE from transformers import PreTrainedTokenizerFast class gLM2Tokenizer(PreTrainedTokenizerFast): VOCAB = [ "", "", "", "", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", "a", "t", "c", "g", "<+>", "<->", "", "", ] def __init__( self, unk_token="", cls_token="", pad_token="", mask_token="", eos_token="", sep_token="", pos_token="<+>", neg_token="<->", **kwargs, ): all_tokens = self.VOCAB token_to_id = {tok: ind for ind, tok in enumerate(all_tokens)} bpe = BPE(token_to_id, merges=[], unk_token=str(unk_token)) tokenizer = Tokenizer(bpe) special_tokens = [cls_token, pad_token, mask_token, eos_token, sep_token, pos_token, neg_token] tokenizer.add_special_tokens( special_tokens, ) super().__init__( tokenizer_object=tokenizer, unk_token=unk_token, cls_token=cls_token, pad_token=pad_token, mask_token=mask_token, eos_token=eos_token, sep_token=sep_token, **kwargs, ) class gLM2BGCTokenizer(gLM2Tokenizer): BGC_DOMAINS = [ "1-cysPrx_C", "2-Hacid_dh", "2-Hacid_dh_C", "2-oxoacid_dh", "2-ph_phosp", "2Fe-2S_thioredx", "2OG-FeII_Oxy_2", "2OG-FeII_Oxy_3", "2OG-Fe_Oxy_2", "3-HAO", "3Beta_HSD", "3HBOH", "3HCDH", "3HCDH_N", "4HBT", "4HBT_2", "5-FTHF_cyc-lig", "5-nucleotidase", "5_nucleotid_C", "6PF2K", "6PGD", "AAL_decarboxy", "AAT", "AATase", "AA_kinase", "ABM", "ACBP", "ACCA", "ACC_central", "ACOX", "ACPS", "ACP_PD", "ACP_syn_III", "ACP_syn_III_C", "ACT", "ACT_7", "ADC", "ADH_N", "ADH_N_2", "ADH_N_assoc", "ADH_zinc_N", "ADH_zinc_N_2", "ADK", "ADK_lid", "ADP_PFK_GK", "ADP_ribosyl_GH", "ADPrib_exo_Tox", "ADSL_C", "AFOR_C", "AICARFT_IMPCHas", "AIG2_2", "AIRC", "AIRS", "AIRS_C", "ALAD", "ALG3", "ALIX_LYPXL_bnd", "ALO", "ALS_ss_C", "AMIN", "AMNp_N", "AMP-binding_C", "AMP-binding_C_2", "AMP_N", "ANF_receptor", "AOX", "APH", "APH_6_hur", "APS_kinase", "AP_endonuc_2", "ARD", "ART", "AS-2OG-FeII_Oxy", "AS-A-OX", "AS-AMP-binding", "AS-APE_KS1", "AS-APE_KS2", "AS-ATP-grasp", "AS-ATd", "AS-Abhydrolase_6", "AS-Abi", "AS-Acyl-CoA_dh_1", "AS-Acyl-CoA_dh_M", "AS-Acyl-CoA_dh_N", "AS-AfsA", "AS-Alpha-amylase", "AS-Alpha-amylase_C", "AS-Antimicr18", "AS-Antimicrobial17", "AS-Asn_synthase", "AS-Autoind_synth", "AS-AvrD", "AS-BLS", "AS-Bac_transf", "AS-BacteriocIIc_cy", "AS-Bacteriocin_II", "AS-Bacteriocin_IIc", "AS-Bacteriocin_IId", "AS-Bacteriocin_IIi", "AS-CAS", "AS-CBM_48", "AS-CDPS", "AS-CaiA", "AS-Capsule_synth", "AS-Chal_sti_synt_C", "AS-Chal_sti_synt_N", "AS-Chor_lyase", "AS-Cloacin", "AS-Condensation", "AS-DOIS", "AS-DUF1205", "AS-DUF3328", "AS-DUF4135", "AS-DUF692", "AS-DUF98", "AS-DarB", "AS-DegT_DnrJ_EryC1", "AS-Epimerase_2", "AS-FMO-like", "AS-FabF", "AS-Fer4_12", "AS-Flavoprotein", "AS-GATase_7", "AS-GST_C", "AS-GST_N_3", "AS-Gallidermin", "AS-Glyco_tran_28_C", "AS-Glyco_transf_28", "AS-Glyco_transf_8", "AS-Glyco_transf_8C", "AS-Glyco_transf_9", "AS-Glycos_transf_1", "AS-Glycos_transf_2", "AS-Glycos_transf_4", "AS-HMGL-like", "AS-IucA_IucC", "AS-LANC_like", "AS-LD_lanti_pre", "AS-L_biotic_A", "AS-Lactococcin", "AS-Lactococcin_972", "AS-Lant_dehydr_C", "AS-Lant_dehydr_N", "AS-LcnG-beta", "AS-Linocin_M18", "AS-LipM", "AS-LipU", "AS-LipV", "AS-LmbU", "AS-Lycopene_cycl", "AS-MGT", "AS-MannoseP_isomer", "AS-MoeO5", "AS-NAD_binding_4", "AS-NapT7", "AS-NasY", "AS-NeoL", "AS-Neocarzinostat", "AS-PCMT", "AS-PF00561", "AS-PF00881", "AS-PF04055", "AS-PF05402", "AS-PF06968", "AS-PF07366", "AS-PF13186", "AS-PF13471", "AS-PIG-L", "AS-PKS_AT", "AS-PKS_KS", "AS-PP-binding", "AS-PUFA_KS", "AS-Peptidase_C39", "AS-Peptidase_M16", "AS-Peptidase_M16_C", "AS-Peptidase_M42", "AS-Peptidase_M50", "AS-Peptidase_S28", "AS-Peptidase_S41", "AS-Peptidase_S8", "AS-Peptidase_S9", "AS-Pkinase", "AS-Poly_export", "AS-Polysacc_synt", "AS-Polysacc_synt_2", "AS-PoyD", "AS-PpyS", "AS-RimK", "AS-RmlD_sub_bind", "AS-SpcDK_cou", "AS-SpcDK_glyc", "AS-SpcFG", "AS-Subtilosin_A", "AS-TIGR01193", "AS-TIGR01716", "AS-TIGR02278", "AS-TIGR03103", "AS-TIGR03104", "AS-TIGR03106", "AS-TIGR03601", "AS-TIGR03602", "AS-TIGR03603", "AS-TIGR03604", "AS-TIGR03605", "AS-TIGR03651", "AS-TIGR03678", "AS-TIGR03693", "AS-TIGR03731", "AS-TIGR03793", "AS-TIGR03795", "AS-TIGR03798", "AS-TIGR03882", "AS-TIGR03973", "AS-TIGR03975", "AS-TIGR04404", "AS-TIGR04424", "AS-TP_methylase", "AS-TRI5", "AS-Terpene_synth", "AS-Terpene_synth_C", "AS-TfuA", "AS-ThiF", "AS-ToyB", "AS-Trp_halogenase", "AS-TunD", "AS-ValA", "AS-Wzy_C", "AS-YcaO", "AS-adh_short", "AS-adh_short_C2", "AS-bacilysin", "AS-bcpB", "AS-botH", "AS-bt1fas", "AS-cyanobactin_synth", "AS-cycdipepsynth", "AS-cypI", "AS-cypemycin", "AS-dTDP_sugar_isom", "AS-dmat", "AS-ectoine_synt", "AS-ene_KS", "AS-fabH", "AS-fom1", "AS-frbD", "AS-ft1fas", "AS-fung_ggpps", "AS-fung_ggpps2", "AS-glycocin", "AS-goadsporin_like", "AS-hglD", "AS-hglE", "AS-hyb_KS", "AS-indole_PTase", "AS-indsynth", "AS-itr_KS", "AS-ksIII", "AS-lacticin_l", "AS-lacticin_mat", "AS-ladderane", "AS-lasso", "AS-leader_d", "AS-leader_eh", "AS-mature_a", "AS-mature_ab", "AS-mature_b", "AS-mature_d", "AS-mature_ha", "AS-mcjC", "AS-melC", "AS-micJ25", "AS-micKC", "AS-mitE", "AS-mmyO", "AS-mod_KS", "AS-moeGT", "AS-mvd", "AS-mvnA", "AS-nikJ", "AS-nikO", "AS-novH", "AS-novI", "AS-novJ", "AS-novK", "AS-orf2_PTase", "AS-p450", "AS-pgm1", "AS-pgm_amidino", "AS-phosphonates", "AS-phytoene_synt", "AS-phzB", "AS-pop", "AS-prnB", "AS-pur10", "AS-pur6", "AS-salQ", "AS-skfc", "AS-strH", "AS-strK1", "AS-strK2", "AS-strep_PEQAXS", "AS-strepbact", "AS-sublancin", "AS-subtilosin", "AS-t2clf", "AS-t2fas", "AS-t2ks", "AS-t2pks2", "AS-tabtoxin", "AS-terpene_cyclase", "AS-thio_amide", "AS-thiostrepton", "AS-thuricin", "AS-tra_KS", "AS-trichodiene_synth", "AS-truD", "AS-ubiA", "AS-ustH", "AS-vlmB", "ASL_C", "ASL_C2", "ATP-grasp_2", "ATP-grasp_3", "ATP-grasp_4", "ATP-grasp_5", "ATP-gua_Ptrans", "ATP-sulfurylase", "ATP-synt_ab", "ATP_bind_3", "ATP_transf", "ATPase", "ATPgrasp_Ter", "AXE1", "A_deamin", "A_deaminase", "AbfB", "Abhydro_lipase", "Abhydrolase_2", "Abhydrolase_3", "Abhydrolase_4", "Abhydrolase_5", "Abhydrolase_8", "AceK", "AcetDehyd-dimer", "Acetate_kinase", "Acetone_carb_G", "AcetylCoA_hyd_C", "Acetyltransf_1", "Acetyltransf_10", "Acetyltransf_11", "Acetyltransf_2", "Acetyltransf_3", "Acetyltransf_4", "Acetyltransf_5", "Acetyltransf_6", "Acetyltransf_7", "Acetyltransf_8", "Acetyltransf_9", "Acetyltransf_CG", "Acid_PPase", "Acid_phosphat_B", "Aconitase", "Aconitase_2_N", "Aconitase_B_N", "Aconitase_C", "Actino_peptide", "Acyl-ACP_TE", "Acyl-CoA_dh_2", "Acyl-CoA_dh_C", "Acyl-CoA_ox_N", "Acyl_CoA_thio", "Acyl_transf_1", "Acyl_transf_3", "Acylphosphatase", "Acyltransferase", "Ad_cyc_g-alpha", "Adenine_deam_C", "Adenine_glyco", "Adenylate_cycl", "Adenylsucc_synt", "AdoHcyase", "AdoHcyase_NAD", "AdoMet_MTase", "AdoMet_Synthase", "AdoMet_dc", "AftA_C", "AftA_N", "AhpC-TSA", "AlaDh_PNT_C", "AlaDh_PNT_N", "Ala_racemase_C", "Ala_racemase_N", "Ald_Xan_dh_C", "Ald_Xan_dh_C2", "Ald_deCOase", "Aldedh", "Aldo_ket_red", "Aldolase", "Aldolase_II", "Aldose_epim", "Alg6_Alg8", "Alginate_lyase", "Alginate_lyase2", "AlkA_N", "Alk_phosphatase", "Alkyl_sulf_C", "Allantoicase", "Alliinase_C", "Alpha-L-AF_C", "Alpha-amyl_C2", "Alpha-mann_mid", "Alpha_L_fucos", "Alpha_kinase", "Amidase", "Amidase02_C", "Amidase_2", "Amidase_3", "Amidinotransf", "Amido_AtzD_TrzD", "Amidohydro_1", "Amidohydro_2", "Amidohydro_3", "Amino_oxidase", "Aminotran_1_2", "Aminotran_3", "Aminotran_4", "Aminotran_5", "An_peroxidase", "AnfG_VnfG", "AnmK", "Anth_synt_I_N", "Antibiotic_NAT", "ApbA", "ApbA_C", "ApbE", "ArabFuran-catal", "Arabinose_Iso_C", "Arabinose_Isome", "Arabinose_trans", "ArgJ", "Arginase", "Arginosuc_synth", "Aromatic_hydrox", "ArsC", "Arylesterase", "Arylsulfotran_2", "Arylsulfotrans", "AsnA", "Asp", "Asp_Arg_Hydrox", "Asp_Glu_race", "Asp_decarbox", "Asparaginase", "Asparaginase_2", "Asparaginase_II", "Aspzincin_M35", "AstB", "AstE_AspA", "Astacin", "B12-binding", "B12-binding_2", "BAAT_C", "BAH", "BATS", "BBE", "BCD", "BCDHK_Adom3", "BLUF", "BPL_C", "BPL_LplA_LipB", "BPL_N", "BRO1", "B_lectin", "BacA", "Bac_GDH", "Bac_luciferase", "Bac_rhamnosid", "Bac_rhamnosid_N", "BcrAD_BadFG", "BcsB", "Bet_v_1", "Beta-lactamase", "BetaGal_dom2", "BetaGal_dom3", "BetaGal_dom4_5", "Beta_elim_lyase", "Bgal_small_N", "Bile_Hydr_Trans", "BioW", "Biopterin_H", "Biotin_carb_C", "Biotin_carb_N", "Biotin_lipoyl", "Branch", "BtrH_N", "C1_1", "C1_2", "C1q", "CARDB", "CAT", "CBAH", "CBM-like", "CBM9_1", "CBM_1", "CBM_10", "CBM_14", "CBM_19", "CBM_2", "CBM_21", "CBM_3", "CBM_4_9", "CBM_5_12", "CBM_6", "CBM_X2", "CBS", "CCG", "CCP_MauG", "CDH", "CDO_I", "CDP-OH_P_transf", "CHAP", "CHB_HEX_C", "CHMI", "CHU_C", "CLP_protease", "CMAS", "CMD", "CM_1", "CM_2", "CNH", "CN_hydrolase", "COX1", "COX2", "COX2_TM", "COX3", "COX4", "COX4_pro", "COX7C", "CO_deh_flav_C", "COesterase", "CP12", "CPSase_L_D2", "CPSase_L_D3", "CPSase_sm_chain", "CP_ATPgrasp_1", "CRAL_TRIO", "CS", "CSS-motif", "CTP_synth_N", "CTP_transf_1", "CTP_transf_3", "CTP_transf_like", "CT_A_B", "CT_C_D", "CYTH", "Cadherin-like", "Caleosin", "Calpain_III", "CarS-like", "Carb_anhydrase", "Carb_bind", "Carb_kinase", "Carbam_trans_N", "Carboxyl_trans", "Carn_acyltransf", "Catalase", "Catalase-rel", "CbiA", "CbiC", "CbiD", "CbiG_C", "CbiG_N", "CbiG_mid", "CbiJ", "CbiK", "CbiX", "CbiZ", "CcmE", "Cellulase", "Cellulose_synt", "Ceramidase", "Ceramidase_alk", "CheB_methylest", "CheD", "CheR", "CheR_N", "ChiC", "Chitin_bind_1", "Chitin_synth_1", "Chitin_synth_1N", "ChitinaseA_N", "Chlor_dismutase", "Chloroa_b-bind", "Chlorophyllase", "Chlorophyllase2", "Choline_kinase", "Choline_sulf_C", "Chondroitinas_B", "Chorismate_bind", "Chorismate_synt", "CinA", "CitF", "CitG", "CitX", "Citrate_ly_lig", "Citrate_synt", "ClpB_D2-small", "CoA_binding", "CoA_binding_2", "CoA_trans", "CoA_transf_3", "CoaE", "CobA_CobO_BtuR", "CobD_Cbib", "CobN-Mg_chel", "CobS", "CobT_C", "CobU", "Cob_adeno_trans", "CofC", "ComA", "Complex1_30kDa", "Complex1_49kDa", "Complex1_51K", "Cons_hypoth95", "Coprogen_oxidas", "CpcD", "Cpn10", "Cpn60_TCP1", "Creatinase_N", "Creatininase", "CrtC", "Cu-oxidase", "Cu-oxidase_2", "Cu-oxidase_3", "Cu2_monoox_C", "Cu_amine_oxid", "Cu_amine_oxidN1", "Cu_amine_oxidN2", "Cu_amine_oxidN3", "Cutinase", "Cyanate_lyase", "Cyc-maltodext_C", "Cyclase", "Cyclase_polyket", "CysG_dimeriser", "Cys_Met_Meta_PP", "Cyt-b5", "Cyt_c_ox_IV", "Cytidylate_kin", "Cyto_heme_lyase", "Cytochrom_B561", "Cytochrom_C", "Cytochrom_C552", "Cytochrom_D1", "Cytochrome_C7", "Cytochrome_CBB3", "Cytokin-bind", "D-ser_dehydrat", "DAD", "DAGAT", "DAGK_acc", "DAGK_cat", "DAGK_prokar", "DAHP_synth_1", "DAHP_synth_2", "DAO", "DAP_B", "DAP_epimerase", "DBI_PRT", "DCD", "DDOST_48kD", "DEK_C", "DFP", "DGOK", "DHBP_synthase", "DHDPS", "DHFR_1", "DHH", "DHHA2", "DHHC", "DHODB_Fe-S_bind", "DHO_dh", "DHQS", "DHQ_synthase", "DHquinase_I", "DHquinase_II", "DIE2_ALG10", "DIOX_N", "DJ-1_PfpI", "DKCLD", "DLH", "DMRL_synthase", "DOMON", "DPPIV_N", "DPRP", "DRAT", "DS", "DSBA", "DSPc", "DSPn", "DUF1045", "DUF108", "DUF1100", "DUF123", "DUF1279", "DUF1298", "DUF1349", "DUF1624", "DUF1702", "DUF1729", "DUF1730", "DUF1731", "DUF1768", "DUF1772", "DUF1926", "DUF1929", "DUF1933", "DUF1939", "DUF1957", "DUF1965", "DUF1966", "DUF1995", "DUF2156", "DUF2183", "DUF2384", "DUF2431", "DUF294", "DUF3291", "DUF3336", "DUF3340", "DUF3367", "DUF3372", "DUF3381", "DUF3385", "DUF3410", "DUF3417", "DUF3458", "DUF3459", "DUF3479", "DUF3488", "DUF3543", "DUF4033", "DUF4111", "DUF4118", "DUF4129", "DUF43", "DUF4332", "DUF4440", "DUF452", "DUF563", "DUF608", "DUF756", "DUF89", "DUF953", "DUF971", "DUSP", "DXPR_C", "DXP_reductoisom", "DXP_synthase_N", "Dabb", "Dak1", "Dak2", "Dala_Dala_lig_C", "Dala_Dala_lig_N", "DapB_C", "DapB_N", "DehI", "Dehydratase_MU", "Dehydratase_SU", "Dehydratase_hem", "DeoC", "Desulfoferrodox", "DiS_P_DiS", "Dimerisation", "Dioxygenase_C", "Dioxygenase_N", "Diphthami_syn_2", "Diphthamide_syn", "Disintegrin", "DmpG_comm", "Dockerin_1", "Docking", "DoxA", "DoxD", "DsbC", "DsbC_N", "DsbD", "DsrC", "Dus", "Dynamin_M", "Dynamin_N", "Dyp_perox", "E1_dh", "E3_binding", "EAL", "EBP", "EB_dh", "ECH_1", "EF-hand_like", "EGF_CA", "EHN", "EIIBC-GUT_C", "ELFV_dehydrog", "ELFV_dehydrog_N", "ELO", "EMG1", "EPSP_synthase", "ERAP1_C", "ERG4_ERG24", "ERp29", "ETF_QO", "Ectoine_synth", "End_N_terminal", "Eno-Rase_FAD_bd", "Enolase_C", "Enolase_N", "Enoyl_reductase", "Enterotoxin_a", "Epimerase", "Erythro-docking", "Esterase", "Esterase_phd", "EutB", "EutC", "Exostosin", "F-box", "F-box-like", "F420_ligase", "F420_oxidored", "F5_F8_type_C", "FAA_hydrolase", "FAA_hydrolase_N", "FAD-oxidase_C", "FAD_binding_1", "FAD_binding_2", "FAD_binding_3", "FAD_binding_4", "FAD_binding_5", "FAD_binding_6", "FAD_binding_8", "FAD_binding_9", "FAD_oxidored", "FAD_syn", "FAE1_CUT1_RppA", "FAT", "FATC", "FA_desaturase", "FA_desaturase_2", "FA_hydroxylase", "FA_synthesis", "FBPase", "FBPase_2", "FBPase_glpX", "FCH", "FCSD-flav_bind", "FDX-ACB", "FGGY_C", "FGGY_N", "FGase", "FIST", "FKBP_C", "FKBP_N", "FMN_bind_2", "FMN_dh", "FMN_red", "FR47", "FRB_dom", "FSH1", "FTCD", "FTCD_C", "FTHFS", "F_bP_aldolase", "FabA", "Fae", "Fasciclin", "FdtA", "Fe-ADH", "FeS", "FeThRed_B", "Fe_bilin_red", "Fe_hyd_SSU", "Fe_hyd_lg_C", "FemAB", "Fer2", "Fer2_2", "Fer2_3", "Fer2_BFD", "Fer4", "Fer4_10", "Fer4_11", "Fer4_14", "Fer4_18", "Fer4_2", "Fer4_20", "Fer4_21", "Fer4_6", "Fer4_7", "Fer4_8", "Fer4_9", "Ferric_reduct", "Ferritin", "Ferrochelatase", "Flavin_Reduct", "Flavodoxin_1", "Flavodoxin_2", "Flavodoxin_5", "Flavokinase", "FlpD", "FmdA_AmdA", "FolB", "Formyl_trans_C", "Formyl_trans_N", "Frataxin_Cyay", "FrhB_FdhB_C", "Fringe", "Fructosamin_kin", "FtsJ", "Fucose_iso_C", "Fucose_iso_N1", "FumaraseC_C", "Fumble", "Fumerase", "Fumerase_C", "G3P_acyltransf", "G5", "G6PD_C", "G6PD_N", "GAD", "GAF", "GAF_2", "GAF_3", "GARS_A", "GARS_C", "GARS_N", "GATase", "GATase_2", "GATase_3", "GCHY-1", "GCS", "GCS2", "GCV_T", "GCV_T_C", "GDA1_CD39", "GDC-P", "GDE_C", "GDPD", "GDP_Man_Dehyd", "GD_AH_C", "GED", "GFA", "GFO_IDH_MocA", "GFO_IDH_MocA_C", "GGACT", "GGDEF", "GH97_C", "GHMP_kinases_C", "GHMP_kinases_N", "GIDA", "GLF", "GMC_oxred_C", "GMC_oxred_N", "GMP_synt_C", "GNAT_acetyltr_2", "GNT-I", "GPI2", "GRAM", "GRDB", "GSDH", "GSH-S_ATP", "GSHPx", "GSH_synth_ATP", "GSH_synthase", "GSP_synth", "GST_C_3", "GST_N", "GST_N_2", "GT87", "GTP_cyclohydro2", "GTP_cyclohydroI", "GXGXG", "G_glu_transpept", "GalKase_gal_bdg", "GalP_UDP_tr_C", "GalP_UDP_transf", "Gal_Lectin", "Gal_mutarotas_2", "Galactosyl_T", "GatB_Yqey", "GcpE", "GlcNAc", "Gln-synt_C", "Gln-synt_N", "GlnD_UR_UTase", "Globin", "Glu_cyclase_2", "Glu_cys_ligase", "Glu_syn_central", "Glu_synthase", "Glucan_synthase", "Glucokinase", "Gluconate_2-dh3", "Glucosamine_iso", "Glucosaminidase", "Glug", "Glutaminase", "Glutaredoxin", "Gly_kinase", "Gly_radical", "Gly_transf_sug", "Glyco_hyd_65N_2", "Glyco_hydr_116N", "Glyco_hydro_1", "Glyco_hydro_10", "Glyco_hydro_100", "Glyco_hydro_11", "Glyco_hydro_12", "Glyco_hydro_127", "Glyco_hydro_130", "Glyco_hydro_14", "Glyco_hydro_15", "Glyco_hydro_16", "Glyco_hydro_17", "Glyco_hydro_18", "Glyco_hydro_19", "Glyco_hydro_2", "Glyco_hydro_20", "Glyco_hydro_20b", "Glyco_hydro_25", "Glyco_hydro_26", "Glyco_hydro_28", "Glyco_hydro_2_C", "Glyco_hydro_2_N", "Glyco_hydro_3", "Glyco_hydro_30", "Glyco_hydro_31", "Glyco_hydro_32C", "Glyco_hydro_32N", "Glyco_hydro_35", "Glyco_hydro_38", "Glyco_hydro_38C", "Glyco_hydro_39", "Glyco_hydro_3_C", "Glyco_hydro_42", "Glyco_hydro_42C", "Glyco_hydro_42M", "Glyco_hydro_43", "Glyco_hydro_44", "Glyco_hydro_45", "Glyco_hydro_46", "Glyco_hydro_47", "Glyco_hydro_48", "Glyco_hydro_49", "Glyco_hydro_4C", "Glyco_hydro_52", "Glyco_hydro_53", "Glyco_hydro_56", "Glyco_hydro_57", "Glyco_hydro_59", "Glyco_hydro_6", "Glyco_hydro_61", "Glyco_hydro_62", "Glyco_hydro_63", "Glyco_hydro_65C", "Glyco_hydro_65N", "Glyco_hydro_65m", "Glyco_hydro_66", "Glyco_hydro_67C", "Glyco_hydro_68", "Glyco_hydro_7", "Glyco_hydro_71", "Glyco_hydro_75", "Glyco_hydro_76", "Glyco_hydro_77", "Glyco_hydro_79n", "Glyco_hydro_8", "Glyco_hydro_81", "Glyco_hydro_85", "Glyco_hydro_88", "Glyco_hydro_9", "Glyco_hydro_97", "Glyco_tran_WecB", "Glyco_tranf_2_3", "Glyco_tranf_2_5", "Glyco_trans_1_4", "Glyco_trans_4_2", "Glyco_trans_4_4", "Glyco_trans_A_1", "Glyco_transf_15", "Glyco_transf_17", "Glyco_transf_20", "Glyco_transf_21", "Glyco_transf_22", "Glyco_transf_25", "Glyco_transf_29", "Glyco_transf_34", "Glyco_transf_36", "Glyco_transf_4", "Glyco_transf_41", "Glyco_transf_5", "Glyco_transf_54", "Glyco_transf_64", "Glyco_transf_7C", "Glyco_transf_90", "Glycogen_syn", "Glycohydro_20b2", "Glycolytic", "Glycos_trans_3N", "Glycos_transf_3", "Glycos_transf_N", "Glyoxalase", "Glyoxalase_2", "Glyoxalase_4", "Glyphos_transf", "Gp_dh_C", "Gp_dh_N", "Gpi1", "GrpB", "Guanylate_cyc", "Guanylate_kin", "HAD_2", "HD", "HD_3", "HD_assoc", "HEM4", "HGD-D", "HI0933_like", "HIT", "HK", "HMA", "HMG-CoA_red", "HMG_CoA_synt_C", "HMG_CoA_synt_N", "HNOB", "HPIH", "HPPK", "HR1", "HTS", "H_PPase", "Haem_bd", "Ham1p_like", "HemN_C", "HemX", "Heme_oxygenase", "Hemocyanin_C", "Hepar_II_III", "Hexapep_2", "Hexokinase_1", "Hexokinase_2", "Hexose_dehydrat", "HgmA", "HisG", "HisG_C", "His_Phos_1", "His_Phos_2", "His_biosynth", "Histidinol_dh", "Homoserine_dh", "HpaB", "HpaB_N", "HpcH_HpaI", "Hpt", "HsdM_N", "HycI", "Hydant_A_N", "Hydantoinase_A", "Hydantoinase_B", "Hydrolase", "Hydrolase_3", "Hydrolase_4", "Hydrolase_6", "ICL", "ICMT", "IDH", "IDO", "IF-2B", "IGPD", "IGPS", "ILVD_EDD", "IMPDH", "IMP_cyclohyd", "IMS", "IPK", "IPPT", "IPT", "IU_nuc_hydro", "IlvC", "IlvN", "Indigoidine_A", "Inhibitor_I78", "Inhibitor_I9", "Inos-1-P_synth", "Inositol_P", "Ins134_P3_kin", "Ins_P5_2-kin", "Intron_maturas2", "Iso_dh", "Isochorismatase", "IspD", "JAB", "JmjC", "KA1", "KAsynt_C_assoc", "KDGP_aldolase", "KR", "KTI12", "K_oxygenase", "Kdo", "KduI", "Ketoacyl-synt_2", "Ketoacyl-synt_C", "Kinase-PPPase", "Ku_N", "LAB_N", "LAM_C", "LCAT", "LCCL", "LCM", "LEH", "LIP", "LMWPc", "LNS2", "LON_substr_bdg", "LPG_synthase_TM", "LRAT", "LRRNT_2", "LYTB", "LacAB_rpiB", "Lact-deh-memb", "Lactamase_B", "Lactamase_B_3", "Lactonase", "Laminin_G_2", "Ldh_1_C", "Ldh_1_N", "Ldh_2", "Lectin_legB", "LeuA_dimer", "Leuk-A4-hydro_C", "LigA", "LigB", "Ligase_CoA", "Lip_A_acyltrans", "Lip_prot_lig_C", "Lipase3_N", "Lipase_2", "Lipase_3", "Lipase_GDSL", "Lipin_N", "Lipocalin", "Lipoxygenase", "Lon_C", "LpxB", "LpxC", "LpxD", "LpxK", "Lum_binding", "LuxC", "LuxE", "LuxS", "Lyase_1", "Lyase_8", "Lyase_8_C", "Lyase_aromatic", "Lyase_catalyt", "Lys-AminoMut_A", "Lysyl_oxidase", "M20_dimer", "MAAL_C", "MAF_flag10", "MAPEG", "MASE1", "MASE2", "MATH", "MBOAT", "MCD", "MCH", "MCRA", "MDH", "MDMPI_N", "MGDG_synth", "MGS", "MIOX", "MIR", "MIase", "MM_CoA_mutase", "MOFRL", "MOSC", "MOSC_N", "MOZ_SAS", "MR_MLE_C", "MR_MLE_N", "MT-A70", "MTHFR", "MTHFR_C", "MTS", "MTTB", "Mac", "Macro", "Macro_2", "Malate_synthase", "Malectin_like", "Malic_M", "Mannitol_dh", "Mannitol_dh_C", "MaoC_dehydratas", "MazG", "MdcE", "MdcG", "Me-amine-dh_H", "Me-amine-dh_L", "MeMO_Hyd_G", "Melibiase", "MerB", "Met_10", "Met_asp_mut_E", "Met_synt_B12", "Metalloenzyme", "Metallophos", "Metallophos_2", "Metallophos_C", "Meth_synt_1", "Meth_synt_2", "Methyltrans_SAM", "Methyltransf_10", "Methyltransf_11", "Methyltransf_12", "Methyltransf_13", "Methyltransf_14", "Methyltransf_16", "Methyltransf_19", "Methyltransf_2", "Methyltransf_21", "Methyltransf_23", "Methyltransf_24", "Methyltransf_25", "Methyltransf_3", "Methyltransf_30", "Methyltransf_31", "Methyltransf_33", "Methyltransf_34", "Methyltransf_4", "Methyltransf_5", "Methyltransf_7", "Methyltransf_8", "Methyltransf_PK", "Mg-por_mtran_C", "Mg_chelatase", "MgtC", "MmgE_PrpD", "MmlI", "MmoB_DmpM", "Mo-co_dimer", "MoCF_biosynth", "MoaC", "MoaE", "MobB", "Mob_synth_C", "MoeA_C", "MoeA_N", "Molybdop_Fe4S4", "Molybdopterin", "Molydop_binding", "Mqo", "Multi-haem_cyto", "MurB_C", "Mur_ligase", "Mur_ligase_C", "Mur_ligase_M", "Muraidase", "Myotub-related", "N-glycanase_C", "NAAA-beta", "NAD-GH", "NADH-u_ox-rdase", "NADH5_C", "NADH_4Fe-4S", "NADH_dhqG_C", "NADH_u_ox_C", "NADHdh", "NAD_Gly3P_dh_C", "NAD_Gly3P_dh_N", "NAD_binding_1", "NAD_binding_10", "NAD_binding_11", "NAD_binding_2", "NAD_binding_3", "NAD_binding_5", "NAD_binding_6", "NAD_binding_7", "NAD_binding_8", "NAD_binding_9", "NAD_kinase", "NAD_synthase", "NAGLU_C", "NAGLU_N", "NAGPA", "NAGidase", "NAPRTase", "NAS", "NAT", "NDK", "NHase_alpha", "NHase_beta", "NIF", "NLPC_P60", "NMO", "NMT", "NMT_C", "NO_synthase", "NPCBM", "NPCBM_assoc", "NQR2_RnfD_RnfE", "NQRA_SLBB", "NRDD", "NTP_transf_2", "NTP_transf_3", "NTP_transf_9", "NTP_transferase", "NTPase_1", "NUDIX", "NUDIX-like", "NadA", "NanE", "NeuB", "NiFeSe_Hases", "NiFe_hyd_SSU_C", "Nitr_red_bet_C", "Nitrate_red_del", "Nitrate_red_gam", "NmrA", "NnrU", "NosD", "Nuc_deoxyrib_tr", "Nucleotid_trans", "Nudix_N_2", "NusB", "O-FucT", "OAD_gamma", "OB_NTP_bind", "OCD_Mu_crystall", "OHCU_decarbox", "OKR_DC_1", "OKR_DC_1_C", "OMPdecase", "OSR1_C", "OTCace", "OTCace_N", "OTU", "Octopine_DH", "Omptin", "OrfB_IS605", "Orn_Arg_deC_N", "Orn_DAP_Arg_deC", "OsmC", "Osmo_MPGsynth", "Ost4", "Oxidored-like", "Oxidored_FMN", "Oxidored_molyb", "Oxidored_nitro", "Oxidored_q2", "Oxidored_q3", "Oxidored_q4", "Oxidored_q6", "P-mevalo_kinase", "P5CR_dimer", "PA", "PA14", "PAD", "PAD_porph", "PAF-AH_p_II", "PALP", "PAP2", "PAPS_reduct", "PAP_assoc", "PAP_central", "PARG_cat", "PB1", "PBD", "PBP5_C", "PBP_dimer", "PCO_ADO", "PCP_red", "PDEase_I", "PDEase_II", "PDH", "PDT", "PEMT", "PEP-utilisers_N", "PEP-utilizers", "PEP-utilizers_C", "PEPCK_ATP", "PEPCK_C", "PEPcase", "PFK", "PFL-like", "PGAP1", "PGI", "PGK", "PGM_PMM_I", "PGM_PMM_II", "PGM_PMM_III", "PGM_PMM_IV", "PGP_phosphatase", "PG_binding_1", "PH", "PHB_depo_C", "PHP", "PI-PLC-X", "PI-PLC-Y", "PI3_PI4_kinase", "PIG-H", "PIG-P", "PIGA", "PIP5K", "PK", "PKD", "PK_C", "PLA1", "PLA2G12", "PLA2_B", "PLAT", "PLDc", "PLDc_2", "PMI_typeI", "PMM", "PMSR", "PMT", "PNGaseA", "PNK3P", "PNP_UDP_1", "PNP_phzG_C", "PNPase_C", "PNTB", "PNTB_4TM", "POR", "POR_N", "PP2C", "PPC", "PPDK_N", "PPK2", "PPO1_DWL", "PP_kinase", "PP_kinase_C", "PP_kinase_N", "PRA-CH", "PRA-PH", "PRAI", "PRK", "PRMT5", "PS-DH", "PSS", "PS_Dcarbxylase", "PS_pyruv_trans", "PTAC", "PTA_PTB", "PTE", "PTPA", "PTPLA", "PTPS", "PTase_Orf2", "PUA", "PUD", "PYC_OADA", "PYNP_C", "P_proprotein", "PaO", "PaaA_PaaC", "PagL", "Palm_thioest", "Pan_kinase", "Pantoate_ligase", "Pantoate_transf", "Patatin", "PcrB", "PdxA", "PdxJ", "Pec_lyase", "Pec_lyase_C", "Pectate_lyase", "Pectinesterase", "Penicil_amidase", "PepSY", "PepX_C", "Pep_deformylase", "Peptidase_A24", "Peptidase_A4", "Peptidase_A8", "Peptidase_C1", "Peptidase_C11", "Peptidase_C12", "Peptidase_C13", "Peptidase_C14", "Peptidase_C15", "Peptidase_C1_2", "Peptidase_C2", "Peptidase_C25", "Peptidase_C26", "Peptidase_C48", "Peptidase_C50", "Peptidase_C65", "Peptidase_M1", "Peptidase_M10", "Peptidase_M10_C", "Peptidase_M13", "Peptidase_M13_N", "Peptidase_M14", "Peptidase_M15", "Peptidase_M15_3", "Peptidase_M17", "Peptidase_M17_N", "Peptidase_M18", "Peptidase_M19", "Peptidase_M2", "Peptidase_M20", "Peptidase_M22", "Peptidase_M23", "Peptidase_M24", "Peptidase_M28", "Peptidase_M29", "Peptidase_M3", "Peptidase_M32", "Peptidase_M35", "Peptidase_M4", "Peptidase_M43", "Peptidase_M48", "Peptidase_M49", "Peptidase_M4_C", "Peptidase_M7", "Peptidase_M8", "Peptidase_M9", "Peptidase_S10", "Peptidase_S11", "Peptidase_S13", "Peptidase_S15", "Peptidase_S24", "Peptidase_S26", "Peptidase_S32", "Peptidase_S51", "Peptidase_S55", "Peptidase_S58", "Peptidase_S6", "Peptidase_S66", "Peptidase_S7", "Peptidase_S9_N", "Peripla_BP_6", "Peroxidase_2", "Peroxidase_ext", "PfkB", "PgpA", "Phe_hydrox_dim", "Phenol_Hydrox", "PhnG", "PhnH", "PhnI", "PhnJ", "PhoD", "Phos_pyr_kin", "Phosphodiest", "Phosphoesterase", "Phospholip_A2_1", "Phosphorylase", "Photo_RC", "PhyH", "Phytase", "Phytochelatin", "PhzC-PhzF", "PilZ", "Pkinase_C", "PolyA_pol", "PolyA_pol_arg_C", "Polyketide_cyc", "Polyketide_cyc2", "Polysacc_deac_1", "Polysacc_lyase", "Polysacc_syn_2C", "Polysacc_synt_4", "Porphobil_deam", "Porphobil_deamC", "Ppx-GppA", "Prenylcys_lyase", "Prenyltrans", "Prenyltransf", "Pribosyl_synth", "Pribosyltran", "Pribosyltran_N", "Prismane", "Pro-kuma_activ", "Pro_3_hydrox_C", "Pro_Al_protease", "Pro_CA", "Pro_dh", "Pro_isomerase", "Pro_racemase", "Propep_M14", "Proteasome", "Proteasome_A_N", "Proton_antipo_N", "PrpF", "Pterin_4a", "Pterin_bind", "PurS", "Put_Phosphatase", "Putative_PNPOx", "PvlArgDC", "Pyr_redox", "Pyr_redox_2", "Pyr_redox_3", "Pyr_redox_dim", "Pyridox_ox_2", "Pyridox_oxase_2", "Pyridoxal_deC", "Pyrophosphatase", "QRPTase_C", "QRPTase_N", "QueC", "QueF", "QueF_N", "Queuosine_synth", "RAP", "RCC_reductase", "RGP", "RNR_N", "ROK", "RPE65", "RVP", "RVT_thumb", "Rad4", "Rad60-SLD", "Radical_SAM", "Raffinose_syn", "RasGEF", "Redoxin", "Reductase_C", "RelA_SpoT", "Reprolysin_3", "Reprolysin_5", "Reticulon", "RhgB_N", "Rhodanese", "Rhomboid", "Rhomboid_N", "RibD_C", "Rib_5-P_isom_A", "Ribonuc_red_lgC", "Ribonuc_red_lgN", "Ribonuc_red_sm", "Ribophorin_I", "Ribophorin_II", "Ribul_P_3_epim", "Ricin_B_lectin", "Rieske", "Ring_hydroxyl_A", "Ring_hydroxyl_B", "Rnf-Nqr", "Rotamase", "Rotamase_2", "RraA-like", "RuBisCO_large", "RuBisCO_small", "Rubredoxin", "Rubrerythrin", "S-AdoMet_synt_C", "S-AdoMet_synt_M", "S-AdoMet_synt_N", "S-methyl_trans", "S6PP", "SAICAR_synt", "SAM_MT", "SAM_adeno_trans", "SAM_decarbox", "SASA", "SATase_N", "SBP_bac_1", "SCP2", "SDH_alpha", "SDH_beta", "SE", "SET", "SGL", "SH2", "SH3_2", "SH3_3", "SH3_5", "SH3_9", "SHMT", "SIP", "SIS", "SIS_2", "SKI", "SLBB", "SLT", "SNO", "SOR_SNZ", "SPOR", "SPOUT_MTase", "SQHop_cyclase_C", "SQHop_cyclase_N", "SQS_PSY", "START", "STT3", "SUA5", "Sacchrp_dh_C", "Sacchrp_dh_NADP", "Salt_tol_Pase", "SapB_1", "Scytalone_dh", "Se-cys_synth_N", "SelR", "Semialdhyde_dh", "Semialdhyde_dhC", "Shikimate_DH", "Shikimate_dh_N", "SicP-binding", "Sirohm_synth_C", "Sirohm_synth_M", "SnoaL", "SnoaL_2", "SnoaL_4", "Sod_Cu", "Sod_Fe_C", "Sod_Fe_N", "Sod_Ni", "Sortase", "SoxD", "SoxG", "Spb1_C", "Spermine_synth", "SpoIIE", "Stealth_CR2", "Sterile", "Steroid_dh", "Sterol_MT_C", "Str_synth", "Succ_DH_flav_C", "Sucrose_synth", "SufE", "Sugar-bind", "Sugar_tr", "Sulfatase", "Sulfotransfer_1", "Sulfotransfer_2", "Sulfotransfer_3", "SurA_N", "SurE", "Syja_N", "Synthase_beta", "TAL_FSA", "TENA_THI-4", "TGS", "THDPS_N_2", "THF_DHG_CYH", "THF_DHG_CYH_C", "THUMP", "TIM", "TK", "TMP-TENI", "TOBE", "TPK_B1_binding", "TPK_catalytic", "TPMT", "TPP_enzyme_C", "TPP_enzyme_M", "TPP_enzyme_N", "TRAM_LAG1_CLN8", "TSCPD", "TTL", "Tagatose_6_P_K", "Tannase", "TaqI_C", "TauD", "Tautomerase", "TehB", "Thaumatin", "ThiC-associated", "ThiC_Rad_SAM", "ThiG", "ThiI", "ThiP_synth", "ThiS", "Thioesterase", "Thiolase_C", "Thiolase_N", "Thioredoxin", "Thioredoxin_2", "Thioredoxin_3", "Thioredoxin_8", "Thr_synth_N", "Thy1", "Thymidylat_synt", "Thymidylate_kin", "TilS_C", "Tim17", "Tocopherol_cycl", "Topo-VIb_trans", "Topo_C_assoc", "Topo_Zn_Ribbon", "Transferase", "Transglut_core", "Transglut_prok", "Transgly", "Transket_pyr", "Transketolase_C", "Transketolase_N", "Transpeptidase", "Transthyretin", "Trehalase", "Trehalase_Ca-bi", "Trehalose_PPase", "Trm112p", "Trp_DMAT", "Trp_dioxygenase", "Trp_syntA", "TruB-C_2", "TruB_N", "TrwC", "Trypsin", "TylF", "Tyrosinase", "UBA_3", "UCH", "UCH_1", "UDPGP", "UDPGT", "UDPG_MGDP_dh", "UDPG_MGDP_dh_C", "UDPG_MGDP_dh_N", "UME", "UPF0004", "UPF0052", "UPRTase", "UQ_con", "URO-D", "USP7_C2", "USP7_ICP0_bdg", "USP8_dimer", "UbiA", "UbiD", "Ubie_methyltran", "UnbV_ASPIC", "Urease_alpha", "Urease_beta", "Urease_gamma", "Ureidogly_lyase", "Uricase", "Urocanase", "Usp", "UxaC", "UxuA", "VCBS", "VIT", "VKG_Carbox", "VKOR", "VanY", "VitK2_biosynth", "Vps4_C", "WBS_methylT", "WES_acyltransf", "WSC", "WW", "Wyosine_form", "X8", "XFP", "XFP_C", "XFP_N", "XdhC_C", "XdhC_CoxI", "YHS", "Y_phosphatase", "Y_phosphatase2", "Y_phosphatase3", "Yae1_N", "YbgT_YccB", "YdjC", "YgbB", "YhhN", "YjeF_N", "YmdB", "YqaJ", "Zeta_toxin", "Zn_dep_PLPC", "bact-PGI_C", "dCMP_cyt_deam_1", "dCMP_cyt_deam_2", "dNK", "dUTPase", "dUTPase_2", "fn3_3", "gag_pre-integrs", "hGDE_N", "iPGM_N", "ketoacyl-synt", "malic", "peroxidase", "polyprenyl_synt", "rhaM", "tRNA-synt_2", "zf-UBP", ] def __init__(self, **kwargs): super().__init__(**kwargs) # NOTE: self.add_tokens(domain_tokens, special_tokens=True) does not work # with get_special_tokens_mask(). # Instead use self.add_special_tokens(). # https://github.com/huggingface/transformers/issues/12936 domain_tokens = [f"<{domain}>" for domain in self.BGC_DOMAINS] domain_tokens_dict = {"additional_special_tokens": domain_tokens} self.add_special_tokens(domain_tokens_dict)