diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/data/whitelist/tts.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/data/whitelist/tts.tsv new file mode 100644 index 0000000000000000000000000000000000000000..758691676048f0894e2648c9732fdec92da735d9 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/data/whitelist/tts.tsv @@ -0,0 +1,3890 @@ +Ph.D. PHD +Hon. honorable +Mt. Mount +Maj. Major +Rev. Reverend +w/o without +a/c number account number +c/o care of +Gov. governor +vs. versus +vs versus +dept. department +prof. professor +est. estimated +vol volume +vol. volume +bldg. building +Bldg. Building +apt. apartment +Apt. Apartment +World War I World War one +World War II World War two +etc. etcetera. +SnO2 tin four oxide +dept department +HVAC H-vac +SPDR spider +ZIP zip +~ approximately +κ kappa +ω omega +α alpha +ν nu +δ delta +Δ delta +Α alpha +β beta +Β beta +χ chi +Χ chi +ε epsilon +Ε epsilon +φ phi +Φ phi +γ gamma +Γ gamma +η eta +Η eta +ι iota +Ι iota +κ kappa +Κ kappa +λ lambda +Λ lambda +μ mu +Μ mu +ν nu +Ν nu +ο omicron +Ο omicron +π pi +Π pi +θ theta +Θ theta +ρ rho +Ρ rho +σ sigma +Σ sigma +τ tau +Τ tau +υ upsilon +Υ upsilon +ω omega +Ω omega +ξ xi +Ξ xi +ψ psi +Ψ psi +ζ zeta +Ζ zeta +ltd limited +int'l international +A. D AD +A.D AD +a. d AD +a.d AD +a. d. AD +a.d. AD +B. C BC +B.C BC +b. c BC +b.c BC +A. D. AD +A.D. AD +B. C. BC +B.C. BC +b. c. BC +b.c. BC +A. A. a a +A.A. AA +A&A AANDA +A100 AONEHUNDRED +AAAs AAA's +AaB AAB +Aabo AABO +aac AAC +AACNo AACNO +AADTs AADT's +Aadu AADU +AAFld AAFLD +Aap AAP +A. B. AB +A.B. AB +Abhi ABHI +abl ABL +Abl ABL +Abp ABP +Abr ABR +abv ABV +Abz ABZ +A&C AANDC +A. C. AC +A.C. AC +A/C AC +acac ACAC +Acad ACAD +ACC&S ACC and S +Achs ACH's +Acis ACI's +acls ACLS +ACMs ACM's +AcpA ACPA +AcpB ACPB +acq ACQ +A&CR AANDCR +AcSOC ACSOC +acu ACU +AC&W AC and W +Acy ACY +A&D AANDD +Adab ADAB +A. D. AD +A.D. AD +A.D.A.M. adam +AdaSL ADASL +AdC ADC +ADCs ADC's +AD&D AD and D +AdK ADK +Adl ADL +ADLs ADL's +adm ADM +Adm ADM +admn ADMN +ADNs ADN's +AdP ADP +ADRs ADR's +ADSRs ADSR's +ADSs ADS's +Adwa ADWA +A&E AANDE +A. E. AE +A.E. AE +Aed AED +aegte AEGTE +Aeka AEKA +ael AEL +Aema AEMA +aen AEN +aere AERE +AEre AERE +AErn AERN +Aert AERT +Aes AE's +aet AET +Afa AFA +af AF +Af AF +A. F. AF +A.F. AF +Afaf AFAF +afc AFC +AFCSThe AFCSTHE +Afd AFD +AfD AFD +AfDB AFDB +Afe AFE +Aff AFF +Afi AFI +Afiq AFIQ +AFMs AFM's +AfNS AFNS +Afo AFO +Afr AFR +A. G. AG +A.G. AG +Agco AGCO +Agda AGDA +AG&G AG and G +agg AGG +Agi AGI +Agis AGI's +agli AGLI +Agn AGN +AgN AGN +AGNs AGN's +AGRs AGR's +Agsu AGSU +agt AGT +Agte AGTE +Agu AGU +A. H. AH +A.H. AH +Ahdhra AHDHRA +ahe AHE +Ahe AHE +Ahirs AHIR's +Ahk AHK +Ahsa'i AHSAI +Ahta AHTA +aht AHT +Ahu AHU +Ahva AHVA +A&I AANDI +A. I. AI +A.I. AI +Aias AIA's +aici AICI +Aicme AICME +AICs AIC's +Aiel AIEL +Aife AIFE +AiG AIG +Aigis AIGI's +Aija AIJA +Aik AIK +AIs AI's +Aist AIST +Aitu AITU +Aiud AIUD +aius AIUS +A. J. AJ +A.J. AJ +Ajba AJBA +Ajmi AJMI +Ajok AJOK +Ajsa AJSA +Aju AJU +Akaa AKAA +a.k.a. AKA +aka. AKA +aka AKA +Aka. AKA +Aka AKA +A&K AANDK +A. K. AK +A.K. AK +AKAPs AKAP's +akas AKA's +AKAs AKA's +akh AKH +Akpa AKPA +Aku AKU +A&L AANDL +A. L. AL +A.L. AL +Alh ALH +ALUs ALU's +A'ma AMA +A&M AANDM +a.m. AM +a.m AM +A. M. AM +A.M. AM +AMAs AMA's +amb AMB +Amb AMB +AmBX AMBX +AMCs AMC's +Amda AMDA +Amd AMD +AM&FM AM and FM +Amha AMHA +aml AML +Amli AMLI +Amlwch AMLWCH +Amm AMM +AM&O AM and O +Ampt AMPT +amr AMR +A&M's AANDM's +Ams' AM's +amsl AMSL +Amta AMTA +amt AMT +Amt AMT +Amts AMT's +Amu AMU +A. N. AN +A.N. AN +andFHL ANDFHL +andPPP ANDPPP +Anej ANEJ +Angti ANGTI +Angu ANGU +Anhe ANHE +Anr ANR +Anrep ANREP +A&O AANDO +A. O. AO +A.O. AO +AOCCs AOCC's +Aodh AODH +Aodla AODLA +Aogo AOGO +Aoke AOKE +Aoko AOKO +Aola AOLA +Aol AOL +Aone AONE +Aoni AONI +Aoos AOO's +aor AOR +AORs AOR's +aos AOS +AoS AOS +Aotus AOTU's +aov AOV +aovf AOVF +A&P AANDP +A. P. AP +A.P. AP +Apc APC +APCh APCH +APCs APC's +apg APG +Aph APH +Apiao APIAO +Api API +APi API +Apic APIC +APICv APICV +Apl APL +Aplu APLU +Apphttp APPHTTP +Appl APPL +appr APPR +Appts APPT's +appu APPU +Appu APPU +A&P's AANDP's +Aps AP's +APs AP's +AP&T AP and T +Aqa AQA +Aql AQL +Aqr AQR +A&R AANDR +A. R. AR +A.R. AR +ArgR ARGR +ARGs ARG's +ArH ARH +ARMv ARMV +Arnd ARND +arv ARV +Arv ARV +Arwi ARWI +As'ad ASAD +Asai ASAI +Asao ASAO +A.s A's +A. S. AS +A.S. AS +Asasp ASASP +asbl ASBL +asci ASCI +Asci ASCI +Asdis ASDI's +Asfi ASFI +Asn ASN +Aso ASO +Aspe ASPE +Asr ASR +Assn ASSN +assoc ASSOC +Assoc ASSOC +ASSPs ASSP's +Asst ASST +Aste ASTE +Astt ASTT +Aswa ASWA +A&T AANDT +atac ATAC +Atac ATAC +Atad ATAD +Ata's ATA's +Atas ATA's +A. T. AT +A.T. AT +ATBs ATB's +atc ATC +atcc ATCC +Atg ATG +ath ATH +Ath ATH +Atia's ATIA's +Ati ATI +ATi ATI +Atid ATID +Atiiq ATIIQ +Atil ATIL +Atiq ATIQ +Ativ ATIV +Atka ATKA +Atl ATL +Atli ATLI +atm ATM +Atm ATM +ATMs ATM's +Atos ATO's +ATOs ATO's +atpB ATPB +atri ATRI +Atri ATRI +A&T's AANDT's +A.T.s AT's +AT&SF AT and SF +AT&T ATANDT +attd ATTD +AT&T's ATANDTS +atv ATV +ATVs ATV's +AtxA ATXA +A. U. AU +A.U. AU +Aub AUB +Aubl AUBL +AUVs AUV's +Auw AUW +avab AVAB +A. V. AV +A.V. AV +AVCs AVC's +avg AVG +Avn AVN +Avo AVO +AvP AVP +AVRs AVR's +Avs' AV's +Avs AV's +avvo AVVO +Awa AWA +AWAs AWA's +A. W. AW +A.W. AW +Awo AWO +Awwa AWWA +A. X. AX +A. Y. AY +A.Y. AY +A. Z. AZ +A.Z. AZ +Azg AZG +Azi AZI +azm AZM +AzTV AZTV +Baad BAAD +Ba'al BAAL +Baal BAAL +Baam BAAM +Baap BAAP +ba'as BAA's +Baat BAAT +Ba'ath BAATH +B. A. BA +B.A. BA +B&A B and A +Bac BAC +bae BAE +Bae BAE +BAe BAE +Bauw BAUW +B'Av BAV +B&B B and B +bb BB +B. B. BB +B.B. BB +bbc BBC +BB&CI BB and CI +BBC&PJR BBC and PJR +BBQ'er BBQER +B&Bs B and B's +BB&S BB and S +BBSes BBSE's +BB&T BB and T +BbvCI BBVCI +bc BC +B. C. BC +B.C. BC +Bcci BCCI +B.C.'s BC's +BCs BC's +BCSic BCSIC +BdA BDA +bd BD +B. D. BD +B.D. BD +Bde BDE +bds BDS +BdU BDU +B. E. BE +B.E. BE +Bedw BEDW +bef BEF +bei BEI +Bei BEI +Beih BEIH +Beis BEI's +bej BEJ +BEMs BEM's +Ber BER +Bes BE's +bf BF +B. F. BF +B.F. BF +BFFs BFF's +bfy BFY +B. G. BG +B.G. BG +BglII BGLII +bgs BGS +Bgy BGY +Bha BHA +B&H B and H +B. H. BH +B.H. BH +BH&E BH and E +Bhe BHE +BHMs BHM's +bhttp BHTTP +B. I. BI +B.I. BI +Bie BIE +biedt BIEDT +Biem BIEM +B.I.G.'s BIG's +BiH BIH +B. J. BJ +B.J. BJ +BJPs BJP's +B.J.'s BJ's +B&K B and K +B. K. BK +B.K. BK +BKCa BKCA +B. L. BL +B.L. BL +ble BLE +bleg BLEG +blev BLEV +bli BLI +Blla BLLA +BLR&D BLR and D +BLTs BLT's +B&M B and M +B. M. BM +B.M. BM +bmd BMD +bmi BMI +BMPs BMP's +BMRs BMR's +bmt BMT +B&N B and N +bnb BNB +B. N. BN +B.N. BN +Boac BOAC +B&O B and O +B. O. BO +B.O. BO +Boc BOC +BoC BOC +BOCs BOC's +boj BOJ +Boj BOJ +B&O's B and O's +bp BP +B. P. BP +B.P. BP +bpl BPL +B&Q B and Q +B. Q. BQ +Brbic BRBIC +B. R. BR +B.R. BR +BRBs BRB's +Brne BRNE +Brno BRNO +BRTs BRT's +Brza BRZA +brzu BRZU +brzy BRZY +Brzyk BRZYK +BsaL BSAL +B. s B's +B.'s B's +Bs B's +B. S. BS +B.S. BS +bsd BSD +bt BT +B. T. BT +B.T. BT +Btry BTRY +btsan BTSAN +btus BTUS +BTUs BTU's +B.U. BU +B. V. BV +B.V. BV +Bwa BWA +B&W B and W +B. W. BW +B.W. BW +BWTs BWT's +bwwtv BWWTV +Bxa BXA +Bxe BXE +B. Y. BY +byc BYC +bygd BYGD +Byk BYK +byn BYN +Byo BYO +Byou BYOU +Byrl BYRL +bySLC BYSLC +byt BYT +B.Z. BZ +bzhed BZHED +Bzik BZIK +Bzyb BZYB +Caat CAAT +C. A. CA +C.A. CA +C&A CANDA +CA&CC CA and CC +Cadw CADW +caeca CAECA +Cae CAE +C.A.M.'s CAM's +Capt. captain +cas CAS +Cas CA's +Casc CASC +cb CB +C. B. CB +C.B. CB +cbc CBC +CB&CNS CB and CNS +CBDs CBD's +CBGBs CBGB's +CbiXS CBIXS +CB&Q CB and Q +CBs CB's +'c C +CcaA CCAA +cca CCA +Cca CCA +C&C C and C +C. C. CC +C.C. CC +ccd CCD +CCDev CCDEV +CCDs CCD's +CCEd CCED +CCi CCI +ccm CCM +CCRCs CCRC's +cct CCT +ccus CCUS +C&D C and D +cdc CDC +cd CD +C. D. CD +C.D. CD +CD&DR CD and DR +CDi CDI +cDNA CDNA +cdnas CDNAS +cDNAs CDNA's +CDOs CDO's +CDPs CDP's +CD&R CD and R +cds CDS +CDs CD's +CDSPCo CDSPCO +CdtA CDTA +CdTe CDTE +CDTi CDTI +CD&V CD and V +Cec CEC +c'e CE +ce CE +C'e CE +Ce CE +C. E. CE +C.E. CE +Ceel CEEL +Cefn CEFN +C&EI's C and EI's +C&EN C and EN +cen CEN +Cen CEN +ceo CEO +Ceol CEOL +Ceorl CEORL +Ceos CEO's +CEOs' CEO's +CEOs CEO's +cer CER +Cer CER +ces CES +Ces CE's +Cex CEX +CeX CEX +Cfa CFA +C&F C and F +cf CF +C. F. CF +C.F. CF +CFOs CFO's +CFTs CFT's +C&G C and G +C. G. CG +C.G. CG +cgl CGL +C&H C and H +C. H. CH +C.H. CH +Chhu CHHU +Chirs CHIR's +CHP'den CHPDEN +CHPs CHP's +Chy CHY +Cia CIA +ci CI +Ci CI +C. I. CI +C.I. CI +Cicic CICIC +cii CII +cim CIM +Cim CIM +Cio CIO +CIOs CIO's +Cip CIP +cis CIS +CIs CI's +cit CIT +CiTD CITD +citS CITS +CitX CITX +Ciuc CIUC +Ciu CIU +CiU CIU +Cixi CIXI +C. J. CJ +C.J. CJ +C. K. CK +C.K. CK +ckx CKX +CLBs CLB's +clc CLC +cl CL +C. L. CL +C.L. CL +Cle CLE +cllr CLLR +CL&N CL and N +Clo CLO +CL&P CL and P +CLs CL's +CLTs CLT's +Clu CLU +Clwyd CLWYD +C&MA C and MA +cm CM +C. M. CM +C.M. CM +CMCs CMC's +Cmde CMDE +cmdr CMDR +cml CML +CMMs CMM's +cmn CMN +Cmte CMTE +cn CN +C. N. CN +C.N. CN +CNMs CNM's +cnr CNR +CNs CN's +CNTs CNT's +Cnut's CNUT's +CNVs CNV's +C&O C and O +C. O. CO +C.O. CO +Co. company +Col. colonel +C&P C and P +cp CP +C. P. CP +C.P. CP +CPCs CPC's +cpo CPO +CPUs CPU's +cq CQ +CQDs CQD's +cr CR +C. R. CR +C.R. CR +Cre CRE +crk CRK +Crkva CRKVA +crkve CRKVE +CRLs CRL's +crm CRM +Crne CRNE +Crni CRNI +Crno CRNO +CRs CR's +csa CSA +Csa CSA +Csak CSAK +Csaky CSAKY +C&S C and S +cs CS +C.s C's +Cs C's +C. S. CS +C.S. CS +C&SF C and SF +csg CSG +CsI CSI +CSis CSI's +CSIs CSI's +CSOs CSO's +cspA CSPA +csp CSP +CSPs CSP's +CSSNCs CSSNC's +Csuz CSUZ +CTAs CTA's +C&T C and T +cte CTE +CTLs CTL's +ctr CTR +Ctrip CTRIP +ctv CTV +cu CU +C. U. CU +Cuyp CUYP +C&V C and V +cvcp CVCP +'cv CV +cv CV +C. V. CV +C.V. CV +CVEs CVE's +CVOs CVO's +CVs CV's +CVTs CVT's +C. W. CW +C.W. CW +Cwej CWEJ +Cwele CWELE +cwm CWM +cwrt CWRT +C.X. CX +C. Y. CY +C.Y. CY +cyl CYL +Cyn CYN +CyP CYP +Cyrl CYRL +Cys CY's +czci CZCI +cz CZ +C.Z. CZ +czy CZY +Daai DAAI +D. A. DA +D.A. DA +D&AD DANDAD +D.A.N.C.E. dance +Dav DAV +dBa DBA +db DB +D. B. DB +D.B. DB +DCCs DCC's +D&C D and C +dc DC +D. C. DC +D.C. DC +DCom DCOM +D.C.'s DC's +DCs DC's +DCUs DCU's +D&D D and D +D. D. DD +D.D. DD +DDi DDI +Ddoc DDOC +Ddu DDU +Deah DEAH +D. E. DE +D.E. DE +deg DEG +dei DEI +Dei DEI +DeI DEI +Deijm DEIJM +Deip DEIP +dez DEZ +Dez DEZ +Dfa DFA +DFCs DFC's +D. F. DF +D.F. DF +DfE DFE +DfES DFES +DFPs DFP's +DFs DF's +dfx DFX +D&G D and G +D. G. DG +D.G. DG +Dha DHA +DHBs DHB's +D&H D and H +dh DH +D. H. DH +D.H. DH +d'HEC DHEC +dhe DHE +Dheku DHEKU +Dhod DHOD +dhol DHOL +Dhol DHOL +Dhoo DHOO +Dhor DHOR +Dhou DHOU +dhr DHR +DHSs DHS's +D.I. DI +diy DIY +DiY DIY +Diyn DIYN +DIYs DIY's +Djam DJAM +dj DJ +D. J. DJ +D.J. DJ +DJ'ed DJED +Djem DJEM +Dji DJI +Djo DJO +D'Jok DJOK +djr DJR +D.J.s DJ's +DJs DJ's +DjVu DJVU +DjVus DJVU's +D. K. DK +D.K. DK +dla DLA +dl DL +D. L. DL +D.L. DL +dlia DLIA +DL&LR DL and LR +DLLs DLL's +dlo DLO +DL&W DL and W +D.M.C.'s DMC's +D&M D and M +D. M. DM +D.M. DM +DMed DMED +DM&E DM and E +DMs DM's +DMUs DMU's +Dna DNA +DnA DNA +DnaJ DNAJ +D. N. DN +D.N. DN +Dnepr DNEPR +dnes DNES +Dnes DNE's +DNFs DNF's +DNSBLs DNSBL's +dns DNS +D.O.A.'s DOA's +dod DOD +Dod DOD +DoD DOD +D. O. DO +D.O. DO +D'oh DOH +dok DOK +Dok DOK +domt DOMT +Domt DOMT +Dop DOP +dotCMS DOTCMS +DotSVN DOTSVN +Douw DOUW +dozd DOZD +d'OZ DOZ +Dozhd DOZHD +dp DP +D. P. DP +D.P. DP +dpon DPON +DProf DPROF +DPs DP's +D.Q. DQ +drc DRC +Dr. doctor +D. R. DR +D.R. DR +dri DRI +Dri DRI +DRIs DRI's +Driu DRIU +Driv DRIV +Drnis DRNI's +Dro DRO +Drs. doctors +Drska DRSKA +druj DRUJ +DsbA DSBA +DS&BB DS and BB +dsDNA DSDNA +ds DS +Ds D's +D. S. DS +D.S. DS +DSGi DSGI +dsi DSI +DSi DSI +dsl DSL +DSMs DSM's +dsn DSN +D&SNGRR D and SNGRR +dsu' DSU +DTCs DTC's +DTDs DTD's +D. T. DT +D.T. DT +DT&E DT and E +DT&I DT and I +dtl DTL +DTs DT's +dty DTY +D.U. DU +dva DVA +Dva DVA +DVCs DVC's +dvd DVD +DVDs DVD's +dv DV +D. V. DV +D.V. DV +dve DVE +dvfb DVFB +dvi DVI +dvije DVIJE +Dvin DVIN +Dvir DVIR +Dwa DWA +D&W D and W +D. W. DW +D.W. DW +Dwedw DWEDW +Dwi DWI +dwr DWR +dwur DWUR +dxa DXA +dx DX +D.X. DX +DXers DXER's +Dydd DYDD +dy DY +Dy DY +D. Y. DY +dypl DYPL +dyr DYR +dz DZ +dzis DZIS +Dzog DZOG +Dzor DZOR +Dzus DZU's +E. A. EA +E.A. EA +EAMs EAM's +eas EAS +Eas EA's +Eav EAV +E. B. EB +E.B. EB +Ebn EBN +E&BR E and BR +Ebru EBRU +EBWs EBW's +ecc ECC +Eccl ECCL +ec EC +Ec EC +E. C. EC +E.C. EC +ece ECE +Ece ECE +Ecem ECEM +ecg ECG +ecma ECMA +ECNs ECN's +ECP&DA's ECP and DA's +eCRM ECRM +ect ECT +ECVs ECV's +edb EDB +E. D. ED +E.D. ED +Edh EDH +edhe EDHE +EDMs EDM's +Edmx EDMX +edn EDN +Edn EDN +eds EDS +Eds ED's +Edw EDW +Eeb EEB +E. E. EE +E.E. EE +Eef EEF +Eega EEGA +eene EENE +eep EEP +Ees EE's +eeuw EEUW +Eeuw EEUW +EEZs EEZ's +efc EFC +ef EF +Ef EF +E. F. EF +E.F. EF +eFS EFS +Efs EF's +Efu EFU +Egba EGBA +egfl EGFL +e.g. for example +e. g. for example +EG&G EG and G +Egi EGI +E&GR E and GR +egy EGY +Egyl EGYL +EHAs EHA's +E. H. EH +E.H. EH +ehf EHF +Ehttp EHTTP +EiCs EIC's +E. I. EI +E.I. EI +eIF EIF +Eitr EITR +EITs EIT's +EJBs EJB's +ej EJ +E. J. EJ +E.J. EJ +E. K. EK +E.K. EK +eki EKI +Ekma EKMA +Ekow EKOW +eks EKS +Eks EK's +E. L. EL +E.L. EL +ELTs ELT's +ELUs ELU's +Embd EMBD +eMC EMC +EMDs EMD's +Emea EMEA +E&M E and M +E. M. EM +E.M. EM +Emge EMGE +emra EMRA +E&NA E and NA +Enas ENA's +EnBW ENBW +EncFS ENCFS +Encyc ENCYC +E. N. EN +E.N. EN +Enes ENE's +engl ENGL +Engl ENGL +Engr ENGR +Eni ENI +enn ENN +Enn ENN +ens ENS +Ens EN's +Enso ENSO +ENTJs ENTJ's +EoD EOD +E. O. EO +E.O. EO +Eois EOI's +eok EOK +Eole EOLE +Eol EOL +Eolss EOLS's +Eom EOM +eop EOP +eoptta EOPTTA +Eora EORA +Eorl EORL +Eors EOR's +E&P E and P +Epe EPE +ep EP +Ep EP +E. P. EP +E.P. EP +Eph EPH +Ephs EPH's +epi EPI +Epi EPI +ePO EPO +epos' EPO's +epos EPOS +Epos EPO's +E.P.s EP's +eq EQ +Eq EQ +E. R. ER +E.R. ER +ERJs ERJ's +Eru ERU +Eruv ERUV +Erv ERV +ES3 ES three +Esa ESA +Esam ESAM +Esa's ESA's +Esat ESAT +ESCs ESC's +Esd ESD +E. S. ES +E.S. ES +esi ESI +espn ESPN +ESPNhttp ESPNHTTP +esq ESQ +Esq ESQ +Esq. esquire +Esraa ESRAA +Esra ESRA +Esref ESREF +Esri ESRI +estd ESTD +Estd ESTD +esu ESU +Esva ESVA +Etad ETAD +ETBs ETB's +E. T. ET +E.T. ET +ETFs ETF's +Et'hem ETHEM +Eti ETI +etj ETJ +Eto ETO +ets ETS +Ets ET's +ETs ET's +ett ETT +Ett ETT +Ettre ETTRE +Etts ETT's +Etz ETZ +EUBs EUB's +E. U. EU +E.U. EU +eup EUP +Eurwg EURWG +Euse EUSE +Eusi EUSI +EvaGT EVAGT +E. V. EV +E.V. EV +evl EVL +Evna EVNA +evnt EVNT +EVs EV's +evv EVV +Ewa's EWA's +E. W. EW +E.W. EW +Ewha EWHA +EWOs EWO's +Ewu EWU +excl EXCL +exd EXD +E.X. EX +exhb EXHB +exh EXH +Exptl EXPTL +exsul EXSUL +E&Y E and Y +E. Y. EY +Eyk EYK +eyu EYU +Ezaa EZAA +ez EZ +Ez EZ +Ezh EZH +Ezu EZU +Ezy EZY +Faaa FAAA +faa FAA +Faa FAA +faama FAAMA +FabH FABH +FabR FABR +F.A.B.'s FAB's +Fadl FADL +fa FA +Fa FA +F. A. FA +F.A. FA +Faf FAF +Fafhrd FAFHRD +fai FAI +Fai FAI +FAIPs FAIP's +Faiq FAIQ +fajn FAJN +Fal FAL +farw FARW +fasc FASC +Fasc FASC +fas FAS +Fas FA's +FAs FA's +FasL FASL +F. B. FB +F.B. FB +F.B.G.s FBG's +FBOs FBO's +fcb FCB +FCBs FCB's +F&C F and C +fc FC +F. C. FC +F.C. FC +F.C.'s FC's +FD&C FD and C +FDCPAs FDCPA's +F. D. FD +F.D. FD +FDICs FDIC's +FdI FDI +F. E. FE +F.E. FE +FEGs FEG's +Fes FE's +ffc FFC +ff FF +F. F. FF +F.F. FF +ffm FFM +FFs FF's +F. G. FG +F.G. FG +fgk FGK +FGs FG's +F. H. FH +F.H. FH +fhm FHM +fiadh FIADH +fia FIA +FiBL FIBL +Fi'd FID +F.I F +F&I F and I +F. I. FI +F.I. FI +Figl FIGL +fija FIJA +Fio FIO +FiO FIO +F. J. FJ +F.J. FJ +F. K. FK +F.K. FK +Fla FLA +Fles FLE's +fl FL +F. L. FL +F.L. FL +fli FLI +flyr FLYR +fm FM +F. M. FM +F.M. FM +fMRI FMRI +FMs FM's +F. N. FN +F.N. FN +F. O. FO +FPDs FPD's +fp FP +F. P. FP +F.P. FP +FPGAs FPGA's +FPM&SA FPM and SA +fps FPS +FPSs FPS's +F.Q. FQ +fr FR +F. R. FR +F.R. FR +frs FRS +Fru FRU +Frwydr FRWYDR +fsf FSF +fs FS +F. s F's +Fs F's +F. S. FS +F.S. FS +fsn FSN +FSSs FSS's +FTAs FTA's +Ft. Fort +F. T. FT +F.T. FT +ftp FTP +FtsA FTSA +Ftuh FTUH +F. V. FV +F.V. FV +F&W F and W +F. W. FW +F.W. FW +fwr FWR +fx FX +Fyb FYB +F.Y. FY +Fyn FYN +fyi FYI +fyr FYR +Fyw FYW +F.Z. FZ +Gaac GAAC +G. A. GA +G.A. GA +Gbe GBE +GbE GBE +G. B. GB +G.B. GB +GB&NDR GB and NDR +G&C G and C +G. C. GC +G.C. GC +GCSEs GCSE's +GC&SF GC and SF +Gdal GDAL +Gde GDE +gd GD +G. D. GD +G.D. GD +Gdow GDOW +GDPs GDP's +gdr GDR +Gebr GEBR +Gebt GEBT +Ged GED +ge GE +G. E. GE +G.E. GE +Gek GEK +Gen. general +geq GEQ +Geu GEU +Gev GEV +GeV GEV +Gfa GFA +GF&A GFANDA +G. F. GF +G.F. GF +G. G. GG +G.G. GG +Ghe GHE +G. H. GH +G.H. GH +GHGs GHG's +Ghir GHIR +ghra GHRA +GH&SA GH and SA +Giei GIEI +G. I. GI +G.I. GI +G. J. GJ +G.J. GJ +G. K. GK +G.K. GK +Gla GLA +G&L G and L +G. L. GL +G.L. GL +gli GLI +Gli GLI +Glis GLI's +glnA GLNA +Glos GLO's +gma GMA +GmbH GMBH +Gmel GMEL +gm GM +G. M. GM +G.M. GM +gmin GMIN +GMOs GMO's +GMs GM's +gmt GMT +Gmul GMUL +gnb GNB +G. N. GN +G.N. GN +GNP&BR GNP and BR +gnp GNP +GNPs GNP's +G.O. GO +GPCRs GPCR's +GPdI GPDI +G&P G and P +gp GP +G. P. GP +G.P. GP +GpIIb GPIIB +GPMGs GPMG's +GPRs GPR's +GPUs GPU's +G. Q. GQ +grac GRAC +gra GRA +Gra GRA +Grbac GRBAC +GRBs GRB's +grc GRC +GRCs GRC's +gre GRE +Gre GRE +gr GR +G. R. GR +G.R. GR +grrl GRRL +Gryf GRYF +gry GRY +Gry GRY +grz GRZ +G&S G and S +Gs G's +G. S. GS +G.S. GS +GSIs GSI's +GSOp GSOP +GTAs GTA's +gt GT +G. T. GT +G.T. GT +GTi GTI +GTs GT's +G. U. GU +G.U. GU +GUIs GUI's +G. V. GV +G.V. GV +GvpA GVPA +GWe GWE +Gwet GWET +gw GW +G. W. GW +G.W. GW +Gwi GWI +gwr GWR +Gy GY +G. Y. GY +Gyn GYN +Gyps GYP's +gyu GYU +Gyu GYU +H. A. HA +H.A. HA +H&A H and A +HBCo HBCO +HBGAs HBGA's +H. B. HB +H.B. HB +H&BR H and BR +HCEs HCE's +H. C. HC +H.C. HC +hcl HCL +HClO HCLO +HCoV HCOV +HCPs HCP's +Hcy HCY +Hdad HDAD +hDAF HDAF +HDDs HDD's +hd HD +H. D. HD +H.D. HD +hdh HDH +HDi HDI +H. E. HE +H.E. HE +hezb HEZB +Hezb HEZB +H&F H and F +H. F. HF +H.F. HF +hgcA HGCA +hg HG +H. G. HG +H.G. HG +HgO HGO +HGPs HGP's +HgU HGU +HGVs HGV's +H&H H and H +H. H. HH +H.H. HH +H. I. HI +H.I. HI +Hizb HIZB +hja HJA +H. J. HJ +H.J. HJ +Hkam HKAM +H&K H and K +H. K. HK +H.K. HK +Hla HLA +H. L. HL +H.L. HL +Hluk HLUK +hlutr HLUTR +Hly HLY +Hman HMAN +H&M H and M +hm HM +H. M. HM +H.M. HM +HMOs HMO's +hMRE HMRE +H&M's H and M's +H. N. HN +H.N. HN +H. O. HO +H.O. HO +hpc HPC +hp HP +H. P. HP +H.P. HP +H.Q. HQ +H&R H and R +H. R. HR +H.R. HR +hroa HROA +HRo HRO +hrs HRS +HRs HR's +hscy HSCY +Hsee HSEE +hs HS +Hs H's +H. S. HS +H.S. HS +hsi HSI +hSlo HSLO +HSPs HSP's +HSTs HST's +hsv HSV +hTAS HTAS +HTAs HTA's +htc HTC +ht HT +H. T. HT +H.T. HT +hti HTI +Htin HTIN +Htoo HTOO +HtrA HTRA +HTRs HTR's +http HTTP +https HTTPS +Htwa HTWA +H. U. HU +HVCs HVC's +hvcv HVCV +hvem HVEM +Hvem HVEM +H. V. HV +H.V. HV +H. W. HW +H.W. HW +hwy HWY +Hwy HWY +hxt HXT +H. Y. HY +H.Y. HY +H. Z. HZ +HZ&PC HZ and PC +iaaf IAAF +IaaS IAAS +Iaca IACA +Iacob IACOB +IACTs IACT's +Iacub IACUB +iagt IAGT +I. A. IA +I.A. IA +Iapa IAPA +iar IAR +Iar IAR +IAUCs IAUC's +Iax IAX +Ibac IBAC +Ibaes IBAE's +IBCs IBC's +IBDA'A IBDAA +Ibda IBDA +ibdal IBDAL +I. B. IB +I.B. IB +Ibm IBM +Ibne IBNE +ibn IBN +Ibra IBRA +Ibsa IBSA +Ibs IB's +iBT IBT +Ibu IBU +iby IBY +ICBMs ICBM's +Icche ICCHE +icf ICF +I&C I and C +ic IC +Ic IC +I. C. IC +I.C. IC +Ici ICI +iCN ICN +Ico ICO +IcRn ICRN +ics ICS +ICs IC's +ICTs ICT's +ICTVdB ICTVDB +ICv ICV +iCyt ICYT +Iddaa IDDAA +Idd IDD +ID'd IDD +id ID +I. D. ID +I.D. ID +Idi IDI +Idir IDIR +IDLHs IDLH's +Idm IDM +IDPs IDP's +ids IDS +IDs ID's +IDx IDX +i.e. that is +Ifa IFA +ifc IFC +IfF IFF +Ifft IFFT +ifi IFI +Ifi IFI +IFIs IFI's +IfM IFM +Ifni IFNI +IFNs IFN's +Ifop IFOP +IFRSs IFRS's +Iga IGA +IgA IGA +IGFs IGF's +'ig IG +Ig IG +I.G. IG +igi IGI +IgM IGM +ign IGN +Ign IGN +Ih IH +I. H. IH +ihi IHI +ihm IHM +ihn IHN +Ihn IHN +Ihor IHOR +Iht IHT +IIb IIB +IIc IIC +IIfx IIFX +Iiga IIGA +IIga IIGA +IIgs IIG's +IIGs IIG's +IIIb IIIB +IIIBy IIIBY +IIIc IIIC +IIId IIID +I. I. II +I.I. II +ija IJA +Ija IJA +ij IJ +I. J. IJ +Ijok IJOK +Ijui IJUI +Ikh IKH +I.K. IK +IKr IKR +Iksa IKSA +Iku IKU +Ilbe ILBE +IL&FS IL and FS +ili ILI +Ili ILI +I. L. IL +I.L. IL +Ilm ILM +ilu ILU +Ilu ILU +IMbd IMBD +imc IMC +imdb IMDB +IMDb. IMDB +IMDb IMDB +ime IME +I&M I and M +I. M. IM +I.M. IM +Iml IML +Imm IMM +Imms' IMM's +imoa IMOA +impr IMPR +Impr IMPR +iMSNs IMSN's +Imst IMST +Inba INBA +inb INB +incl INCL +ind IND +Ind IND +Infs INF's +inHg INHG +I. N. IN +I.N. IN +InlB INLB +InP INP +Inre INRE +Inspx INSPX +Ints INT's +INTs INT's +Intu INTU +inv INV +Ioba IOBA +IODs IOD's +io. IO +io IO +.I.o. IO +Io. IO +Io IO +.IO IO +I/O IO +Ioka IOKA +Iok IOK +IoM IOM +IOPs IOP's +iOS IOS +Io's IO's +I/Os IO's +I.O.'s IO's +Iosu IOSU +IoT IOT +IoW IOW +IPCs IPC's +ip IP +Ip IP +I.P. IP +Ipo IPO +IPPs IPP's +Ippu IPPU +iPPV IPPV +iPSC IPSC +iPSCs IPSC's +ipse IPSE +iPS IPS +IPs IP's +IPv IPV +iQ IQ +I.Q. IQ +Irgm IRGM +Irig IRIG +Iril IRIL +i'r IR +ir IR +Ir IR +I. R. IR +I.R. IR +Irla IRLA +Irmis IRMI's +iro IRO +Iro IRO +Irra IRRA +IRs IR's +iru IRU +Iru IRU +ISAv ISAV +Isba ISBA +isbn ISBN +ISBNs ISBN's +isCf ISCF +iSC ISC +Isc ISC +Isd ISD +isdn ISDN +Isgec ISGEC +ishq ISHQ +Ishq ISHQ +I. S. IS +I.S. IS +Isl ISL +ISMNs ISMN's +ISPs ISP's +Isra'il ISRAIL +Isra ISRA +iss ISS +Iss IS's +ISSNs ISSN's +Isu ISU +Ite ITE +ITHs ITH's +iti ITI +Iti ITI +I.T. IT +itk ITK +Itk ITK +ITNNs ITNN's +Itoi ITOI +iTP ITP +Itse ITSE +itt ITT +Itu ITU +Itz ITZ +Iucn IUCN +IUDs IUD's +Iuz IUZ +I. V. IV +I.V. IV +IVs IV's +Ivu IVU +Iwas IWA's +Iwaz IWAZ +IWBs IWB's +iwi IWI +iw IW +I. W. IW +iwrg IWRG +iwspy IWSPY +Iwuh IWUH +IXb IXB +IXBs IXB's +IXCs IXC's +IXe IXE +iXL IXL +ixtle IXTLE +Iya IYA +Iyar IYAR +iyem IYEM +Iyi IYI +iy IY +I. Y. IY +I.Y. IY +Jaf JAF +J. A. JA +J&B J and B +jb JB +J. B. JB +J.B. JB +jcis JCIS +jc JC +J. C. JC +J.C. JC +JCRs JCR's +JdeBP JDEBP +jdi JDI +jd JD +J. D. JD +J.D. JD +jdk JDK +J&D's J and D's +J.D.s JD's +J. E. JE +J.E. JE +jf JF +J. F. JF +J.F. JF +J. G. JG +J.G. JG +jh JH +J. H. JH +J.H. JH +JHs JH's +JHSVs JHSV's +JHud JHUD +J. I. JI +J.I. JI +J&J JANDJ +J. J. JJ +J.J. JJ +Jka JKA +jkd JKD +J&K J and K +J. K. JK +J.K. JK +jkx JKX +J. L. JL +J.L. JL +J. M. JM +J.M. JM +jnb JNB +J. N. JN +J.N. JN +Jno JNO +jnr JNR +J. O. JO +J.O. JO +jpg JPG +J&P J and P +jp JP +J.P. JP +J.Q. JQ +J&R J and R +J. R. JR +J.R. JR +Jr. junior +jr. junior +Jr junior +jr junior +J.R.'s JR's +jsb JSB +Js J's +J. S. JS +J.S. JS +J&T J and T +J. T. JT +J.T. JT +JT&KW JT and KW +J. U. JU +J&V J and V +J. V. JV +J.V. JV +Jwa JWA +J. W. JW +J.W. JW +jx JX +J. Y. JY +J.Y. JY +Jym JYM +Jymn JYMN +J. Z. JZ +J.Z. JZ +K8S KUBERNETES +K. A. KA +K.A. KA +kbi KBI +K&B K and B +K. B. KB +K.B. KB +K. C. KC +K.C. KC +K.C.'s KC's +kDa KDA +kdal KDAL +K&D K and D +kd KD +K. D. KD +K.D. KD +K. D.'s KD's +K.D.'s KD's +ke KE +Ke KE +K. E. KE +K.E. KE +kfc KFC +K.F. KF +kgr KGR +kgt KGT +KgU KGU +kgv KGV +Khizr KHIZR +K. H. KH +K.H. KH +Khlav KHLAV +Khmu KHMU +Khri KHRI +Khru KHRU +K. I. KI +K.I. KI +Kjer KJER +kj KJ +K. J. KJ +K.J. KJ +Kjop KJOP +KJo's KJO's +K. K. KK +K.K. KK +Kle KLE +kl KL +K. L. KL +K.L. KL +K&M K and M +km KM +K. M. KM +K.M. KM +K&N K and N +kn KN +K. N. KN +K.N. KN +k'o KO +K. O. KO +K.O. KO +KPIs KPI's +K&P K and P +K. P. KP +K.P. KP +Kppen KPPEN +kptm KPTM +krc KRC +KRCs KRC's +K.R.I.T.'s KRIT's +Krka KRKA +kr KR +K. R. KR +K.R. KR +Krne KRNE +kroz KROZ +Kroz KROZ +Kru KRU +ksa KSA +kset KSET +Kseur KSEUR +Ksevt KSEVT +ks KS +K. S. KS +K.S. KS +Ktav KTAV +KT&K KT and K +kt KT +K. T. KT +K.T. KT +Kuaa KUAA +Kud KUD +kuih KUIH +K. U. KU +K.U. KU +Kutb KUTB +Kvik KVIK +Kvit KVIT +K. V. KV +K.V. KV +Kvyat KVYAT +kwa KWA +kwe KWE +Kyse KYSE +kyt KYT +Kyt KYT +kz KZ +K. Z. KZ +L. A. LA +L.A.'s LA's +Lay's LAY's +LBi LBI +L. B. LB +L.B. LB +LBPs LBP's +LCAs LCA's +lcc LCC +LCCs LCC's +LC&DR LC and DR +LCDs LCD's +LCIs LCI's +L. C. LC +L.C. LC +lcn LCN +LCSs LCS's +LCTs LCT's +LCVPs LCVP's +Lda LDA +LDCs LDC's +L. D. LD +L.D. LD +L. E. LE +L.E. LE +LFFCs LFFC's +L. F. LF +L.F. LF +LFs LF's +LFSRs LFSR's +L. G. LG +L.G. LG +LGMs LGM's +LG&RDD LG and RDD +LGs LG's +LGUs LGU's +LGVs LGV's +Lha LHA +L. H. LH +L.H. LH +L. I. LI +L.I. LI +Lje LJE +L. J. LJ +L.J. LJ +L. K. LK +lks LKS +Llapi LLAPI +lle LLE +lli LLI +L&L L and L +L. L. LL +L.L. LL +llp LLP +llu LLU +LMGs LMG's +lm LM +L. M. LM +L.M. LM +LMPs LMP's +LMQs LMQ's +LnAIB LNAIB +L&N L and N +L. N. LN +L.N. LN +L.O.C.'s LOC's +Lokk LOKK +l'OL LOL +L. O. LO +L.O. LO +LPARs LPAR's +L&P L and P +lp LP +L. P. LP +L.P. LP +L&PM L and PM +LPMud LPMUD +LPs LP's +LPThe LPTHE +lr LR +L. R. LR +L.R. LR +LRTs LRT's +LRVs LRV's +LSDs LSD's +LSi LSI +ls LS +Ls L's +L. S. LS +L.S. LS +LSTs LST's +LSVCCs LSVCC's +Ltda LTDA +Ltd. limited +L&T L and T +Lt. lieutenant +lt LT +L. T. LT +L.T. LT +ltoh LTOH +L. U. LU +lv LV +L. V. LV +L.V. LV +L. W. LW +L.W. LW +L.Y. LY +L&YR L and YR +M. A. MA +M.A. MA +MAX-Q MAXQ +M&A M and A +Mbewu MBEWU +mbi MBI +M. B. MB +M.B. MB +mBo MBO +Mbre MBRE +mbr MBR +MCCs MCC's +M. C. MC +M.C. MC +MCs MC's +mcyG MCYG +mdb MDB +MDCs MDC's +mdDA MDDA +MD&DI MD and DI +Mde MDE +MDGs MDG's +MDHUs MDHU's +mDia MDIA +MDic MDIC +mdla MDLA +md MD +M. D. MD +M.D. MD +mdr MDR +M. E. ME +M.E. ME +MFDs MFD's +MFe MFE +M. F. MF +M.F. MF +MFTs MFT's +Mgadla MGADLA +Mgal MGAL +mga MGA +Mga MGA +Mgbo MGBO +MGen MGEN +M&G M and G +M. G. MG +M.G. MG +M&GN M and GN +M&GR's M and GR's +Mha MHA +MHCs MHC's +MH&L MH and L +M&H M and H +M. H. MH +M.H. MH +Mhor MHOR +Mhow MHOW +M.I.A.'s MIA's +M&I M and I +M. I. MI +M.I. MI +M&J M and J +M. J. MJ +M.J. MJ +M. K. MK +M.K. MK +MKs MK's +MLAs MLA's +mlc MLC +MLCs MLC's +mlg MLG +mli MLI +Mlle MLLE +M&L M and L +M. L. ML +M.L. ML +MLPs MLP's +MM&A MMANDA +M&M M and M +M. M. MM +M.M. MM +M&M's M and M's +MNCs MNC's +M&NF M and NF +M. N. MN +M.N. MN +M.O.G.U.E.R.A.'s MOGUERA's +M. O. MO +M.O. MO +MpA MPA +MPAs MPA's +MPBu MPBU +MP&I MP and I +MPi MPI +MPLMs MPLM's +M&P M and P +M. P. MP +M.P. MP +MPPs MPP's +M.P.'s MP's +MPs MP's +MPThe MPTHE +Mpu MPU +M. Q. MQ +Mra MRA +MRBs MRB's +mre MRE +MR&LE MR and LE +Mr. mister +M. R. MR +M.R. MR +mRNA MRNA +mRNAs MRNA's +Mrs. misses +MRTs MRT's +msd MSD +Mse MSE +M&S M and S +Ms. miss +M. S. MS +M.S. MS +MSPs MSP's +mst MST +Mta MTA +MTAs MTA's +M&T M and T +M. T. MT +M.T. MT +Mtor MTOR +mtvU MTVU +MTVu MTVU +M. U. MU +M.U.s MU's +MVMs MVM's +M. V. MV +M.V. MV +MVPs MVP's +MVs MV's +MWe MWE +M. W. MW +M.W. MW +M. X. MX +myb MYB +myc MYC +Myc MYC +MyDD MYDD +M. Y. MY +M.Y. MY +MySQL MYSQL +M. Z. MZ +N. A. NA +N.A. NA +N.B.A.'s NBA's +NBAs NBA's +N. B. NB +N.B. NB +nbs NBS +NBTwo NBTWO +NCAAs NCAA's +NCCs NCC's +N. C. NC +N.C. NC +NCOs NCO's +N. D. ND +N.D. ND +N. E. NE +N.E. NE +ner NER +NFATc NFATC +nfed NFED +N. F. NF +N.F. NF +Nge NGE +N. G. NG +N.G. NG +NGOs NGO's +Nha NHA +NHCEs NHCE's +NHCs NHC's +nhi NHI +NHM&W NHM and W +N. H. NH +N.H. NH +NHPs NHP's +N. I. NI +N.I. NI +N'I NI +N. J. NJ +N.J. NJ +Njoo NJOO +N.J.'s NJ's +N. K. NK +N.K. NK +nkvd NKVD +nkv NKV +N. L. NL +N.L. NL +NLRs NLR's +N. M. NM +N.M. NM +N. N. NN +N.N. NN +NPCs NPC's +NPMs NPM's +N. P. NP +N.P. NP +NPs NP's +N. Q. NQ +nri NRI +Nri NRI +NRIs NRI's +NRJs NRJ's +nr NR +N. R. NR +N.R. NR +N. S. NS +N.S. NS +NSOs NSO's +Nta NTA +ntb NTB +N. T. NT +N.T. NT +ntw NTW +N.U. NU +NvDA's NVDA's +N. V. NV +N.V. NV +nwa NWA +N&W N and W +N. W. NW +N.W. NW +nyc NYC +N. Y. NY +N.Y. NY +NX-SOC NXSOC +N'Zif NZIF +NZiK NZIK +N'Zi NZI +N.Z. NZ +O. A. OA +O.A. OA +obl OBL +Oblt OBLT +O.B. OB +O. C. OC +O.C. OC +Octl OCTL +O. D. OD +O.E. OE +O.F. OF +ofr OFR +O&G O and G +O. G. OG +O.G. OG +O. H. OH +O.H. OH +O. I. OI +O.I. OI +O. J. OJ +O.J. OJ +OK'd OKD +OKd OKD +oke OKE +Oke OKE +O&K O and K +O. K. OK +O.K. OK +O. L. OL +O.L. OL +Olo OLO +Olov OLOV +ols OLS +OLs OL's +olvwm OLVWM +olwm OLWM +O&MFL O and MFL +omg OMG +OMGs OMG's +O. M. OM +O.M. OM +OmOm OMOM +Om's OM's +oncu ONCU +ond OND +onf ONF +O. N. ON +O.N. ON +O&O O and O +O. O. OO +O.O. OO +Oop OOP +oor OOR +oose OOSE +Oo's OO's +Ootw OOTW +OPMs OPM's +O. P. OP +O.P. OP +opr OPR +Opr OPR +ORFs ORF's +OR&N OR and N +O. R. OR +osaa OSAA +osa OSA +Osa OSA +OSBs OSB's +O. S. OS +O.S. OS +OSTs OST's +osv OSV +O.T. OT +OTs OT's +Otu OTU +otv OTV +O.U. OU +ovca OVCA +Ovca OVCA +Ovda OVDA +Ovo OVO +ov OV +Ov OV +O. V. OV +O.V. OV +OWCs OWC's +O. W. OW +O.W. OW +pa PA +P. A. PA +P.A. PA +pBCE PBCE +pbc PBC +pb PB +P. B. PB +P.B. PB +PCBs PCB's +PCDDs PCDD's +PCeU PCEU +PCIe PCIE +pci PCI +pcl PCL +PcoA PCOA +pc PC +P. C. PC +P.C. PC +PCRev PCREV +pcs PCS +P.C.s PC's +pDAB PDAB +PDAs PDA's +pdbp PDBP +PDBsum PDBSUM +PDCs PDC's +PD&D PD and D +PDEs PDE's +pdf PDF +PDFs PDF's +P&D P and D +P. D. PD +P.D. PD +PDPs PDP's +PDs PD's +P.E.I.'s PEI's +P. E. PE +P.E. PE +PFs PF's +PFW&C PFW and C +PGMs PGM's +PG&N PG and N +P&G P and G +P. G. PG +P.G. PG +Phlo PHLO +P&H P and H +P. H. PH +P.H. PH +php PHP +PHPs PHP's +P&I P and I +P. I. PI +P.I. PI +Piz PIZ +pj PJ +P. J. PJ +P.J. PJ +PJs PJ's +P. K. PK +P.K. PK +plc PLC +PLCs PLC's +P. L. PL +P.L. PL +PMMoV PMMOV +p.m. PM +p.m PM +P. M. PM +P.M. PM +PMs PM's +PNaCl PNACL +pna PNA +Pnau PNAU +pnb PNB +PNEs PNE's +PNETs PNET's +pneus PNEUS +png PNG +PNNs PNN's +PNoy PNOY +pn PN +P. N. PN +P.N. PN +P.O.D.'s POD's +P&O P and O +P. O. PO +P.O. PO +P.O.W.'s POW's +P&PH P and PH +PPi PPI +P. P. PP +P.P. PP +PPVs PPV's +PPy PPY +pr PR +P. R. PR +P.R. PR +P. s P's +P.'s P's +Ps P's +P. S. PS +P.S. PS +PSSAs PSSA's +pTA PTA +PTAs PTA's +Pte PTE +PTEs PTE's +ptf PTF +Ptie PTIE +PTLs PTL's +Ptol PTOL +pt PT +P. T. PT +P.T. PT +PTTs PTT's +P'Twa PTWA +Puiu PUIU +Pul PUL +PVAs PVA's +PvdA PVDA +pve PVE +P. V. PV +P.V. PV +pvr PVR +PVs PV's +pwll PWLL +P&W P and W +P. W. PW +P.W. PW +Pyi PYI +Pyk PYK +Pyl PYL +PyL PYL +P. Y. PY +Pyu PYU +Pyw PYW +Pyx PYX +P. Z. PZ +Q. A. QA +Q&A QANDA +Q&A's QANDA's +Q&As QANDAS +QbA QBA +Q.B. QB +Q.C. QC +Q. E. QE +Q.H. QH +Q.I. QI +Q. J. QJ +Q. N. QN +QPOs QPO's +Q.V. QV +R. A. RA +R.A. RA +R&A R and A +R&AW R and AW +RbAg RBAG +RBCs RBC's +RBIs RBI's +R&B R and B +R. B. RB +R.B. RB +rca RCA +R&C R and C +rc RC +R. C. RC +R.C. RC +RCTs RCT's +RdE RDE +R&D R and D +R. D. RD +R.D. RD +R.E.M.'s REM's +REPLs REPL's +R. E. RE +R.E. RE +R. F. RF +R.F. RF +R&G R and G +R. G. RG +R.G. RG +RHIBs RHIB's +Rhiw RHIW +rhl RHL +RhoG RHOG +Rho's RHO's +R. H. RH +R.H. RH +R&I R and I +R. I. RI +R.I. RI +riu RIU +Riu RIU +rivs RIVS +Rivu RIVU +Rixt RIXT +rjf RJF +R. J. RJ +R.J. RJ +R. K. RK +R.K. RK +R. L. RL +R.L. RL +rly RLY +RMDs RMD's +RMLs RML's +rm RM +R. M. RM +R.M. RM +RMSDs RMSD's +RMs RM's +rna RNA +RNAs RNA's +Rnet RNET +Rnic RNIC +RNNs RNN's +RNPs RNP's +R. N. RN +R.N. RN +rOmpB ROMPB +Rooi ROOI +R. O. RO +R.O. RO +ROVs ROV's +Roxb ROXB +Roxx ROXX +RP&C RP and C +rpc RPC +Rpe RPE +rpg RPG +RPGs RPG's +rpm RPM +R. P. RP +R.P. RP +RPs RP's +R. Q. RQ +R.Q. RQ +rra RRA +rrd RRD +Rreli RRELI +rre RRE +rRNA RRNA +rRNAs RRNA's +R&R R and R +rr RR +R. R. RR +R.R. RR +rsh RSH +Rsis RSI's +rsly RSLY +RsmA RSMA +RSpec RSPEC +rs RS +Rs R's +R. S. RS +R.S. RS +RSu RSU +rtb RTB +RTCs RTC's +RTEjr RTEJR +RTeOR RTEOR +RTOs RTO's +rtPA RTPA +rt RT +R. T. RT +R.T. RT +rts RTS +RTVFBiH RTVFBIH +Ruao RUAO +Rukn RUKN +Ruk RUK +Rupf RUPF +Rupr RUPR +rup RUP +Rup RUP +R. U. RU +R.U. RU +RUN:AI RUN AI +RutB RUTB +Ruu RUU +Ruwa RUWA +Ruy RUY +Ruyt RUYT +Rvat RVAT +Rvo RVO +R. V. RV +R.V. RV +rvs RVS +rwa RWA +Rwa RWA +rwb RWB +RWEs RWE's +rwjf RWJF +R. W. RW +R.W. RW +Rxa RXA +rz RZ +R. Z. RZ +Sa'id SAID +S. A. SA +S.A. SA +Saxl SAXL +Sa'yo SAYO +Sbai SBAI +sbc SBC +SBCs SBC's +sbk SBK +Sborz SBORZ +sb SB +S. B. SB +S.B. SB +Sbu SBU +scr SCR +SCRs SCR's +scry SCRY +sc SC +S. C. SC +S.C. SC +Scuf SCUF +Scymn SCYMN +SD&AE SD and AE +Sdei SDEI +sde SDE +Sde SDE +SDG&E SDG and E +sdk SDK +S. D. SD +S.D. SD +SDSM&T's SDSM and T's +Sejms SEJM's +sejr SEJR +Sekl SEKL +Sek SEK +S. E. SE +S.E. SE +ses SES +Ses SE's +SEs SE's +S.E.S.'s SES's +Seyh SEYH +sfadb SFADB +Sfax SFAX +SFFCo SFFCO +sfn SFN +'sf SF +sf SF +S. F. SF +S.F. SF +sfs SFS +S.F.'s SF's +SG&A SGANDA +sgb SGB +Sgip SGIP +sgml SGML +sgra SGRA +S. G. SG +S.G. SG +SGSNs SGSN's +Sgt. sergeant +sgt SGT +S.H.I.E.L.D.'s SHIELD's +Shma SHMA +S. H. SH +S.H. SH +S. I. SI +S.I. SI +sjef SJEF +Sjon SJON +S. J. SJ +S.J. SJ +S. K. SK +S.K. SK +S.L.A.A.'s SLAA's +sla SLA +SLAs SLA's +S&L S and L +SLS&E SLS and E +S. L. SL +S.L. SL +SMe SME +SMEs SME's +SmI SMI +SMPSs SMPS's +smr SMR +S&M S and M +sm SM +S. M. SM +S.M. SM +S.M.'s SM's +SMs SMS +Smyl SMYL +SNESjr SNESJR +sngle SNGLE +SNPs SNP's +SnSe SNSE +S. N. SN +S.N. SN +S. O. SO +S.O. SO +Sos SO's +Sovn SOVN +Sov SOV +S&P 500 S and P five hundred +SPCAs SPCA's +Spe SPE +Spoa SPOA +SpPIn SPPIN +S&P S and P +S. P. SP +S.P. SP +SPs SP's +SPUs SPU's +sql SQL +sq SQ +S. Q. SQ +Sra SRA +SRAs SRA's +Srba SRBA +Srbi SRBI +SRBs SRB's +SRGs SRG's +Srhir SRHIR +SRLGs SRLG's +SRMs SRM's +Srni SRNI +srp SRP +sr SR +S. R. SR +S.R. SR +Srul SRUL +srx SRX +SSAs SSA's +SSDs SSD's +ssl SSL +SSoSV SSOSV +SSRIs SSRI's +S&S S and S +ss SS +S. s S's +Ss S's +S. S. SS +S.S. SS +Ssu SSU +ssw SSW +Ssy SSY +ST&AJ ST and AJ +STDs STD's +stfv STFV +STG-TWO STGTWO +STGs STG's +STIs STI's +STi STI +stl STL +Stryj STRYJ +S&T S and T +STScI STSCI +S. T. ST +S.T. ST +suo SUO +Suo SUO +S. U. SU +Susz SUSZ +SUTs SUT's +Suu SUU +SUVs SUV's +Suy SUY +Svac SVAC +svar SVAR +sve SVE +Sve SVE +SVMs SVM's +svn SVN +svom SVOM +sv SV +S. V. SV +S.V. SV +SVTs SVT's +Swe SWE +SwRI SWRI +S&W S and W +sw SW +S. W. SW +S.W. SW +sx SX +S.X. SX +SysML SYSML +SysRq SYSRQ +Sys SY's +SysV SYSV +S. Y. SY +S.Y. SY +SyT SYT +syv SYV +Syxx SYXX +Szasz SZASZ +Szer SZER +szkic SZKIC +Szklo SZKLO +Szlak SZLAK +SzMME SZMME +Szpir SZPIR +sz SZ +S. Z. SZ +Szu SZU +Szyk SZYK +taf TAF +Taf TAF +T. A. TA +T.A. TA +T&A T and A +TBCs TBC's +TBMs TBM's +tbh TBH +T. B. TB +T.B. TB +TCiAP TCIAP +TCKs TCK's +tc TC +T. C. TC +T.C. TC +TCUs TCU's +tcu TCU +tdb TDB +TDCi TDCI +TdIF TDIF +TDs TD's +T. D. TD +T.D. TD +Teatr TEATR +T. E. TE +T.E. TE +T&F T and F +tf TF +T. F. TF +T.F. TF +tge TGE +tgf TGF +tg TG +T. G. TG +T.G. TG +TGVs TGV's +tgv TGV +Thok THOK +ThSe THSE +T. H. TH +T.H. TH +T.I.'s TI's +T.I. TI +Tiu TIU +tiv TIV +Tiv TIV +Tiy's TIY's +T. J. TJ +T.J. TJ +t'ju TJU +tjz TJZ +tko TKO +Tko TKO +tk TK +T. K. TK +T.K. TK +Tlas TLA's +TlCu TLCU +TLDs TLD's +tli TLI +tlp TLP +TLRs TLR's +TLs TL's +tl TL +T. L. TL +T.L. TL +Tluk TLUK +Tluszcz TLUSZCZ +tmc TMC +TMGs TMG's +tmos TMOS +tmRNA TMRNA +TMSs TMS's +T.M.s TM's +tm TM +T. M. TM +T.M. TM +tna TNA +tnbc TNBC +TNTAs TNTA's +tn TN +T. N. TN +T.N. TN +T.O.'s TO's +T. O. TO +T.O. TO +tou TOU +Tou TOU +ToU TOU +tPA TPA +tpr TPR +tp TP +T. P. TP +T.P. TP +TPVs TPV's +T.Q. TQ +tra TRA +Tra TRA +Trbic TRBIC +Trcek TRCEK +TrkA TRKA +Trmcic TRMCIC +tRNA TRNA +TRPs TRP's +trs TRS +tr TR +T. R. TR +T.R. TR +truTV TRUTV +TruTV TRUTV +Trve TRVE +Tsa TSA +tsit TSIT +Tsiv TSIV +tso TSO +Tso TSO +tsr TSR +Ts T's +T. S. TS +T.S. TS +tsus TSUS +Tsvi TSVI +Tta TTA +tteok TTEOK +tte TTE +ttp TTP +T&T T and T +T. T. TT +T.T. TT +T'uqu TUQU +T.U. TU +Tuzk TUZK +tvaan TVAAN +Tvam TVAM +tva TVA +TVETs TVET's +TVii TVII +TVIn TVIN +TVi TVI +tvo TVO +Tvo TVO +TVRi TVRI +tvr TVR +TVSpy TVSPY +tvs TVS +TVs TV's +tVTA TVTA +tv TV +T. V. TV +T.V. TV +Twa TWA +Twi TWI +TWTs TWT's +T. W. TW +T.W. TW +TxDOT TXDOT +T.X. TX +tya TYA +Tyk TYK +Tza'ar TZAAR +Tze TZE +Tzrif TZRIF +Tzuh TZUH +Tzvi TZVI +Ua's UA's +UAs UA's +uat UAT +U.A. UA +UAVs UAV's +Uba UBA +UbcM UBCM +Ube UBE +ubi UBI +Ubi UBI +Ucar UCAR +Uca UCA +Ucmak UCMAK +Ucn UCN +uCs UC's +Uc UC +U.C. UC +Ucuncu UCUNCU +Uczta UCZTA +Uda UDA +Udit UDIT +Udny UDNY +UDTs UDT's +ud UD +Ud UD +U. D. UD +udu UDU +Udu UDU +Ueda's UEDA's +Uéda UE acute DA +U. E. UE +UFOs UFO's +Ufot UFOT +ufo UFO +Ufo UFO +U. F. UF +Uga UGA +Uge UGE +Ugni UGNI +Ugra UGRA +Ugrszke UGRSZKE +Ug UG +Uhha UHHA +uhn UHN +UHTCs UHTC's +Uhud UHUD +U.H. UH +uhur UHUR +Uiam UIAM +Uibh UIBH +Uible UIBLE +Uig UIG +uisae UISAE +UiS UIS +UiTM UITM +uit UIT +Uit UIT +Uiy UIY +Ujed UJED +Ujsag UJSAG +uj UJ +U.J. UJ +Uka UKA +Ukhra UKHRA +Ukic UKIC +Uki UKI +Ukoh UKOH +Uko UKO +ukr UKR +Ukr UKR +Ukui UKUI +u'k UK +uk UK +Uk UK +U.K. UK +Ukwu UKWU +Ulic ULIC +Ull ULL +Ulms ULM's +UlSU ULSU +Uluj ULUJ +U.L. UL +ulus ULUS +Ulus ULU's +Ulwa ULWA +Ulwe ULWE +Umbr UMBR +umelcu UMELCU +UMe UME +Umla UMLA +Umme UMME +Umno UMNO +Umri UMRI +U. M. UM +U.M. UM +Unli UNLI +Unlu UNLU +unm UNM +unnd UNND +UNSh UNSH +Unst UNST +Uns UN's +U.N.'s UN's +Unt UNT +U.N. UN +Unz UNZ +UOCl UOCL +UofM UOFM +Uoho UOHO +UoMs UOM's +uORF UORF +UoW UOW +UPBs UPB's +Upd UPD +Upf UPF +Upir UPIR +UPnP UPNP +Uppu UPPU +U.P. UP +UPyD UPYD +uq UQ +Urbz URBZ +URCs URC's +Urdd URDD +Urei UREI +urf URF +Urla URLA +URLhttp URLHTTP +URLs URL's +Usal USAL +usan USAN +Usan USAN +usao USAO +usata USATA +usa USA +.USA USA +USAya USAYA +USBs USB's +Uscie USCIE +U.S.C.'s USC's +USDoE USDOE +usd USD +USFbA USFBA +USF&WS USF and WS +UShs USH's +Usia USIA +usih USIH +Uslu USLU +Usmar USMAR +Usna USNA +Usnic USNIC +Usoi USOI +Usos USO's +Uso USO +USPs USP's +U.S.'s US's +Usti USTI +Ustka USTKA +Usui USUI +usum USUM +U. S. US +U.S. US +Usut USUT +Usvit USVIT +Uta's UTA's +uta UTA +Uta UTA +Utca UTCA +Utd's UTD's +Utd UTD +utea UTEA +Utes UTE's +Uth UTH +uti UTI +Uti UTI +Utne UTNE +Utnur UTNUR +Uto UTO +utrci UTRCI +utsav UTSAV +Utsav UTSAV +Utsu UTSU +U.T. UT +Utu UTU +Utva UTVA +Uuh UUH +Uul UUL +Uulu UULU +Uusi UUSI +UUs UU's +Uuto UUTO +Uvac UVAC +Uvea UVEA +uvnitr UVNITR +Uvo UVO +U.V. UV +Uwasa UWASA +UWFi UWFI +V. A. VA +V.A. VA +V&A V and A +Vav VAV +Vay VAY +vb VB +V. B. VB +V.B. VB +VCDs VCD's +VCRs VCR's +VCSELs VCSEL's +VCs VC's +VCTs VCT's +vc VC +V. C. VC +V.C. VC +vda VDA +Vda VDA +VDCs VDC's +vdiq VDIQ +vdm VDM +V. D. VD +V.D. VD +vez VEZ +Vez VEZ +vfp VFP +vfr VFR +V. F. VF +V.F. VF +V'Ger VGER +vgmdb VGMDB +VGo VGO +VGSCs VGSC's +VGSoM VGSOM +V. G. VG +V.G. VG +V. H. VH +V.I.C.'s VIC's +Vict VICT +viita VIITA +vijf VIJF +vij VIJ +Vij VIJ +V.I.P.s VIP's +VIPs VIP's +V. I. VI +V.I. VI +VJs VJ's +V. J. VJ +V.J. VJ +V. K. VK +V.K. VK +Vlah VLAH +VLCCs VLCC's +vlei VLEI +Vlijt VLIJT +V. L. VL +V.L. VL +VMAs VMA's +vm VM +V. M. VM +V.M. VM +vner VNER +V. N. VN +V.N. VN +V.O. VO +Vov VOV +Voz VOZ +vpis VPIS +VPNs VPN's +vpu VPU +V. P. VP +V.P. VP +vq VQ +vrak VRAK +Vrba VRBA +Vrbuv VRBUV +Vrej VREJ +vrem VREM +Vrin VRIN +vrj VRJ +vrn VRN +vroee VROEE +vrou VROU +vrouw VROUW +Vrouw VROUW +Vrsac VRSAC +Vrtis VRTI's +V&R V and R +vr VR +V. R. VR +V.R. VR +VSANs VSAN's +VSATs VSAT's +Vsekh VSEKH +vse VSE +vso VSO +VSPs VSP's +vs. versus +_vs._ versus +vsyo VSYO +VTE VT eL +Vtic VTIC +VTi VTI +V&T's V and T's +V&T V and T +V.T. VT +Vuur VUUR +VVIPs VVIP's +V. V. VV +V.V. VV +VWs VW's +Vyg VYG +vyr VYR +vy VY +vz VZ +V. Z. VZ +Waay WAAY +Wa'il WAIL +Wakf WAKF +wa'l WAL +waqf WAQF +waqt WAQT +Waqt WAQT +Wasl WASL +Watfa WATFA +wau WAU +Wau WAU +W. A. WA +W.A. WA +waw WAW +waza WAZA +Waza WAZA +WBs WB's +W. B. WB +W.B. WB +W&CBR W and CBR +wc WC +W. C. WC +W.C. WC +W. D. WD +W.D. WD +W. E. WE +W.E. WE +WF&NW WF and NW +W&F W and F +W. F. WF +W.F. WF +wga WGA +wgn WGN +wg WG +W. G. WG +W.G. WG +W&H W and H +W. H. WH +W.H. WH +Whyld WHYLD +Wica WICA +wici WICI +Wif WIF +Wijk WIJK +Wiwa WIWA +W. I. WI +wjaz WJAZ +W&J's W and J's +W&J W and J +W. J. WJ +W.J. WJ +W. K. WK +W.K. WK +W&LE W and LE +Wley WLEY +wlrs WLRS +W. L. WL +W.L. WL +wml WML +W&M W and M +W. M. WM +W.M. WM +W. N. WN +W.N. WN +wnzaa WNZAA +W&OD W and OD +Worh WORH +W. O. WO +W.O. WO +wpt WPT +wp WP +W. P. WP +W.P. WP +Wrec WREC +W. R. WR +W.R. WR +ws WS +W. S. WS +W.S. WS +WTs WT's +WTTs WTT's +W. T. WT +W.T. WT +WUAs WUA's +Wudl WUDL +Wuhr WUHR +Wuhu WUHU +Wu's WU's +wuv WUV +W. U. WU +Wuz WUZ +WVa WVA +W. V. WV +W.V. WV +Wwe WWE +W. W. WW +W.W. WW +wx WX +W.Y. WY +wyzc WYZC +W. Z. WZ +xbg XBG +Xbra XBRA +xCo XCO +X. C. XC +xda XDA +XDRs XDR's +X. D. XD +Xfce XFCE +xf XF +X.F. XF +xh XH +X. H. XH +xk XK +xmc XMC +xml XML +X.M. XM +xO XO +X.O. XO +XPe XPE +X.Q. XQ +XSi XSI +xsr XSR +XTwas XTWA's +xt XT +X. W. XW +xy XY +xyz XYZ +xyZ XYZ +Y. A. YA +Y.A. YA +Y. B. YB +Y.B. YB +ycia YCIA +ycie YCIE +yc YC +Y. C. YC +Y.C. YC +Ydby YDBY +Yde YDE +YdiB YDIB +yd YD +Y. D. YD +Y.D. YD +Y.E. YE +YFCs YFC's +yfle YFLE +YF&R YF and R +yg YG +Y. G. YG +Y.G. YG +Y. H. YH +Y. I. YI +Y.I. YI +yj YJ +Y. J. YJ +Ykt YKT +Y. K. YK +Y.K. YK +Yle YLE +yl YL +Y. L. YL +Ymke YMKE +ym YM +Y. M. YM +yndi YNDI +Yndi YNDI +yne YNE +Yngve YNGVE +Ynis YNI's +yn YN +Yn YN +Y. N. YN +Y.N. YN +Y.O. YO +ypa YPA +Y.P. YP +Y. Q. YQ +yra YRA +Yra YRA +Y&R Y and R +YSK'da YSKDA +YSK'ya YSKYA +Ys Y's +Y.S. YS +yta YTA +Ytre YTRE +Y&T Y and T +Y. T. YT +Y.T. YT +Y. V. YV +Y. W. YW +Y.Y. YY +Z. A. ZA +Z.A. ZA +Zdar ZDAR +zda ZDA +zg ZG +Z.G. ZG +zh ZH +Z. H. ZH +Z.H. ZH +zijn ZIJN +zij ZIJ +Zij ZIJ +Z. I. ZI +Z. J. ZJ +Z.K. ZK +Z.L. ZL +Z.M. ZM +ZnO ZNO +Zpav ZPAV +Z.P. ZP +ZrI ZRI +Zsuzsa ZSUZSA +Z.W. ZW +Z.X. ZX +Z. Y. ZY +Z.Y. ZY +Z. Z. ZZ +Z.Z. ZZ +2.5G two point five g +7-eleven seven eleven +AAA triple a +C# c sharp +DeepMind deep mind +DeepStream deep stream +DevOps dev ops +DisplayPort display port +HuggingFace hugging face +ServiceNow service now +SuperCloud super cloud +Wi-Fi wifi +7-Eleven seven eleven +Deepmind deep mind +Deepstream deep stream +Devops dev ops +Displayport display port +Huggingface hugging face +Servicenow service now +Supercloud super cloud +wi-fi wifi +deepmind deep mind +deepstream deep stream +devops dev ops +displayport display port +huggingface hugging face +servicenow service now +supercloud super cloud +wi-fi wifi +3-D three d +401k four oh one k +401(k) four oh one k +401 (k) four oh one k \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/graph_utils.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..417668871351df66f256035a8289219184036f0e --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/graph_utils.py @@ -0,0 +1,337 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from zonos3.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels +from zonos3.vendor.nemo_text_processing.utils.logging import logger + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = "\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_LOWER_NOT_A = pynini.union( + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", +).optimize() + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(' field_order: "') + NEMO_NOT_QUOTE + pynutil.delete('"')) +) + + +# Common string literals; expand as you see fit. +username_string = "username" +double_quotes = '"' +domain_string = "domain" +protocol_string = "protocol" +slash = "/" +double_slash = "//" +triple_slash = "///" +file = "file" +period = "." +at = "@" +colon = ":" +https = "https" +http = "http" +www = "www" + + +suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv")) +# _v = pynini.union("a", "e", "i", "o", "u") +_c = pynini.union( + "b", + "c", + "d", + "f", + "g", + "h", + "j", + "k", + "l", + "m", + "n", + "p", + "q", + "r", + "s", + "t", + "v", + "w", + "x", + "y", + "z", +) +_ies = NEMO_SIGMA + _c + pynini.cross("y", "ies") +_es = NEMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es") +_s = NEMO_SIGMA + pynutil.insert("s") + +graph_plural = plurals._priority_union( + suppletive, + plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), + NEMO_SIGMA, +).optimize() + +SINGULAR_TO_PLURAL = graph_plural +PLURAL_TO_SINGULAR = pynini.invert(graph_plural) +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) +TO_UPPER = pynini.invert(TO_LOWER) +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("minus", "Minus").optimize() + + +def capitalized_input_graph( + graph: "pynini.FstLike", + original_graph_weight: float = None, + capitalized_graph_weight: float = None, +) -> "pynini.FstLike": + """ + Allow graph input to be capitalized, e.g. for ITN) + + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logger.info(f"Created {file_name}") + + +def get_plurals(fst): + """ + Given singular returns plurals + + Args: + fst: Fst + + Returns plurals to given singular forms + """ + return SINGULAR_TO_PLURAL @ fst + + +def get_singulars(fst): + """ + Given plural returns singulars + + Args: + fst: Fst + + Returns singulars to given plural forms + """ + return PLURAL_TO_SINGULAR @ fst + + +def convert_space(fst) -> "pynini.FstLike": + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [ + written_capitalized, + spoken.capitalize(), + ], # first letter capitalized + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], # # add pairs with the all letters capitalized + ] + ) + + spoken_no_space = spoken.replace(" ", "") + # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" + if len(spoken) == (2 * len(spoken_no_space) - 1): + logger.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [ + [written, spoken_no_space], + [written_capitalized, spoken_no_space.upper()], + ] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far") + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> "pynini.FstLike": + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> "pynini.FstLike": + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> "pynini.FstLike": + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc443be41c4c3fcf0764eb9fd3e1828d63f8438c --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/abbreviation.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/abbreviation.py new file mode 100644 index 0000000000000000000000000000000000000000..da48195fc8b98bc683b0026721207357b6503e57 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/abbreviation.py @@ -0,0 +1,51 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_UPPER, GraphFst, insert_space + + +class AbbreviationFst(GraphFst): + """ + Finite state transducer for classifying electronic: as URLs, email addresses, etc. + e.g. "ABC" -> tokens { abbreviation { value: "A B C" } } + + Args: + whitelist: whitelist FST + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, whitelist: 'pynini.FstLike', deterministic: bool = True): + super().__init__(name="abbreviation", kind="classify", deterministic=deterministic) + + dot = pynini.accep(".") + # A.B.C. -> A. B. C. + graph = NEMO_UPPER + dot + pynini.closure(insert_space + NEMO_UPPER + dot, 1) + # A.B.C. -> A.B.C. + graph |= NEMO_UPPER + dot + pynini.closure(NEMO_UPPER + dot, 1) + # ABC -> A B C + graph |= NEMO_UPPER + pynini.closure(insert_space + NEMO_UPPER, 1) + + # exclude words that are included in the whitelist + graph = pynini.compose( + pynini.difference(pynini.project(graph, "input"), pynini.project(whitelist.graph, "input")), graph + ) + + graph = pynutil.insert("value: \"") + graph.optimize() + pynutil.insert("\"") + graph = self.add_tokens(graph) + self.fst = graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/cardinal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..88ee94b3e2fd04f8b943c41f134b039952e61bbf --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/cardinal.py @@ -0,0 +1,151 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.date import get_four_digit_year_graph +from zonos3.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + -23 -> cardinal { negative: "true" integer: "twenty three" } } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + self.lm = lm + self.deterministic = deterministic + # TODO replace to have "oh" as a default for "0" + graph = pynini.Far(get_abs_path("data/number/cardinal_number_name.far")).get_fst() + graph_au = pynini.Far(get_abs_path("data/number/cardinal_number_name_au.far")).get_fst() + self.graph_hundred_component_at_least_one_none_zero_digit = ( + pynini.closure(NEMO_DIGIT, 2, 3) | pynini.difference(NEMO_DIGIT, pynini.accep("0")) + ) @ graph + + graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + + single_digits_graph = pynini.invert(graph_digit | graph_zero) + self.single_digits_graph = single_digits_graph + pynini.closure(insert_space + single_digits_graph) + + if not deterministic: + # for a single token allow only the same normalization + # "007" -> {"oh oh seven", "zero zero seven"} not {"oh zero seven"} + single_digits_graph_zero = pynini.invert(graph_digit | graph_zero) + single_digits_graph_oh = pynini.invert(graph_digit) | pynini.cross("0", "oh") + + self.single_digits_graph = single_digits_graph_zero + pynini.closure( + insert_space + single_digits_graph_zero + ) + self.single_digits_graph |= single_digits_graph_oh + pynini.closure(insert_space + single_digits_graph_oh) + + single_digits_graph_with_commas = pynini.closure( + self.single_digits_graph + insert_space, 1, 3 + ) + pynini.closure( + pynutil.delete(",") + + single_digits_graph + + insert_space + + single_digits_graph + + insert_space + + single_digits_graph, + 1, + ) + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + graph = ( + pynini.closure(NEMO_DIGIT, 1, 3) + + (pynini.closure(pynutil.delete(",") + NEMO_DIGIT**3) | pynini.closure(NEMO_DIGIT**3)) + ) @ graph + + self.graph = graph + self.graph_with_and = self.add_optional_and(graph) + + if deterministic: + long_numbers = pynini.compose(NEMO_DIGIT ** (5, ...), self.single_digits_graph).optimize() + self.long_numbers = plurals._priority_union(long_numbers, self.graph_with_and, NEMO_SIGMA).optimize() + cardinal_with_leading_zeros = pynini.compose( + pynini.accep("0") + pynini.closure(NEMO_DIGIT), self.single_digits_graph + ) + final_graph = self.long_numbers | cardinal_with_leading_zeros + final_graph |= self.add_optional_and(graph_au) + else: + leading_zeros = pynini.compose(pynini.closure(pynini.accep("0"), 1), self.single_digits_graph) + cardinal_with_leading_zeros = ( + leading_zeros + pynutil.insert(" ") + pynini.compose(pynini.closure(NEMO_DIGIT), self.graph_with_and) + ) + self.long_numbers = self.graph_with_and | pynutil.add_weight(self.single_digits_graph, 0.0001) + # add small weight to non-default graphs to make sure the deterministic option is listed first + final_graph = ( + self.long_numbers + | get_four_digit_year_graph() # allows e.g. 4567 be pronounced as forty five sixty seven + | pynutil.add_weight(single_digits_graph_with_commas, 0.0001) + | cardinal_with_leading_zeros + ).optimize() + + one_to_a_replacement_graph = ( + pynini.cross("one hundred", "a hundred") + | pynini.cross("one thousand", "thousand") + | pynini.cross("one million", "a million") + ) + final_graph |= pynini.compose(final_graph, one_to_a_replacement_graph.optimize() + NEMO_SIGMA).optimize() + # remove commas for 4 digits numbers + four_digit_comma_graph = (NEMO_DIGIT - "0") + pynutil.delete(",") + NEMO_DIGIT**3 + final_graph |= pynini.compose(four_digit_comma_graph.optimize(), final_graph).optimize() + + self.final_graph = final_graph + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + final_graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() + + def add_optional_and(self, graph): + graph_with_and = graph + + if not self.lm: + graph_with_and = pynutil.add_weight(graph, 0.00001) + not_quote = pynini.closure(NEMO_NOT_QUOTE) + no_thousand_million = pynini.difference( + not_quote, not_quote + pynini.union("thousand", "million") + not_quote + ).optimize() + integer = ( + not_quote + pynutil.add_weight(pynini.cross("hundred ", "hundred and ") + no_thousand_million, -0.0001) + ).optimize() + + no_hundred = pynini.difference(NEMO_SIGMA, not_quote + pynini.accep("hundred") + not_quote).optimize() + integer |= ( + not_quote + pynutil.add_weight(pynini.cross("thousand ", "thousand and ") + no_hundred, -0.0001) + ).optimize() + + optional_hundred = pynini.compose((NEMO_DIGIT - "0") ** 3, graph).optimize() + optional_hundred = pynini.compose(optional_hundred, NEMO_SIGMA + pynini.cross(" hundred", "") + NEMO_SIGMA) + graph_with_and |= pynini.compose(graph, integer).optimize() + graph_with_and |= optional_hundred + return graph_with_and diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/date.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..6177ed5a93738e0afeab9a80eeb723102272fa58 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/date.py @@ -0,0 +1,386 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_LOWER, + NEMO_SIGMA, + TO_LOWER, + GraphFst, + delete_extra_space, + delete_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.utils import ( + augment_labels_with_punct_at_end, + get_abs_path, + load_labels, +) + +graph_teen = pynini.invert(pynini.string_file(get_abs_path("data/number/teen.tsv"))).optimize() +graph_digit = pynini.invert(pynini.string_file(get_abs_path("data/number/digit.tsv"))).optimize() +ties_graph = pynini.invert(pynini.string_file(get_abs_path("data/number/ty.tsv"))).optimize() +year_suffix = load_labels(get_abs_path("data/date/year_suffix.tsv")) +year_suffix.extend(augment_labels_with_punct_at_end(year_suffix)) +year_suffix = pynini.string_map(year_suffix).optimize() + + +def get_ties_graph(deterministic: bool = True): + """ + Returns two digit transducer, e.g. + 03 -> o three + 12 -> thirteen + 20 -> twenty + """ + graph = graph_teen | ties_graph + pynutil.delete("0") | ties_graph + insert_space + graph_digit + + if deterministic: + graph = graph | pynini.cross("0", "oh") + insert_space + graph_digit + else: + graph = graph | (pynini.cross("0", "oh") | pynini.cross("0", "zero")) + insert_space + graph_digit + + return graph.optimize() + + +def get_four_digit_year_graph(deterministic: bool = True): + """ + Returns a four digit transducer which is combination of ties/teen or digits + (using hundred instead of thousand format), e.g. + 1219 -> twelve nineteen + 3900 -> thirty nine hundred + """ + graph_ties = get_ties_graph(deterministic) + + graph_with_s = ( + (graph_ties + insert_space + graph_ties) + | (graph_teen + insert_space + (ties_graph | pynini.cross("1", "ten"))) + ) + pynutil.delete("0s") + + graph_with_s |= (graph_teen | graph_ties) + insert_space + pynini.cross("00", "hundred") + pynutil.delete("s") + graph_with_s = graph_with_s @ pynini.cdrewrite( + pynini.cross("y", "ies") | pynutil.insert("s"), "", "[EOS]", NEMO_SIGMA + ) + + graph = graph_ties + insert_space + graph_ties + graph |= (graph_teen | graph_ties) + insert_space + pynini.cross("00", "hundred") + + thousand_graph = ( + graph_digit + + insert_space + + pynini.cross("00", "thousand") + + (pynutil.delete("0") | insert_space + graph_digit) + ) + thousand_graph |= ( + graph_digit + + insert_space + + pynini.cross("000", "thousand") + + pynini.closure(pynutil.delete(" "), 0, 1) + + pynini.accep("s") + ) + + graph |= graph_with_s + if deterministic: + graph = plurals._priority_union(thousand_graph, graph, NEMO_SIGMA) + else: + graph |= thousand_graph + + return graph.optimize() + + +def _get_two_digit_year_with_s_graph(): + # to handle '70s -> seventies + graph = ( + pynini.closure(pynutil.delete("'"), 0, 1) + + pynini.compose( + ties_graph + pynutil.delete("0s"), pynini.cdrewrite(pynini.cross("y", "ies"), "", "[EOS]", NEMO_SIGMA) + ) + ).optimize() + return graph + + +def _get_year_graph(cardinal_graph, deterministic: bool = True): + """ + Transducer for year, only from 1000 - 2999 e.g. + 1290 -> twelve nineteen + 2000 - 2009 will be verbalized as two thousand. + + Transducer for 3 digit year, e.g. 123-> one twenty three + + Transducer for year with suffix + 123 A.D., 4200 B.C + """ + graph = get_four_digit_year_graph(deterministic) + graph = (pynini.union("1", "2") + (NEMO_DIGIT**3) + pynini.closure(pynini.cross(" s", "s") | "s", 0, 1)) @ graph + + graph |= _get_two_digit_year_with_s_graph() + + three_digit_year = (NEMO_DIGIT @ cardinal_graph) + insert_space + (NEMO_DIGIT**2) @ cardinal_graph + year_with_suffix = ( + (get_four_digit_year_graph(deterministic=True) | three_digit_year) + delete_space + insert_space + year_suffix + ) + graph |= year_with_suffix + return graph.optimize() + + +def _get_two_digit_year(cardinal_graph, single_digits_graph): + two_digit_year = NEMO_DIGIT ** (2) @ plurals._priority_union(cardinal_graph, single_digits_graph, NEMO_SIGMA) + return two_digit_year + + +def _get_financial_period_graph(): + # 1H23 -> first half of twenty three + # 3Q22 -> third quarter of twenty two + + h_ordinals = pynini.cross('1', 'first') | pynini.cross('2', 'second') + q_ordinals = h_ordinals | pynini.cross('3', 'third') | pynini.cross('4', 'fourth') + + h_graph = h_ordinals + pynini.cross('H', ' half') + q_graph = q_ordinals + pynini.cross('Q', ' quarter') + period_graph = h_graph | q_graph + + return period_graph + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date, e.g. + jan. 5, 2012 -> date { month: "january" day: "five" year: "twenty twelve" preserve_order: true } + jan. 5 -> date { month: "january" day: "five" preserve_order: true } + 5 january 2012 -> date { day: "five" month: "january" year: "twenty twelve" preserve_order: true } + 2012-01-05 -> date { year: "twenty twelve" month: "january" day: "five" } + 2012.01.05 -> date { year: "twenty twelve" month: "january" day: "five" } + 2012/01/05 -> date { year: "twenty twelve" month: "january" day: "five" } + 2012 -> date { year: "twenty twelve" } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool, lm: bool = False): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + # january + month_graph = pynini.string_file(get_abs_path("data/date/month_name.tsv")).optimize() + # January, JANUARY + month_graph |= pynini.compose(TO_LOWER + pynini.closure(NEMO_CHAR), month_graph) | pynini.compose( + TO_LOWER ** (2, ...), month_graph + ) + + # jan + month_abbr_graph = pynini.string_file(get_abs_path("data/date/month_abbr.tsv")).optimize() + # jan, Jan, JAN + month_abbr_graph = ( + month_abbr_graph + | pynini.compose(TO_LOWER + pynini.closure(NEMO_LOWER, 1), month_abbr_graph).optimize() + | pynini.compose(TO_LOWER ** (2, ...), month_abbr_graph).optimize() + ) + pynini.closure(pynutil.delete("."), 0, 1) + month_graph |= month_abbr_graph.optimize() + + month_numbers_labels = pynini.string_file(get_abs_path("data/date/month_number.tsv")).optimize() + cardinal_graph = cardinal.graph_hundred_component_at_least_one_none_zero_digit + + year_graph = _get_year_graph(cardinal_graph=cardinal_graph, deterministic=deterministic) + + # three_digit_year = (NEMO_DIGIT @ cardinal_graph) + insert_space + (NEMO_DIGIT ** 2) @ cardinal_graph + # year_graph |= three_digit_year + + month_graph = pynutil.insert("month: \"") + month_graph + pynutil.insert("\"") + month_numbers_graph = pynutil.insert("month: \"") + month_numbers_labels + pynutil.insert("\"") + + endings = ["rd", "th", "st", "nd"] + endings += [x.upper() for x in endings] + endings = pynini.union(*endings) + + day_graph = ( + pynutil.insert("day: \"") + + pynini.closure(pynutil.delete("the "), 0, 1) + + ( + ((pynini.union("1", "2") + NEMO_DIGIT) | NEMO_DIGIT | (pynini.accep("3") + pynini.union("0", "1"))) + + pynini.closure(pynutil.delete(endings), 0, 1) + ) + @ cardinal_graph + + pynutil.insert("\"") + ) + + two_digit_year = _get_two_digit_year( + cardinal_graph=cardinal_graph, single_digits_graph=cardinal.single_digits_graph + ) + two_digit_year = pynutil.insert("year: \"") + two_digit_year + pynutil.insert("\"") + + # if lm: + # two_digit_year = pynini.compose(pynini.difference(NEMO_DIGIT, "0") + NEMO_DIGIT ** (3), two_digit_year) + # year_graph = pynini.compose(pynini.difference(NEMO_DIGIT, "0") + NEMO_DIGIT ** (2), year_graph) + # year_graph |= pynini.compose(pynini.difference(NEMO_DIGIT, "0") + NEMO_DIGIT ** (4, ...), year_graph) + + graph_year = pynutil.insert(" year: \"") + pynutil.delete(" ") + year_graph + pynutil.insert("\"") + graph_year |= ( + pynutil.insert(" year: \"") + + pynini.accep(",") + + pynini.closure(pynini.accep(" "), 0, 1) + + year_graph + + pynutil.insert("\"") + ) + optional_graph_year = pynini.closure(graph_year, 0, 1) + + year_graph = pynutil.insert("year: \"") + year_graph + pynutil.insert("\"") + + graph_mdy = month_graph + ( + (delete_extra_space + day_graph) + | (pynini.accep(" ") + day_graph) + | graph_year + | (delete_extra_space + day_graph + graph_year) + ) + + graph_mdy |= ( + month_graph + + pynini.cross("-", " ") + + day_graph + + pynini.closure(((pynini.cross("-", " ") + NEMO_SIGMA) @ graph_year), 0, 1) + ) + + for x in ["-", "/", "."]: + delete_sep = pynutil.delete(x) + graph_mdy |= ( + month_numbers_graph + + delete_sep + + insert_space + + pynini.closure(pynutil.delete("0"), 0, 1) + + day_graph + + delete_sep + + insert_space + + (year_graph | two_digit_year) + ) + + graph_dmy = day_graph + delete_extra_space + month_graph + optional_graph_year + day_ex_month = (NEMO_DIGIT**2 - pynini.project(month_numbers_graph, "input")) @ day_graph + for x in ["-", "/", "."]: + delete_sep = pynutil.delete(x) + graph_dmy |= ( + day_ex_month + + delete_sep + + insert_space + + month_numbers_graph + + delete_sep + + insert_space + + (year_graph | two_digit_year) + ) + + graph_ymd = pynini.accep("") + for x in ["-", "/", "."]: + delete_sep = pynutil.delete(x) + graph_ymd |= ( + (year_graph | two_digit_year) + + delete_sep + + insert_space + + month_numbers_graph + + delete_sep + + insert_space + + pynini.closure(pynutil.delete("0"), 0, 1) + + day_graph + ) + + final_graph = graph_mdy | graph_dmy + + if not deterministic or lm: + final_graph += pynini.closure(pynutil.insert(" preserve_order: true"), 0, 1) + m_sep_d = ( + month_numbers_graph + + pynutil.delete(pynini.union("-", "/")) + + insert_space + + pynini.closure(pynutil.delete("0"), 0, 1) + + day_graph + ) + final_graph |= m_sep_d + else: + final_graph += pynutil.insert(" preserve_order: true") + + period_fy = pynutil.insert("text: \"") + _get_financial_period_graph() + pynutil.insert("\"") + graph_fy = period_fy + insert_space + two_digit_year + + final_graph |= graph_ymd | year_graph | graph_fy + + if not deterministic or lm: + ymd_to_mdy_graph = None + ymd_to_dmy_graph = None + mdy_to_dmy_graph = None + md_to_dm_graph = None + + for month in [x[0] for x in load_labels(get_abs_path("data/date/month_name.tsv"))]: + for day in [x[0] for x in load_labels(get_abs_path("data/date/day.tsv"))]: + ymd_to_mdy_curr = ( + pynutil.insert("month: \"" + month + "\" day: \"" + day + "\" ") + + pynini.accep('year:') + + NEMO_SIGMA + + pynutil.delete(" month: \"" + month + "\" day: \"" + day + "\"") + ) + + # YY-MM-DD -> MM-DD-YY + ymd_to_mdy_curr = pynini.compose(graph_ymd, ymd_to_mdy_curr) + ymd_to_mdy_graph = ( + ymd_to_mdy_curr + if ymd_to_mdy_graph is None + else pynini.union(ymd_to_mdy_curr, ymd_to_mdy_graph) + ) + + ymd_to_dmy_curr = ( + pynutil.insert("day: \"" + day + "\" month: \"" + month + "\" ") + + pynini.accep('year:') + + NEMO_SIGMA + + pynutil.delete(" month: \"" + month + "\" day: \"" + day + "\"") + ) + + # YY-MM-DD -> MM-DD-YY + ymd_to_dmy_curr = pynini.compose(graph_ymd, ymd_to_dmy_curr).optimize() + ymd_to_dmy_graph = ( + ymd_to_dmy_curr + if ymd_to_dmy_graph is None + else pynini.union(ymd_to_dmy_curr, ymd_to_dmy_graph) + ) + + mdy_to_dmy_curr = ( + pynutil.insert("day: \"" + day + "\" month: \"" + month + "\" ") + + pynutil.delete("month: \"" + month + "\" day: \"" + day + "\" ") + + pynini.accep('year:') + + NEMO_SIGMA + ).optimize() + # MM-DD-YY -> verbalize as MM-DD-YY (February fourth 1991) or DD-MM-YY (the fourth of February 1991) + mdy_to_dmy_curr = pynini.compose(graph_mdy, mdy_to_dmy_curr).optimize() + mdy_to_dmy_graph = ( + mdy_to_dmy_curr + if mdy_to_dmy_graph is None + else pynini.union(mdy_to_dmy_curr, mdy_to_dmy_graph).optimize() + ).optimize() + + md_to_dm_curr = pynutil.insert("day: \"" + day + "\" month: \"" + month + "\"") + pynutil.delete( + "month: \"" + month + "\" day: \"" + day + "\"" + ) + md_to_dm_curr = pynini.compose(m_sep_d, md_to_dm_curr).optimize() + + md_to_dm_graph = ( + md_to_dm_curr + if md_to_dm_graph is None + else pynini.union(md_to_dm_curr, md_to_dm_graph).optimize() + ).optimize() + + final_graph |= mdy_to_dmy_graph | md_to_dm_graph | ymd_to_mdy_graph | ymd_to_dmy_graph + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/decimal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..6066bc040e6fb5f0da602e90e826fe04b9110d0c --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/decimal.py @@ -0,0 +1,130 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, TO_UPPER, GraphFst, get_abs_path + +delete_space = pynutil.delete(" ") +quantities = pynini.string_file(get_abs_path("data/number/thousand.tsv")) +quantities_abbr = pynini.string_file(get_abs_path("data/number/quantity_abbr.tsv")) +quantities_abbr |= TO_UPPER @ quantities_abbr + + +def get_quantity( + decimal: 'pynini.FstLike', cardinal_up_to_hundred: 'pynini.FstLike', include_abbr: bool +) -> 'pynini.FstLike': + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, + e.g. 1 million -> integer_part: "one" quantity: "million" + e.g. 1.5 million -> integer_part: "one" fractional_part: "five" quantity: "million" + + Args: + decimal: decimal FST + cardinal_up_to_hundred: cardinal FST + """ + quantity_wo_thousand = pynini.project(quantities, "input") - pynini.union("k", "K", "thousand") + if include_abbr: + quantity_wo_thousand |= pynini.project(quantities_abbr, "input") - pynini.union("k", "K", "thousand") + res = ( + pynutil.insert("integer_part: \"") + + cardinal_up_to_hundred + + pynutil.insert("\"") + + pynini.closure(pynutil.delete(" "), 0, 1) + + pynutil.insert(" quantity: \"") + + (quantity_wo_thousand @ (quantities | quantities_abbr)) + + pynutil.insert("\"") + ) + if include_abbr: + quantity = quantities | quantities_abbr + else: + quantity = quantities + res |= ( + decimal + + pynini.closure(pynutil.delete(" "), 0, 1) + + pynutil.insert("quantity: \"") + + quantity + + pynutil.insert("\"") + ) + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + -12.5006 billion -> decimal { negative: "true" integer_part: "12" fractional_part: "five o o six" quantity: "billion" } + 1 billion -> decimal { integer_part: "one" quantity: "billion" } + + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.graph_with_and + cardinal_graph_hundred_component_at_least_one_none_zero_digit = ( + cardinal.graph_hundred_component_at_least_one_none_zero_digit + ) + + self.graph = cardinal.single_digits_graph.optimize() + + if not deterministic: + self.graph = self.graph | cardinal_graph + + point = pynutil.delete(".") + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + self.graph_fractional = pynutil.insert("fractional_part: \"") + self.graph + pynutil.insert("\"") + self.graph_integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + final_graph_wo_sign = ( + pynini.closure(self.graph_integer + pynutil.insert(" "), 0, 1) + + point + + pynutil.insert(" ") + + self.graph_fractional + ) + + quantity_w_abbr = get_quantity( + final_graph_wo_sign, cardinal_graph_hundred_component_at_least_one_none_zero_digit, include_abbr=True + ) + quantity_wo_abbr = get_quantity( + final_graph_wo_sign, cardinal_graph_hundred_component_at_least_one_none_zero_digit, include_abbr=False + ) + self.final_graph_wo_negative_w_abbr = final_graph_wo_sign | quantity_w_abbr + self.final_graph_wo_negative = final_graph_wo_sign | quantity_wo_abbr + + # reduce options for non_deterministic and allow either "oh" or "zero", but not combination + if not deterministic: + no_oh_zero = pynini.difference( + NEMO_SIGMA, + (NEMO_SIGMA + "oh" + NEMO_SIGMA + "zero" + NEMO_SIGMA) + | (NEMO_SIGMA + "zero" + NEMO_SIGMA + "oh" + NEMO_SIGMA), + ).optimize() + no_zero_oh = pynini.difference( + NEMO_SIGMA, NEMO_SIGMA + pynini.accep("zero") + NEMO_SIGMA + pynini.accep("oh") + NEMO_SIGMA + ).optimize() + + self.final_graph_wo_negative |= pynini.compose( + self.final_graph_wo_negative, + pynini.cdrewrite( + pynini.cross("integer_part: \"zero\"", "integer_part: \"oh\""), NEMO_SIGMA, NEMO_SIGMA, NEMO_SIGMA + ), + ) + self.final_graph_wo_negative = pynini.compose(self.final_graph_wo_negative, no_oh_zero).optimize() + self.final_graph_wo_negative = pynini.compose(self.final_graph_wo_negative, no_zero_oh).optimize() + + final_graph = optional_graph_negative + self.final_graph_wo_negative + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/electronic.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..351becd356946961736e0979f84d9077f0bbb6cd --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/electronic.py @@ -0,0 +1,179 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( # common string literals + MIN_NEG_WEIGHT, + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_SIGMA, + NEMO_UPPER, + TO_UPPER, + GraphFst, + get_abs_path, + insert_space, +) + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying electronic: as URLs, email addresses, etc. + e.g. cdf1@abc.edu -> tokens { electronic { username: "cdf1" domain: "abc.edu" } } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + if deterministic: + numbers = NEMO_DIGIT + else: + numbers = pynutil.insert(" ") + cardinal.long_numbers + pynutil.insert(" ") + + cc_cues = pynutil.add_weight( + pynini.string_file(get_abs_path("data/electronic/cc_cues.tsv")), + MIN_NEG_WEIGHT, + ) + + cc_cues = pynutil.add_weight( + pynini.string_file(get_abs_path("data/electronic/cc_cues.tsv")), + MIN_NEG_WEIGHT, + ) + + accepted_symbols = pynini.project(pynini.string_file(get_abs_path("data/electronic/symbol.tsv")), "input") + + accepted_common_domains = pynini.project( + pynini.string_file(get_abs_path("data/electronic/domain.tsv")), "input" + ) + + dict_words = pynutil.add_weight( + pynini.string_file(get_abs_path("data/electronic/words.tsv")), + MIN_NEG_WEIGHT, + ) + + dict_words_without_delimiter = dict_words + pynini.closure( + pynutil.add_weight(pynutil.insert(" ") + dict_words, MIN_NEG_WEIGHT), + 1, + ) + dict_words_graph = dict_words_without_delimiter | dict_words + + all_accepted_symbols_start = ( + dict_words_graph | pynini.closure(TO_UPPER) | pynini.closure(NEMO_UPPER) | accepted_symbols + ).optimize() + + all_accepted_symbols_end = ( + dict_words_graph | numbers | pynini.closure(TO_UPPER) | pynini.closure(NEMO_UPPER) | accepted_symbols + ).optimize() + + graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")).optimize() + + username = (NEMO_ALPHA | dict_words_graph) + pynini.closure( + NEMO_ALPHA | numbers | accepted_symbols | dict_words_graph + ) + + username = pynutil.insert('username: "') + username + pynutil.insert('"') + pynini.cross("@", " ") + + domain_graph = all_accepted_symbols_start + pynini.closure( + all_accepted_symbols_end | pynutil.add_weight(accepted_common_domains, MIN_NEG_WEIGHT) + ) + + protocol_symbols = pynini.closure((graph_symbols | pynini.cross(":", "colon")) + pynutil.insert(" ")) + + protocol_start = (pynini.cross("https", "HTTPS ") | pynini.cross("http", "HTTP ")) + ( + pynini.accep("://") @ protocol_symbols + ) + + protocol_file_start = pynini.accep("file") + insert_space + (pynini.accep(":///") @ protocol_symbols) + + protocol_end = pynutil.add_weight(pynini.cross("www", "WWW ") + pynini.accep(".") @ protocol_symbols, -1000) + + protocol = protocol_file_start | protocol_start | protocol_end | (protocol_start + protocol_end) + + domain_graph_with_class_tags = ( + pynutil.insert('domain: "') + + pynini.compose( + NEMO_ALPHA + pynini.closure(NEMO_NOT_SPACE) + (NEMO_ALPHA | NEMO_DIGIT | pynini.accep("/")), + domain_graph, + ).optimize() + + pynutil.insert('"') + ) + + protocol = pynutil.insert('protocol: "') + pynutil.add_weight(protocol, MIN_NEG_WEIGHT) + pynutil.insert('"') + + # email + graph = pynini.compose( + NEMO_SIGMA + pynini.accep("@") + NEMO_SIGMA + pynini.accep(".") + NEMO_SIGMA, + username + domain_graph_with_class_tags, + ) + + full_stop_accep = pynini.accep(".") + dollar_accep = pynini.accep("$") # Include for the correct transduction of the money graph + excluded_symbols = full_stop_accep | dollar_accep | pynini.accep(",") + filtered_symbols = pynini.difference(accepted_symbols, excluded_symbols) + accepted_characters = NEMO_ALPHA | NEMO_DIGIT | filtered_symbols + domain_component = full_stop_accep + pynini.closure(accepted_characters, 2) + graph_domain = pynutil.add_weight( + pynutil.insert('domain: "') + + (pynini.closure(accepted_characters, 1) + pynini.closure(domain_component, 1)) + + pynutil.insert('"'), + 0.1, + ).optimize() + + graph |= pynutil.add_weight(graph_domain, MIN_NEG_WEIGHT) + + # www.abc.com/sdafsdf, or https://www.abc.com/asdfad or www.abc.abc/asdfad + graph |= protocol + pynutil.insert(" ") + domain_graph_with_class_tags + + # recursively handles the "/" in strings like: + # update/upgrade -> update slash upgrade + # update/upgrade/downgrade -> update slash upgrade slash downgrade + + slash_string = ( + pynini.accep(" ").ques + pynini.accep("/") + pynini.accep(" ").ques + pynini.closure(NEMO_ALPHA, 1) + ) + + graph |= ( + pynutil.insert('domain: "') + + (pynini.closure(NEMO_ALPHA, 1) - "and") + + pynini.closure(slash_string, 1) + + pynutil.insert('"') + ).optimize() + + if deterministic: + # credit card cues + numbers = pynini.closure(NEMO_DIGIT, 4, 16) + cc_phrases = ( + pynutil.insert('protocol: "') + cc_cues + pynutil.insert('" domain: "') + numbers + pynutil.insert('"') + ) + graph |= cc_phrases + + if deterministic: + # credit card cues + numbers = pynini.closure(NEMO_DIGIT, 4, 16) + cc_phrases = ( + pynutil.insert('protocol: "') + cc_cues + pynutil.insert('" domain: "') + numbers + pynutil.insert('"') + ) + graph |= cc_phrases + + final_graph = self.add_tokens(graph) + + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/fraction.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..ca747d5191fcc6d6a8ed95fbd069f66a9bfa67fe --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/fraction.py @@ -0,0 +1,56 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst, get_abs_path + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fraction + "23 4/5" -> + tokens { fraction { integer: "twenty three" numerator: "four" denominator: "five" } } + "23 4/5th" -> + tokens { fraction { integer: "twenty three" numerator: "four" denominator: "five" } } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + + integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + numerator = ( + pynutil.insert("numerator: \"") + cardinal_graph + (pynini.cross("/", "\" ") | pynini.cross(" / ", "\" ")) + ) + + endings = ["rd", "th", "st", "nd"] + endings += [x.upper() for x in endings] + optional_end = pynini.closure(pynini.cross(pynini.union(*endings), ""), 0, 1) + + denominator = pynutil.insert("denominator: \"") + cardinal_graph + optional_end + pynutil.insert("\"") + + graph = pynini.closure(integer + pynini.accep(" "), 0, 1) + (numerator + denominator) + graph |= pynini.closure(integer + (pynini.accep(" ") | pynutil.insert(" ")), 0, 1) + pynini.compose( + pynini.string_file(get_abs_path("data/number/fraction.tsv")), (numerator + denominator) + ) + + self.graph = graph + final_graph = self.add_tokens(self.graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/measure.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..b4741c493fc1852ed2a03f1d67b55d522d047672 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/measure.py @@ -0,0 +1,332 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NON_BREAKING_SPACE, + NEMO_SIGMA, + NEMO_SPACE, + NEMO_UPPER, + SINGULAR_TO_PLURAL, + TO_LOWER, + GraphFst, + convert_space, + delete_space, + delete_zero_or_one_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.ordinal import OrdinalFst as OrdinalTagger +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.whitelist import get_formats +from zonos3.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst as OrdinalVerbalizer + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure, suppletive aware, e.g. + -12kg -> measure { negative: "true" cardinal { integer: "twelve" } units: "kilograms" } + 1kg -> measure { cardinal { integer: "one" } units: "kilogram" } + .5kg -> measure { decimal { fractional_part: "five" } units: "kilograms" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + fraction: FractionFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__( + self, + cardinal: GraphFst, + decimal: GraphFst, + fraction: GraphFst, + deterministic: bool = True, + ): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph_with_and | self.get_range(cardinal.graph_with_and) + + graph_unit = pynini.string_file(get_abs_path("data/measure/unit.tsv")) + if not deterministic: + graph_unit |= pynini.string_file(get_abs_path("data/measure/unit_alternatives.tsv")) + + graph_unit |= pynini.compose( + pynini.closure(TO_LOWER, 1) + (NEMO_ALPHA | TO_LOWER) + pynini.closure(NEMO_ALPHA | TO_LOWER), + graph_unit, + ).optimize() + + graph_unit_plural = convert_space(graph_unit @ SINGULAR_TO_PLURAL) + graph_unit = convert_space(graph_unit) + + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + + graph_unit2 = ( + pynini.cross("/", "per") + delete_zero_or_one_space + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit + ) + + optional_graph_unit2 = pynini.closure( + delete_zero_or_one_space + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit2, + 0, + 1, + ) + + unit_plural = ( + pynutil.insert('units: "') + (graph_unit_plural + optional_graph_unit2 | graph_unit2) + pynutil.insert('"') + ) + + unit_singular = ( + pynutil.insert('units: "') + (graph_unit + optional_graph_unit2 | graph_unit2) + pynutil.insert('"') + ) + + subgraph_decimal = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + decimal.final_graph_wo_negative + + delete_space + + pynutil.insert(" } ") + + unit_plural + ) + + # support radio FM/AM + subgraph_decimal |= ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + delete_space + + pynutil.insert(" } ") + + pynutil.insert('units: "') + + pynini.union("AM", "FM") + + pynutil.insert('"') + ) + + subgraph_cardinal = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert('integer: "') + + ((NEMO_SIGMA - "1") @ cardinal_graph) + + delete_space + + pynutil.insert('"') + + pynutil.insert(" } ") + + unit_plural + ) + + subgraph_cardinal |= ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert('integer: "') + + pynini.cross("1", "one") + + delete_space + + pynutil.insert('"') + + pynutil.insert(" } ") + + unit_singular + ) + + unit_graph = ( + pynutil.insert('cardinal { integer: "-" } units: "') + + ((pynini.cross("/", "per") + delete_zero_or_one_space) | (pynini.accep("per") + pynutil.delete(" "))) + + pynutil.insert(NEMO_NON_BREAKING_SPACE) + + graph_unit + + pynutil.insert('" preserve_order: true') + ) + + decimal_dash_alpha = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynini.cross("-", "") + + pynutil.insert(' } units: "') + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert('"') + ) + + decimal_times = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(' } units: "') + + (pynini.cross(pynini.union("x", "X"), "x") | pynini.cross(pynini.union("x", "X"), " times")) + + pynutil.insert('"') + ) + + alpha_dash_decimal = ( + pynutil.insert('units: "') + + pynini.closure(NEMO_ALPHA, 1) + + pynini.accep("-") + + pynutil.insert('"') + + pynutil.insert(" decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(" } preserve_order: true") + ) + + subgraph_fraction = ( + pynutil.insert("fraction { ") + fraction.graph + delete_space + pynutil.insert(" } ") + unit_plural + ) + + address = self.get_address_graph(cardinal) + address = ( + pynutil.insert('units: "address" cardinal { integer: "') + + address + + pynutil.insert('" } preserve_order: true') + ) + + math_operations = pynini.string_file(get_abs_path("data/measure/math_operation.tsv")) + delimiter = pynini.accep(" ") | pynutil.insert(" ") + + math = ( + (cardinal_graph | NEMO_ALPHA) + + delimiter + + math_operations + + (delimiter | NEMO_ALPHA) + + cardinal_graph + + delimiter + + pynini.cross("=", "equals") + + delimiter + + (cardinal_graph | NEMO_ALPHA) + ) + + math |= ( + (cardinal_graph | NEMO_ALPHA) + + delimiter + + pynini.cross("=", "equals") + + delimiter + + (cardinal_graph | NEMO_ALPHA) + + delimiter + + math_operations + + delimiter + + cardinal_graph + ) + + math = ( + pynutil.insert('units: "math" cardinal { integer: "') + math + pynutil.insert('" } preserve_order: true') + ) + final_graph = ( + subgraph_decimal + | subgraph_cardinal + | unit_graph + | decimal_dash_alpha + | decimal_times + | alpha_dash_decimal + | subgraph_fraction + | address + | math + ) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() + + def get_range(self, cardinal: GraphFst): + """ + Returns range forms for measure tagger, e.g. 2-3, 2x3, 2*2 + + Args: + cardinal: cardinal GraphFst + """ + range_graph = cardinal + pynini.cross(pynini.union("-", " - "), " to ") + cardinal + + for x in [" x ", "x"]: + range_graph |= cardinal + pynini.cross(x, " by ") + cardinal + if not self.deterministic: + range_graph |= cardinal + pynini.cross(x, " times ") + pynini.closure(cardinal, 0, 1) + + for x in ["*", " * "]: + range_graph |= cardinal + pynini.cross(x, " times ") + cardinal + return range_graph.optimize() + + def get_address_graph(self, cardinal): + """ + Finite state transducer for classifying serial. + The serial is a combination of digits, letters and dashes, e.g.: + 2788 San Tomas Expy, Santa Clara, CA 95051 -> + units: "address" cardinal + { integer: "two seven eight eight San Tomas Expressway Santa Clara California nine five zero five one" } + preserve_order: true + """ + ordinal_verbalizer = OrdinalVerbalizer().graph + ordinal_tagger = OrdinalTagger(cardinal=cardinal).graph + ordinal_num = pynini.compose( + pynutil.insert('integer: "') + ordinal_tagger + pynutil.insert('"'), + ordinal_verbalizer, + ) + + address_num = NEMO_DIGIT ** (1, 2) @ cardinal.graph_hundred_component_at_least_one_none_zero_digit + address_num += insert_space + NEMO_DIGIT**2 @ ( + pynini.closure(pynini.cross("0", "zero "), 0, 1) + + cardinal.graph_hundred_component_at_least_one_none_zero_digit + ) + # to handle the rest of the numbers + address_num = pynini.compose(NEMO_DIGIT ** (3, 4), address_num) + address_num = plurals._priority_union(address_num, cardinal.graph, NEMO_SIGMA) + + direction = ( + pynini.cross("E", "East") + | pynini.cross("S", "South") + | pynini.cross("W", "West") + | pynini.cross("N", "North") + ) + pynini.closure(pynutil.delete("."), 0, 1) + + direction = pynini.closure(pynini.accep(NEMO_SPACE) + direction, 0, 1) + address_words = get_formats(get_abs_path("data/address/address_word.tsv")) + address_words = ( + pynini.accep(NEMO_SPACE) + + (pynini.closure(ordinal_num, 0, 1) | NEMO_UPPER + pynini.closure(NEMO_ALPHA, 1)) + + NEMO_SPACE + + pynini.closure(NEMO_UPPER + pynini.closure(NEMO_ALPHA) + NEMO_SPACE) + + address_words + ) + + city = pynini.closure(NEMO_ALPHA | pynini.accep(NEMO_SPACE), 1) + city = pynini.closure(pynini.accep(",") + pynini.accep(NEMO_SPACE) + city, 0, 1) + + states = load_labels(get_abs_path("data/address/state.tsv")) + + additional_options = [] + for x, y in states: + additional_options.append((x, f"{y[0]}.{y[1:]}")) + states.extend(additional_options) + state_graph = pynini.string_map(states) + state = pynini.invert(state_graph) + state = pynini.closure(pynini.accep(",") + pynini.accep(NEMO_SPACE) + state, 0, 1) + + zip_code = pynini.compose(NEMO_DIGIT**5, cardinal.single_digits_graph) + zip_code = pynini.closure( + pynini.closure(pynini.accep(","), 0, 1) + pynini.accep(NEMO_SPACE) + zip_code, + 0, + 1, + ) + + address = address_num + direction + address_words + pynini.closure(city + state + zip_code, 0, 1) + + # Handles sentences ending in terminal period abbreviations, when followed by another sentence + # (e.g. Nancy lived on 1428 Elm St. It was a strange place-> Nancy lived on fourteen twenty eight Elm Street. It was a strange place.) + + hyphen = pynini.accep("-") + alpha_chars = NEMO_ALPHA | hyphen + + address_eos = ( + address_num + + direction + + address_words + + pynini.accep(".") + + pynini.closure(NEMO_SPACE, 1, 2) + + NEMO_UPPER + + pynini.closure(alpha_chars) + ) + + address |= pynutil.add_weight(address_eos, -0.001) + + address |= address_num + direction + address_words + pynini.closure(pynini.cross(".", ""), 0, 1) + + return address diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/money.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..ac4d416d22290e84b201271374da21b70e29c228 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/money.py @@ -0,0 +1,199 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + SINGULAR_TO_PLURAL, + GraphFst, + convert_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + +min_singular = pynini.string_file(get_abs_path("data/money/currency_minor_singular.tsv")) +min_plural = pynini.string_file(get_abs_path("data/money/currency_minor_plural.tsv")) +maj_singular = pynini.string_file((get_abs_path("data/money/currency_major.tsv"))) +per_units = pynini.string_file(get_abs_path("data/money/per_unit.tsv")) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money, suppletive aware, e.g. + $12.05 -> money { integer_part: "twelve" currency_maj: "dollars" fractional_part: "five" currency_min: "cents" preserve_order: true } + $12.0500 -> money { integer_part: "twelve" currency_maj: "dollars" fractional_part: "five" currency_min: "cents" preserve_order: true } + $1 -> money { currency_maj: "dollar" integer_part: "one" } + $1.00 -> money { currency_maj: "dollar" integer_part: "one" } + $0.05 -> money { fractional_part: "five" currency_min: "cents" preserve_order: true } + $1 million -> money { currency_maj: "dollars" integer_part: "one" quantity: "million" } + $1.2 million -> money { currency_maj: "dollars" integer_part: "one" fractional_part: "two" quantity: "million" } + $1.2320 -> money { currency_maj: "dollars" integer_part: "one" fractional_part: "two three two" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph_with_and + graph_decimal_final = decimal.final_graph_wo_negative_w_abbr + + maj_singular_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) + maj_unit_plural = convert_space(maj_singular @ SINGULAR_TO_PLURAL) + maj_unit_singular = convert_space(maj_singular) + + graph_maj_singular = pynutil.insert('currency_maj: "') + maj_unit_singular + pynutil.insert('"') + graph_maj_plural = pynutil.insert('currency_maj: "') + maj_unit_plural + pynutil.insert('"') + + optional_delete_fractional_zeros = pynini.closure( + pynutil.delete(".") + pynini.closure(pynutil.delete("0"), 1), 0, 1 + ) + + graph_integer_one = pynutil.insert('integer_part: "') + pynini.cross("1", "one") + pynutil.insert('"') + # only for decimals where third decimal after comma is non-zero or with quantity + decimal_delete_last_zeros = ( + pynini.closure(NEMO_DIGIT | pynutil.delete(",")) + + pynini.accep(".") + + pynini.closure(NEMO_DIGIT, 2) + + (NEMO_DIGIT - "0") + + pynini.closure(pynutil.delete("0")) + ) + decimal_with_quantity = NEMO_SIGMA + NEMO_ALPHA + + graph_decimal = ( + graph_maj_plural + insert_space + (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final + ) + + graph_integer = pynutil.insert('integer_part: "') + ((NEMO_SIGMA - "1") @ cardinal_graph) + pynutil.insert('"') + + graph_integer_only = graph_maj_singular + insert_space + graph_integer_one + graph_integer_only |= graph_maj_plural + insert_space + graph_integer + + final_graph = (graph_integer_only + optional_delete_fractional_zeros) | graph_decimal + + # remove trailing zeros of non zero number in the first 2 digits and fill up to 2 digits + # e.g. 2000 -> 20, 0200->02, 01 -> 01, 10 -> 10 + # not accepted: 002, 00, 0, + two_digits_fractional_part = ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(pynutil.delete("0")) + ) @ ( + (pynutil.delete("0") + (NEMO_DIGIT - "0")) + | ((NEMO_DIGIT - "0") + pynutil.insert("0")) + | ((NEMO_DIGIT - "0") + NEMO_DIGIT) + ) + + graph_min_singular = pynutil.insert(' currency_min: "') + min_singular + pynutil.insert('"') + graph_min_plural = pynutil.insert(' currency_min: "') + min_plural + pynutil.insert('"') + # format ** dollars ** cent + decimal_graph_with_minor = None + integer_graph_reordered = None + decimal_default_reordered = None + for curr_symbol, _ in maj_singular_labels: + preserve_order = pynutil.insert(" preserve_order: true") + integer_plus_maj = graph_integer + insert_space + pynutil.insert(curr_symbol) @ graph_maj_plural + integer_plus_maj |= graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular + + integer_plus_maj_with_comma = pynini.compose( + NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT | pynutil.delete(",")), + integer_plus_maj, + ) + integer_plus_maj = pynini.compose(pynini.closure(NEMO_DIGIT) - "0", integer_plus_maj) + integer_plus_maj |= integer_plus_maj_with_comma + + graph_fractional_one = two_digits_fractional_part @ pynini.cross("1", "one") + graph_fractional_one = pynutil.insert('fractional_part: "') + graph_fractional_one + pynutil.insert('"') + graph_fractional = ( + two_digits_fractional_part + @ (pynini.closure(NEMO_DIGIT, 1, 2) - "1") + @ cardinal.graph_hundred_component_at_least_one_none_zero_digit + ) + graph_fractional = pynutil.insert('fractional_part: "') + graph_fractional + pynutil.insert('"') + + fractional_plus_min = graph_fractional + insert_space + pynutil.insert(curr_symbol) @ graph_min_plural + fractional_plus_min |= ( + graph_fractional_one + insert_space + pynutil.insert(curr_symbol) @ graph_min_singular + ) + + decimal_graph_with_minor_curr = integer_plus_maj + pynini.cross(".", " ") + fractional_plus_min + + if not deterministic: + decimal_graph_with_minor_curr |= pynutil.add_weight( + integer_plus_maj + + pynini.cross(".", " ") + + pynutil.insert('fractional_part: "') + + two_digits_fractional_part @ cardinal.graph_hundred_component_at_least_one_none_zero_digit + + pynutil.insert('"'), + weight=0.0001, + ) + default_fraction_graph = (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final + decimal_graph_with_minor_curr |= ( + pynini.closure(pynutil.delete("0"), 0, 1) + pynutil.delete(".") + fractional_plus_min + ) + decimal_graph_with_minor_curr = ( + pynutil.delete(curr_symbol) + decimal_graph_with_minor_curr + preserve_order + ) + + decimal_graph_with_minor = ( + decimal_graph_with_minor_curr + if decimal_graph_with_minor is None + else pynini.union(decimal_graph_with_minor, decimal_graph_with_minor_curr).optimize() + ) + + if not deterministic: + integer_graph_reordered_curr = ( + pynutil.delete(curr_symbol) + integer_plus_maj + preserve_order + ).optimize() + + integer_graph_reordered = ( + integer_graph_reordered_curr + if integer_graph_reordered is None + else pynini.union(integer_graph_reordered, integer_graph_reordered_curr).optimize() + ) + decimal_default_reordered_curr = ( + pynutil.delete(curr_symbol) + + default_fraction_graph + + insert_space + + pynutil.insert(curr_symbol) @ graph_maj_plural + ) + + decimal_default_reordered = ( + decimal_default_reordered_curr + if decimal_default_reordered is None + else pynini.union(decimal_default_reordered, decimal_default_reordered_curr) + ).optimize() + + # weight for SH + final_graph |= pynutil.add_weight(decimal_graph_with_minor, -0.0001) + + # utilizes morphosyntactic features to append "per units" + graph_per_units = pynutil.insert(' morphosyntactic_features: "') + per_units + pynutil.insert('"') + + if not deterministic: + final_graph |= integer_graph_reordered | decimal_default_reordered + # to handle "$2.00" cases + final_graph |= pynini.compose( + NEMO_SIGMA + pynutil.delete(".") + pynini.closure(pynutil.delete("0"), 1), + integer_graph_reordered, + ) + + final_graph += graph_per_units.ques + final_graph = self.add_tokens(final_graph.optimize()) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/ordinal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..5271105850882277aef4b42dbda3a40180653c48 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/ordinal.py @@ -0,0 +1,62 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal, e.g. + 13th -> ordinal { integer: "thirteen" } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.graph + cardinal_format = pynini.closure(NEMO_DIGIT | pynini.accep(",")) + st_format = ( + pynini.closure(cardinal_format + (NEMO_DIGIT - "1"), 0, 1) + + pynini.accep("1") + + pynutil.delete(pynini.union("st", "ST", "ˢᵗ")) + ) + nd_format = ( + pynini.closure(cardinal_format + (NEMO_DIGIT - "1"), 0, 1) + + pynini.accep("2") + + pynutil.delete(pynini.union("nd", "ND", "ⁿᵈ")) + ) + rd_format = ( + pynini.closure(cardinal_format + (NEMO_DIGIT - "1"), 0, 1) + + pynini.accep("3") + + pynutil.delete(pynini.union("rd", "RD", "ʳᵈ")) + ) + th_format = pynini.closure( + (NEMO_DIGIT - "1" - "2" - "3") + | (cardinal_format + "1" + NEMO_DIGIT) + | (cardinal_format + (NEMO_DIGIT - "1") + (NEMO_DIGIT - "1" - "2" - "3")), + 1, + ) + pynutil.delete(pynini.union("th", "TH", "ᵗʰ")) + self.graph = (st_format | nd_format | rd_format | th_format) @ cardinal_graph + final_graph = pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/punctuation.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/punctuation.py new file mode 100644 index 0000000000000000000000000000000000000000..9cd1f18ef65098787afc1b257c812cd881220dfd --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/punctuation.py @@ -0,0 +1,66 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from unicodedata import category + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + s = "!#%&\'()*+,-./:;<=>?@^_`{|}~\"" + + punct_symbols_to_exclude = ["[", "]"] + punct_unicode = [ + chr(i) + for i in range(sys.maxunicode) + if category(chr(i)).startswith("P") and chr(i) not in punct_symbols_to_exclude + ] + [r"\[", r"\]"] + + whitelist_symbols = load_labels(get_abs_path("data/whitelist/symbol.tsv")) + whitelist_symbols = [x[0] for x in whitelist_symbols] + self.punct_marks = [p for p in punct_unicode + list(s) if p not in whitelist_symbols] + + punct = pynini.union(*self.punct_marks) + punct = pynini.closure(punct, 1) + + emphasis = ( + pynini.accep("<") + + ( + (pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1) + pynini.closure(pynini.accep("/"), 0, 1)) + | (pynini.accep("/") + pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1)) + ) + + pynini.accep(">") + ) + punct = plurals._priority_union(emphasis, punct, NEMO_SIGMA) + + self.graph = punct + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/range.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/range.py new file mode 100644 index 0000000000000000000000000000000000000000..a9508fae4216e7e18d1eb5f3db29b489eca41b3d --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/range.py @@ -0,0 +1,128 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, convert_space + + +class RangeFst(GraphFst): + """ + This class is a composite class of two other class instances + + Args: + time: composed tagger and verbalizer + date: composed tagger and verbalizer + cardinal: tagger + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + lm: whether to use for hybrid LM + """ + + def __init__( + self, + time: GraphFst, + date: GraphFst, + cardinal: GraphFst, + deterministic: bool = True, + lm: bool = False, + ): + super().__init__(name="range", kind="classify", deterministic=deterministic) + + delete_space = pynini.closure(pynutil.delete(" "), 0, 1) + + approx = pynini.cross("~", "approximately") + + # TIME + time_graph = time + delete_space + pynini.cross("-", " to ") + delete_space + time + self.graph = time_graph | (approx + time) + + cardinal = cardinal.graph_with_and + # YEAR + date_year_four_digit = (NEMO_DIGIT**4 + pynini.closure(pynini.accep("s"), 0, 1)) @ date + date_year_two_digit = (NEMO_DIGIT**2 + pynini.closure(pynini.accep("s"), 0, 1)) @ date + year_to_year_graph = ( + date_year_four_digit + + delete_space + + pynini.cross("-", " to ") + + delete_space + + (date_year_four_digit | date_year_two_digit | (NEMO_DIGIT**2 @ cardinal)) + ) + mid_year_graph = pynini.accep("mid") + pynini.cross("-", " ") + (date_year_four_digit | date_year_two_digit) + + self.graph |= year_to_year_graph + self.graph |= mid_year_graph + + # ADDITION + range_graph = cardinal + pynini.closure(pynini.cross("+", " plus ") + cardinal, 1) + range_graph |= cardinal + pynini.closure(pynini.cross(" + ", " plus ") + cardinal, 1) + range_graph |= approx + cardinal + range_graph |= cardinal + (pynini.cross("...", " ... ") | pynini.accep(" ... ")) + cardinal + + if not deterministic or lm: + # cardinal ---- + cardinal_to_cardinal_graph = ( + cardinal + delete_space + pynini.cross("-", pynini.union(" to ", " minus ")) + delete_space + cardinal + ) + + range_graph |= cardinal_to_cardinal_graph | ( + cardinal + delete_space + pynini.cross(":", " to ") + delete_space + cardinal + ) + + # MULTIPLY + for x in [" x ", "x"]: + range_graph |= cardinal + pynini.cross(x, pynini.union(" by ", " times ")) + cardinal + + # 40x -> "40 times" ("40 x" cases is covered in serial) + for x in [" x", "x"]: + range_graph |= cardinal + pynini.cross(x, " times") + + # 5x to 7x-> five to seven x/times + range_graph |= ( + cardinal + + pynutil.delete(x) + + pynini.union(" to ", "-", " - ") + + cardinal + + pynini.cross(x, pynini.union(" x", " times")) + ) + + for x in ["*", " * "]: + range_graph |= cardinal + pynini.closure(pynini.cross(x, " times ") + cardinal, 1) + + # supports "No. 12" -> "Number 12" + range_graph |= ( + (pynini.cross(pynini.union("NO", "No"), "Number") | pynini.cross("no", "number")) + + pynini.closure(pynini.union(". ", " "), 0, 1) + + cardinal + ) + + for x in ["/", " / "]: + range_graph |= cardinal + pynini.closure(pynini.cross(x, " divided by ") + cardinal, 1) + + # 10% to 20% -> ten to twenty percent + range_graph |= ( + cardinal + + pynini.closure(pynini.cross("%", " percent") | pynutil.delete("%"), 0, 1) + + pynini.union(" to ", "-", " - ") + + cardinal + + pynini.cross("%", " percent") + ) + + self.graph |= range_graph + + self.graph = self.graph.optimize() + graph = pynutil.insert("name: \"") + convert_space(self.graph).optimize() + pynutil.insert("\"") + self.fst = graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/roman.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/roman.py new file mode 100644 index 0000000000000000000000000000000000000000..bafc35ea8fdb3b2ea63f70c51e247d6808391062 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/roman.py @@ -0,0 +1,115 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_SIGMA, GraphFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + + +class RomanFst(GraphFst): + """ + Finite state transducer for classifying roman numbers: + e.g. "IV" -> tokens { roman { integer: "four" } } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="roman", kind="classify", deterministic=deterministic) + + roman_dict = load_labels(get_abs_path("data/roman/roman_to_spoken.tsv")) + default_graph = pynini.string_map(roman_dict).optimize() + default_graph = pynutil.insert("integer: \"") + default_graph + pynutil.insert("\"") + ordinal_limit = 19 + + if deterministic: + # exclude "I" + start_idx = 1 + else: + start_idx = 0 + + graph_teens = pynini.string_map([x[0] for x in roman_dict[start_idx:ordinal_limit]]).optimize() + + # roman numerals up to ordinal_limit with a preceding name are converted to ordinal form + names = get_names() + graph = ( + pynutil.insert("key_the_ordinal: \"") + + names + + pynutil.insert("\"") + + pynini.accep(" ") + + graph_teens @ default_graph + ).optimize() + + # single symbol roman numerals with preceding key words (multiple formats) are converted to cardinal form + key_words = [] + for k_word in load_labels(get_abs_path("data/roman/key_word.tsv")): + key_words.append(k_word) + key_words.append([k_word[0][0].upper() + k_word[0][1:]]) + key_words.append([k_word[0].upper()]) + + key_words = pynini.string_map(key_words).optimize() + graph |= ( + pynutil.insert("key_cardinal: \"") + key_words + pynutil.insert("\"") + pynini.accep(" ") + default_graph + ).optimize() + + if deterministic or lm: + # two digit roman numerals up to 49 + roman_to_cardinal = pynini.compose( + pynini.closure(NEMO_ALPHA, 2), + ( + pynutil.insert("default_cardinal: \"default\" ") + + (pynini.string_map([x[0] for x in roman_dict[:50]]).optimize()) @ default_graph + ), + ) + graph |= roman_to_cardinal + elif not lm: + # two or more digit roman numerals + roman_to_cardinal = pynini.compose( + pynini.difference(NEMO_SIGMA, "I"), + ( + pynutil.insert("default_cardinal: \"default\" integer: \"") + + pynini.string_map(roman_dict).optimize() + + pynutil.insert("\"") + ), + ).optimize() + graph |= roman_to_cardinal + + # convert three digit roman or up with suffix to ordinal + roman_to_ordinal = pynini.compose( + pynini.closure(NEMO_ALPHA, 3), + (pynutil.insert("default_ordinal: \"default\" ") + graph_teens @ default_graph + pynutil.delete("th")), + ) + + graph |= roman_to_ordinal + graph = self.add_tokens(graph.optimize()) + + self.fst = graph.optimize() + + +def get_names(): + """ + Returns the graph that matched common male and female names. + """ + male_labels = load_labels(get_abs_path("data/roman/male.tsv")) + female_labels = load_labels(get_abs_path("data/roman/female.tsv")) + male_labels.extend([[x[0].upper()] for x in male_labels]) + female_labels.extend([[x[0].upper()] for x in female_labels]) + names = pynini.string_map(male_labels).optimize() + names |= pynini.string_map(female_labels).optimize() + return names diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/serial.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/serial.py new file mode 100644 index 0000000000000000000000000000000000000000..04c23baf18bf7e7374b8b112b884fd73e2c90f6a --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/serial.py @@ -0,0 +1,143 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_SIGMA, + GraphFst, + convert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + + +class SerialFst(GraphFst): + """ + This class is a composite class of two other class instances + + Args: + time: composed tagger and verbalizer + date: composed tagger and verbalizer + cardinal: tagger + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + lm: whether to use for hybrid LM + """ + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True, lm: bool = False): + super().__init__(name="integer", kind="classify", deterministic=deterministic) + + """ + Finite state transducer for classifying serial (handles only cases without delimiters, + values with delimiters are handled by default). + The serial is a combination of digits, letters and dashes, e.g.: + c325b -> tokens { cardinal { integer: "c three two five b" } } + """ + if deterministic: + num_graph = pynini.compose(NEMO_DIGIT ** (6, ...), cardinal.single_digits_graph).optimize() + num_graph |= pynini.compose(NEMO_DIGIT ** (1, 5), cardinal.graph).optimize() + # to handle numbers starting with zero + num_graph |= pynini.compose( + pynini.accep("0") + pynini.closure(NEMO_DIGIT), cardinal.single_digits_graph + ).optimize() + else: + num_graph = cardinal.final_graph + + # TODO: "#" doesn't work from the file + symbols_graph = pynini.string_file(get_abs_path("data/whitelist/symbol.tsv")).optimize() | pynini.cross( + "#", "hash" + ) + num_graph |= symbols_graph + + if not self.deterministic and not lm: + num_graph |= cardinal.single_digits_graph + num_graph |= pynini.compose(num_graph, NEMO_SIGMA + pynutil.delete("hundred ") + NEMO_SIGMA) + # also allow double digits to be pronounced as integer in serial number + num_graph |= pynutil.add_weight( + NEMO_DIGIT**2 @ cardinal.graph_hundred_component_at_least_one_none_zero_digit, weight=0.0001 + ) + + # add space between letter and digit/symbol + symbols = [x[0] for x in load_labels(get_abs_path("data/whitelist/symbol.tsv"))] + symbols = pynini.union(*symbols) + digit_symbol = NEMO_DIGIT | symbols + + graph_with_space = pynini.compose( + pynini.cdrewrite(pynutil.insert(" "), NEMO_ALPHA | symbols, digit_symbol, NEMO_SIGMA), + pynini.cdrewrite(pynutil.insert(" "), digit_symbol, NEMO_ALPHA | symbols, NEMO_SIGMA), + ) + + # serial graph with delimiter + delimiter = pynini.accep("-") | pynini.accep("/") | pynini.accep(" ") + if not deterministic: + delimiter |= pynini.cross("-", " dash ") | pynini.cross("/", " slash ") + + alphas = pynini.closure(NEMO_ALPHA, 1) + letter_num = alphas + delimiter + num_graph + num_letter = pynini.closure(num_graph + delimiter, 1) + alphas + next_alpha_or_num = pynini.closure(delimiter + (alphas | num_graph)) + next_alpha_or_num |= pynini.closure( + delimiter + + num_graph + + plurals._priority_union(pynini.accep(" "), pynutil.insert(" "), NEMO_SIGMA).optimize() + + alphas + ) + + serial_graph = letter_num + next_alpha_or_num + serial_graph |= num_letter + next_alpha_or_num + # numbers only with 2+ delimiters + serial_graph |= ( + num_graph + delimiter + num_graph + delimiter + num_graph + pynini.closure(delimiter + num_graph) + ) + # 2+ symbols + serial_graph |= pynini.compose(NEMO_SIGMA + symbols + NEMO_SIGMA, num_graph + delimiter + num_graph) + + # exclude ordinal numbers from serial options + serial_graph = pynini.compose( + pynini.difference(NEMO_SIGMA, pynini.project(ordinal.graph, "input")), serial_graph + ).optimize() + + serial_graph = pynutil.add_weight(serial_graph, 0.0001) + serial_graph |= ( + pynini.closure(NEMO_NOT_SPACE, 1) + + (pynini.cross("^2", " squared") | pynini.cross("^3", " cubed")).optimize() + ) + + # at least one serial graph with alpha numeric value and optional additional serial/num/alpha values + serial_graph = ( + pynini.closure((serial_graph | num_graph | alphas) + delimiter) + + serial_graph + + pynini.closure(delimiter + (serial_graph | num_graph | alphas)) + ) + + serial_graph |= pynini.compose(graph_with_space, serial_graph.optimize()).optimize() + serial_graph = pynini.compose(pynini.closure(NEMO_NOT_SPACE, 2), serial_graph).optimize() + + # this is not to verbolize "/" as "slash" in cases like "import/export" + serial_graph = pynini.compose( + pynini.difference( + NEMO_SIGMA, pynini.closure(NEMO_ALPHA, 1) + pynini.accep("/") + pynini.closure(NEMO_ALPHA, 1) + ), + serial_graph, + ) + + self.graph = serial_graph.optimize() + graph = pynutil.insert("name: \"") + convert_space(self.graph).optimize() + pynutil.insert("\"") + self.fst = graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/telephone.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/telephone.py new file mode 100644 index 0000000000000000000000000000000000000000..78f41d828b40c4dc9363c8a4972f8239c2cf535a --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/telephone.py @@ -0,0 +1,134 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + GraphFst, + delete_extra_space, + delete_space, + insert_space, + plurals, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying telephone, and IP, and SSN which includes country code, number part and extension + country code optional: +*** + number part: ***-***-****, or (***) ***-**** + extension optional: 1-9999 + E.g + +1 123-123-5678-1 -> telephone { country_code: "one" number_part: "one two three, one two three, five six seven eight" extension: "one" } + 1-800-GO-U-HAUL -> telephone { country_code: "one" number_part: "one, eight hundred GO U HAUL" } + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="classify", deterministic=deterministic) + + add_separator = pynutil.insert(", ") # between components + zero = pynini.cross("0", "zero") + if not deterministic: + zero |= pynini.cross("0", pynini.union("o", "oh")) + digit = pynini.invert(pynini.string_file(get_abs_path("data/number/digit.tsv"))).optimize() | zero + + telephone_prompts = pynini.string_file(get_abs_path("data/telephone/telephone_prompt.tsv")) + country_code = ( + pynini.closure(telephone_prompts + delete_extra_space, 0, 1) + + pynini.closure(pynini.cross("+", "plus "), 0, 1) + + pynini.closure(digit + insert_space, 0, 2) + + digit + + pynutil.insert(",") + ) + country_code |= telephone_prompts + country_code = pynutil.insert("country_code: \"") + country_code + pynutil.insert("\"") + country_code = country_code + pynini.closure(pynutil.delete("-"), 0, 1) + delete_space + insert_space + + area_part_default = pynini.closure(digit + insert_space, 2, 2) + digit + area_part = pynini.cross("800", "eight hundred") | pynini.compose( + pynini.difference(NEMO_SIGMA, "800"), area_part_default + ) + + area_part = ( + (area_part + (pynutil.delete("-") | pynutil.delete("."))) + | ( + pynutil.delete("(") + + area_part + + ((pynutil.delete(")") + pynini.closure(pynutil.delete(" "), 0, 1)) | pynutil.delete(")-")) + ) + ) + add_separator + + del_separator = pynini.closure(pynini.union("-", " ", "."), 0, 1) + number_length = ((NEMO_DIGIT + del_separator) | (NEMO_ALPHA + del_separator)) ** 7 + number_words = pynini.closure( + (NEMO_DIGIT @ digit) + (insert_space | (pynini.cross("-", ', '))) + | NEMO_ALPHA + | (NEMO_ALPHA + pynini.cross("-", ' ')) + ) + number_words |= pynini.closure( + (NEMO_DIGIT @ digit) + (insert_space | (pynini.cross(".", ', '))) + | NEMO_ALPHA + | (NEMO_ALPHA + pynini.cross(".", ' ')) + ) + number_words = pynini.compose(number_length, number_words) + number_part = area_part + number_words + number_part = pynutil.insert("number_part: \"") + number_part + pynutil.insert("\"") + extension = ( + pynutil.insert("extension: \"") + pynini.closure(digit + insert_space, 0, 3) + digit + pynutil.insert("\"") + ) + extension = pynini.closure(insert_space + extension, 0, 1) + + graph = plurals._priority_union(country_code + number_part, number_part, NEMO_SIGMA).optimize() + graph = plurals._priority_union(country_code + number_part + extension, graph, NEMO_SIGMA).optimize() + graph = plurals._priority_union(number_part + extension, graph, NEMO_SIGMA).optimize() + + # ip + ip_prompts = pynini.string_file(get_abs_path("data/telephone/ip_prompt.tsv")) + digit_to_str_graph = digit + pynini.closure(pynutil.insert(" ") + digit, 0, 2) + ip_graph = digit_to_str_graph + (pynini.cross(".", " dot ") + digit_to_str_graph) ** 3 + graph |= ( + pynini.closure( + pynutil.insert("country_code: \"") + ip_prompts + pynutil.insert("\"") + delete_extra_space, 0, 1 + ) + + pynutil.insert("number_part: \"") + + ip_graph.optimize() + + pynutil.insert("\"") + ) + # ssn + ssn_prompts = pynini.string_file(get_abs_path("data/telephone/ssn_prompt.tsv")) + three_digit_part = digit + (pynutil.insert(" ") + digit) ** 2 + two_digit_part = digit + pynutil.insert(" ") + digit + four_digit_part = digit + (pynutil.insert(" ") + digit) ** 3 + ssn_separator = pynini.cross("-", ", ") + ssn_graph = three_digit_part + ssn_separator + two_digit_part + ssn_separator + four_digit_part + + graph |= ( + pynini.closure( + pynutil.insert("country_code: \"") + ssn_prompts + pynutil.insert("\"") + delete_extra_space, 0, 1 + ) + + pynutil.insert("number_part: \"") + + ssn_graph.optimize() + + pynutil.insert("\"") + ) + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/time.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..80fbca56cd30f3a5e1b9ee3216ad05e530586adc --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/time.py @@ -0,0 +1,133 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + GraphFst, + convert_space, + delete_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.utils import ( + augment_labels_with_punct_at_end, + get_abs_path, + load_labels, +) + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + 12:30 a.m. est -> time { hours: "twelve" minutes: "thirty" suffix: "a m" zone: "e s t" } + 2.30 a.m. -> time { hours: "two" minutes: "thirty" suffix: "a m" } + 02.30 a.m. -> time { hours: "two" minutes: "thirty" suffix: "a m" } + 2.00 a.m. -> time { hours: "two" suffix: "a m" } + 2 a.m. -> time { hours: "two" suffix: "a m" } + 02:00 -> time { hours: "two" } + 2:00 -> time { hours: "two" } + 10:00:05 a.m. -> time { hours: "ten" minutes: "zero" seconds: "five" suffix: "a m" } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + suffix_labels = load_labels(get_abs_path("data/time/suffix.tsv")) + suffix_labels.extend(augment_labels_with_punct_at_end(suffix_labels)) + suffix_graph = pynini.string_map(suffix_labels) + + time_zone_graph = pynini.string_file(get_abs_path("data/time/zone.tsv")) + + # only used for < 1000 thousand -> 0 weight + cardinal = cardinal.graph + + labels_hour = [str(x) for x in range(0, 24)] + labels_minute_single = [str(x) for x in range(1, 10)] + labels_minute_double = [str(x) for x in range(10, 60)] + + delete_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | ( + pynini.closure(pynutil.delete("0"), 0, 1) + NEMO_DIGIT + ) + + graph_hour = delete_leading_zero_to_double_digit @ pynini.union(*labels_hour) @ cardinal + + graph_minute_single = pynini.union(*labels_minute_single) @ cardinal + graph_minute_double = pynini.union(*labels_minute_double) @ cardinal + + final_graph_hour = pynutil.insert("hours: \"") + graph_hour + pynutil.insert("\"") + final_graph_minute = ( + pynutil.insert("minutes: \"") + + (pynini.cross("0", "o") + insert_space + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + final_graph_second = ( + pynutil.insert("seconds: \"") + + (pynini.cross("0", "o") + insert_space + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + final_suffix = pynutil.insert("suffix: \"") + convert_space(suffix_graph) + pynutil.insert("\"") + final_suffix_optional = pynini.closure(delete_space + insert_space + final_suffix, 0, 1) + final_time_zone_optional = pynini.closure( + delete_space + + insert_space + + pynutil.insert("zone: \"") + + convert_space(time_zone_graph) + + pynutil.insert("\""), + 0, + 1, + ) + + # 2:30 pm, 02:30, 2:00 + graph_hm = ( + final_graph_hour + + pynutil.delete(":") + + (pynutil.delete("00") | insert_space + final_graph_minute) + + final_suffix_optional + + final_time_zone_optional + ) + + # 10:30:05 pm, + graph_hms = ( + final_graph_hour + + pynutil.delete(":") + + (pynini.cross("00", " minutes: \"zero\"") | insert_space + final_graph_minute) + + pynutil.delete(":") + + (pynini.cross("00", " seconds: \"zero\"") | insert_space + final_graph_second) + + final_suffix_optional + + final_time_zone_optional + ) + + # 2.xx pm/am + graph_hm2 = ( + final_graph_hour + + pynutil.delete(".") + + (pynutil.delete("00") | insert_space + final_graph_minute) + + delete_space + + insert_space + + final_suffix + + final_time_zone_optional + ) + # 2 pm est + graph_h = final_graph_hour + delete_space + insert_space + final_suffix + final_time_zone_optional + final_graph = (graph_hm | graph_h | graph_hm2 | graph_hms).optimize() + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify.py new file mode 100644 index 0000000000000000000000000000000000000000..3451aebc20bc2790b3a90b450592b71dfc17f76c --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify.py @@ -0,0 +1,241 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.abbreviation import AbbreviationFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.cardinal import CardinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.date import DateFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.decimal import DecimalFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.electronic import ElectronicFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.fraction import FractionFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.measure import MeasureFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.money import MoneyFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.ordinal import OrdinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.range import RangeFst as RangeFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.roman import RomanFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.serial import SerialFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.telephone import TelephoneFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.time import TimeFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.whitelist import WhiteListFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.word import WordFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.date import DateFst as vDateFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst as vOrdinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.time import TimeFst as vTimeFst +from zonos3.vendor.nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"en_tn_{deterministic}_deterministic_{input_case}_{whitelist_file}_tokenize.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating ClassifyFst grammars.") + + start_time = time.time() + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + logger.debug(f"cardinal: {time.time() - start_time: .2f}s -- {cardinal_graph.num_states()} nodes") + + start_time = time.time() + ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + ordinal_graph = ordinal.fst + logger.debug(f"ordinal: {time.time() - start_time: .2f}s -- {ordinal_graph.num_states()} nodes") + + start_time = time.time() + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + decimal_graph = decimal.fst + logger.debug(f"decimal: {time.time() - start_time: .2f}s -- {decimal_graph.num_states()} nodes") + + start_time = time.time() + fraction = FractionFst(deterministic=deterministic, cardinal=cardinal) + fraction_graph = fraction.fst + logger.debug(f"fraction: {time.time() - start_time: .2f}s -- {fraction_graph.num_states()} nodes") + + start_time = time.time() + measure = MeasureFst( + cardinal=cardinal, + decimal=decimal, + fraction=fraction, + deterministic=deterministic, + ) + measure_graph = measure.fst + logger.debug(f"measure: {time.time() - start_time: .2f}s -- {measure_graph.num_states()} nodes") + + start_time = time.time() + date_graph = DateFst(cardinal=cardinal, deterministic=deterministic).fst + logger.debug(f"date: {time.time() - start_time: .2f}s -- {date_graph.num_states()} nodes") + + start_time = time.time() + time_graph = TimeFst(cardinal=cardinal, deterministic=deterministic).fst + logger.debug(f"time: {time.time() - start_time: .2f}s -- {time_graph.num_states()} nodes") + + start_time = time.time() + telephone_graph = TelephoneFst(deterministic=deterministic).fst + logger.debug(f"telephone: {time.time() - start_time: .2f}s -- {telephone_graph.num_states()} nodes") + + start_time = time.time() + electonic_graph = ElectronicFst(cardinal=cardinal, deterministic=deterministic).fst + logger.debug(f"electronic: {time.time() - start_time: .2f}s -- {electonic_graph.num_states()} nodes") + + start_time = time.time() + money_graph = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic).fst + logger.debug(f"money: {time.time() - start_time: .2f}s -- {money_graph.num_states()} nodes") + + start_time = time.time() + whitelist_graph = WhiteListFst( + input_case=input_case, deterministic=deterministic, input_file=whitelist + ).fst + logger.debug(f"whitelist: {time.time() - start_time: .2f}s -- {whitelist_graph.num_states()} nodes") + + start_time = time.time() + punctuation = PunctuationFst(deterministic=deterministic) + punct_graph = punctuation.fst + logger.debug(f"punct: {time.time() - start_time: .2f}s -- {punct_graph.num_states()} nodes") + + start_time = time.time() + word_graph = WordFst(punctuation=punctuation, deterministic=deterministic).fst + logger.debug(f"word: {time.time() - start_time: .2f}s -- {word_graph.num_states()} nodes") + + start_time = time.time() + serial_graph = SerialFst(cardinal=cardinal, ordinal=ordinal, deterministic=deterministic).fst + logger.debug(f"serial: {time.time() - start_time: .2f}s -- {serial_graph.num_states()} nodes") + + start_time = time.time() + v_time_graph = vTimeFst(deterministic=deterministic).fst + v_ordinal_graph = vOrdinalFst(deterministic=deterministic) + v_date_graph = vDateFst(ordinal=v_ordinal_graph, deterministic=deterministic).fst + time_final = pynini.compose(time_graph, v_time_graph) + date_final = pynini.compose(date_graph, v_date_graph) + range_graph = RangeFst( + time=time_final, + date=date_final, + cardinal=cardinal, + deterministic=deterministic, + ).fst + logger.debug(f"range: {time.time() - start_time: .2f}s -- {range_graph.num_states()} nodes") + + # A quick fix to address money ranges: + # $150-$200 -> one hundred and fifty dollars to two hundred dollars + + dash = (pynutil.insert('name: "') + pynini.cross("-", "to") + pynutil.insert('"')).optimize() + + graph_range_money = pynini.closure( + money_graph + + pynutil.insert(" }") + + pynutil.insert(" tokens { ") + + dash + + pynutil.insert(" } ") + + pynutil.insert("tokens { ") + + money_graph, + 1, + ) + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(date_graph, 1.09) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(telephone_graph, 1.1) + | pynutil.add_weight(electonic_graph, 1.11) + | pynutil.add_weight(fraction_graph, 1.1) + | pynutil.add_weight(range_graph, 1.1) + | pynutil.add_weight(serial_graph, 1.12) # should be higher than the rest of the classes + | pynutil.add_weight(graph_range_money, 1.1) + ) + + # roman_graph = RomanFst(deterministic=deterministic).fst + # classify |= pynutil.add_weight(roman_graph, 1.1) + + if not deterministic: + abbreviation_graph = AbbreviationFst(deterministic=deterministic).fst + classify |= pynutil.add_weight(abbreviation_graph, 100) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") + punct = pynini.closure( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct) + | punct, + 1, + ) + + classify |= pynutil.add_weight(word_graph, 100) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure( + ( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct + pynutil.insert(" ")) + ) + + token_plus_punct + ) + + graph = delete_space + graph + delete_space + graph |= punct + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_lm.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_lm.py new file mode 100644 index 0000000000000000000000000000000000000000..5ac152656f0ff45cb53a988b0106485b0e6ec676 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_lm.py @@ -0,0 +1,227 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_SIGMA, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.cardinal import CardinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.date import DateFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.decimal import DecimalFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.electronic import ElectronicFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.fraction import FractionFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.measure import MeasureFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.money import MoneyFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.ordinal import OrdinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.range import RangeFst as RangeFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.roman import RomanFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.serial import SerialFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.telephone import TelephoneFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.time import TimeFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.whitelist import WhiteListFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.word import WordFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.cardinal import CardinalFst as vCardinal +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.date import DateFst as vDate +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.decimal import DecimalFst as vDecimal +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.electronic import ElectronicFst as vElectronic +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.fraction import FractionFst as vFraction +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.measure import MeasureFst as vMeasure +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.money import MoneyFst as vMoney +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst as vOrdinal +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.roman import RomanFst as vRoman +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.telephone import TelephoneFst as vTelephone +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.time import TimeFst as vTime +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.word import WordFst as vWord +from zonos3.vendor.nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = True, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != 'None': + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, f"_{input_case}_en_tn_{deterministic}_deterministic{whitelist_file}_lm.far" + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode='r')['tokenize_and_classify'] + no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) + self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() + logger.info(f'ClassifyFst.fst was restored from {far_file}.') + else: + logger.info(f'Creating ClassifyFst grammars. This might take some time...') + # TAGGERS + cardinal = CardinalFst(deterministic=True, lm=True) + cardinal_tagger = cardinal + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst(cardinal=cardinal, deterministic=True) + ordinal_graph = ordinal.fst + + decimal = DecimalFst(cardinal=cardinal, deterministic=True) + decimal_graph = decimal.fst + fraction = FractionFst(deterministic=True, cardinal=cardinal) + fraction_graph = fraction.fst + + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=True) + measure_graph = measure.fst + date = DateFst(cardinal=cardinal, deterministic=True, lm=True) + date_graph = date.fst + punctuation = PunctuationFst(deterministic=True) + punct_graph = punctuation.graph + word_graph = WordFst(punctuation=punctuation, deterministic=deterministic).graph + time_graph = TimeFst(cardinal=cardinal, deterministic=True).fst + telephone_graph = TelephoneFst(deterministic=True).fst + electronic_graph = ElectronicFst(cardinal=cardinal, deterministic=True).fst + money_graph = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=False).fst + whitelist = WhiteListFst(input_case=input_case, deterministic=False, input_file=whitelist) + whitelist_graph = whitelist.graph + serial_graph = SerialFst(cardinal=cardinal, ordinal=ordinal, deterministic=deterministic, lm=True).fst + + # VERBALIZERS + cardinal = vCardinal(deterministic=True) + v_cardinal_graph = cardinal.fst + decimal = vDecimal(cardinal=cardinal, deterministic=True) + v_decimal_graph = decimal.fst + ordinal = vOrdinal(deterministic=True) + v_ordinal_graph = ordinal.fst + fraction = vFraction(deterministic=True, lm=True) + v_fraction_graph = fraction.fst + v_telephone_graph = vTelephone(deterministic=True).fst + v_electronic_graph = vElectronic(deterministic=True).fst + measure = vMeasure(decimal=decimal, cardinal=cardinal, fraction=fraction, deterministic=False) + v_measure_graph = measure.fst + v_time_graph = vTime(deterministic=True).fst + v_date_graph = vDate(ordinal=ordinal, deterministic=deterministic, lm=True).fst + v_money_graph = vMoney(decimal=decimal, deterministic=deterministic).fst + v_roman_graph = vRoman(deterministic=deterministic).fst + v_word_graph = vWord(deterministic=deterministic).fst + + cardinal_or_date_final = plurals._priority_union(date_graph, cardinal_graph, NEMO_SIGMA) + cardinal_or_date_final = pynini.compose(cardinal_or_date_final, (v_cardinal_graph | v_date_graph)) + + time_final = pynini.compose(time_graph, v_time_graph) + ordinal_final = pynini.compose(ordinal_graph, v_ordinal_graph) + sem_w = 1 + word_w = 100 + punct_w = 2 + classify_and_verbalize = ( + pynutil.add_weight(time_final, sem_w) + | pynutil.add_weight(pynini.compose(decimal_graph, v_decimal_graph), sem_w) + | pynutil.add_weight(pynini.compose(measure_graph, v_measure_graph), sem_w) + | pynutil.add_weight(ordinal_final, sem_w) + | pynutil.add_weight(pynini.compose(telephone_graph, v_telephone_graph), sem_w) + | pynutil.add_weight(pynini.compose(electronic_graph, v_electronic_graph), sem_w) + | pynutil.add_weight(pynini.compose(fraction_graph, v_fraction_graph), sem_w) + | pynutil.add_weight(pynini.compose(money_graph, v_money_graph), sem_w) + | pynutil.add_weight(cardinal_or_date_final, sem_w) + | pynutil.add_weight(whitelist_graph, sem_w) + | pynutil.add_weight( + pynini.compose(serial_graph, v_word_graph), 1.1001 + ) # should be higher than the rest of the classes + ).optimize() + + roman_graph = RomanFst(deterministic=deterministic, lm=True).fst + # the weight matches the word_graph weight for "I" cases in long sentences with multiple semiotic tokens + classify_and_verbalize |= pynutil.add_weight(pynini.compose(roman_graph, v_roman_graph), sem_w) + + date_final = pynini.compose(date_graph, v_date_graph) + range_graph = RangeFst( + time=time_final, cardinal=cardinal_tagger, date=date_final, deterministic=deterministic + ).fst + classify_and_verbalize |= pynutil.add_weight(pynini.compose(range_graph, v_word_graph), sem_w) + classify_and_verbalize = pynutil.insert("< ") + classify_and_verbalize + pynutil.insert(" >") + classify_and_verbalize |= pynutil.add_weight(word_graph, word_w) + + punct_only = pynutil.add_weight(punct_graph, weight=punct_w) + punct = pynini.closure( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct_only), + 1, + ) + + def get_token_sem_graph(classify_and_verbalize): + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + + classify_and_verbalize + + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure( + ( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct + pynutil.insert(" ")) + ) + + token_plus_punct + ) + + graph |= punct_only + pynini.closure(punct) + graph = delete_space + graph + delete_space + + remove_extra_spaces = pynini.closure(NEMO_NOT_SPACE, 1) + pynini.closure( + delete_extra_space + pynini.closure(NEMO_NOT_SPACE, 1) + ) + remove_extra_spaces |= ( + pynini.closure(pynutil.delete(" "), 1) + + pynini.closure(NEMO_NOT_SPACE, 1) + + pynini.closure(delete_extra_space + pynini.closure(NEMO_NOT_SPACE, 1)) + ) + + graph = pynini.compose(graph.optimize(), remove_extra_spaces).optimize() + return graph + + self.fst = get_token_sem_graph(classify_and_verbalize) + no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) + self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_with_audio.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_with_audio.py new file mode 100644 index 0000000000000000000000000000000000000000..1ca1d651f355e8b19ebb0e96950c4960eeea4c53 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_with_audio.py @@ -0,0 +1,229 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.abbreviation import AbbreviationFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.cardinal import CardinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.date import DateFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.decimal import DecimalFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.electronic import ElectronicFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.fraction import FractionFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.measure import MeasureFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.money import MoneyFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.ordinal import OrdinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.range import RangeFst as RangeFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.roman import RomanFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.serial import SerialFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.telephone import TelephoneFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.time import TimeFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.whitelist import WhiteListFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.word import WordFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.abbreviation import AbbreviationFst as vAbbreviation +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.cardinal import CardinalFst as vCardinal +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.date import DateFst as vDate +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.decimal import DecimalFst as vDecimal +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.electronic import ElectronicFst as vElectronic +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.fraction import FractionFst as vFraction +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.measure import MeasureFst as vMeasure +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.money import MoneyFst as vMoney +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst as vOrdinal +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.roman import RomanFst as vRoman +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.telephone import TelephoneFst as vTelephone +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.time import TimeFst as vTime +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.word import WordFst as vWord +from zonos3.vendor.nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = True, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != 'None': + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, f"_{input_case}_en_tn_{deterministic}_deterministic{whitelist_file}.far" + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode='r')['tokenize_and_classify'] + logger.info(f'ClassifyFst.fst was restored from {far_file}.') + else: + logger.info(f'Creating ClassifyFst grammars. This might take some time...') + # TAGGERS + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + deterministic_ordinal = OrdinalFst(cardinal=cardinal, deterministic=True) + ordinal_graph = ordinal.fst + + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + decimal_graph = decimal.fst + fraction = FractionFst(deterministic=deterministic, cardinal=cardinal) + fraction_graph = fraction.fst + + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + measure_graph = measure.fst + date_graph = DateFst(cardinal=cardinal, deterministic=deterministic).fst + punctuation = PunctuationFst(deterministic=True) + punct_graph = punctuation.graph + word_graph = WordFst(punctuation=punctuation, deterministic=deterministic).graph + time_graph = TimeFst(cardinal=cardinal, deterministic=deterministic).fst + telephone_graph = TelephoneFst(deterministic=deterministic).fst + electronic_graph = ElectronicFst(cardinal=cardinal, deterministic=deterministic).fst + money_graph = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic).fst + whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) + whitelist_graph = whitelist.graph + serial_graph = SerialFst(cardinal=cardinal, ordinal=deterministic_ordinal, deterministic=deterministic).fst + + # VERBALIZERS + cardinal = vCardinal(deterministic=deterministic) + v_cardinal_graph = cardinal.fst + decimal = vDecimal(cardinal=cardinal, deterministic=deterministic) + v_decimal_graph = decimal.fst + ordinal = vOrdinal(deterministic=deterministic) + v_ordinal_graph = ordinal.fst + fraction = vFraction(deterministic=deterministic) + v_fraction_graph = fraction.fst + v_telephone_graph = vTelephone(deterministic=deterministic).fst + v_electronic_graph = vElectronic(deterministic=deterministic).fst + measure = vMeasure(decimal=decimal, cardinal=cardinal, fraction=fraction, deterministic=deterministic) + v_measure_graph = measure.fst + v_time_graph = vTime(deterministic=deterministic).fst + v_date_graph = vDate(ordinal=ordinal, deterministic=deterministic).fst + v_money_graph = vMoney(decimal=decimal, deterministic=deterministic).fst + v_roman_graph = vRoman(deterministic=deterministic).fst + v_abbreviation = vAbbreviation(deterministic=deterministic).fst + + det_v_time_graph = vTime(deterministic=True).fst + det_v_date_graph = vDate(ordinal=vOrdinal(deterministic=True), deterministic=True).fst + time_final = pynini.compose(time_graph, det_v_time_graph) + date_final = pynini.compose(date_graph, det_v_date_graph) + range_graph = RangeFst( + time=time_final, date=date_final, cardinal=CardinalFst(deterministic=True), deterministic=deterministic + ).fst + v_word_graph = vWord(deterministic=deterministic).fst + + sem_w = 1 + word_w = 100 + punct_w = 2 + classify_and_verbalize = ( + pynutil.add_weight(whitelist_graph, sem_w) + | pynutil.add_weight(pynini.compose(time_graph, v_time_graph), sem_w) + | pynutil.add_weight(pynini.compose(decimal_graph, v_decimal_graph), sem_w) + | pynutil.add_weight(pynini.compose(measure_graph, v_measure_graph), sem_w) + | pynutil.add_weight(pynini.compose(cardinal_graph, v_cardinal_graph), sem_w) + | pynutil.add_weight(pynini.compose(ordinal_graph, v_ordinal_graph), sem_w) + | pynutil.add_weight(pynini.compose(telephone_graph, v_telephone_graph), sem_w) + | pynutil.add_weight(pynini.compose(electronic_graph, v_electronic_graph), sem_w) + | pynutil.add_weight(pynini.compose(fraction_graph, v_fraction_graph), sem_w) + | pynutil.add_weight(pynini.compose(money_graph, v_money_graph), sem_w) + | pynutil.add_weight(word_graph, word_w) + | pynutil.add_weight(pynini.compose(date_graph, v_date_graph), sem_w - 0.01) + | pynutil.add_weight(pynini.compose(range_graph, v_word_graph), sem_w) + | pynutil.add_weight( + pynini.compose(serial_graph, v_word_graph), 1.1001 + ) # should be higher than the rest of the classes + ).optimize() + + if not deterministic: + roman_graph = RomanFst(deterministic=deterministic).fst + # the weight matches the word_graph weight for "I" cases in long sentences with multiple semiotic tokens + classify_and_verbalize |= pynutil.add_weight(pynini.compose(roman_graph, v_roman_graph), word_w) + + abbreviation_graph = AbbreviationFst(whitelist=whitelist, deterministic=deterministic).fst + classify_and_verbalize |= pynutil.add_weight( + pynini.compose(abbreviation_graph, v_abbreviation), word_w + ) + + punct_only = pynutil.add_weight(punct_graph, weight=punct_w) + punct = pynini.closure( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct_only), + 1, + ) + + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + + classify_and_verbalize + + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure( + ( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct + pynutil.insert(" ")) + ) + + token_plus_punct + ) + + graph |= punct_only + pynini.closure(punct) + graph = delete_space + graph + delete_space + + remove_extra_spaces = pynini.closure(NEMO_NOT_SPACE, 1) + pynini.closure( + delete_extra_space + pynini.closure(NEMO_NOT_SPACE, 1) + ) + remove_extra_spaces |= ( + pynini.closure(pynutil.delete(" "), 1) + + pynini.closure(NEMO_NOT_SPACE, 1) + + pynini.closure(delete_extra_space + pynini.closure(NEMO_NOT_SPACE, 1)) + ) + + graph = pynini.compose(graph.optimize(), remove_extra_spaces).optimize() + self.fst = graph + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + + # to remove normalization options that still contain digits and some special symbols + # e.g., "P&E" -> {P and E, P&E}, "P & E" will be removed from the list of normalization options + no_digits = pynini.closure(pynini.difference(NEMO_CHAR, pynini.union(NEMO_DIGIT, "&"))) + self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/whitelist.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..b24e64a2fb6830eada3ef2ffedbe09d0c32d9f67 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/whitelist.py @@ -0,0 +1,153 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + NEMO_CHAR, + NEMO_NOT_SPACE, + NEMO_SIGMA, + NEMO_UPPER, + SINGULAR_TO_PLURAL, + GraphFst, + convert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.roman import get_names +from zonos3.vendor.nemo_text_processing.text_normalization.en.utils import ( + augment_labels_with_punct_at_end, + get_abs_path, + load_labels, +) + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist, e.g. + misses -> tokens { name: "mrs" } + for non-deterministic case: "Dr. Abc" -> + tokens { name: "drive" } tokens { name: "Abc" } + tokens { name: "doctor" } tokens { name: "Abc" } + tokens { name: "Dr." } tokens { name: "Abc" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + input_file: path to a file with whitelist replacements + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(input_case, file, keep_punct_add_end: bool = False): + whitelist = load_labels(file) + if input_case == INPUT_LOWER_CASED: + whitelist = [[x.lower(), y] for x, y in whitelist] + else: + whitelist = [[x, y] for x, y in whitelist] + + if keep_punct_add_end: + whitelist.extend(augment_labels_with_punct_at_end(whitelist)) + + graph = pynini.string_map(whitelist) + return graph + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist/tts.tsv")) + graph |= pynini.compose( + pynini.difference(NEMO_SIGMA, pynini.accep("/")).optimize(), + _get_whitelist_graph(input_case, get_abs_path("data/whitelist/symbol.tsv")), + ).optimize() + + if deterministic: + names = get_names() + graph |= ( + pynini.cross(pynini.union("st", "St", "ST"), "Saint") + + pynini.closure(pynutil.delete(".")) + + pynini.accep(" ") + + names + ) + else: + graph |= _get_whitelist_graph( + input_case, get_abs_path("data/whitelist/alternatives.tsv"), keep_punct_add_end=True + ) + + for x in [".", ". "]: + graph |= ( + NEMO_UPPER + + pynini.closure(pynutil.delete(x) + NEMO_UPPER, 2) + + pynini.closure(pynutil.delete("."), 0, 1) + ) + + if not deterministic: + multiple_forms_whitelist_graph = get_formats(get_abs_path("data/whitelist/alternatives_all_format.tsv")) + graph |= multiple_forms_whitelist_graph + + graph_unit = pynini.string_file(get_abs_path("data/measure/unit.tsv")) | pynini.string_file( + get_abs_path("data/measure/unit_alternatives.tsv") + ) + graph_unit_plural = graph_unit @ SINGULAR_TO_PLURAL + units_graph = pynini.compose(NEMO_CHAR ** (3, ...), convert_space(graph_unit | graph_unit_plural)) + graph |= units_graph + + # convert to states only if comma is present before the abbreviation to avoid converting all caps words, + # e.g. "IN", "OH", "OK" + # TODO or only exclude above? + states = load_labels(get_abs_path("data/address/state.tsv")) + additional_options = [] + for x, y in states: + if input_case == INPUT_LOWER_CASED: + x = x.lower() + additional_options.append((x, f"{y[0]}.{y[1:]}")) + if not deterministic: + additional_options.append((x, f"{y[0]}.{y[1:]}.")) + + states.extend(additional_options) + state_graph = pynini.string_map(states) + graph |= pynini.closure(NEMO_NOT_SPACE, 1) + pynini.union(", ", ",") + pynini.invert(state_graph).optimize() + + if input_file: + whitelist_provided = _get_whitelist_graph(input_case, input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + self.graph = (convert_space(graph)).optimize() + + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() + + +def get_formats(input_f, input_case=INPUT_CASED, is_default=True): + """ + Adds various abbreviation format options to the list of acceptable input forms + """ + multiple_formats = load_labels(input_f) + additional_options = [] + for x, y in multiple_formats: + if input_case == INPUT_LOWER_CASED: + x = x.lower() + additional_options.append((f"{x}.", y)) # default "dr" -> doctor, this includes period "dr." -> doctor + additional_options.append((f"{x[0].upper() + x[1:]}", f"{y[0].upper() + y[1:]}")) # "Dr" -> Doctor + additional_options.append((f"{x[0].upper() + x[1:]}.", f"{y[0].upper() + y[1:]}")) # "Dr." -> Doctor + multiple_formats.extend(additional_options) + + if not is_default: + multiple_formats = [(x, f"|raw_start|{x}|raw_end||norm_start|{y}|norm_end|") for (x, y) in multiple_formats] + + multiple_formats = pynini.string_map(multiple_formats) + return multiple_formats diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/word.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..87aec0aea85330424f248795d5d1d50477956bd7 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/taggers/word.py @@ -0,0 +1,91 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_SIGMA, + GraphFst, + convert_space, + get_abs_path, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying word. Considers sentence boundary exceptions. + e.g. sleep -> tokens { name: "sleep" } + + Args: + punctuation: PunctuationFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, punctuation: GraphFst, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + + punct = PunctuationFst().graph + default_graph = pynini.closure(pynini.difference(NEMO_NOT_SPACE, punct.project("input")), 1) + symbols_to_exclude = (pynini.union("$", "€", "₩", "£", "¥", "#", "%") | NEMO_DIGIT).optimize() + graph = pynini.closure(pynini.difference(NEMO_NOT_SPACE, symbols_to_exclude), 1) + graph = pynutil.add_weight(graph, MIN_NEG_WEIGHT) | default_graph + + # leave phones of format [HH AH0 L OW1] untouched + phoneme_unit = pynini.closure(NEMO_ALPHA, 1) + pynini.closure(NEMO_DIGIT) + phoneme = ( + pynini.accep(pynini.escape("[")) + + pynini.closure(phoneme_unit + pynini.accep(" ")) + + phoneme_unit + + pynini.accep(pynini.escape("]")) + ) + + # leave IPA phones of format [ˈdoʊv] untouched, single words and sentences with punctuation marks allowed + punct_marks = pynini.union(*punctuation.punct_marks).optimize() + stress = pynini.union("ˈ", "'", "ˌ") + ipa_phoneme_unit = pynini.string_file(get_abs_path("data/whitelist/ipa_symbols.tsv")) + # word in ipa form + ipa_phonemes = ( + pynini.closure(stress, 0, 1) + + pynini.closure(ipa_phoneme_unit, 1) + + pynini.closure(stress | ipa_phoneme_unit) + ) + # allow sentences of words in IPA format separated with spaces or punct marks + delim = (punct_marks | pynini.accep(" ")) ** (1, ...) + ipa_phonemes = ipa_phonemes + pynini.closure(delim + ipa_phonemes) + pynini.closure(delim, 0, 1) + ipa_phonemes = (pynini.accep(pynini.escape("[")) + ipa_phonemes + pynini.accep(pynini.escape("]"))).optimize() + + if not deterministic: + phoneme = ( + pynini.accep(pynini.escape("[")) + + pynini.closure(pynini.accep(" "), 0, 1) + + pynini.closure(phoneme_unit + pynini.accep(" ")) + + phoneme_unit + + pynini.closure(pynini.accep(" "), 0, 1) + + pynini.accep(pynini.escape("]")) + ).optimize() + ipa_phonemes = ( + pynini.accep(pynini.escape("[")) + ipa_phonemes + pynini.accep(pynini.escape("]")) + ).optimize() + + phoneme |= ipa_phonemes + self.graph = plurals._priority_union(convert_space(phoneme.optimize()), graph, NEMO_SIGMA) + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/utils.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a2a765a06cb6ddba97575cba8662f2498140f606 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/utils.py @@ -0,0 +1,60 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels + + +def augment_labels_with_punct_at_end(labels): + """ + augments labels: if key ends on a punctuation that value does not have, add a new label + where the value maintains the punctuation + + Args: + labels : input labels + Returns: + additional labels + """ + res = [] + for label in labels: + if len(label) > 1: + if label[0][-1] == "." and label[1][-1] != ".": + res.append([label[0], label[1] + "."] + label[2:]) + return res diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc443be41c4c3fcf0764eb9fd3e1828d63f8438c --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/abbreviation.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/abbreviation.py new file mode 100644 index 0000000000000000000000000000000000000000..d8c9854532ffdefcbe71fd273fa622f73902a149 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/abbreviation.py @@ -0,0 +1,36 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class AbbreviationFst(GraphFst): + """ + Finite state transducer for verbalizing abbreviations + e.g. tokens { abbreviation { value: "A B C" } } -> "ABC" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="abbreviation", kind="verbalize", deterministic=deterministic) + + graph = pynutil.delete("value: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/cardinal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..8d5bb9811187c99bc2ebd20a59690164bc74037e --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/cardinal.py @@ -0,0 +1,48 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal, e.g. + cardinal { negative: "true" integer: "23" } -> minus twenty three + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + self.optional_sign = pynini.cross("negative: \"true\"", "minus ") + if not deterministic: + self.optional_sign |= pynini.cross("negative: \"true\"", "negative ") + self.optional_sign |= pynini.cross("negative: \"true\"", "dash ") + + self.optional_sign = pynini.closure(self.optional_sign + delete_space, 0, 1) + + integer = pynini.closure(NEMO_NOT_QUOTE) + + self.integer = delete_space + pynutil.delete("\"") + integer + pynutil.delete("\"") + integer = pynutil.delete("integer:") + self.integer + + self.numbers = self.optional_sign + integer + delete_tokens = self.delete_tokens(self.numbers) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/date.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..bf42b7a3f0a580bff1706fc259c1d6bbaa63ffab --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/date.py @@ -0,0 +1,107 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_extra_space, + delete_space, +) + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + date { month: "february" day: "five" year: "twenty twelve" preserve_order: true } -> february fifth twenty twelve + date { day: "five" month: "february" year: "twenty twelve" preserve_order: true } -> the fifth of february twenty twelve + + Args: + ordinal: OrdinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, ordinal: GraphFst, deterministic: bool = True, lm: bool = False): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + phrase = pynini.closure(NEMO_NOT_QUOTE, 1) + day_cardinal = ( + pynutil.delete("day:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + day = day_cardinal @ ordinal.suffix + period = pynutil.delete("text:") + delete_space + pynutil.delete("\"") + phrase + pynutil.delete("\"") + month = pynutil.delete("month:") + delete_space + pynutil.delete("\"") + phrase + pynutil.delete("\"") + + year = ( + pynutil.delete("year:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + delete_space + + pynutil.delete("\"") + ) + + # financial period + graph_fy = ( + pynutil.insert("the ") + period + pynutil.insert(" of ") + pynini.closure(delete_extra_space + year, 0, 1) + ) + + # month (day) year + graph_mdy = ( + month + pynini.closure(delete_extra_space + day, 0, 1) + pynini.closure(delete_extra_space + year, 0, 1) + ) + # may 5 -> may five + if not deterministic and not lm: + graph_mdy |= ( + month + + pynini.closure(delete_extra_space + day_cardinal, 0, 1) + + pynini.closure(delete_extra_space + year, 0, 1) + ) + + # day month year + graph_dmy = ( + pynutil.insert("the ") + + day + + delete_extra_space + + pynutil.insert("of ") + + month + + pynini.closure(delete_extra_space + year, 0, 1) + ) + + optional_preserve_order = pynini.closure( + pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + | pynutil.delete("field_order:") + + delete_space + + pynutil.delete("\"") + + NEMO_NOT_QUOTE + + pynutil.delete("\"") + + delete_space + ) + + final_graph = ( + (plurals._priority_union(graph_mdy, pynutil.add_weight(graph_dmy, 0.0001), NEMO_SIGMA) | year | graph_fy) + + delete_space + + optional_preserve_order + ) + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/decimal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..efbb42de9404f62206bc3d0dc2d2b5f0918a0f2f --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/decimal.py @@ -0,0 +1,85 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimal, e.g. + decimal { negative: "true" integer_part: "twelve" fractional_part: "five o o six" quantity: "billion" } -> minus twelve point five o o six billion + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal, deterministic: bool = True): + super().__init__(name="decimal", kind="verbalize", deterministic=deterministic) + self.optional_sign = pynini.cross("negative: \"true\"", "minus ") + if not deterministic: + self.optional_sign |= pynini.cross("negative: \"true\"", "negative ") + self.optional_sign = pynini.closure(self.optional_sign + delete_space, 0, 1) + self.integer = pynutil.delete("integer_part:") + cardinal.integer + self.optional_integer = pynini.closure(self.integer + delete_space + insert_space, 0, 1) + self.fractional_default = ( + pynutil.delete("fractional_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + self.fractional = pynutil.insert("point ") + self.fractional_default + + self.quantity = ( + delete_space + + insert_space + + pynutil.delete("quantity:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + self.optional_quantity = pynini.closure(self.quantity, 0, 1) + + graph = self.optional_sign + ( + self.integer + | (self.integer + self.quantity) + | (self.optional_integer + self.fractional + self.optional_quantity) + ) + + self.numbers = graph + delete_tokens = self.delete_tokens(graph) + if not deterministic: + delete_tokens |= pynini.compose( + delete_tokens, + NEMO_SIGMA + + ( + pynini.cross(" point five", " and a half") + | pynini.cross("zero point five", "half") + | pynini.cross(" point two five", " and a quarter") + | pynini.cross("zero point two five", "quarter") + ), + ).optimize() + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/electronic.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..9f9bb17b3723c131bf9253f01a32b2998f4564e3 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/electronic.py @@ -0,0 +1,122 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_ALPHA, + NEMO_CHAR, + NEMO_LOWER, + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + TO_LOWER, + GraphFst, + delete_extra_space, + delete_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for verbalizing electronic + e.g. tokens { electronic { username: "cdf1" domain: "abc.edu" } } -> c d f one at a b c dot e d u + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) + graph_digit_no_zero = pynini.invert(pynini.string_file(get_abs_path("data/number/digit.tsv"))).optimize() + graph_zero = pynini.cross("0", "zero") + long_numbers = pynutil.add_weight(graph_digit_no_zero + pynini.cross("000", " thousand"), MIN_NEG_WEIGHT) + + if not deterministic: + graph_zero |= pynini.cross("0", "o") | pynini.cross("0", "oh") + + graph_digit = graph_digit_no_zero | graph_zero + graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")).optimize() + + NEMO_NOT_BRACKET = pynini.difference(NEMO_CHAR, pynini.union("{", "}")).optimize() + dict_words = pynini.project(pynini.string_file(get_abs_path("data/electronic/words.tsv")), "output") + default_chars_symbols = pynini.cdrewrite( + pynutil.insert(" ") + (graph_symbols | graph_digit | long_numbers) + pynutil.insert(" "), + "", + "", + NEMO_SIGMA, + ) + default_chars_symbols = pynini.compose( + pynini.closure(NEMO_NOT_BRACKET), default_chars_symbols.optimize() + ).optimize() + + # this is far cases when user name was split by dictionary words, i.e. "sevicepart@ab.com" -> "service part" + space_separated_dict_words = pynutil.add_weight( + NEMO_ALPHA + + pynini.closure(NEMO_ALPHA | NEMO_SPACE) + + NEMO_SPACE + + pynini.closure(NEMO_ALPHA | NEMO_SPACE), + MIN_NEG_WEIGHT, + ) + + user_name = ( + pynutil.delete("username:") + + delete_space + + pynutil.delete("\"") + + (default_chars_symbols | space_separated_dict_words).optimize() + + pynutil.delete("\"") + ) + + domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv")) + + # this will be used for a safe fallback + domain_all = pynini.compose( + default_chars_symbols, + pynini.closure(TO_LOWER | NEMO_LOWER | NEMO_SPACE | pynutil.add_weight(dict_words, MIN_NEG_WEIGHT)), + ) + + domain = ( + domain_all + + insert_space + + plurals._priority_union( + domain_common, pynutil.add_weight(pynini.cross(".", "dot"), weight=0.0001), NEMO_SIGMA + ) + + pynini.closure(insert_space + default_chars_symbols, 0, 1) + ) + + domain = ( + pynutil.delete("domain:") + + delete_space + + pynutil.delete("\"") + + (domain | pynutil.add_weight(domain_all, weight=100)).optimize() + + delete_space + + pynutil.delete("\"") + ).optimize() + + protocol = pynutil.delete("protocol: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + graph = ( + pynini.closure(protocol + delete_space, 0, 1) + + pynini.closure(user_name + delete_space + pynutil.insert(" at ") + delete_space, 0, 1) + + domain + + delete_space + ).optimize() @ pynini.cdrewrite(delete_extra_space, "", "", NEMO_SIGMA) + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/fraction.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..7c98c8a6c279cbf2f70815499e2b5ac4ee364302 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/fraction.py @@ -0,0 +1,89 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, insert_space +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fraction + e.g. tokens { fraction { integer: "twenty three" numerator: "four" denominator: "five" } } -> + twenty three and four fifth + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + suffix = OrdinalFst().suffix + + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\" ") + denominator_one = pynini.cross("denominator: \"one\"", "over one") + denominator_half = pynini.cross("denominator: \"two\"", "half") + denominator_quarter = pynini.cross("denominator: \"four\"", "quarter") + + denominator_rest = ( + pynutil.delete("denominator: \"") + pynini.closure(NEMO_NOT_QUOTE) @ suffix + pynutil.delete("\"") + ) + + denominators = plurals._priority_union( + denominator_one, + plurals._priority_union( + denominator_half, + plurals._priority_union(denominator_quarter, denominator_rest, NEMO_SIGMA), + NEMO_SIGMA, + ), + NEMO_SIGMA, + ).optimize() + if not deterministic: + denominators |= pynutil.delete("denominator: \"") + (pynini.accep("four") @ suffix) + pynutil.delete("\"") + + numerator_one = pynutil.delete("numerator: \"") + pynini.accep("one") + pynutil.delete("\" ") + numerator_one = numerator_one + insert_space + denominators + numerator_rest = ( + pynutil.delete("numerator: \"") + + (pynini.closure(NEMO_NOT_QUOTE) - pynini.accep("one")) + + pynutil.delete("\" ") + ) + numerator_rest = numerator_rest + insert_space + denominators + numerator_rest @= pynini.cdrewrite( + plurals._priority_union(pynini.cross("half", "halves"), pynutil.insert("s"), NEMO_SIGMA), + "", + "[EOS]", + NEMO_SIGMA, + ) + + graph = numerator_one | numerator_rest + + conjunction = pynutil.insert("and ") + if not deterministic and not lm: + conjunction = pynini.closure(conjunction, 0, 1) + + integer = pynini.closure(integer + insert_space + conjunction, 0, 1) + + graph = integer + graph + graph @= pynini.cdrewrite( + pynini.cross("and one half", "and a half") | pynini.cross("over ones", "over one"), "", "[EOS]", NEMO_SIGMA + ) + + self.graph = graph + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/measure.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..076cb3a4f4c31a70537218c830bd5288728c09a0 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/measure.py @@ -0,0 +1,122 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { negative: "true" cardinal { integer: "twelve" } units: "kilograms" } -> minus twelve kilograms + measure { decimal { integer_part: "twelve" fractional_part: "five" } units: "kilograms" } -> twelve point five kilograms + tokens { measure { units: "covid" decimal { integer_part: "nineteen" fractional_part: "five" } } } -> covid nineteen point five + + Args: + decimal: DecimalFst + cardinal: CardinalFst + fraction: FractionFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + optional_sign = cardinal.optional_sign + unit = ( + pynutil.delete("units: \"") + + pynini.difference(pynini.closure(NEMO_NOT_QUOTE, 1), pynini.union("address", "math")) + + pynutil.delete("\"") + + delete_space + ) + + if not deterministic: + unit |= pynini.compose(unit, pynini.cross(pynini.union("inch", "inches"), "\"")) + + graph_decimal = ( + pynutil.delete("decimal {") + + delete_space + + optional_sign + + delete_space + + decimal.numbers + + delete_space + + pynutil.delete("}") + ) + + if not deterministic: + graph_decimal |= pynini.compose( + graph_decimal, + NEMO_SIGMA + + ( + pynini.cross(" point five", " and a half") + | pynini.cross("zero point five", "half") + | pynini.cross(" point two five", " and a quarter") + | pynini.cross("zero point two five", "quarter") + ), + ).optimize() + + graph_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + optional_sign + + delete_space + + cardinal.numbers + + delete_space + + pynutil.delete("}") + ) + + graph_fraction = ( + pynutil.delete("fraction {") + delete_space + fraction.graph + delete_space + pynutil.delete("}") + ) + + graph = (graph_cardinal | graph_decimal | graph_fraction) + delete_space + insert_space + unit + + # SH adds "preserve_order: true" by default + preserve_order = pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + graph |= unit + insert_space + (graph_cardinal | graph_decimal) + delete_space + pynini.closure(preserve_order) + # for only unit + graph |= ( + pynutil.delete("cardinal { integer: \"-\"") + + delete_space + + pynutil.delete("}") + + delete_space + + unit + + pynini.closure(preserve_order) + ) + address = ( + pynutil.delete("units: \"address\" ") + + delete_space + + graph_cardinal + + delete_space + + pynini.closure(preserve_order) + ) + math = ( + pynutil.delete("units: \"math\" ") + + delete_space + + graph_cardinal + + delete_space + + pynini.closure(preserve_order) + ) + graph |= address | math + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/money.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..7a6321bb0789f97fbf1f2f20c7d81637bdf144df --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/money.py @@ -0,0 +1,85 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_extra_space, + delete_preserve_order, +) + +from zonos3.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path + +per_units = pynini.string_file(get_abs_path("data/money/per_unit.tsv")) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { integer_part: "twelve" fractional_part: "o five" currency: "dollars" } -> twelve o five dollars + + Args: + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + keep_space = pynini.accep(" ") + maj = pynutil.delete('currency_maj: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + min = pynutil.delete('currency_min: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + fractional_part = ( + pynutil.delete('fractional_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + ) + + integer_part = decimal.integer + + # *** currency_maj + graph_integer = integer_part + keep_space + maj + + # *** currency_maj + (***) | ((and) *** current_min) + fractional = fractional_part + delete_extra_space + min + + if not deterministic: + fractional |= pynutil.insert("and ") + fractional + + graph_integer_with_minor = integer_part + keep_space + maj + keep_space + fractional + delete_preserve_order + + # *** point *** currency_maj + graph_decimal = decimal.numbers + keep_space + maj + + # *** current_min + graph_minor = fractional_part + delete_extra_space + min + delete_preserve_order + + graph = graph_integer | graph_integer_with_minor | graph_decimal | graph_minor + + if not deterministic: + graph |= graph_integer + delete_preserve_order + + per_units_normalized = pynini.project(per_units, "output") + remove_per_units_normalized = ( + pynutil.delete(' morphosyntactic_features: "') + + pynutil.insert(" ") + + per_units_normalized + + pynutil.delete('" ') + ) + graph += remove_per_units_normalized.ques + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/ordinal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..1b63cabd3de4db8c7568c06c6ee8446621da7661 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/ordinal.py @@ -0,0 +1,57 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, delete_space +from zonos3.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinal, e.g. + ordinal { integer: "thirteen" } } -> thirteenth + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + graph_digit = pynini.string_file(get_abs_path("data/ordinal/digit.tsv")).invert() + graph_teens = pynini.string_file(get_abs_path("data/ordinal/teen.tsv")).invert() + + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + convert_rest = pynutil.insert("th") + + suffix = pynini.cdrewrite( + graph_digit | graph_teens | pynini.cross("ty", "tieth") | convert_rest, + "", + "[EOS]", + NEMO_SIGMA, + ).optimize() + self.graph = pynini.compose(graph, suffix) + self.suffix = suffix + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/post_processing.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/post_processing.py new file mode 100644 index 0000000000000000000000000000000000000000..726411d2e920bbf0e6d5ac956c85ff564e6e5092 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/post_processing.py @@ -0,0 +1,182 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_ALPHA, + NEMO_CHAR, + NEMO_SIGMA, + NEMO_SPACE, + generator_main, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from zonos3.vendor.nemo_text_processing.utils.logging import logger + + +class PostProcessingFst: + """ + Finite state transducer that post-processing an entire sentence after verbalization is complete, e.g. + removes extra spaces around punctuation marks " ( one hundred and twenty three ) " -> "(one hundred and twenty three)" + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "en_tn_post_processing.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] + logger.info(f'Post processing graph was restored from {far_file}.') + else: + self.set_punct_dict() + self.fst = self.get_punct_postprocess_graph() + + if far_file: + generator_main(far_file, {"post_process_graph": self.fst}) + + def set_punct_dict(self): + self.punct_marks = { + "'": [ + "'", + '´', + 'ʹ', + 'ʻ', + 'ʼ', + 'ʽ', + 'ʾ', + 'ˈ', + 'ˊ', + 'ˋ', + '˴', + 'ʹ', + '΄', + '՚', + '՝', + 'י', + '׳', + 'ߴ', + 'ߵ', + 'ᑊ', + 'ᛌ', + '᾽', + '᾿', + '`', + '´', + '῾', + '‘', + '’', + '‛', + '′', + '‵', + 'ꞌ', + ''', + '`', + '𖽑', + '𖽒', + ], + } + + def get_punct_postprocess_graph(self): + """ + Returns graph to post process punctuation marks. + + {``} quotes are converted to {"}. Note, if there are spaces around single quote {'}, they will be kept. + By default, a space is added after a punctuation mark, and spaces are removed before punctuation marks. + """ + punct_marks_all = PunctuationFst().punct_marks + + # no_space_before_punct assume no space before them + quotes = ["'", "\"", "``", "«"] + dashes = ["-", "—"] + brackets = ["<", "{", "(", r"\["] + open_close_single_quotes = [ + ("`", "`"), + ] + + open_close_double_quotes = [('"', '"'), ("``", "``"), ("“", "”")] + open_close_symbols = open_close_single_quotes + open_close_double_quotes + allow_space_before_punct = ["&"] + quotes + dashes + brackets + [k[0] for k in open_close_symbols] + + no_space_before_punct = [m for m in punct_marks_all if m not in allow_space_before_punct] + no_space_before_punct = pynini.union(*no_space_before_punct) + no_space_after_punct = pynini.union(*brackets) + delete_space = pynutil.delete(" ") + delete_space_optional = pynini.closure(delete_space, 0, 1) + + # non_punct allows space + # delete space before no_space_before_punct marks, if present + non_punct = pynini.difference(NEMO_CHAR, no_space_before_punct).optimize() + graph = ( + pynini.closure(non_punct) + + pynini.closure( + no_space_before_punct | pynutil.add_weight(delete_space + no_space_before_punct, MIN_NEG_WEIGHT) + ) + + pynini.closure(non_punct) + ) + graph = pynini.closure(graph).optimize() + graph = pynini.compose( + graph, pynini.cdrewrite(pynini.cross("``", '"'), "", "", NEMO_SIGMA).optimize() + ).optimize() + + # remove space after no_space_after_punct (even if there are no matching closing brackets) + no_space_after_punct = pynini.cdrewrite(delete_space, no_space_after_punct, NEMO_SIGMA, NEMO_SIGMA).optimize() + graph = pynini.compose(graph, no_space_after_punct).optimize() + + # remove space around text in quotes + single_quote = pynutil.add_weight(pynini.accep("`"), MIN_NEG_WEIGHT) + double_quotes = pynutil.add_weight(pynini.accep('"'), MIN_NEG_WEIGHT) + quotes_graph = ( + single_quote + delete_space_optional + NEMO_ALPHA + NEMO_SIGMA + delete_space_optional + single_quote + ).optimize() + + # this is to make sure multiple quotes are tagged from right to left without skipping any quotes in the left + not_alpha = pynini.difference(NEMO_CHAR, NEMO_ALPHA).optimize() | pynutil.add_weight( + NEMO_SPACE, MIN_NEG_WEIGHT + ) + end = pynini.closure(pynutil.add_weight(not_alpha, MIN_NEG_WEIGHT)) + quotes_graph |= ( + double_quotes + + delete_space_optional + + NEMO_ALPHA + + NEMO_SIGMA + + delete_space_optional + + double_quotes + + end + ) + + quotes_graph = pynutil.add_weight(quotes_graph, MIN_NEG_WEIGHT) + quotes_graph = NEMO_SIGMA + pynini.closure(NEMO_SIGMA + quotes_graph + NEMO_SIGMA) + + graph = pynini.compose(graph, quotes_graph).optimize() + + # remove space between a word and a single quote followed by s + remove_space_around_single_quote = pynini.cdrewrite( + delete_space_optional + pynini.union(*self.punct_marks["'"]) + delete_space, + NEMO_ALPHA, + pynini.union("s ", "s[EOS]"), + NEMO_SIGMA, + ) + + graph = pynini.compose(graph, remove_space_around_single_quote).optimize() + return graph diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/roman.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/roman.py new file mode 100644 index 0000000000000000000000000000000000000000..38837108f33ac99f15bbe713c98578ef2e97494b --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/roman.py @@ -0,0 +1,69 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst + + +class RomanFst(GraphFst): + """ + Finite state transducer for verbalizing roman numerals + e.g. tokens { roman { integer: "one" } } -> one + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="roman", kind="verbalize", deterministic=deterministic) + suffix = OrdinalFst().suffix + + cardinal = pynini.closure(NEMO_NOT_QUOTE) + ordinal = pynini.compose(cardinal, suffix) + + graph = ( + pynutil.delete("key_cardinal: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + pynini.accep(" ") + + pynutil.delete("integer: \"") + + cardinal + + pynutil.delete("\"") + ).optimize() + + graph |= ( + pynutil.delete("default_cardinal: \"default\" integer: \"") + cardinal + pynutil.delete("\"") + ).optimize() + + graph |= ( + pynutil.delete("default_ordinal: \"default\" integer: \"") + ordinal + pynutil.delete("\"") + ).optimize() + + graph |= ( + pynutil.delete("key_the_ordinal: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + pynini.accep(" ") + + pynutil.delete("integer: \"") + + pynini.closure(pynutil.insert("the "), 0, 1) + + ordinal + + pynutil.delete("\"") + ).optimize() + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/telephone.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/telephone.py new file mode 100644 index 0000000000000000000000000000000000000000..e01ccb982902e7c151f2c725e53f4f970baa97b7 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/telephone.py @@ -0,0 +1,64 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing telephone numbers, e.g. + telephone { country_code: "one" number_part: "one two three, one two three, five six seven eight" extension: "one" } + -> one, one two three, one two three, five six seven eight, one + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="verbalize", deterministic=deterministic) + + optional_country_code = pynini.closure( + pynutil.delete("country_code: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + delete_space + + insert_space, + 0, + 1, + ) + + number_part = ( + pynutil.delete("number_part: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynini.closure(pynutil.add_weight(pynutil.delete(" "), -0.0001), 0, 1) + + pynutil.delete("\"") + ) + + optional_extension = pynini.closure( + delete_space + + insert_space + + pynutil.delete("extension: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\""), + 0, + 1, + ) + + graph = optional_country_code + number_part + optional_extension + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/time.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..7f8e88f384acf3462cbf6af7817823fd1789cdd4 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/time.py @@ -0,0 +1,103 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g. + time { hours: "twelve" minutes: "thirty" suffix: "a m" zone: "e s t" } -> twelve thirty a m e s t + time { hours: "twelve" } -> twelve o'clock + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + hour = ( + pynutil.delete("hours:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + minute = ( + pynutil.delete("minutes:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + suffix = ( + pynutil.delete("suffix:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_suffix = pynini.closure(delete_space + insert_space + suffix, 0, 1) + zone = ( + pynutil.delete("zone:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_zone = pynini.closure(delete_space + insert_space + zone, 0, 1) + second = ( + pynutil.delete("seconds:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + graph_hms = ( + hour + + pynutil.insert(" hours ") + + delete_space + + minute + + pynutil.insert(" minutes and ") + + delete_space + + second + + pynutil.insert(" seconds") + + optional_suffix + + optional_zone + ) + graph_hms @= pynini.cdrewrite(pynini.cross("one hours", "one hour"), "[BOS]", "", NEMO_SIGMA) + graph_hms @= pynini.cdrewrite( + pynutil.delete("o ") + | pynutil.add_weight(pynini.cross("o one minutes", "one minute"), -0.01) + | pynutil.add_weight(pynini.cross("o one seconds", "one second"), -0.01), + " ", + "", + NEMO_SIGMA, + ) + graph = hour + delete_space + insert_space + minute + optional_suffix + optional_zone + graph |= hour + insert_space + pynutil.insert("o'clock") + optional_zone + graph |= hour + delete_space + insert_space + suffix + optional_zone + graph |= graph_hms + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/verbalize.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/verbalize.py new file mode 100644 index 0000000000000000000000000000000000000000..8a4ed1ca382aaedfdc94b3e2e28cb6e1526678b5 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/verbalize.py @@ -0,0 +1,82 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.abbreviation import AbbreviationFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.cardinal import CardinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.date import DateFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.decimal import DecimalFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.electronic import ElectronicFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.fraction import FractionFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.measure import MeasureFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.money import MoneyFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.roman import RomanFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.telephone import TelephoneFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.time import TimeFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + decimal_graph = decimal.fst + ordinal = OrdinalFst(deterministic=deterministic) + ordinal_graph = ordinal.fst + fraction = FractionFst(deterministic=deterministic) + fraction_graph = fraction.fst + telephone_graph = TelephoneFst(deterministic=deterministic).fst + electronic_graph = ElectronicFst(deterministic=deterministic).fst + measure = MeasureFst(decimal=decimal, cardinal=cardinal, fraction=fraction, deterministic=deterministic) + measure_graph = measure.fst + time_graph = TimeFst(deterministic=deterministic).fst + date_graph = DateFst(ordinal=ordinal, deterministic=deterministic).fst + money_graph = MoneyFst(decimal=decimal, deterministic=deterministic).fst + whitelist_graph = WhiteListFst(deterministic=deterministic).fst + + graph = ( + time_graph + | date_graph + | money_graph + | measure_graph + | ordinal_graph + | decimal_graph + | cardinal_graph + | telephone_graph + | electronic_graph + | fraction_graph + | whitelist_graph + ) + + roman_graph = RomanFst(deterministic=deterministic).fst + graph |= roman_graph + + if not deterministic: + abbreviation_graph = AbbreviationFst(deterministic=deterministic).fst + graph |= abbreviation_graph + + self.fst = graph diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/verbalize_final.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/verbalize_final.py new file mode 100644 index 0000000000000000000000000000000000000000..38902457a954a82d28e85eecfaa960dca9073be6 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/verbalize_final.py @@ -0,0 +1,75 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.verbalize import VerbalizeFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.word import WordFst +from zonos3.vendor.nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "its" } tokens { time { hours: "twelve" minutes: "thirty" } } tokens { name: "now" } tokens { name: "." } -> its twelve thirty now . + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"en_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + + if deterministic: + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + else: + graph = delete_space + types + delete_space + + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/whitelist.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..1cda0fbc51daa603ce7ce1b5e26dd5549c64e0a2 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/whitelist.py @@ -0,0 +1,40 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for verbalizing whitelist + e.g. tokens { name: "misses" } } -> misses + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + self.fst = graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/word.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..426c9b9510bf2919b9b8e1fcc0e12c93157dcc35 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/en/verbalizers/word.py @@ -0,0 +1,36 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing word + e.g. tokens { name: "sleep" } -> sleep + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="verbalize", deterministic=deterministic) + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/README.md b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ec946a614da8661eb38cf96d7b4b2145fd367e2f --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/README.md @@ -0,0 +1,22 @@ +# Localization Note + +Depending on locale, Spanish number strings will vary in formatting. In the EU and South American countries, it is common to use a period (".") or space to delineate groupings of three +digits. e.g. + `1.000.000` -> "un millón" + `1 000 000` -> "un millón" + +and commas (",") to seperate cardinal and decimal strings. e.g. + + `1,00` -> "uno coma cero cero" + +While Central and Northern America will use commas (",") to delineate groupings of three digits, e.g. + `1,000,000` -> "un millón" + +and periods (".") to seperate cardinal and decimal strings. e.g. + + `1.00` -> "uno coma cero cero" + +As inclusion of both forms will create inherrent ambiguity for verbalization, this module defaults to the former formatting (periods for cardinal delineation and commas for decimals). + +To toggle the alternate formatting, you may edit the `LOCALIZATION` variable in `nemo_text_processing.text_normalization.es.__init__` with the value of `'am'`. This will perform necessary +adjustments to all affected classes. \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bd1fc7d433efa6efa41a9182829300774b1c233d --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +LOCALIZATION = "eu" # Set to am for alternate formatting diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/dates/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/dates/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/dates/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/dates/days_abbr.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/dates/days_abbr.tsv new file mode 100644 index 0000000000000000000000000000000000000000..bb4a9e73bcfb58181d1f6cdf030880093c031987 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/dates/days_abbr.tsv @@ -0,0 +1,27 @@ +LU lunes +lun. lunes +LUN lunes +ma. martes +MA martes +mar. martes +mi. miércoles +mié. miércoles +miér. miércoles +MIÉ miércoles +mié miércoles +miér miércoles +ju. jueves +JU jueves +jue jueves +JUE jueves +vie. viernes +vier. viernes +vie viernes +vier. viernes +sá. sábado +sáb sábado +sáb. sábado +SÁB sábado +do. domingo +dom. domingo +DOM domingo diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/dates/months.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/dates/months.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7cfdf329aa5b2f13f94c650d7cbab5d5fe616ddd --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/dates/months.tsv @@ -0,0 +1,12 @@ +1 enero +2 febrero +3 marzo +4 abril +5 mayo +6 junio +7 julio +8 agosto +9 septiembre +10 octubre +11 noviembre +12 diciembre \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/dates/months_abbr.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/dates/months_abbr.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d08bba36c5432b1e69e9f8e68a078a2434e09e85 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/dates/months_abbr.tsv @@ -0,0 +1,44 @@ +ene. enero +en. enero +ENE enero +febr. febrero +FEB febrero +feb. febrero +mzo. marzo +mzo marzo +MAR marzo +mar. marzo +abr. abril +ABR abril +abr abril +my. mayo +MAY mayo +may. mayo +may mayo +jun. junio +JUN junio +jun junio +jul. julio +JUL julio +jul julio +AGO agosto +ag. agosto +agt. agosto +ago. agosto +set. septiembre +sept. septiembre +setbre. septiembre +sept septiembre +setbre septiembre +SEP septiembre +oct. octubre +OCT octubre +oct octubre +nov. noviembre +novbre. noviembre +NOV noviembre +nov noviembre +novbre noviembre +dic. diciembre +dicbre. diciembre +DIC diciembre diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/dates/year_suffix.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/dates/year_suffix.tsv new file mode 100644 index 0000000000000000000000000000000000000000..455b05d3e1b48388a88c8084c70a8de8af44094f --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/dates/year_suffix.tsv @@ -0,0 +1,51 @@ +ane antes de nuestra era +a.n.e. antes de nuestra era +a. n. e. antes de nuestra era +aec antes de la era común +a.e.c antes de la era común +a. e. c. antes de la era común +ac antes de cristo +a.c. antes de cristo +a. c. antes de cristo +adc antes de cristo +a.d.c. antes de cristo +a. d. c. antes de cristo +a. de c. antes de cristo +ajc antes de jesucristo +a.j.c. antes de jesucristo +a. j. c. antes de jesucristo +adjc antes de jesucristo +a.d.j.c. antes de jesucristo +a. d. j. c. antes de jesucristo +a. de j.c. antes de jesucristo +a. de j. c. antes de jesucristo +ap antes del presente +a.p. antes del presente +a. p. antes del presente +bp antes del presente +b.p. antes del presente +b. p. antes del presente +ne nuestra era +n.e. nuestra era +n. e. nuestra era +ec era común +e.c. era común +e. c. era común +dc después de cristo +d.c. después de cristo +d. c. después de cristo +ddc después de cristo +d.d.c después de cristo +d. d. c. después de cristo +d. de c. después de cristo +ddjc después de jesucristo +d.d.j.c después de jesucristo +d. d. j. c. después de jesucristo +d. de j.c. después de jesucristo +d. de j. c. después de jesucristo +bc b c +b.c. b c +b. c. b c +ad a d +a.d. a d +a. d. a d diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/electronic/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/electronic/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2db92b2574167fb3436c079fd7941450ae0c316b --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/electronic/domain.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/electronic/domain.tsv new file mode 100644 index 0000000000000000000000000000000000000000..06581ba1395c8606e04c1d82e2c56feafb19f186 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/electronic/domain.tsv @@ -0,0 +1,6 @@ +.com punto com +.gov punto gov +.net punto net +.in punto in +.ru punto ru +.gob punto gob diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/electronic/server_name.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/electronic/server_name.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9a0c7562948684993372a2c389c9f32297550e5f --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/electronic/server_name.tsv @@ -0,0 +1,11 @@ +gmail +nvidia +outlook +hotmail +yahoo +live +yandex +orange +wanadoo +web +comcast \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/electronic/symbols.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/electronic/symbols.tsv new file mode 100644 index 0000000000000000000000000000000000000000..77f5d50a4b9068cbb3b3056fc94f22ae227d316e --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/electronic/symbols.tsv @@ -0,0 +1,22 @@ +. punto +- guión +_ barra baja +! signo de exclamación +# almohadilla +$ signo de dólar +% signo de porcentaje +& et +' comilla +* asterisco ++ signo más +/ barra += signo igual +? signo de interrogación +^ acento circunflejo +` acento grave +{ llave izquierda +| barra vertical +} llave derecha +~ tilde +, coma +: dos puntos diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/fractions/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/fractions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/fractions/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/fractions/fraction_symbols.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/fractions/fraction_symbols.tsv new file mode 100644 index 0000000000000000000000000000000000000000..ffebd210552da18e3be34229f81195342c978e7f --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/fractions/fraction_symbols.tsv @@ -0,0 +1,15 @@ +⅓ 1/3 +⅔ 2/3 +⅕ 1/5 +⅖ 2/5 +⅗ 3/5 +⅘ 4/5 +⅙ 1/6 +⅚ 5/6 +⅛ 1/8 +⅜ 3/8 +⅝ 5/8 +⅞ 7/8 +¼ 1/4 +½ 1/2 +¾ 3/4 diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/fractions/ordinal_exceptions.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/fractions/ordinal_exceptions.tsv new file mode 100644 index 0000000000000000000000000000000000000000..32cc3930a0e6e77cddf3fdb01e9da87a63071fdc --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/fractions/ordinal_exceptions.tsv @@ -0,0 +1,2 @@ +segundo medio +tercero tercio \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/fractions/powers_of_ten.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/fractions/powers_of_ten.tsv new file mode 100644 index 0000000000000000000000000000000000000000..021dc0a101e03ba7815d43f4076a340af4dc37a7 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/fractions/powers_of_ten.tsv @@ -0,0 +1,9 @@ +millón millonésimo +millones millonésimo +billón billonésimo +billones billonésimo +trillón trillonésimo +trillones trillonésimo +mil millones mil millonésimo +mil billones mil billonésimo +mil trillones mil trillonésimo \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/fractions/powers_of_ten_fem.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/fractions/powers_of_ten_fem.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9e1a4ca4ce9f21ccef7f1d06a1bb068adb1f4561 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/fractions/powers_of_ten_fem.tsv @@ -0,0 +1,6 @@ +décimo décima +centésimo centésima +milésimo milésima +millonésimo millonésima +billonésimo billonésima +trillonésimo trillonésima \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/math_symbols.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/math_symbols.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9a2a150c8939351d5f8786cc997cd639253a86f8 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/math_symbols.tsv @@ -0,0 +1,7 @@ ++ más +- menos +* por +× por +· por +/ entre +÷ entre diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements.tsv new file mode 100644 index 0000000000000000000000000000000000000000..2107c4691370f6121220b8f3b907ac16b685935c --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements.tsv @@ -0,0 +1,18 @@ +cm centímetro +g gramo +h hora +kg kilogramo +km kilómetro +km² kilómetro cuadrado +l litro +m metro +m² metro cuadrado +m³ metro cúbico +ml mililitro +mm milímetro +ms milisegundo +min minuto +s segundo +° c grado centígrado +° f grado farenheit +° k grado kelvin diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_complex.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_complex.tsv new file mode 100644 index 0000000000000000000000000000000000000000..8cc7dae9982fa7899278895ac7d20b8ee91f6399 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_complex.tsv @@ -0,0 +1,5 @@ +% por ciento +mph milla por hora +kph kilómetro por hora +gsm gramo por metro cuadrado +psi p s i diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_plural_fem.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_plural_fem.tsv new file mode 100644 index 0000000000000000000000000000000000000000..eb3ae2322e7a06e88c4abae5cf3cee6847ea1e10 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_plural_fem.tsv @@ -0,0 +1,3 @@ +hora horas +milla por hora millas por hora +milla millas diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_plural_masc.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_plural_masc.tsv new file mode 100644 index 0000000000000000000000000000000000000000..a7733c2558ad644245a0686f1d52d0530cf55b92 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/measures/measurements_plural_masc.tsv @@ -0,0 +1,21 @@ +centímetro centímetros +gramo gramos +kilogramo kilogramos +kilómetro kilómetros +kilómetro cuadrado kilómetros cuadrados +litro litros +metro metros +metro cuadrado metros cuadrados +metro cúbico metros cúbicos +mililitro mililitros +milímetro milímetros +milisegundo milisegundos +minuto minutos +segundo segundos +por ciento por ciento +kilómetro por hora kilómetros por hora +gramo por metro cuadrado gramos por metro cuadrado +p s i p s i +grado centígrado grados centígrados +grado farenheit grados farenheit +grado kelvin grados kelvin diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_major.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_major.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9dfd19ba570e5dac0cfafb0c1ce2e26677965997 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_major.tsv @@ -0,0 +1,64 @@ +ars peso argentino +ar$ peso argentino +awg florín arubeño +bob boliviano +brl real brasileño +cad dólar canadiense +chf franco suizo +clp peso chileno +cny yuan chino +cop peso colombiano +crc colón costarricense +₡ colón +cup peso cubano +dop peso dominicano +rd$ peso dominicano +eur euro +gtq quetzal +gyd dólar guyanés +hkd dólar de hong kong +hnl lempira +htg gourde +inr rupia india +jpy yen +kr corona +kr. corona +krw won +mxn peso mexicano +nio córdoba +pab balboa +pen sol +s/ sol +pyg guaraní +₲ guaraní +rs rupia +rs. rupia +svc colón salvadoreño +ttd dólar de trinidad y tobago +usd dólar estadounidense +uyu peso uruguayo +uyw unidad previsional uruguaya +ves​ bolívar soberano +arp peso argentino +bop peso boliviano +cle escudo chileno +ecs sucre ecuatoriano +mxp peso mexicano +peh sol de oro +svc colón salvadoreño +veb bolívar venezolano +vef bolívar fuerte +€ euro +$ dólar +£ libra +₩ won +fr franco +mk marco +¥ yen +r$ real +₧ peseta +ptas peseta +us$ dólar estadounidense +mex$ peso mexicano +q quetzal +us $ dólar estadounidense diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_major_ext.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_major_ext.tsv new file mode 100644 index 0000000000000000000000000000000000000000..092b50202e49dd91cbb7ff71d0c24b7a28f6ce1d --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_major_ext.tsv @@ -0,0 +1,215 @@ +# Additional currencies are listed below. +# To use them, simply copy them to the main currency_major.tsv file. + +aed dírham +dh dírham +dhs dírham +afn afgani +all lek +amd dram armenio +ang florín antillano neerlandés +aoa kwanza +aok primer kwanza +aon nuevo kwanza +aor kwanza reajustado +azn manat azerbaiyano +bam marco convertible +bbd dólar barbadense +bdt taka +bgn leva búlgara +bhd dinar bareiní +bif franco de burundi +bmd dólar bermudeño +bd$ dólar bermudeño +bnd dólar de brunéi +bsd dólar bahameño +b$ dólar bahameño +btn ngultrum +bwp pula +byn rublo bielorruso +bzd dólar beliceño +bz$ dólar beliceño +cdf franco congoleño +cou unidad de valor real +cve escudo caboverdiano +czk corona checa +djf franco yibutiano +dkk corona danesa +dzd dinar argelino +egp libra egipcia +ern nakfa +etb birr etíope +fjd dólar fiyiano +fj$ dólar fiyiano +fkp libra malvinense +gbp libra esterlina +gel lari +ghs cedi ghanés +gip libra de gibraltar +gmd dalasi +gnf franco guineano +hrk kuna +huf forinto +idr rupia indonesia +ils nuevo séquel israelí +iqd dinar iraquí +irr rial iraní +isk corona islandesa +jmd dólar jamaiquino +jod dinar jordano +kes chelín keniano +kgs som +khr riel +kmf franco comorense +kpw won norcoreano +kwd dinar kuwaití +kyd dólar de las islas caimán +kzt tenge +lak kip +lbp libra libanesa +lkr rupia de sri lanka +lrd dólar liberiano +lsl loti +lyd dinar libio +mad dírham marroquí +mdl leu moldavo +mga ariary malgache +mkd denar +mmk kyat +mnt tugrik +₮ tugrik +mop pataca +mop$ pataca +mru uguiya +mur rupia de mauricio +mvr rufiyaa +mwk kwacha +myr ringgit malayo +mzn metical mozambiqueño +nad dólar namibio +ngn naira +₦ naira +nok corona noruega +npr rupia nepalí +nzd dólar neozelandés +nz$ dólar neozelandés +omr rial omaní +pgk kina +php peso filipino +₱ peso filipino +P peso filipino +pkr rupia pakistaní +pln esloti +zł esloti +zł. esloti +qar rial catarí +ron leu rumano +rsd dinar serbio +rub rublo ruso +rwf franco ruandés +sar rial saudí +sbd dólar de las islas salomón +scr rupia seychelense +sdg libra sudanesa +sek corona sueca +shp libra de santa elena +sll leone +sos chelín somalí +srd dólar surinamés +ssp libra sursudanesa +stn dobra +syp libra siria +szl lilangeni +thb baht +฿ baht +tjs somoni tayiko +tmt manat turcomano +tnd dinar tunecino +top paʻanga +try lira turca +twd nuevo dólar taiwanés +tzs chelín tanzano +uah grivna +ugx chelín ugandés +uzs som uzbeko +vnd dong vietnamita +₫ dong vietnamita +vuv vatu +wst tala +xaf franco c f a de áfrica central +xof franco c f a de áfrica occidental +xpf franco c f p +xsu sistema unitario de compensación regional +xua unidad de cuenta del banco africano de desarrollo +yer rial yemení +zar rand +zmw kwacha zambiano +zwl dólar zimbabuense +adf franco andorrano +adp peseta andorrana +afa afgani +aon kwanza angoleño +aor kwanza angoleño reajustado +ats chelín austríaco +bec franco belga +bef franco belga +bel franco belga +bgl lev búlgaro +csd dinar serbio +csk corona checoslovaca +cyp libra chipriota +ddm marco de rda +dem marco alemán +eek corona estonia +ehp peseta saharaui +fim marco finlandés +frf franco francés +grd dracma griega +iep libra irlandesa +itl lira italiana +luf franco luxemburgués +mtl lira maltesa +nlg florín neerlandés +pei inti +plz esloti polaco +pte escudo portugués +rur rublo ruso +sit tólar esloveno +skk corona eslovaca +srg guilder surinamés +sur rublo de la unión soviética +trl lira turca +yud nuevo dinar yugoslavo +yum dinar yugoslavo +zal rand surafricano código de fondos +zrn nuevo zaire +zrz zaire +zml kwacha zambiano +kč corona checa +aud dólar australiano +bov boliviano con mantenimiento de valor respecto al dólar estadounidense +che euro wir +chw franco wir +clf unidad de fomento +cuc peso convertible +mxv unidad de inversión mexicana +sgd dólar de singapur +usn dólar estadounidense siguiente día +uyi peso uruguayo en unidades indexadas +xag plata +xau oro +xcd dólar del caribe oriental +xdr derecho especial de giro +xpd paladio +xpt platino +ara austral argentino +arc peso moneda corriente argentino +arf peso fuerte argentino +arg peso oro sellado argentino +arl peso ley argentino +arm peso moneda nacional argentino +ecv unidad de valor constante ecuatoriana +esa peseta española +esb peseta española +esp peseta española +xeu unidad monetaria europea diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_minor.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_minor.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f61f345f60dd7ade187b65fd14e2ef00d91a1ca9 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_minor.tsv @@ -0,0 +1,64 @@ +ars centavo +ar$ centavo +awg centavo +bob centavo +brl centavo +cad centavo +chf céntimo +clp centavo +cny jiao +cop centavo +crc céntimo +₡ céntimo +cup centavo +dop centavo +rd$ centavo +eur centavo +gtq centavo +gyd centavo +hkd centavo +hnl centavo +htg céntimo +inr paisa +jpy centavo +kr øre +kr. øre +krw chon +mxn centavo +nio centavo +pab centésimo +pen céntimo +s/ céntimo +pyg céntimo +₲ céntimo +rs paisa +rs. paisa +svc centavo +ttd céntimo +usd centavo +uyu centésimo +uyw centésimo +ves​ céntimo +arp centavo +bop centavo +cle centésimo +ecs centésimo +mxp centavo +peh centavo +svc centavo +veb céntimo +vef céntimo +€ céntimo +$ centavo +£ penique +₩ chon +fr céntimo +mk céntimo +¥ centavo +r$ centavo +₧ céntimo +ptas céntimo +us$ centavo +mex$ centavo +q centavo +us $ centavo diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_minor_ext.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_minor_ext.tsv new file mode 100644 index 0000000000000000000000000000000000000000..80cda16e94f12d19b34c93b99bbff788baa50b74 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_minor_ext.tsv @@ -0,0 +1,181 @@ +# Additional currencies are listed below. +# To use them, simply copy them to the main currency_minor.tsv file. + +aed fils +dh fils +dhs fils +afn pul +all qindarke +amd luma +ang centavo +aoa céntimo +aok lwei +aon lwei +aor lwei +aud centavo +azn qəpik +bam feninga +bbd centavo +bdt poisha +bgn stotinka +bhd fils +bif céntimo +bmd centavo +bd$ centavo +bnd centavo +bov centavo con mantenimiento de valor respecto al Dólar estadounidense +r$ centavo +bsd centavo +b$ centavo +btn chertum +bwp thebe +byn kap +bzd centavo +bz$ centavo +cdf céntimo +che centavo +chw céntimo +cuc centavo +cve centavo +czk haléř +djf céntimo +dkk øre +dzd céntimo +egp piastra +ern céntimo +etb santim +fjd centavo +fj$ centavo +fkp penique +gbp penique +gel tetri +ghs pesewa +gip penique +gmd butut +gnf céntimo +hrk lipa +huf krajczár +idr sen +ils agorot +iqd fils +irr dinar +isk aurar +jmd céntimo +jod qirsh +kes centavo +kgs tiyin +khr sen +kmf céntimo +kpw chon +kwd fils +kyd centavo +kzt tiyin +lak att +lbp piastra +lkr céntimo +lrd centavo +lsl sente +mad céntimo +mdl ban +mkd deni +mmk pya +mnt mongo +₮ mongo +mop avo +mop$ avo +mru jum +mur centavo +mvr laari +mwk támbala +myr sen +nad centavo +ngn kobo +₦ kobo +nok øre +npr paisa +nzd centavo +nz$ centavo +omr baisa +pgk toea +php céntimo +₱ céntimo +P céntimo +pkr paisa +pln groszy +zł groszy +zł. groszy +qar dirham +ron ban +rsd para +rub kopek +rwf céntimo +sar halalá +sbd centavo +scr centavo +sdg piastra +sek öre +sgd centavo +shp penique +sos céntimo +srd centavo +ssp piastra +stn céntimo +syp piastra +szl centavo +thb satang +฿ satang +tjs diram +tmt teňňe +tnd millime +top seniti +try kuruş +tzs céntimo +uah kopek +ugx centavo +usn centavo +uyi centésimo +uzs tiyin +vnd xu +₫ xu +wst sene +xcd centavo +xpf céntimo +yer fils +zmw ngwee +zwl centavo +adf céntimo +adp céntimo +afa pul +ara centavo +arc centavo +arf centavo +arg centavo +arl centavo +arm centavo +bec céntimo +bef céntimo +bel céntimo +bgl stotinki +csd para +csk heller +cyp céntimo +ddm pfennig +dem pfennig +eek céntimo +esa céntimo +esb céntimo +esp céntimo +frf céntimo +iep penique +luf céntimo +mtl céntimo +nlg céntimo +pei céntimo +plz groszy +rur kopek +sit stotin +skk heller +sur kopek +trl kuruş +zml ngwee +kč haléř diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_fem.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_fem.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c1d94fd87be3fa2e4034be40f90b84b658f439b3 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_fem.tsv @@ -0,0 +1,8 @@ +gourde gourdes +rupia india rupias indias +corona coronas +córdoba córdobas +rupia rupias +unidad previsional uruguaya unidades previsionales uruguayas +libra libras +peseta pesetas diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_fem_ext.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_fem_ext.tsv new file mode 100644 index 0000000000000000000000000000000000000000..3d7c27559d78164e672bbe848d509ce412de2163 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_fem_ext.tsv @@ -0,0 +1,56 @@ +# Additional currencies are listed below. +# To use them, simply copy them to the main currency_plural_fem.tsv file. + +leva búlgara levas búlgaras +pula pula +unidad de fomento unidades de fomento +unidad de valor real unidades de valor real +corona checa coronas checas +corona danesa coronas danesas +libra egipcia libras egipcias +libra malvinense libras malvinenses +libra esterlina libras esterlinas +libra de gibraltar libras de gibraltar +kuna kuna +rupia indonesia rupias indonesias +corona islandesa coronas islandesas +libra libanesa libras libanesas +rupia de sri lanka rupias de sri lanka +pataca patacas +uguiya uguiya +rupia de mauricio rupias de mauricio +unidad de inversión mexicana unidades de inversión mexicana +rufiyaa rufiyaa +corona noruega coronas noruegas +rupia nepalí rupias nepalíes +rupia pakistaní rupias pakistaníes +rupia seychelense rupias seychelenses +libra sudanesa libras sudanesas +corona sueca coronas suecas +libra de santa elena libras de santa elena +libra siria libras sirias +lira turca liras turcas +grivna grivnas +unidad de cuenta del banco africano de desarrollo unidades de cuenta del banco africano de desarrollo +peseta andorrana pesetas andorranas +corona checoslovaca coronas checoslovacas +libra chipriota libras chipriotas +unidad de valor constante ecuatoriana unidades de valor constante ecuatorianas +corona estonia coronas estonias +peseta saharaui pesetas saharauies +peseta española pesetas españolas +dracma griega dracmas griegas +libra irlandesa libras irlandesas +lira italiana liras italianas +lira maltesa liras maltesas +corona eslovaca coronas eslovacas +lira turca liras turcas +unidad monetaria europea unidades monetaria europea +feninga feninga +poisha poisha +stotinka stotinki +piastra piastras +pesewa pesewas +pya pyas +támbala támbala +corona checa coronas checas diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_masc.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_masc.tsv new file mode 100644 index 0000000000000000000000000000000000000000..76954cc0d5614f6a8bdd695264efef45dbb44e4a --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_masc.tsv @@ -0,0 +1,51 @@ +real brasileño reales brasileños +peso argentino pesos argentinos +florín arubeño florines arubeños +boliviano bolivianos +dólar canadiense dólares canadienses +franco suizo francos suizos +peso chileno pesos chilenos +yuan chino yuan chinos +peso colombiano pesos colombianos +colón costarricense colones costarricenses +peso cubano pesos cubanos +peso dominicano pesos dominicanos +euro euros +quetzal quetzales +dólar guyanés dólares guyaneses +dólar de hong kong dólares de hong kong +lempira lempiras +yen yenes +won wones +balboa balboas +sol soles +guaraní guaraníes +dólar de trinidad y tobago dólares de trinidad y tobago +dólar estadounidense dólares estadounidenses +peso uruguayo pesos uruguayos +bolívar soberano bolívares soberanos +peso argentino pesos argentinos +peso boliviano pesos bolivianos +escudo chileno escudos chilenos +sucre ecuatoriano sucres ecuatorianos +peso mexicano pesos mexicanos +sol de oro soles de oro +colón salvadoreño colones salvadoreños +bolívar venezolano bolívares venezolanos +bolívar fuerte bolívares fuertes +centavo centavos +céntimo céntimos +øre øre +penique peniques +paisa paise +chon chon +øre øre +centésimo centésimos +dólar dólares +won wones +franco francos +marco marcos +yen yenes +real reales +colón colones +quetzal quetzales diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_masc_ext.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_masc_ext.tsv new file mode 100644 index 0000000000000000000000000000000000000000..548558b3463bb06e5105186204da1234367ca366 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/money/currency_plural_masc_ext.tsv @@ -0,0 +1,199 @@ +# Additional currencies are listed below. +# To use them, simply copy them to the main currency_plural_masc.tsv file. + +dírham dírhams +afgani afganis +lek lekë +dram armenio dram armenios +florín antillano neerlandés florines antillanos neerlandeses +kwanza kwanzas +primer kwanza primeros kwanzas +nuevo kwanza nuevos kwanzas +kwanza reajustado kwanzas reajustados +dólar australiano dólares australianos +manat azerbaiyano manat azerbaiyanos +marco convertible marcos convertibles +dólar barbadense dólares barbadenses +taka taka +dinar bareiní dinares bareinís +franco de burundi francos de burundi +dólar bermudeño dólares bermudeños +dólar de brunéi dólares de brunéi +boliviano con mantenimiento de valor respecto al dólar estadounidense bolivianos con mantenimiento de valor respecto al dólar estadounidense +dólar bahameño dólares bahameños +ngultrum ngultrum +rublo bielorruso rublos bielorrusos +dólar beliceño dólares beliceños +franco congoleño francos congoleños +euro wir euros wir +franco wir francos wir +peso convertible pesos convertibles +escudo caboverdiano escudos caboverdianos +franco yibutiano francos yibutianos +dinar argelino dinares argelinos +nakfa nakfa +birr etíope birr etíopes +dólar fiyiano dólares fiyianos +lari lari +cedi ghanés cedis ghaneses +dalasi dalasis +franco guineano francos guineanos +forinto forintos +nuevo séquel israelí nuevos séqueles israelíes +dinar iraquí dinares iraquíes +rial iraní riales iraníes +dólar jamaiquino dólares jamaiquinos +dinar jordano dinares jordanos +chelín keniano chelines kenianos +som som +riel riel +franco comorense francos comorenses +won norcoreano wones norcoreanos +dinar kuwaití dinares kuwaitíes +dólar de las islas caimán dólares de las islas caimán +tenge tenge +kip kip +dólar liberiano dólares liberianos +loti maloti +dinar libio dinares libios +dírham marroquí dírhams marroquíes +leu moldavo lei moldavos +ariary malgache ariary +denar denares +kyat kyats +tugrik tugrik +kwacha kwacha +ringgit malayo ringgit malayos +metical mozambiqueño meticales mozambiqueños +dólar namibio dólares namibios +naira naira +dólar neozelandés dólares neozelandeses +rial omaní riales omaníes +kina kina +peso filipino pesos filipinos +esloti eslotis +rial catarí riales cataríes +leu rumano lei rumanos +dinar serbio dinares serbios +rublo ruso rublos rusos +franco ruandés francos ruandeses +rial saudí riales saudíes +dólar de las islas salomón dólares de las islas salomón +dólar de singapur dólares de singapur +leone leones +chelín somalí chelines somalíes +dólar surinamés dólares surinameses +libra sursudanesa libras sursudanesas +dobra dobras +lilangeni emalangeni +baht baht +somoni tayiko somoni tayikos +manat turcomano manat turcomanos +dinar tunecino dinares tunecinos +paʻanga paʻanga +nuevo dólar taiwanés nuevos dólares taiwaneses +chelín tanzano chelines tanzanos +chelín ugandés chelines ugandeses +dólar estadounidense siguiente día dólares estadounidenses siguiente día +peso uruguayo en unidades indexadas pesos uruguayos en unidades indexadas +som uzbeko som uzbekos +dong vietnamita dong vietnamitas +vatu vatu +tala tala +franco c f a de áfrica central francos c f a de áfrica central +plata plata +oro oro +dólar del caribe oriental dólares del caribe oriental +derecho especial de giro derechos especiales de giro +franco c f a de áfrica occidental francos c f a de áfrica occidental +paladio paladio +franco c f p francos c f p +platino platino +sistema unitario de compensación regional sistemas unitarios de compensación regional +rial yemení riales yemeníes +rand rand +kwacha zambiano kwacha zambianos +dólar zimbabuense dólares zimbabuenses +franco andorrano francos andorranos +kwanza angoleño kwanzas angoleños +kwanza angoleño reajustado kwanzas angoleños reajustados +austral argentino australes argentinos +peso moneda corriente argentino pesos moneda corriente argentinos +peso fuerte argentino pesos fuertes argentinos +peso oro sellado argentino pesos oro sellado argentinos +peso ley argentino pesos ley argentinos +peso moneda nacional argentino pesos moneda nacional argentinos +chelín austríaco chelines austríacos +franco belga francos belgas +lev búlgaro lev búlgaros +dinar serbio dinares serbios +marco de rda marcos de rda +marco alemán marcos alemanes +marco finlandés marcos finlandeses +franco francés francos franceses +franco luxemburgués francos luxemburgueses +florín neerlandés florines neerlandeses +inti intis +esloti polaco eslotis polacos +escudo portugués escudos portugueses +rublo ruso rublos rusos +tólar esloveno tólares eslovenos +guilder surinamés guilder surinameses +rublo de la unión soviética rublos de la unión soviética +nuevo dinar yugoslavo nuevos dinares yugoslavos +dinar yugoslavo dinares yugoslavos +rand surafricano código de fondos rands surafricanos código de fondos +nuevo zaire nuevos zaires +zaire zaires +kwacha zambiano kwacha zambianos +fils fils +pul pul +qindarke qindarka +luma luma +lwei lwei +qəpik qəpik +centavo con mantenimiento de valor respecto al dólar estadounidense centavos con mantenimiento de valor respecto al dólar estadounidense +chertum chertums +thebe thebe +kap kopeks +haléř haléř +santim santim +tetri tetri +butut bututs +lipa lipa +krajczár krajczár +sen sen +agorot agorot +dinar dinares +aurar aurar +qirsh qirsh +tiyin tiyin +att att +sente lisente +ban bani +deni deni +mongo mongo +avo avos +jum jums +laari laari +kobo kobo +baisa baise +toea toea +groszy groszy +dirham dirhams +öre öre +diram dirams +pfennig pfennigs +kopek kopeks +para para +halalá halalás +satang satang +teňňe teňňe +millime millimes +seniti seniti +kuruş kuruş +xu xu +sene sene +ngwee ngwee +heller hellers +stotin stotinov diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/digit.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..462803637ed571078f22949c75a146655ceff0ad --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +un 1 +dos 2 +tres 3 +cuatro 4 +cinco 5 +seis 6 +siete 7 +ocho 8 +nueve 9 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/hundreds.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/hundreds.tsv new file mode 100644 index 0000000000000000000000000000000000000000..bf651cd0a9786151f7b26e02a2f5ba09b0c5c3f8 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/hundreds.tsv @@ -0,0 +1,8 @@ +doscientos 2 +trescientos 3 +cuatrocientos 4 +quinientos 5 +seiscientos 6 +setecientos 7 +ochocientos 8 +novecientos 9 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/quantities.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/quantities.tsv new file mode 100644 index 0000000000000000000000000000000000000000..58bce3e5dcff0019538bb75996b0822c7827fbf0 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/quantities.tsv @@ -0,0 +1,6 @@ +millón +millones +billón +billones +trillón +trillones \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/teen.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/teen.tsv new file mode 100644 index 0000000000000000000000000000000000000000..370526ea57c3edce5fd6e7e4f8647a1715dc76e4 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +diez 10 +once 11 +doce 12 +trece 13 +catorce 14 +quince 15 +dieciséis 16 +diecisiete 17 +dieciocho 18 +diecinueve 19 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/ties.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/ties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..51b21081b3e35e10d6efbc87b4b274e20e183af8 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/ties.tsv @@ -0,0 +1,7 @@ +treinta 3 +cuarenta 4 +cincuenta 5 +sesenta 6 +setenta 7 +ochenta 8 +noventa 9 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/twenties.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/twenties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c378a3ad9817d72e60c7b9a762f6c5a3dd804098 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/twenties.tsv @@ -0,0 +1,10 @@ +veinte 20 +veintiún 21 +veintidós 22 +veintitrés 23 +veinticuatro 24 +veinticinco 25 +veintiséis 26 +veintisiete 27 +veintiocho 28 +veintinueve 29 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/zero.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/zero.tsv new file mode 100644 index 0000000000000000000000000000000000000000..945785283a516d40edc75a693777ec6ef4a72c01 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/numbers/zero.tsv @@ -0,0 +1 @@ +cero 0 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/digit.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..08d86ac928b4ce0bb7220acba13173af6f998816 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/digit.tsv @@ -0,0 +1,11 @@ +primero uno +primera una +primero un +segundo dos +tercero tres +cuarto cuatro +quinto cinco +sexto seis +séptimo siete +octavo ocho +noveno nueve \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/gender_suffix.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/gender_suffix.tsv new file mode 100644 index 0000000000000000000000000000000000000000..10eabb11de3ca51a0fbc508a8b88129d626b0e7a --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/gender_suffix.tsv @@ -0,0 +1,20 @@ +ᵉʳ apocope +er apocope +º gender_masc +° gender_masc +ro gender_masc +ero gender_masc +do gender_masc +o gender_masc +to gender_masc +mo gender_masc +vo gender_masc +ª gender_fem +ra gender_fem +era gender_fem +a gender_fem +da gender_fem +ta gender_fem +ma gender_fem +va gender_fem +na gender_fem diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/hundreds.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/hundreds.tsv new file mode 100644 index 0000000000000000000000000000000000000000..65867fd84ecacd446bc23ba1572c9c921d1d186c --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/hundreds.tsv @@ -0,0 +1,18 @@ +centésimo ciento +centésimo cien +ducentésimo doscientos +ducentésima doscientas +tricentésimo trescientos +tricentésima trescientas +cuadringentésimo cuatrocientos +cuadringentésima cuatrocientas +quingentésimo quinientos +quingentésima quinientas +sexcentésimo seiscientos +sexcentésima seiscientas +septingentésimo setecientos +septingentésima setecientas +octingentésimo ochocientos +octingentésima ochocientas +noningentésimo novecientos +noningentésima novecientas \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/roman_exceptions.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/roman_exceptions.tsv new file mode 100644 index 0000000000000000000000000000000000000000..688da504d26484ca584567da6c63e96781ac74c9 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/roman_exceptions.tsv @@ -0,0 +1,10 @@ +di +mi +vi +i +v +x +l +c +d +m \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/teen.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/teen.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1ffcfb2697d45d0a583925ee927bdd2b5b775a62 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/teen.tsv @@ -0,0 +1,10 @@ +décimo diez +undécimo once +duodécimo doce +decimotercero trece +decimocuarto catorce +decimoquinto quince +decimosexto dieciséis +decimoséptimo diecisiete +decimooctavo dieciocho +decimonoveno diecinueve \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/ties.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/ties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..21c488ef2ab870a98d54119013cb068b56d66d35 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/ties.tsv @@ -0,0 +1,8 @@ +vigésimo veinte +trigésimo treinta +cuadragésimo cuarenta +quincuagésimo cincuenta +sexagésimo sesenta +septuagésimo setenta +octogésimo ochenta +nonagésimo noventa \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/twenties.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/twenties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..8fd7634438a375c10b86d4e0681e8e82a71a044f --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/ordinals/twenties.tsv @@ -0,0 +1,11 @@ +vigesimoprimero veintiuno +vigesimoprimera veintiuna +vigesimoprimero veintiún +vigesimosegundo veintidós +vigesimotercero veintitrés +vigesimocuarto veinticuatro +vigesimoquinto veinticinco +vigesimosexto veintiséis +vigesimoséptimo veintisiete +vigesimoctavo veintiocho +vigesimonoveno veintinueve diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/roman/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/roman/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2db92b2574167fb3436c079fd7941450ae0c316b --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/roman/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/roman/digit.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/roman/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..0610b4a54ff3c9c90ad16a4ae9a9428a2a5484f4 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/roman/digit.tsv @@ -0,0 +1,9 @@ +i 1 +ii 2 +iii 3 +iv 4 +v 5 +vi 6 +vii 7 +viii 8 +ix 9 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/roman/hundreds.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/roman/hundreds.tsv new file mode 100644 index 0000000000000000000000000000000000000000..cdbdb68147261048298134373f77c3ade2689659 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/roman/hundreds.tsv @@ -0,0 +1,9 @@ +c 1 +cc 2 +ccc 3 +cd 4 +d 5 +dc 6 +dcc 7 +dccc 8 +cm 9 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/roman/thousands.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/roman/thousands.tsv new file mode 100644 index 0000000000000000000000000000000000000000..19e96b9c6decb7c9760ad34e21a9602bc6393d7d --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/roman/thousands.tsv @@ -0,0 +1,3 @@ +m 1 +mm 2 +mmm 3 diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/roman/ties.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/roman/ties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..ac043aa146bfeb1600a76b37ec59ef5254dc60b8 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/roman/ties.tsv @@ -0,0 +1,9 @@ +x 1 +xx 2 +xxx 3 +xl 4 +l 5 +lx 6 +lxx 7 +lxxx 8 +xc 9 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/telephone/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/telephone/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/telephone/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/telephone/ip_prompt.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/telephone/ip_prompt.tsv new file mode 100644 index 0000000000000000000000000000000000000000..fc226eb30488da403e43c8b43dffa434b06805a7 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/telephone/ip_prompt.tsv @@ -0,0 +1,2 @@ +la dirección de ip es la dirección de i p es +el ip es el i p es diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/telephone/telephone_prompt.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/telephone/telephone_prompt.tsv new file mode 100644 index 0000000000000000000000000000000000000000..33efe4550791ae50613088c4696813e263a23281 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/telephone/telephone_prompt.tsv @@ -0,0 +1,10 @@ +llámame al +llámeme al +llamar al +llama al +llame al +marque +mi número es +mi teléfono es +mi número de teléfono es +el número de teléfono es diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/afternoon_times.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/afternoon_times.tsv new file mode 100644 index 0000000000000000000000000000000000000000..0c266580597bdc279fe0ad53c60d7e9f7c866484 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/afternoon_times.tsv @@ -0,0 +1,6 @@ +una +dos +tres +cuatro +cinco +seis diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/alt_minutes.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/alt_minutes.tsv new file mode 100644 index 0000000000000000000000000000000000000000..6bdff30993d42bdc60725e9a507a83d3df537213 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/alt_minutes.tsv @@ -0,0 +1,2 @@ +quince cuarto +treinta media \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/evening_times.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/evening_times.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d10160d0673f43006e0151c641fd7ff7b024e914 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/evening_times.tsv @@ -0,0 +1,5 @@ +siete +ocho +nueve +diez +once diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_12.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_12.tsv new file mode 100644 index 0000000000000000000000000000000000000000..e681afc6c25c71eef187fc7a300df91cb411149b --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_12.tsv @@ -0,0 +1,12 @@ +1 12 +2 1 +3 2 +4 3 +5 4 +6 5 +7 6 +8 7 +9 8 +10 9 +11 10 +12 11 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_24.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_24.tsv new file mode 100644 index 0000000000000000000000000000000000000000..13f43dc5b9b94144846e164611b8391010b94c22 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_24.tsv @@ -0,0 +1,25 @@ +1 0 +2 1 +3 2 +4 3 +5 4 +6 5 +7 6 +8 7 +9 8 +10 9 +11 10 +12 11 +13 12 +14 13 +15 14 +16 15 +17 16 +18 17 +19 18 +20 19 +21 20 +22 21 +23 22 +0 23 +1 24 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_night.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_night.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b7765ac2a3c6158197b46e5886345e0e08b6b3ac --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/hour_to_night.tsv @@ -0,0 +1,13 @@ +12 0 +1 13 +2 14 +3 15 +4 16 +5 17 +6 18 +7 19 +8 20 +9 21 +10 22 +11 23 +12 24 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/minute_to.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/minute_to.tsv new file mode 100644 index 0000000000000000000000000000000000000000..edab4d5b03d6fd6c8187ea76523f61f0df3410b2 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/minute_to.tsv @@ -0,0 +1,59 @@ +1 59 +2 58 +3 57 +4 56 +5 55 +6 54 +7 53 +8 52 +9 51 +10 50 +11 49 +12 48 +13 47 +14 46 +15 45 +16 44 +17 43 +18 42 +19 41 +20 40 +21 39 +22 38 +23 37 +24 36 +25 35 +26 34 +27 33 +28 32 +29 31 +30 30 +31 29 +32 28 +33 27 +34 26 +35 25 +36 24 +37 23 +38 22 +39 21 +40 20 +41 19 +42 18 +43 17 +44 16 +45 15 +46 14 +47 13 +48 12 +49 11 +50 10 +51 9 +52 8 +53 7 +54 6 +55 5 +56 4 +57 3 +58 2 +59 1 diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/morning_times.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/morning_times.tsv new file mode 100644 index 0000000000000000000000000000000000000000..e329184d2bd0c46ed5c8fa10609b77a1c553197c --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/morning_times.tsv @@ -0,0 +1,12 @@ +un +una +dos +tres +cuatro +cinco +seis +siete +ocho +nueve +diez +once \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/time_suffix.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/time_suffix.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9ba6b1ceb1704b78d07af0f3c5567bbebe5f8b34 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/time_suffix.tsv @@ -0,0 +1,20 @@ +p.m. pm +p. m. pm +p.m pm +p m. pm +pm pm +P.M. pm +P.M pm +P M. pm +P. M. pm +PM pm +a.m. am +a.m. am +a.m am +a m. am +am am +A.M. am +A.M am +A M. am +A. M. am +AM am \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/time_zone.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/time_zone.tsv new file mode 100644 index 0000000000000000000000000000000000000000..6c86a3e4e40c94dbdf734d4599f6396f648fdf39 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/time_zone.tsv @@ -0,0 +1,42 @@ +utc u t c +cst c s t +cet c e t +pst p s t +est e s t +mdt m d t +mst m s t +pt p t +et e t +mt m t +gmt g m t +adt hora de verano del atlántico +amt hora estándar del amazonas +art hora estándar de argentina +ast hora estándar del atlántico +bot hora de bolivia +brt hora estándar de brasilia +clst hora de verano de chile +clt hora estándar de chile +cot hora estándar de colombia +east hora estándar de la isla de pascua +ect hora de ecuador +eeast hora de verano de la isla de pascua +eest hora de verano de europa oriental +eet hora estándar de europa oriental +fkst hora de verano de las malvinas +fnt hora estándar de fernando de noronha +galt hora de galápagos +gft hora de la guayana francesa +gyt hora de guyana +hkt hora estándar de hong kong +jst hora estándar de japón +kst hora estándar de corea +pet hora estándar de perú +pyst hora de verano de paraguay +pyt hora estándar de paraguay +sgt hora de singapur +uyst hora de verano de uruguay +uyt hora de uruguay +vet hora de venezuela +west hora de verano de europa oriental +wet hora estándar de europa oriental diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/time_zone_ext.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/time_zone_ext.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7f94b24b7205549e279303188871b6dba1f620c9 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/time/time_zone_ext.tsv @@ -0,0 +1,105 @@ +# Additional time zones are listed below. +# To use them, simply copy them to the main time_zone.tsv file. + +acdt hora de verano de australia central +acst hora estándar de australia central +act hora de acre +acwst hora estándar de australia centroccidental +aedt hora de verano de australia oriental +aest hora estándar de australia oriental +aft hora de afganistán +akdt tiempo de verano de alaska +akst tiempo estándar de alaska +almt hora de almá-atá +aqtt hora de aktobé +awst hora estándar de australia occidental +azost hora de verano de las azores +azot hora estándar de las azores +azt hora estándar de azerbaiyán +bdt hora de brunéi +bnt hora de brunéi darussalam +bst hora de verano británica +btt hora de bután +cast hora de casey +cat hora de áfrica central +cct hora de las islas cocos +cest hora de verano de europa central +chut hora de chuuk +cvt hora estándar de cabo verde +cxt hora de la isla de navidad +chst hora estándar de chamorro +eat hora de áfrica oriental +egst hora de verano de groenlandia oriental +egt hora estándar de groenlandia oriental +fjst hora de verano de fiyi +fjt hora estándar de fiyi +gamt hora de gambier +ict hora de indochina +idt hora de verano de israel +iot hora del océano índico +irdt hora de verano de irán +irst hora estándar de irán +ist i s t +kgt hora de kirguistán +krat hora estándar de krasnoyarsk +lint hora de las espóradas ecuatoriales +magt hora estándar de magadán +mart hora de las islas marquesas +mmt hora de birmania +msk hora estándar de moscú +mut hora estándar de mauricio +mvt hora de maldivas +myt hora de malasia +nct hora estándar de nueva caledonia +ndt hora de verano de terranova +nft hora de las islas norfolk +vot hora estándar de novosibirsk +npt hora de nepal +nrt hora de nauru +nst hora estándar de terranova +nzdt hora de verano de nueva zelanda +hzst hora estándar de nueva zelanda +omst hora estándar de omsk +orat hora de kazajistán occidental +pett hora de kamchatka +pgt hora de papúa nueva guinea +phot hora de las islas fénix +pkt hora estándar de pakistán +pmdt hora de verano de san pedro y miquelón +pmst hora estándar de san predo y miquelón +pont hora de pohnpei +pwt hora de palaos +qyzt hora de quizilordá +ret hora de reunión +sakt hora estándar de sajalín +samt hora de samara +sast hora de sudáfrica +sbt hora de las islas salomón +sret hora de srednekolimsk +srt hora de surinam +sst hora estándar de samoa +taht hora de tahití +tft hora de las tierras australes y antárticas francesas +tjt hora de tayikistán +tkt hora de tokelau +tlt hora de timor oriental +tmt hora estándar de turkmenistán +tot hora estándar de tonga +trt hora estándar de turquía +tvt hora de tuvalu +ulat hora estándar de ulán bator +uzt hora estándar de uzbekistán +vlat hora estándar de vladivostok +volt hora estándar de volgogrado +vost hora de vostok +vut hora estándar de vanuatu +wakt hora de la isla wake +wat hora estándar de áfrica occidental +wgst hora de verano de groenlandia occidental +wgt hora estándar de groenlandia occidental +wib hora de indonesia occidental +wit hora de indonesia oriental +wita hora de indonesia central +wst hora de samoa occidental +yakt hora estándar de yakutsk +yekt hora estándar de ekaterimburgo diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/whitelist.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/whitelist.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f86e6d1a5d78f40fec101c35e04fcbfd3337256b --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/data/whitelist.tsv @@ -0,0 +1,62 @@ +ud. usted +Ud. usted +uds. ustedes +Uds. ustedes +Vd. vosotros +Vds. vosotros +vd. vosotros +vds. vosotros +D. Don +Da. Doña +Dr. Doctor +Dra. Doctora +dr. doctor +dra. doctora +d. don +da. doña +EE. UU. Estados Unidos +ee. uu. estados unidos +Gob. gobierno +gob. gobierno +esq. esquina +Av. avenida +Avda. avenida +av. avenida +avda. avenida +avd. avenida +pág. página +p.ej. por ejemplo +p. ej. por ejemplo +Prof. profesor +Profa. profesora +prof. profesor +profa. profesora +S sur +N norte +Sr. señor +sr. señor +Sra. señora +sra. señora +Srta. señorita +srta. señorita +tel. teléfono +arr.º arrollo +blvr. bulevar +blvd. bulevar +c/ calle +c.° camino +c.º a. camino alto +c.º b. camino bajo +c.º v. camino viejo +carr.ª carrera +ctra. carretera +carret. carretera +cta. cuenta +pje. pasaje +p.º paseo +p.º a. paseo alto +p.º b. paseo bajo +pza. plaza +plza. plaza +rbla. rambla +vda. vereda diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/graph_utils.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b8622c2176d3a76bb3068ad311ce06561e8a80bf --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/graph_utils.py @@ -0,0 +1,218 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, NEMO_SPACE +from zonos3.vendor.nemo_text_processing.text_normalization.es import LOCALIZATION +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path, load_labels + +digits = pynini.project(pynini.string_file(get_abs_path("data/numbers/digit.tsv")), "input") +tens = pynini.project(pynini.string_file(get_abs_path("data/numbers/ties.tsv")), "input") +teens = pynini.project(pynini.string_file(get_abs_path("data/numbers/teen.tsv")), "input") +twenties = pynini.project(pynini.string_file(get_abs_path("data/numbers/twenties.tsv")), "input") +hundreds = pynini.project(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv")), "input") + +accents = pynini.string_map([("á", "a"), ("é", "e"), ("í", "i"), ("ó", "o"), ("ú", "u")]) + +if LOCALIZATION == "am": # Setting localization for central and northern america formatting + cardinal_separator = pynini.string_map([",", NEMO_SPACE]) + decimal_separator = pynini.accep(".") +else: + cardinal_separator = pynini.string_map([".", NEMO_SPACE]) + decimal_separator = pynini.accep(",") + +ones = pynini.union("un", "ún") +fem_ones = pynini.union(pynini.cross("un", "una"), pynini.cross("ún", "una"), pynini.cross("uno", "una")) +one_to_one_hundred = pynini.union(digits, "uno", tens, teens, twenties, tens + pynini.accep(" y ") + digits) +fem_hundreds = hundreds @ pynini.cdrewrite(pynini.cross("ientos", "ientas"), "", "", NEMO_SIGMA) + + +ES_MINUS = pynini.union("menos", "Menos", "MENOS").optimize() +ES_PLUS = pynini.union("más", "Más", "MÁS").optimize() + + +def strip_accent(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Converts all accented vowels to non-accented equivalents + + Args: + fst: Any fst. Composes vowel conversion onto fst's output strings + """ + return fst @ pynini.cdrewrite(accents, "", "", NEMO_SIGMA) + + +def shift_cardinal_gender(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Applies gender conversion rules to a cardinal string. These include: rendering all masculine forms of "uno" (including apocopated forms) as "una" and + Converting all gendered numbers in the hundreds series (200,300,400...) to feminine equivalent (e.g. "doscientos" -> "doscientas"). Conversion only applies + to value place for <1000 and multiple of 1000. (e.g. "doscientos mil doscientos" -> "doscientas mil doscientas".) For place values greater than the thousands, there + is no gender shift as the higher powers of ten ("millones", "billones") are masculine nouns and any conversion would be formally + ungrammatical. + e.g. + "doscientos" -> "doscientas" + "doscientos mil" -> "doscientas mil" + "doscientos millones" -> "doscientos millones" + "doscientos mil millones" -> "doscientos mil millones" + "doscientos millones doscientos mil doscientos" -> "doscientos millones doscientas mil doscientas" + + Args: + fst: Any fst. Composes conversion onto fst's output strings + """ + before_mil = ( + NEMO_SPACE + + (pynini.accep("mil") | pynini.accep("milésimo")) + + pynini.closure(NEMO_SPACE + hundreds, 0, 1) + + pynini.closure(NEMO_SPACE + one_to_one_hundred, 0, 1) + + pynini.union(pynini.accep("[EOS]"), pynini.accep('"'), decimal_separator) + ) + before_double_digits = pynini.closure(NEMO_SPACE + one_to_one_hundred, 0, 1) + pynini.union( + pynini.accep("[EOS]"), pynini.accep('"') + ) + + fem_allign = pynini.cdrewrite(fem_hundreds, "", before_mil, NEMO_SIGMA) # doscientas mil dosciento + fem_allign @= pynini.cdrewrite(fem_hundreds, "", before_double_digits, NEMO_SIGMA) # doscientas mil doscienta + + fem_allign @= pynini.cdrewrite( + fem_ones, "", pynini.union("[EOS]", '"', decimal_separator), NEMO_SIGMA + ) # If before a quote or EOS, we know it's the end of a string + + return fst @ fem_allign + + +def shift_number_gender(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Performs gender conversion on all verbalized numbers in output. All values in the hundreds series (200,300,400) are changed to + feminine gender (e.g. "doscientos" -> "doscientas") and all forms of "uno" (including apocopated forms) are converted to "una". + This has no boundary restriction and will perform shift across all values in output string. + e.g. + "doscientos" -> "doscientas" + "doscientos millones" -> "doscientas millones" + "doscientos millones doscientos" -> "doscientas millones doscientas" + + Args: + fst: Any fst. Composes conversion onto fst's output strings + """ + fem_allign = pynini.cdrewrite(fem_hundreds, "", "", NEMO_SIGMA) + fem_allign @= pynini.cdrewrite( + fem_ones, + "", + pynini.union(NEMO_SPACE, pynini.accep("[EOS]"), pynini.accep('"')), + NEMO_SIGMA, + ) # If before a quote or EOS, we know it's the end of a string + + return fst @ fem_allign + + +def strip_cardinal_apocope(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Reverts apocope on cardinal strings in line with formation rules. e.g. "un" -> "uno". Due to cardinal formation rules, this in effect only + affects strings where the final value is a variation of "un". + e.g. + "un" -> "uno" + "veintiún" -> "veintiuno" + + Args: + fst: Any fst. Composes conversion onto fst's output strings + """ + # Since cardinals use apocope by default for large values (e.g. "millón"), this only needs to act on the last instance of one + strip = pynini.cross("un", "uno") | pynini.cross("ún", "uno") + strip = pynini.cdrewrite(strip, "", pynini.union("[EOS]", '"'), NEMO_SIGMA) + return fst @ strip + + +def add_cardinal_apocope_fem(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Adds apocope on cardinal strings in line with stressing rules. e.g. "una" -> "un". This only occurs when "una" precedes a stressed "a" sound in formal speech. This is not predictable + with text string, so is included for non-deterministic cases. + e.g. + "una" -> "un" + "veintiuna" -> "veintiun" + + Args: + fst: Any fst. Composes conversion onto fst's output strings + """ + # Since the stress trigger follows the cardinal string and only affects the preceding sound, this only needs to act on the last instance of one + strip = pynini.cross("una", "un") | pynini.cross("veintiuna", "veintiún") + strip = pynini.cdrewrite(strip, "", pynini.union("[EOS]", '"'), NEMO_SIGMA) + return fst @ strip + + +def roman_to_int(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Alters given fst to convert Roman integers (lower and upper cased) into Arabic numerals. Valid for values up to 1000. + e.g. + "V" -> "5" + "i" -> "1" + + Args: + fst: Any fst. Composes fst onto Roman conversion outputs. + """ + + def _load_roman(file: str, upper_casing: bool): + roman = load_labels(get_abs_path(file)) + if upper_casing: + roman_numerals = [(x.upper(), y) for x, y in roman] + else: + roman_numerals = [(x, y) for x, y in roman] + return pynini.string_map(roman_numerals) + + # A split between all upper-case and all lower-case Roman numerals is introduced in order to preserve orthographic accuracy, + # and to prevent cases in which certain proper nouns e.g. (Li, Xi, Yi, etc.) are transduced to Roman numerals. + + digit_lower = _load_roman("data/roman/digit.tsv", False) + digit_upper = _load_roman("data/roman/digit.tsv", True) + ties_lower = _load_roman("data/roman/ties.tsv", False) + ties_upper = _load_roman("data/roman/ties.tsv", True) + hundreds_lower = _load_roman("data/roman/hundreds.tsv", False) + hundreds_upper = _load_roman("data/roman/hundreds.tsv", True) + thousands_lower = _load_roman("data/roman/thousands.tsv", False) + thousands_upper = _load_roman("data/roman/thousands.tsv", True) + + graph = ( + (digit_upper | digit_lower) + | ( + (ties_upper + (digit_upper | pynutil.add_weight(pynutil.insert("0"), 0.01))) + | (ties_lower + (digit_lower | pynutil.add_weight(pynutil.insert("0"), 0.01))) + ) + | ( + ( + hundreds_upper + + (ties_upper | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (digit_upper | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ) + | ( + hundreds_lower + + (ties_lower | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (digit_lower | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ) + ) + | ( + ( + thousands_upper + + (hundreds_upper | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (ties_upper | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (digit_upper | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ) + | ( + thousands_lower + + (hundreds_lower | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (ties_lower | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (digit_lower | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ) + ) + ).optimize() + + return graph @ fst diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/cardinal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..e44a8bf09c81b1bc2c851377b2bc9432e597275d --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/cardinal.py @@ -0,0 +1,177 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import cardinal_separator +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) +digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) +teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv"))) +ties = pynini.invert(pynini.string_file(get_abs_path("data/numbers/ties.tsv"))) +twenties = pynini.invert(pynini.string_file(get_abs_path("data/numbers/twenties.tsv"))) +hundreds = pynini.invert(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv"))) + + +def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Helper function for parsing number strings. Converts common cardinal strings (groups of three digits delineated by 'cardinal_separator' - see graph_utils) + and converts to a string of digits: + "1 000" -> "1000" + "1.000.000" -> "1000000" + Args: + fst: Any pynini.FstLike object. Function composes fst onto string parser fst + + Returns: + fst: A pynini.FstLike object + """ + exactly_three_digits = NEMO_DIGIT**3 # for blocks of three + up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) # for start of string + + cardinal_string = pynini.closure( + NEMO_DIGIT, 1 + ) # For string w/o punctuation (used for page numbers, thousand series) + + cardinal_string |= ( + up_to_three_digits + + pynutil.delete(cardinal_separator) + + pynini.closure(exactly_three_digits + pynutil.delete(cardinal_separator)) + + exactly_three_digits + ) + + return cardinal_string @ fst + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + "1000" -> cardinal { integer: "mil" } + "2.000.000" -> cardinal { integer: "dos millones" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + # Any single digit + graph_digit = digit + digits_no_one = (NEMO_DIGIT - "1") @ graph_digit + + # Any double digit + graph_tens = teen + graph_tens |= ties + (pynutil.delete('0') | (pynutil.insert(" y ") + graph_digit)) + graph_tens |= twenties + + self.tens = graph_tens.optimize() + + self.two_digit_non_zero = pynini.union( + graph_digit, graph_tens, (pynini.cross("0", NEMO_SPACE) + graph_digit) + ).optimize() + + # Three digit strings + graph_hundreds = hundreds + pynini.union( + pynutil.delete("00"), (insert_space + graph_tens), (pynini.cross("0", NEMO_SPACE) + graph_digit) + ) + graph_hundreds |= pynini.cross("100", "cien") + graph_hundreds |= ( + pynini.cross("1", "ciento") + insert_space + pynini.union(graph_tens, pynutil.delete("0") + graph_digit) + ) + + self.hundreds = graph_hundreds.optimize() + + # For all three digit strings with leading zeroes (graph appends '0's to manage place in string) + graph_hundreds_component = pynini.union(graph_hundreds, pynutil.delete("0") + graph_tens) + + graph_hundreds_component_at_least_one_none_zero_digit = graph_hundreds_component | ( + pynutil.delete("00") + graph_digit + ) + graph_hundreds_component_at_least_one_none_zero_digit_no_one = graph_hundreds_component | ( + pynutil.delete("00") + digits_no_one + ) + + graph_thousands_component_at_least_one_none_zero_digit = pynini.union( + pynutil.delete("000") + graph_hundreds_component_at_least_one_none_zero_digit, + graph_hundreds_component_at_least_one_none_zero_digit_no_one + + pynutil.insert(" mil") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + pynini.cross("001", "mil") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + ) + + graph_thousands_component_at_least_one_none_zero_digit_no_one = pynini.union( + pynutil.delete("000") + graph_hundreds_component_at_least_one_none_zero_digit_no_one, + graph_hundreds_component_at_least_one_none_zero_digit_no_one + + pynutil.insert(" mil") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + pynini.cross("001", "mil") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + ) + + graph_million = pynutil.add_weight(pynini.cross("000001", "un millón"), -0.001) + graph_million |= graph_thousands_component_at_least_one_none_zero_digit_no_one + pynutil.insert(" millones") + graph_million |= pynutil.delete("000000") + graph_million += insert_space + + graph_billion = pynutil.add_weight(pynini.cross("000001", "un billón"), -0.001) + graph_billion |= graph_thousands_component_at_least_one_none_zero_digit_no_one + pynutil.insert(" billones") + graph_billion |= pynutil.delete("000000") + graph_billion += insert_space + + graph_trillion = pynutil.add_weight(pynini.cross("000001", "un trillón"), -0.001) + graph_trillion |= graph_thousands_component_at_least_one_none_zero_digit_no_one + pynutil.insert(" trillones") + graph_trillion |= pynutil.delete("000000") + graph_trillion += insert_space + + graph = ( + graph_trillion + + graph_billion + + graph_million + + (graph_thousands_component_at_least_one_none_zero_digit | pynutil.delete("000000")) + ) + + self.graph = ( + ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT**24 + @ graph + @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) + @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) + @ pynini.cdrewrite( + pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 2), NEMO_SPACE), NEMO_ALPHA, NEMO_ALPHA, NEMO_SIGMA + ) + ) + self.graph |= zero + + self.graph = filter_punctuation(self.graph).optimize() + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/date.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..8312ec1a0fd8531e9338645b16e833fd638eebae --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/date.py @@ -0,0 +1,128 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst, delete_extra_space +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import roman_to_int, strip_cardinal_apocope +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +articles = pynini.union("de", "del", "el", "del año", "año") +delete_leading_zero = (pynutil.delete("0") | (NEMO_DIGIT - "0")) + NEMO_DIGIT +month_numbers = pynini.string_file(get_abs_path("data/dates/months.tsv")) +month_abbr = pynini.string_file(get_abs_path("data/dates/months_abbr.tsv")) +weekdays = pynini.string_file(get_abs_path("data/dates/days_abbr.tsv")) +year_suffix = pynini.string_file(get_abs_path("data/dates/year_suffix.tsv")) + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date, e.g. + "01.04.2010" -> date { day: "un" month: "enero" year: "dos mil diez" preserve_order: true } + "marzo 4 2000" -> date { month: "marzo" day: "cuatro" year: "dos mil" } + "1990-20-01" -> date { year: "mil novecientos noventa" day: "veinte" month: "enero" } + + Args: + cardinal: cardinal GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + number_to_month = month_numbers.optimize() + month_graph = pynini.project(number_to_month, "output") + + abbr_to_month = month_abbr.optimize() + month_graph |= abbr_to_month + + numbers = cardinal.graph + optional_leading_zero = delete_leading_zero | NEMO_DIGIT + + # 01, 31, 1 + digit_day = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 32)]) @ numbers + + # weekdays + optional_weekdays = pynini.closure(weekdays + NEMO_SPACE, 0, 1) + day = (pynutil.insert("day: \"") + optional_weekdays + digit_day + pynutil.insert("\"")).optimize() + + digit_month = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 13)]) + number_to_month = digit_month @ number_to_month + + month_name = (pynutil.insert("month: \"") + month_graph + pynutil.insert("\"")).optimize() + month_number = (pynutil.insert("month: \"") + number_to_month + pynutil.insert("\"")).optimize() + + # prefer cardinal over year + year = (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 1, 3) # 90, 990, 1990 + year = strip_cardinal_apocope(year @ numbers) + self.year = year + + # handle roman centuries and years + roman_numbers = roman_to_int(numbers) + roman_centuries = pynini.union("siglo ", "año ") + roman_numbers + roman_centuries_graph = ( + pynutil.insert("year: \"") + + roman_centuries + + pynutil.insert("\"") + + pynutil.insert(" preserve_order: true") + ).optimize() + + year_only = pynutil.insert("year: \"") + year + pynutil.insert("\"") + year_with_articles = ( + pynutil.insert("year: \"") + pynini.closure(articles + NEMO_SPACE, 0, 1) + year + pynutil.insert("\"") + ) + + # handle years with eras + year_suffix_graph = ( + pynutil.insert("year: \"") + + (numbers | roman_centuries) + + pynini.closure(NEMO_SPACE, 0, 1) + + year_suffix + + pynutil.insert("\"") + ) + + graph_dmy = ( + day + + pynini.closure(pynutil.delete(" de")) + + NEMO_SPACE + + month_name + + pynini.closure(NEMO_SPACE + year_with_articles, 0, 1) + ) + + graph_mdy = ( # English influences on language + month_name + delete_extra_space + day + pynini.closure(NEMO_SPACE + year_with_articles, 0, 1) + ) + + separators = [".", "-", "/"] + for sep in separators: + year_optional = pynini.closure(pynini.cross(sep, NEMO_SPACE) + year_only, 0, 1) + new_graph = day + pynini.cross(sep, NEMO_SPACE) + month_number + year_optional + graph_dmy |= new_graph + if not deterministic: + new_graph = month_number + pynini.cross(sep, NEMO_SPACE) + day + year_optional + graph_mdy |= new_graph + + dash = "-" + day_optional = pynini.closure(pynini.cross(dash, NEMO_SPACE) + day, 0, 1) + graph_ymd = NEMO_DIGIT**4 @ year_only + pynini.cross(dash, NEMO_SPACE) + month_number + day_optional + + final_graph = graph_dmy + pynutil.insert(" preserve_order: true") + final_graph |= graph_ymd + final_graph |= graph_mdy + final_graph |= roman_centuries_graph + final_graph |= year_suffix_graph + + self.final_graph = final_graph.optimize() + self.fst = self.add_tokens(self.final_graph).optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/decimals.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/decimals.py new file mode 100644 index 0000000000000000000000000000000000000000..5a9eb0aa2595546318469dad1a06b35875867bc6 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/decimals.py @@ -0,0 +1,133 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import ( + cardinal_separator, + decimal_separator, + strip_cardinal_apocope, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv")) +digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) +zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) + + +def get_quantity(decimal_graph: "pynini.FstLike", cardinal_graph: "pynini.FstLike") -> "pynini.FstLike": + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, + e.g. 2 millones -> integer_part: "dos" quantity: "millones" + e.g. 2,4 millones -> integer_part: "dos" fractional_part: "cuatro" quantity: "millones" + + Args: + decimal_graph: DecimalFST + cardinal_graph: CardinalFST + """ + numbers = pynini.closure(NEMO_DIGIT, 1, 6) @ cardinal_graph + numbers = pynini.cdrewrite(pynutil.delete(cardinal_separator), "", "", NEMO_SIGMA) @ numbers + + res = ( + pynutil.insert('integer_part: "') + + numbers # The cardinal we're passing only produces 'un' for one, so gender agreement is safe (all quantities are masculine). Limit to 10^6 power. + + pynutil.insert('"') + + NEMO_SPACE + + pynutil.insert('quantity: "') + + quantities + + pynutil.insert('"') + ) + res |= decimal_graph + NEMO_SPACE + pynutil.insert('quantity: "') + quantities + pynutil.insert('"') + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + -11,4006 billones -> decimal { negative: "true" integer_part: "once" fractional_part: "cuatro cero cero seis" quantity: "billones" preserve_order: true } + 1 billón -> decimal { integer_part: "un" quantity: "billón" preserve_order: true } + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + graph_digit = digit | zero + + if not deterministic: + graph = pynini.union(graph_digit, cardinal.hundreds, cardinal.tens) + graph += pynini.closure(insert_space + graph) + + else: + # General pattern is 1-3 digits: map as cardinal, default to tens followed by digits otherwise \ + graph = pynini.union( + pynutil.add_weight(graph_digit + pynini.closure(insert_space + zero), -0.00001), + pynutil.add_weight(cardinal.tens + pynini.closure(insert_space + zero), -0.00002), + pynutil.add_weight(cardinal.hundreds + pynini.closure(insert_space + zero), 0.00001), + pynutil.add_weight( + cardinal.tens + + pynini.closure(insert_space + cardinal.tens, 1) + + pynini.closure(insert_space + zero, 0, 1) + + (pynini.closure(insert_space + graph_digit, 0, 1) | pynini.closure(insert_space + zero, 0)), + -0.00002, + ), # Read out as tens and a possible trailing digit or zeroes + zero + + pynini.closure(insert_space + zero) + + pynini.closure(insert_space + graph_digit), # For cases such as "1,010" + ) + + # Need to strip apocope everywhere BUT end of string + reverse_apocope = pynini.string_map([("un", "uno"), ("ún", "uno")]) + apply_reverse_apocope = pynini.cdrewrite(reverse_apocope, "", NEMO_SPACE, NEMO_SIGMA) + graph @= apply_reverse_apocope + + # Technically decimals should be space delineated groups of three, e.g. (1,333 333). This removes any possible spaces + strip_formatting = pynini.cdrewrite(delete_space, "", "", NEMO_SIGMA) + graph = strip_formatting @ graph + + self.graph = graph.optimize() + + graph_separator = pynutil.delete(decimal_separator) + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + + self.graph_fractional = pynutil.insert('fractional_part: "') + self.graph + pynutil.insert('"') + + # Integer graph maintains apocope except for ones place + graph_integer = ( + strip_cardinal_apocope(cardinal.graph) + if deterministic + else pynini.union(cardinal.graph, strip_cardinal_apocope(cardinal.graph)) + ) # Gives us forms w/ and w/o apocope + self.graph_integer = pynutil.insert('integer_part: "') + graph_integer + pynutil.insert('"') + final_graph_wo_sign = self.graph_integer + graph_separator + insert_space + self.graph_fractional + + self.final_graph_wo_negative = ( + final_graph_wo_sign | get_quantity(final_graph_wo_sign, cardinal.graph).optimize() + ) + final_graph = optional_graph_negative + self.final_graph_wo_negative + + final_graph += pynutil.insert(" preserve_order: true") + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/electronic.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..2b3f36d5ba8fbe2e801f11e0ea69a1404d57eb98 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/electronic.py @@ -0,0 +1,107 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SPACE, + GraphFst, + at, + colon, + domain_string, + double_quotes, + double_slash, + http, + https, + protocol_string, + username_string, + www, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path, load_labels + +common_domains = [x[0] for x in load_labels(get_abs_path("data/electronic/domain.tsv"))] + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying electronic: email addresses + e.g. "abc@hotmail.com" -> electronic { username: "abc" domain: "hotmail.com" preserve_order: true } + e.g. "www.abc.com/123" -> electronic { protocol: "www." domain: "abc.com/123" preserve_order: true } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + full_stop_accep = pynini.accep(".") + full_stop = "." + + symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] + symbols = pynini.union(*symbols) + symbols_no_full_stop = pynini.difference(symbols, full_stop_accep) + accepted_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols_no_full_stop), 1) + all_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols), 1) + + # domains + domain = full_stop_accep + accepted_characters + domain_graph = ( + pynutil.insert(domain_string + colon + NEMO_SPACE + double_quotes) + + (accepted_characters + pynini.closure(domain, 1)) + + pynutil.insert(double_quotes) + ) + + # email + username = ( + pynutil.insert(username_string + colon + NEMO_SPACE + double_quotes) + + all_characters + + pynutil.insert(double_quotes) + + pynini.cross(at, NEMO_SPACE) + ) + email = username + domain_graph + + # social media tags + tag = ( + pynini.cross(at, "") + + pynutil.insert(username_string + colon + NEMO_SPACE + double_quotes) + + (accepted_characters | (accepted_characters + pynini.closure(domain, 1))) + + pynutil.insert(double_quotes) + ) + + # url + protocol_start = pynini.accep(https + colon + double_slash) | pynini.accep(http + colon + double_slash) + # protocol_end = pynini.accep("www.") + protocol_end = ( + pynini.accep(www + full_stop) + if deterministic + else pynini.accep(www + full_stop) | pynini.cross(www + full_stop, "doble ve doble ve doble ve.") + ) + protocol = protocol_start | protocol_end | (protocol_start + protocol_end) + protocol = ( + pynutil.insert(protocol_string + colon + NEMO_SPACE + double_quotes) + + protocol + + pynutil.insert(double_quotes) + ) + url = protocol + pynutil.insert(NEMO_SPACE) + (domain_graph) + + graph = url | domain_graph | email | tag + self.graph = graph + + final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true")) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/fraction.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..a610bb65238317229b9e8583f016180e8d371d27 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/fraction.py @@ -0,0 +1,164 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +ordinal_exceptions = pynini.string_file(get_abs_path("data/fractions/ordinal_exceptions.tsv")) +higher_powers_of_ten = pynini.string_file(get_abs_path("data/fractions/powers_of_ten.tsv")) +fraction_symbols = pynini.string_file(get_abs_path("data/fractions/fraction_symbols.tsv")) + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fraction + "23 4/5" -> + tokens { fraction { integer: "veintitrés" numerator: "cuatro" denominator: "quinto" mophosyntactic_features: "ordinal" } } + + Args: + cardinal: CardinalFst + ordinal: OrdinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + ordinal_graph = ordinal.graph + + # 2-10 are all ordinals + three_to_ten = pynini.string_map( + [ + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + ] + ) + block_three_to_ten = pynutil.delete(three_to_ten) # To block cardinal productions + if not deterministic: # Multiples of tens are sometimes rendered as ordinals + three_to_ten |= pynini.string_map( + [ + "20", + "30", + "40", + "50", + "60", + "70", + "80", + "90", + ] + ) + graph_three_to_ten = three_to_ten @ ordinal_graph + graph_three_to_ten @= pynini.cdrewrite(ordinal_exceptions, "", "", NEMO_SIGMA) + + # Higher powers of tens (and multiples) are converted to ordinals. + hundreds = pynini.string_map( + [ + "100", + "200", + "300", + "400", + "500", + "600", + "700", + "800", + "900", + ] + ) + graph_hundreds = hundreds @ ordinal_graph + + multiples_of_thousand = ordinal.multiples_of_thousand # So we can have X milésimos + + graph_higher_powers_of_ten = ( + pynini.closure(ordinal.one_to_one_thousand + NEMO_SPACE, 0, 1) + + pynini.closure("mil ", 0, 1) + + pynini.closure(ordinal.one_to_one_thousand + NEMO_SPACE, 0, 1) + ) # x millones / x mil millones / x mil z millones + graph_higher_powers_of_ten += higher_powers_of_ten + graph_higher_powers_of_ten = cardinal_graph @ graph_higher_powers_of_ten + graph_higher_powers_of_ten @= pynini.cdrewrite( + pynutil.delete("un "), + pynini.accep("[BOS]"), + pynini.project(higher_powers_of_ten, "output"), + NEMO_SIGMA, + ) # we drop 'un' from these ordinals (millionths, not one-millionths) + + graph_higher_powers_of_ten = multiples_of_thousand | graph_hundreds | graph_higher_powers_of_ten + block_higher_powers_of_ten = pynutil.delete( + pynini.project(graph_higher_powers_of_ten, "input") + ) # For cardinal graph + + graph_fractions_ordinals = graph_higher_powers_of_ten | graph_three_to_ten + graph_fractions_ordinals += pynutil.insert( + '" morphosyntactic_features: "ordinal"' + ) # We note the root for processing later + + # Blocking the digits and hundreds from Cardinal graph + graph_fractions_cardinals = pynini.cdrewrite( + block_three_to_ten | block_higher_powers_of_ten, + pynini.accep("[BOS]"), + pynini.accep("[EOS]"), + NEMO_SIGMA, + ) + graph_fractions_cardinals @= NEMO_CHAR.plus @ pynini.cdrewrite( + pynutil.delete("0"), + pynini.accep("[BOS]"), + pynini.accep("[EOS]"), + NEMO_SIGMA, + ) # Empty characters become '0' for NEMO_CHAR fst, so need to block + graph_fractions_cardinals @= cardinal_graph + graph_fractions_cardinals += pynutil.insert( + '" morphosyntactic_features: "add_root"' + ) # blocking these entries to reduce erroneous possibilities in debugging + + if deterministic: + graph_fractions_cardinals = ( + pynini.closure(NEMO_DIGIT, 1, 2) @ graph_fractions_cardinals + ) # Past hundreds the conventional scheme can be hard to read. For determinism we stop here + + graph_denominator = pynini.union( + graph_fractions_ordinals, + graph_fractions_cardinals, + pynutil.add_weight(cardinal_graph + pynutil.insert('"'), 0.001), + ) # Last form is simply recording the cardinal. Weighting so last resort + + integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"') + NEMO_SPACE + numerator = ( + pynutil.insert('numerator: "') + cardinal_graph + (pynini.cross("/", '" ') | pynini.cross(" / ", '" ')) + ) + denominator = pynutil.insert('denominator: "') + graph_denominator + + fraction = (fraction_symbols @ (numerator + denominator)) | (numerator + denominator) + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + + self.graph = optional_minus_graph + pynini.closure(integer, 0, 1) + fraction + + final_graph = self.add_tokens(self.graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/measure.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..9226bc6d6f1e6260445e552615a2f7ee1daec2f0 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/measure.py @@ -0,0 +1,216 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_NON_BREAKING_SPACE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + convert_space, + delete_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import strip_cardinal_apocope +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +unit = pynini.string_file(get_abs_path("data/measures/measurements.tsv")) +unit_complex = pynini.string_file(get_abs_path("data/measures/measurements_complex.tsv")) +unit_plural_fem = pynini.string_file(get_abs_path("data/measures/measurements_plural_fem.tsv")) +unit_plural_masc = pynini.string_file(get_abs_path("data/measures/measurements_plural_masc.tsv")) +math_symbols = pynini.string_file(get_abs_path("data/measures/math_symbols.tsv")) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure, e.g. + "2,4 g" -> measure { cardinal { integer_part: "dos" fractional_part: "cuatro" units: "gramos" preserve_order: true } } + "1 g" -> measure { cardinal { integer: "un" units: "gramo" preserve_order: true } } + "1 millón g" -> measure { cardinal { integer: "un quantity: "millón" units: "gramos" preserve_order: true } } + This class also converts words containing numbers and letters + e.g. "a-8" —> "a ocho" + e.g. "1,2-a" —> "uno coma dos a" + + + Args: + cardinal: CardinalFst + decimal: DecimalFst + fraction: FractionFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + + unit_singular = unit + unit_plural = unit_singular @ (unit_plural_fem | unit_plural_masc) + + graph_unit_singular = convert_space(unit_singular) + graph_unit_plural = convert_space(unit_plural) + + optional_graph_negative = pynini.closure("-", 0, 1) + + unit_plural = pynutil.insert('units: "') + graph_unit_plural + pynutil.insert('"') + + unit_singular_graph = pynutil.insert('units: "') + graph_unit_singular + pynutil.insert('"') + + unit_complex_plural = unit_complex @ (unit_plural_fem | unit_plural_masc) + + graph_unit_complex = convert_space(unit_complex) + graph_unit_complex_plural = convert_space(unit_complex_plural) + + graph_unit_denominator = ( + pynini.cross("/", "por") + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_singular + ) + + optional_unit_denominator = pynini.closure( + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, + 0, + 1, + ) + + complex_unit_singular_graph = ( + pynutil.insert('units: "') + + ((graph_unit_singular + optional_unit_denominator) | graph_unit_denominator | graph_unit_complex) + + pynutil.insert('"') + ) + + complex_unit_plural_graph = ( + pynutil.insert('units: "') + + ((graph_unit_plural + optional_unit_denominator) | graph_unit_denominator | graph_unit_complex_plural) + + pynutil.insert('"') + ) + + subgraph_decimal = ( + decimal.fst + insert_space + pynini.closure(NEMO_SPACE, 0, 1) + (unit_plural | complex_unit_plural_graph) + ) + subgraph_cardinal = ( + (optional_graph_negative + (NEMO_SIGMA - "1")) @ cardinal.fst + + insert_space + + pynini.closure(delete_space, 0, 1) + + (unit_plural | complex_unit_plural_graph) + ) + + subgraph_cardinal |= ( + (optional_graph_negative + pynini.accep("1")) @ cardinal.fst + + insert_space + + pynini.closure(delete_space, 0, 1) + + (unit_singular_graph | complex_unit_singular_graph) + ) + + subgraph_fraction = ( + fraction.fst + + insert_space + + pynini.closure(delete_space, 0, 1) + + (unit_plural | complex_unit_singular_graph) + ) + + decimal_times = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(' } units: "') + + pynini.union("x", "X") + + pynutil.insert('"') + ) + + cardinal_times = ( + pynutil.insert('cardinal { integer: "') + + strip_cardinal_apocope(cardinal_graph) + + pynutil.insert('" } units: "') + + pynini.union("x", "X") + + pynutil.insert('"') + ) + + cardinal_dash_alpha = ( + pynutil.insert('cardinal { integer: "') + + strip_cardinal_apocope(cardinal_graph) + + pynutil.delete("-") + + pynutil.insert('" } units: "') + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert('"') + ) + + decimal_dash_alpha = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynutil.delete("-") + + pynutil.insert(' } units: "') + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert("\"") + ) + + alpha_dash_cardinal = ( + pynutil.insert('units: "') + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.delete("-") + + pynutil.insert('"') + + pynutil.insert(' cardinal { integer: "') + + cardinal_graph + + pynutil.insert('" } preserve_order: true') + ) + + alpha_dash_decimal = ( + pynutil.insert('units: "') + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.delete("-") + + pynutil.insert('"') + + pynutil.insert(" decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(" } preserve_order: true") + ) + + delimiter = pynini.accep(" ") | pynutil.insert(" ") + math_numbers = cardinal_graph | pynutil.add_weight(cardinal_graph @ pynini.cross("un", "uno"), -1) | NEMO_ALPHA + math = ( + math_numbers + + pynini.closure(delimiter + math_symbols + delimiter + math_numbers, 1) + + delimiter + + pynini.cross("=", "es igual a") + + delimiter + + math_numbers + ) + + math |= ( + math_numbers + + delimiter + + pynini.cross("=", "es igual a") + + math_numbers + + pynini.closure(delimiter + math_symbols + delimiter + math_numbers, 1) + ) + + math = ( + pynutil.insert("units: \"math\" cardinal { integer: \"") + + math + + pynutil.insert("\" } preserve_order: true") + ) + + final_graph = ( + subgraph_decimal + | subgraph_cardinal + | cardinal_dash_alpha + | alpha_dash_cardinal + | decimal_dash_alpha + | subgraph_fraction + | decimal_times + | cardinal_times + | alpha_dash_decimal + | math + ) + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/money.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..7823fd8e4a9470516a309242ad6fb0329986c951 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/money.py @@ -0,0 +1,182 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import decimal_separator +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path, load_labels + +maj_singular_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) +maj_singular = pynini.string_file((get_abs_path("data/money/currency_major.tsv"))) +min_singular = pynini.string_file(get_abs_path("data/money/currency_minor.tsv")) +fem_plural = pynini.string_file((get_abs_path("data/money/currency_plural_fem.tsv"))) +masc_plural = pynini.string_file((get_abs_path("data/money/currency_plural_masc.tsv"))) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money, e.g. + "€1" -> money { currency_maj: "euro" integer_part: "un"} + "€1,000" -> money { currency_maj: "euro" integer_part: "un" } + "€1,001" -> money { currency_maj: "euro" integer_part: "un" fractional_part: "cero cero un" } + "£1,4" -> money { integer_part: "una" currency_maj: "libra" fractional_part: "cuarenta" preserve_order: true } + -> money { integer_part: "una" currency_maj: "libra" fractional_part: "cuarenta" currency_min: "penique" preserve_order: true } + "0,01 £" -> money { fractional_part: "un" currency_min: "penique" preserve_order: true } + "0,02 £" -> money { fractional_part: "dos" currency_min: "peniques" preserve_order: true } + "£0,01 million" -> money { currency_maj: "libra" integer_part: "cero" fractional_part: "cero un" quantity: "million" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + graph_decimal_final = decimal.final_graph_wo_negative + + maj_singular_graph = maj_singular + min_singular_graph = min_singular + maj_plural_graph = maj_singular @ (fem_plural | masc_plural) + min_plural_graph = min_singular @ (fem_plural | masc_plural) + + graph_maj_singular = pynutil.insert("currency_maj: \"") + maj_singular_graph + pynutil.insert("\"") + graph_maj_plural = pynutil.insert("currency_maj: \"") + maj_plural_graph + pynutil.insert("\"") + + graph_integer_one = pynutil.insert("integer_part: \"") + pynini.cross("1", "un") + pynutil.insert("\"") + + decimal_with_quantity = (NEMO_SIGMA + NEMO_ALPHA) @ graph_decimal_final + + graph_decimal_plural = pynini.union( + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + graph_decimal_final, # $1,05 + graph_decimal_final + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_plural, # 1,05 $ + ) + graph_decimal_plural = ( + (NEMO_SIGMA - "1") + decimal_separator + NEMO_SIGMA + ) @ graph_decimal_plural # Can't have "un euros" + + graph_decimal_singular = pynini.union( + graph_maj_singular + pynini.closure(delete_space, 0, 1) + insert_space + graph_decimal_final, # $1,05 + graph_decimal_final + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_singular, # 1,05 $ + ) + graph_decimal_singular = (pynini.accep("1") + decimal_separator + NEMO_SIGMA) @ graph_decimal_singular + + graph_decimal = pynini.union( + graph_decimal_singular, + graph_decimal_plural, + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + decimal_with_quantity, + ) + + graph_integer = ( + pynutil.insert("integer_part: \"") + ((NEMO_SIGMA - "1") @ cardinal_graph) + pynutil.insert("\"") + ) + + graph_integer_only = pynini.union( + graph_maj_singular + pynini.closure(delete_space, 0, 1) + insert_space + graph_integer_one, + graph_integer_one + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_singular, + ) + graph_integer_only |= pynini.union( + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + graph_integer, + graph_integer + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_plural, + ) + + graph = graph_integer_only | graph_decimal + + # remove trailing zeros of non zero number in the first 2 digits and fill up to 2 digits + # e.g. 2000 -> 20, 0200->02, 01 -> 01, 10 -> 10 + # not accepted: 002, 00, 0, + two_digits_fractional_part = ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(pynutil.delete("0")) + ) @ ( + (pynutil.delete("0") + (NEMO_DIGIT - "0")) + | ((NEMO_DIGIT - "0") + pynutil.insert("0")) + | ((NEMO_DIGIT - "0") + NEMO_DIGIT) + ) + + graph_min_singular = pynutil.insert("currency_min: \"") + min_singular_graph + pynutil.insert("\"") + graph_min_plural = pynutil.insert("currency_min: \"") + min_plural_graph + pynutil.insert("\"") + + # format ** euro ** cent + decimal_graph_with_minor = None + for curr_symbol, _ in maj_singular_labels: + preserve_order = pynutil.insert(" preserve_order: true") + + integer_plus_maj = pynini.union( + graph_integer + insert_space + pynutil.insert(curr_symbol) @ graph_maj_plural, + graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular, + ) + # non zero integer part + integer_plus_maj = (pynini.closure(NEMO_DIGIT) - "0") @ integer_plus_maj + + graph_fractional_one = ( + pynutil.insert("fractional_part: \"") + + two_digits_fractional_part @ pynini.cross("1", "un") + + pynutil.insert("\"") + ) + + graph_fractional = ( + two_digits_fractional_part @ (pynini.closure(NEMO_DIGIT, 1, 2) - "1") @ cardinal.two_digit_non_zero + ) + graph_fractional = pynutil.insert("fractional_part: \"") + graph_fractional + pynutil.insert("\"") + + fractional_plus_min = pynini.union( + graph_fractional + insert_space + pynutil.insert(curr_symbol) @ graph_min_plural, + graph_fractional_one + insert_space + pynutil.insert(curr_symbol) @ graph_min_singular, + ) + + decimal_graph_with_minor_curr = ( + integer_plus_maj + pynini.cross(decimal_separator, NEMO_SPACE) + fractional_plus_min + ) + decimal_graph_with_minor_curr |= pynutil.add_weight( + integer_plus_maj + + pynini.cross(decimal_separator, NEMO_SPACE) + + pynutil.insert("fractional_part: \"") + + two_digits_fractional_part @ cardinal.two_digit_non_zero + + pynutil.insert("\""), + weight=0.0001, + ) + + decimal_graph_with_minor_curr |= pynutil.delete("0,") + fractional_plus_min + decimal_graph_with_minor_curr = pynini.union( + pynutil.delete(curr_symbol) + + pynini.closure(delete_space, 0, 1) + + decimal_graph_with_minor_curr + + preserve_order, + decimal_graph_with_minor_curr + + preserve_order + + pynini.closure(delete_space, 0, 1) + + pynutil.delete(curr_symbol), + ) + + decimal_graph_with_minor = ( + decimal_graph_with_minor_curr + if decimal_graph_with_minor is None + else pynini.union(decimal_graph_with_minor, decimal_graph_with_minor_curr) + ) + + final_graph = graph | pynutil.add_weight(decimal_graph_with_minor, -0.001) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/ordinal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..07a41e5aef258bb29329763f31b436a8b39ce717 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/ordinal.py @@ -0,0 +1,172 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import roman_to_int, strip_accent +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +digit = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/digit.tsv"))) +teens = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/teen.tsv"))) +twenties = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/twenties.tsv"))) +ties = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/ties.tsv"))) +hundreds = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/hundreds.tsv"))) +gender_suffix = pynini.string_file(get_abs_path("data/ordinals/gender_suffix.tsv")) + + +def get_one_to_one_thousand(cardinal: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Produces an acceptor for verbalizations of all numbers from 1 to 1000. Needed for ordinals and fractions. + + Args: + cardinal: CardinalFst + + Returns: + fst: A pynini.FstLike object + """ + numbers = pynini.string_map([str(_) for _ in range(1, 1000)]) @ cardinal + return pynini.project(numbers, "output").optimize() + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal + "21.º" -> ordinal { integer: "vigésimo primero" morphosyntactic_features: "gender_masc" } + This class converts ordinal up to the millionth (millonésimo) order (exclusive). + + This FST also records the ending of the ordinal (called "morphosyntactic_features"): + either as gender_masc, gender_fem, or apocope. Also introduces plural feature for non-deterministic graphs. + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify") + cardinal_graph = cardinal.graph + + graph_digit = digit.optimize() + graph_teens = teens.optimize() + graph_ties = ties.optimize() + graph_twenties = twenties.optimize() + graph_hundreds = hundreds.optimize() + + if not deterministic: + # Some alternative derivations + graph_ties = graph_ties | pynini.cross("sesenta", "setuagésimo") + + graph_teens = graph_teens | pynini.cross("once", "decimoprimero") + graph_teens |= pynini.cross("doce", "decimosegundo") + + graph_digit = graph_digit | pynini.cross("nueve", "nono") + graph_digit |= pynini.cross("siete", "sétimo") + + graph_tens_component = ( + graph_teens + | (graph_ties + pynini.closure(pynini.cross(" y ", NEMO_SPACE) + graph_digit, 0, 1)) + | graph_twenties + ) + + graph_hundred_component = pynini.union( + graph_hundreds + pynini.closure(NEMO_SPACE + pynini.union(graph_tens_component, graph_digit), 0, 1), + graph_tens_component, + graph_digit, + ) + + # Need to go up to thousands for fractions + self.one_to_one_thousand = get_one_to_one_thousand(cardinal_graph) + + thousands = pynini.cross("mil", "milésimo") + + graph_thousands = ( + strip_accent(self.one_to_one_thousand) + NEMO_SPACE + thousands + ) # Cardinals become prefix for thousands series. Snce accent on the powers of ten we strip accent from leading words + graph_thousands @= pynini.cdrewrite(delete_space, "", "milésimo", NEMO_SIGMA) # merge as a prefix + graph_thousands |= thousands + + self.multiples_of_thousand = (cardinal_graph @ graph_thousands).optimize() + + if ( + not deterministic + ): # Formally the words preceding the power of ten should be a prefix, but some maintain word boundaries. + graph_thousands |= (self.one_to_one_thousand @ graph_hundred_component) + NEMO_SPACE + thousands + + graph_thousands += pynini.closure(NEMO_SPACE + graph_hundred_component, 0, 1) + + ordinal_graph = graph_thousands | graph_hundred_component + ordinal_graph = cardinal_graph @ ordinal_graph + + if not deterministic: + # The 10's and 20's series can also be two words + split_words = pynini.cross("decimo", "décimo ") | pynini.cross("vigesimo", "vigésimo ") + split_words = pynini.cdrewrite(split_words, "", NEMO_CHAR, NEMO_SIGMA) + ordinal_graph |= ordinal_graph @ split_words + + # If "octavo" is preceeded by the "o" within string, it needs deletion + ordinal_graph @= pynini.cdrewrite(pynutil.delete("o"), "", "octavo", NEMO_SIGMA) + + self.graph = ordinal_graph.optimize() + + delete_period = pynini.closure(pynutil.delete("."), 0, 1) # Sometimes the period is omitted + convert_gender_suffix = delete_period + gender_suffix + + # Managing Romanization, excluding words that may be ambiguous + roman_ordinals = roman_to_int(ordinal_graph) + # exceptions = pynini.accep("vi") | pynini.accep("di") | pynini.accep("mi") + exceptions = pynini.string_file(get_abs_path("data/ordinals/roman_exceptions.tsv")) + graph_exception = pynini.project(exceptions, 'input') + roman_ordinals = (pynini.project(roman_ordinals, "input") - graph_exception.arcsort()) @ roman_ordinals + + graph_roman = pynutil.insert("integer: \"") + roman_ordinals + pynutil.insert("\"") + + if not deterministic: + # Introduce plural + plural = pynini.closure(pynutil.insert("/plural"), 0, 1) + convert_gender_suffix += plural + + # Romanizations have no morphology marker, so in non-deterministic case we provide option for all + insert_morphology = pynutil.insert(convert_gender_suffix) + plural + insert_morphology = ( + pynutil.insert(" morphosyntactic_features: \"") + insert_morphology + pynutil.insert("\"") + ) + + graph_roman += insert_morphology + + else: + # We insert both genders as default + graph_roman += pynutil.insert(" morphosyntactic_features: \"gender_masc\"") | pynutil.insert( + " morphosyntactic_features: \"gender_fem\"" + ) + + graph = ( + pynutil.insert("integer: \"") + + ordinal_graph + + pynutil.insert("\"") + + pynutil.insert(" morphosyntactic_features: \"") + + convert_gender_suffix + + pynutil.insert("\"") + ) + graph = pynini.union(graph, graph_roman) + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/telephone.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/telephone.py new file mode 100644 index 0000000000000000000000000000000000000000..b778a180113ad1d3ee1a17bb0cea21263be1c9d6 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/telephone.py @@ -0,0 +1,194 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import ones +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) +graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) +graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) +graph_twenties = pynini.string_file(get_abs_path("data/numbers/twenties.tsv")) + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying telephone numbers, e.g. + 123-123-5678 -> { number_part: "uno dos tres uno dos tres cinco seis siete ocho" }. + In Spanish, digits are generally read individually, or as 2-digit numbers, + eg. "123" = "uno dos tres", + "1234" = "doce treinta y cuatro". + This will verbalize sequences of 10 (3+3+4 e.g. 123-456-7890). + 9 (3+3+3 e.g. 123-456-789) and 8 (4+4 e.g. 1234-5678) digits. + + (we ignore more complicated cases such as "doscientos y dos" or "tres nueves"). + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="classify") + + # create `single_digits` and `double_digits` graphs as these will be + # the building blocks of possible telephone numbers + single_digits = pynini.invert(graph_digit).optimize() | pynini.cross("0", "cero") + single_digits @= pynini.cdrewrite(pynini.cross(ones, "uno"), "", "", NEMO_SIGMA) + + # Any double digit + teen = pynini.invert(graph_teen) + ties = pynini.invert(graph_ties) + twenties = pynini.invert(graph_twenties) + + double_digits = teen + double_digits |= ties + (pynutil.delete('0') | (pynutil.insert(" y ") + single_digits)) + double_digits |= twenties + + # define separators + separators = pynini.union("-", " ", ".") + delete_separator = pynini.closure(pynutil.delete(separators), 0, 1) + + # process country codes as '+1' -> country_code: "one" + triple_number = pynini.closure(single_digits + insert_space, 2, 2) + single_digits + country_code = pynini.closure(pynini.cross("+", "más "), 0, 1) + ( + single_digits | double_digits | triple_number + ) + + # add ip and telephone prompts to this tag (as is in EN) + ip_prompts = pynini.string_file(get_abs_path("data/telephone/ip_prompt.tsv")) + telephone_prompts = pynini.string_file(get_abs_path("data/telephone/telephone_prompt.tsv")) + tel_prompt_sequence = telephone_prompts + NEMO_SPACE + pynini.closure(country_code, 0, 1) + + country_code_graph = ( + pynutil.insert("country_code: \"") + + (country_code | ip_prompts | tel_prompt_sequence) + + delete_separator + + pynutil.insert("\"") + ) + + # process IP addresses + digit_to_str_graph = single_digits + pynini.closure(pynutil.insert(" ") + single_digits, 0, 2) + ip_graph = digit_to_str_graph + (pynini.cross(".", " punto ") + digit_to_str_graph) ** 3 + + # process area codes with or without parentheses i.e. "212" or (212) + area_code = ( + pynini.closure(pynutil.delete("("), 0, 1) + + pynini.closure(single_digits + insert_space, 3, 3) + + pynini.closure(pynutil.delete(")"), 0, 1) + ) + + # process extensions + delete_ext = pynini.closure(pynutil.delete("ext."), 0, 1) + ext_graph = ( + pynutil.insert("extension: \"") + + delete_separator + + delete_ext + + delete_separator + + pynutil.insert("extensión ") + + pynini.closure(single_digits + insert_space, 1, 3) + + single_digits + + pynutil.insert("\"") + ) + + # define `ten_digit_graph`, `nine_digit_graph`, `eight_digit_graph` + # which produces telephone numbers spoken (1) only with single digits, + # or (2) spoken with double digits (and sometimes single digits) + + # 10-digit option (1): all single digits + ten_digit_graph = ( + area_code + + delete_separator + + pynini.closure(single_digits + insert_space, 3, 3) + + delete_separator + + pynini.closure(single_digits + insert_space, 3, 3) + + single_digits + ) + + # 9-digit option (1): all single digits + nine_digit_graph = ( + area_code + + delete_separator + + pynini.closure(single_digits + insert_space, 3, 3) + + delete_separator + + pynini.closure(single_digits + insert_space, 2, 2) + + single_digits + ) + + # 8-digit option (1): all single digits + eight_digit_graph = ( + pynini.closure(area_code, 0, 1) + + delete_separator + + pynini.closure(single_digits + insert_space, 4, 4) + + delete_separator + + pynini.closure(single_digits + insert_space, 3, 3) + + single_digits + ) + + if not deterministic: + # 10-digit option (2): (1+2) + (1+2) + (2+2) digits + ten_digit_graph |= ( + pynini.closure(single_digits + insert_space + double_digits + insert_space + delete_separator, 2, 2) + + double_digits + + insert_space + + double_digits + ) + + # 9-digit option (2): (1+2) + (1+2) + (1+2) digits + nine_digit_graph |= ( + pynini.closure(single_digits + insert_space + double_digits + insert_space + delete_separator, 2, 2) + + single_digits + + insert_space + + double_digits + ) + + # 8-digit option (2): (2+2) + (2+2) digits + eight_digit_graph |= ( + double_digits + + insert_space + + double_digits + + insert_space + + delete_separator + + double_digits + + insert_space + + double_digits + ) + + # handle numbers with letters like "1-800-go-u-haul" + num_letter_area_code = area_code @ pynini.cross("ocho cero cero ", "ochocientos ") + number_word = pynini.closure(single_digits | NEMO_ALPHA, 1, 8) + number_words = pynini.closure(number_word + pynini.cross(separators, " "), 0, 2) + number_word + + nums_w_letters_graph = pynutil.add_weight(num_letter_area_code + delete_separator + number_words, 0.01) + + number_part = pynini.union( + ten_digit_graph, nine_digit_graph, eight_digit_graph, nums_w_letters_graph, ip_graph + ) + number_part = pynutil.insert("number_part: \"") + number_part + pynutil.insert("\"") + + graph = ( + pynini.closure(country_code_graph + delete_separator + insert_space, 0, 1) + + number_part + + pynini.closure(delete_separator + insert_space + ext_graph, 0, 1) + ) + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/time.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..752c8126398d97fd70d4a59354cd844e487966e0 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/time.py @@ -0,0 +1,226 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +time_zones = pynini.string_file(get_abs_path("data/time/time_zone.tsv")) +suffix = pynini.string_file(get_abs_path("data/time/time_suffix.tsv")) + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + "02:15 est" -> time { hours: "dos" minutes: "quince" zone: "e s t"} + "2 h" -> time { hours: "dos" } + "9 h" -> time { hours: "nueve" } + "02:15:10 h" -> time { hours: "dos" minutes: "quince" seconds: "diez"} + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + + delete_time_delimiter = pynutil.delete(pynini.union(".", ":")) + + fem_one = pynini.string_map([("un", "una"), ("ún", "una")]) + change_one = pynini.cdrewrite(fem_one, "", "", NEMO_SIGMA) + cardinal_graph_fem = cardinal.graph @ change_one + cardinal_graph_masc = cardinal.graph + + day_suffix = pynutil.insert("suffix: \"") + suffix + pynutil.insert("\"") + day_suffix = delete_space + insert_space + day_suffix + + delete_hora_suffix = delete_space + insert_space + pynutil.delete("h") + delete_minute_suffix = delete_space + insert_space + (pynutil.delete("min") | pynutil.delete("m")) + delete_second_suffix = delete_space + insert_space + (pynutil.delete("seg") | pynutil.delete("s")) + + labels_hour_24 = [ + str(x) for x in range(0, 25) + ] # Can see both systems. Twelve hour requires am/pm for ambiguity resolution + labels_hour_12 = [str(x) for x in range(1, 13)] + labels_minute_or_second = [str(x) for x in range(1, 60)] + + delete_leading_zero_to_double_digit = ( + pynini.closure(pynutil.delete("0") | (NEMO_DIGIT - "0"), 0, 1) + NEMO_DIGIT + ) + + graph_24 = ( + pynini.closure(NEMO_DIGIT, 1, 2) @ delete_leading_zero_to_double_digit @ pynini.union(*labels_hour_24) + ) + graph_12 = ( + pynini.closure(NEMO_DIGIT, 1, 2) @ delete_leading_zero_to_double_digit @ pynini.union(*labels_hour_12) + ) + graph_minute_or_second = ( + pynini.closure(NEMO_DIGIT, 1, 2) + @ delete_leading_zero_to_double_digit + @ pynini.union(*labels_minute_or_second) + ) + graph_hour_24 = graph_24 @ cardinal_graph_fem + graph_hour_12 = graph_12 @ cardinal_graph_fem + graph_minute_or_second @= cardinal_graph_masc + + final_graph_hour_only_24 = ( + pynutil.insert("hours: \"") + graph_hour_24 + pynutil.insert("\"") + delete_hora_suffix + ) + final_graph_hour_only_12 = pynutil.insert("hours: \"") + graph_hour_12 + pynutil.insert("\"") + day_suffix + + final_graph_hour_24 = pynutil.insert("hours: \"") + graph_hour_24 + pynutil.insert("\"") + final_graph_hour_12 = pynutil.insert("hours: \"") + graph_hour_12 + pynutil.insert("\"") + + final_graph_minute = pynutil.insert("minutes: \"") + graph_minute_or_second + pynutil.insert("\"") + final_graph_second = pynutil.insert("seconds: \"") + graph_minute_or_second + pynutil.insert("\"") + + # handle suffixes like gmt or utc+1 + utc_one = pynini.string_map([("un", "uno"), ("ún", "uno")]) + change_utc_one = pynini.cdrewrite(utc_one, "", "", NEMO_SIGMA) + utc_cardinal_graph = cardinal.graph @ change_utc_one + + utc_or_gmt_numbers = (NEMO_DIGIT @ utc_cardinal_graph) | ( + graph_hour_24 + delete_time_delimiter + pynutil.insert(" ") + graph_minute_or_second + ) + + utc_or_gmt_diff = ( + pynutil.delete(pynini.closure(" ", 0, 1)) + + pynutil.insert(" ") + + pynini.string_map([("+", "más"), ("-", "menos")]) + + pynutil.delete(pynini.closure(" ", 0, 1)) + + pynutil.insert(" ") + + utc_or_gmt_numbers + ) + + time_zone_graph = time_zones + pynini.closure(utc_or_gmt_diff, 0, 1) + + final_time_zone_optional = pynini.closure( + delete_space + insert_space + pynutil.insert("zone: \"") + time_zone_graph + pynutil.insert("\""), + 0, + 1, + ) + + # 02.30 h + graph_hm = ( + final_graph_hour_24 + + delete_time_delimiter + + (pynutil.delete("00") | (insert_space + final_graph_minute)) + + pynini.closure( + (delete_time_delimiter + (pynutil.delete("00") | (insert_space + final_graph_second))), 0, 1 + ) + + pynini.closure(delete_hora_suffix, 0, 1) # 2.30 is valid if unambiguous + + final_time_zone_optional + ) + + # 2 h 30 min + graph_hm |= ( + final_graph_hour_24 + + delete_hora_suffix + + delete_space + + (pynutil.delete("00") | (insert_space + final_graph_minute)) + + delete_minute_suffix + + pynini.closure( + (delete_space + (pynutil.delete("00") | (insert_space + final_graph_second)) + delete_second_suffix), + 0, + 1, + ) + + final_time_zone_optional + ) + + # 2.30 a. m. (Only for 12 hour clock) + graph_hm |= ( + final_graph_hour_12 + + delete_time_delimiter + + (pynutil.delete("00") | (insert_space + final_graph_minute)) + + pynini.closure( + (delete_time_delimiter + (pynutil.delete("00") | (insert_space + final_graph_second))), 0, 1 + ) + + day_suffix + + final_time_zone_optional + ) + + graph_h = ( + pynini.union(final_graph_hour_only_24, final_graph_hour_only_12) + final_time_zone_optional + ) # Should always have a time indicator, else we'll pass to cardinals + + if not deterministic: + # This includes alternate vocalization (hour menos min, min para hour), here we shift the times and indicate a `style` tag + hour_shift_24 = pynini.invert(pynini.string_file(get_abs_path("data/time/hour_to_24.tsv"))) + hour_shift_12 = pynini.invert(pynini.string_file(get_abs_path("data/time/hour_to_12.tsv"))) + minute_shift = pynini.string_file(get_abs_path("data/time/minute_to.tsv")) + + graph_hour_to_24 = graph_24 @ hour_shift_24 @ cardinal_graph_fem + graph_hour_to_12 = graph_12 @ hour_shift_12 @ cardinal_graph_fem + + graph_minute_to = minute_shift @ graph_minute_or_second + + final_graph_hour_to_24 = pynutil.insert("hours: \"") + graph_hour_to_24 + pynutil.insert("\"") + final_graph_hour_to_12 = pynutil.insert("hours: \"") + graph_hour_to_12 + pynutil.insert("\"") + + final_graph_minute_to = pynutil.insert("minutes: \"") + graph_minute_to + pynutil.insert("\"") + + graph_menos = pynutil.insert(" style: \"1\"") + graph_para = pynutil.insert(" style: \"2\"") + + final_graph_style = graph_menos | graph_para + + # 02.30 h (omitting seconds since a bit awkward) + graph_hm |= ( + final_graph_hour_to_24 + + delete_time_delimiter + + insert_space + + final_graph_minute_to + + pynini.closure(delete_hora_suffix, 0, 1) # 2.30 is valid if unambiguous + + final_time_zone_optional + + final_graph_style + ) + + # 2 h 30 min + graph_hm |= ( + final_graph_hour_to_24 + + delete_hora_suffix + + delete_space + + insert_space + + final_graph_minute_to + + delete_minute_suffix + + final_time_zone_optional + + final_graph_style + ) + + # 2.30 a. m. (Only for 12 hour clock) + graph_hm |= ( + final_graph_hour_to_12 + + delete_time_delimiter + + insert_space + + final_graph_minute_to + + day_suffix + + final_time_zone_optional + + final_graph_style + ) + + final_graph = graph_hm | graph_h + if deterministic: + final_graph = final_graph + pynutil.insert(" preserve_order: true") + final_graph = final_graph.optimize() + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/tokenize_and_classify.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/tokenize_and_classify.py new file mode 100644 index 0000000000000000000000000000000000000000..5e700e784d40ee444748c1cc467a58f7302a5e63 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/tokenize_and_classify.py @@ -0,0 +1,161 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.taggers.cardinal import CardinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.taggers.date import DateFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.taggers.decimals import DecimalFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.taggers.electronic import ElectronicFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.taggers.fraction import FractionFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.taggers.measure import MeasureFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.taggers.money import MoneyFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.taggers.ordinal import OrdinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.taggers.telephone import TelephoneFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.taggers.time import TimeFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.taggers.whitelist import WhiteListFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.taggers.word import WordFst +from zonos3.vendor.nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. + For deployment, this grammar will be compiled and exported to OpenFst Finite State aRchive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = False, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"_{input_case}_es_tn_{deterministic}_deterministic{whitelist_file}.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating ClassifyFst grammars. This might take some time...") + + self.cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = self.cardinal.fst + + self.ordinal = OrdinalFst(cardinal=self.cardinal, deterministic=deterministic) + ordinal_graph = self.ordinal.fst + + self.decimal = DecimalFst(cardinal=self.cardinal, deterministic=deterministic) + decimal_graph = self.decimal.fst + + self.fraction = FractionFst( + cardinal=self.cardinal, + ordinal=self.ordinal, + deterministic=deterministic, + ) + fraction_graph = self.fraction.fst + self.measure = MeasureFst( + cardinal=self.cardinal, + decimal=self.decimal, + fraction=self.fraction, + deterministic=deterministic, + ) + measure_graph = self.measure.fst + self.date = DateFst(cardinal=self.cardinal, deterministic=deterministic) + date_graph = self.date.fst + word_graph = WordFst(deterministic=deterministic).fst + self.time = TimeFst(self.cardinal, deterministic=deterministic) + time_graph = self.time.fst + self.telephone = TelephoneFst(deterministic=deterministic) + telephone_graph = self.telephone.fst + self.electronic = ElectronicFst(deterministic=deterministic) + electronic_graph = self.electronic.fst + self.money = MoneyFst( + cardinal=self.cardinal, + decimal=self.decimal, + deterministic=deterministic, + ) + money_graph = self.money.fst + self.whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) + whitelist_graph = self.whitelist.fst + punct_graph = PunctuationFst(deterministic=deterministic).fst + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(time_graph, 1.09) + | pynutil.add_weight(measure_graph, 1.08) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(fraction_graph, 1.09) + | pynutil.add_weight(date_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(money_graph, 1.09) + | pynutil.add_weight(telephone_graph, 1.11) + | pynutil.add_weight(electronic_graph, 1.11) + | pynutil.add_weight(word_graph, 200) + ) + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") + punct = pynini.closure( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct), + 1, + ) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure( + ( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct + pynutil.insert(" ")) + ) + + token_plus_punct + ) + + graph = delete_space + graph + delete_space + graph |= punct + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/whitelist.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..cc9689a65bc935aea10349f33c896ca6afcb417c --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/whitelist.py @@ -0,0 +1,63 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path, load_labels + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist, e.g. + "sr." -> tokens { name: "señor" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + input_file: path to a file with whitelist replacements + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(input_case, file): + whitelist = load_labels(file) + if input_case == "lower_cased": + whitelist = [[x[0].lower()] + x[1:] for x in whitelist] + graph = pynini.string_map(whitelist) + return graph + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv")) + if not deterministic and input_case != "lower_cased": + graph |= pynutil.add_weight( + _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist.tsv")), weight=0.0001 + ) + + if input_file: + whitelist_provided = _get_whitelist_graph(input_case, input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + if not deterministic: + units_graph = _get_whitelist_graph(input_case, file=get_abs_path("data/measures/measurements.tsv")) + graph |= units_graph + + self.graph = graph + self.final_graph = convert_space(self.graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.final_graph + pynutil.insert("\"")).optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/word.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..803d4d1907c8f4dce7dd9b070a1ecde0e879a5f1 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/taggers/word.py @@ -0,0 +1,33 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying word. + e.g. dormir -> tokens { name: "dormir" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/utils.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..375bc1fb1f3e7593371e8f56ffa1cb0c543ff5c2 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/utils.py @@ -0,0 +1,42 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/cardinal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..5ddb0b4b3a5a01f0c3d283c45ebf4a0376815e50 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/cardinal.py @@ -0,0 +1,61 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import ( + add_cardinal_apocope_fem, + shift_cardinal_gender, + strip_cardinal_apocope, +) + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinals + e.g. cardinal { integer: "dos" } -> "dos" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + optional_sign = pynini.closure(pynini.cross("negative: \"true\" ", "menos "), 0, 1) + self.optional_sign = optional_sign + + integer = pynini.closure(NEMO_NOT_QUOTE, 1) + self.integer = pynutil.delete(" \"") + integer + pynutil.delete("\"") + + integer = pynutil.delete("integer:") + self.integer + + graph_masc = optional_sign + integer + graph_fem = shift_cardinal_gender(graph_masc) + + self.graph_masc = pynini.optimize(graph_masc) + self.graph_fem = pynini.optimize(graph_fem) + + # Adding adjustment for fem gender (choice of gender will be random) + graph = graph_masc | graph_fem + + if not deterministic: + # For alternate renderings when apocope is omitted (i.e. cardinal stands alone) + graph |= strip_cardinal_apocope(graph_masc) + # "una" will drop to "un" in unique contexts + graph |= add_cardinal_apocope_fem(graph_fem) + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/date.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..f80c18ca9e7b608df6521f0df768e2142ab0d7cd --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/date.py @@ -0,0 +1,83 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_preserve_order, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import strip_cardinal_apocope +from zonos3.vendor.nemo_text_processing.text_normalization.es.taggers.date import articles + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + date { day: "treinta y uno" month: "marzo" year: "dos mil" } -> "treinta y uno de marzo de dos mil" + date { day: "uno" month: "mayo" year: "del mil novecientos noventa" } -> "primero de mayo del mil novecientos noventa" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + day_cardinal = pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + day = strip_cardinal_apocope(day_cardinal) + + primero = pynini.cdrewrite(pynini.cross("uno", "primero"), "[BOS]", "[EOS]", NEMO_SIGMA) + day = ( + (day @ primero) if deterministic else pynini.union(day, day @ primero) + ) # Primero for first day is traditional, but will vary depending on region + + month = pynutil.delete("month: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + year = ( + pynutil.delete("year: \"") + + articles + + NEMO_SPACE + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + # Insert preposition if wasn't originally with the year. This would mean a space was present + year = pynutil.add_weight(year, -0.001) + year |= ( + pynutil.delete("year: \"") + + pynutil.insert("de ") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + # Roman centuries + graph_roman_centuries = pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + # day month year + graph_dmy = day + pynini.cross(NEMO_SPACE, " de ") + month + pynini.closure(pynini.accep(" ") + year, 0, 1) + + graph_mdy = month + NEMO_SPACE + day + pynini.closure(NEMO_SPACE + year, 0, 1) + if deterministic: + graph_mdy += pynutil.delete(" preserve_order: true") # Only accepts this if was explicitly passed + + self.graph = graph_dmy | graph_mdy | graph_roman_centuries + final_graph = self.graph + delete_preserve_order + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/decimals.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/decimals.py new file mode 100644 index 0000000000000000000000000000000000000000..9f8f71564b102b8d990ad1cc833122e5c56b1d31 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/decimals.py @@ -0,0 +1,89 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es import LOCALIZATION +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import ( + add_cardinal_apocope_fem, + shift_cardinal_gender, + shift_number_gender, + strip_cardinal_apocope, +) + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + decimal { negative: "true" integer_part: "dos" fractional_part: "cuatro cero" quantity: "billones" } -> menos dos coma quatro cero billones + decimal { integer_part: "un" quantity: "billón" } -> un billón + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "menos ") + delete_space, 0, 1) + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + fractional_default = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + conjunction = pynutil.insert(" punto ") if LOCALIZATION == "am" else pynutil.insert(" coma ") + if not deterministic: + conjunction |= pynutil.insert(pynini.union(" con ", " y ")) + fractional_default |= strip_cardinal_apocope(fractional_default) + fractional = conjunction + fractional_default + + quantity = ( + delete_space + + insert_space + + pynutil.delete("quantity: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_quantity = pynini.closure(quantity, 0, 1) + + graph_masc = optional_sign + pynini.union( + (integer + quantity), (integer + delete_space + fractional + optional_quantity) + ) + + # Allowing permutation for fem gender, don't include quantity since "million","billion", etc.. are masculine + graph_fem = optional_sign + (shift_cardinal_gender(integer) + delete_space + shift_number_gender(fractional)) + if not deterministic: # "una" will drop to "un" in certain cases + graph_fem |= add_cardinal_apocope_fem(graph_fem) + + self.numbers_only_quantity = ( + optional_sign + + pynini.union((integer + quantity), (integer + delete_space + fractional + quantity)).optimize() + ) + + self.graph_masc = (graph_masc + delete_preserve_order).optimize() + self.graph_fem = (graph_fem + delete_preserve_order).optimize() + + graph = graph_masc | graph_fem + + graph += delete_preserve_order + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/electronic.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..39cef28db45a1ac259c8995a0197d5926096eb1e --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/electronic.py @@ -0,0 +1,92 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + colon, + delete_preserve_order, + domain_string, + double_quotes, + protocol_string, + username_string, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +digit_no_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) +zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) + +graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")) +server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv")) +domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv")) +arroba = "arroba" + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for verbalizing electronic + e.g. electronic { username: "abc" domain: "hotmail.com" } -> "a b c arroba hotmail punto com" + -> "a b c arroba h o t m a i l punto c o m" + -> "a b c arroba hotmail punto c o m" + -> "a b c at h o t m a i l punto com" + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) + graph_digit_no_zero = ( + digit_no_zero @ pynini.cdrewrite(pynini.cross("un", "uno"), "", "", NEMO_SIGMA).optimize() + ) + graph_digit = graph_digit_no_zero | zero + + def add_space_after_char(): + return pynini.closure(NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) + pynutil.insert(NEMO_SPACE)) + ( + NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) + ) + + verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", NEMO_SIGMA) + + user_name = ( + pynutil.delete(username_string + colon + NEMO_SPACE + double_quotes) + + add_space_after_char() + + pynutil.delete(double_quotes) + ) + user_name @= verbalize_characters + + convert_defaults = pynutil.add_weight(NEMO_NOT_QUOTE, weight=0.0001) | domain_common | server_common + domain = convert_defaults + pynini.closure(pynutil.insert(NEMO_SPACE) + convert_defaults) + domain @= verbalize_characters + + domain = ( + pynutil.delete(domain_string + colon + NEMO_SPACE + double_quotes) + domain + pynutil.delete(double_quotes) + ) + protocol = ( + pynutil.delete(protocol_string + colon + NEMO_SPACE + double_quotes) + + add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", NEMO_SIGMA) + + pynutil.delete(double_quotes) + ) + + self.graph = (pynini.closure(protocol + NEMO_SPACE, 0, 1) + domain) | ( + user_name + NEMO_SPACE + pynutil.insert(arroba + NEMO_SPACE) + domain + | (pynutil.insert(arroba + NEMO_SPACE) + user_name) + ) + + delete_tokens = self.delete_tokens(self.graph + delete_preserve_order) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/fraction.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..fbd5857871c84ac0e1bf45a73330273abb81fe83 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/fraction.py @@ -0,0 +1,195 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_NOT_QUOTE, + NEMO_NOT_SPACE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import ( + accents, + shift_cardinal_gender, + strip_cardinal_apocope, +) + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fraction + e.g. tokens { fraction { integer: "treinta y tres" numerator: "cuatro" denominator: "quinto" } } -> + treinta y tres y cuatro quintos + + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + optional_sign = pynini.closure(pynini.cross('negative: "true" ', "menos "), 0, 1) + self.optional_sign = optional_sign + + # Derivational strings append 'avo' as a suffix. Adding space for processing aid + fraction_stem = pynutil.insert(" avo") + plural = pynutil.insert("s") + conjunction = pynutil.insert(" y ") + + integer = ( + pynutil.delete('integer_part: "') + + strip_cardinal_apocope(pynini.closure(NEMO_NOT_QUOTE)) + + pynutil.delete('"') + ) + + numerator_one = pynutil.delete('numerator: "') + pynini.accep("un") + pynutil.delete('" ') + numerator = ( + pynutil.delete('numerator: "') + + pynini.difference(pynini.closure(NEMO_NOT_QUOTE), "un") + + pynutil.delete('" ') + ) + + denominator_add_stem = pynutil.delete('denominator: "') + ( + pynini.closure(NEMO_NOT_QUOTE) + fraction_stem + pynutil.delete('" morphosyntactic_features: "add_root"') + ) + denominator_ordinal = pynutil.delete('denominator: "') + ( + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete('" morphosyntactic_features: "ordinal"') + ) + denominator_cardinal = pynutil.delete('denominator: "') + ( + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete('"') + ) + + denominator_singular = pynini.union(denominator_add_stem, denominator_ordinal) + if not deterministic: + # Occasional exceptions + denominator_singular |= denominator_add_stem @ pynini.string_map( + [("once avo", "undécimo"), ("doce avo", "duodécimo")] + ) + denominator_plural = denominator_singular + plural + + # Merging operations + merge = pynini.cdrewrite( + pynini.cross(" y ", "i"), "", "", NEMO_SIGMA + ) # The denominator must be a single word, with the conjunction "y" replaced by i + merge @= pynini.cdrewrite(delete_space, "", pynini.difference(NEMO_CHAR, "parte"), NEMO_SIGMA) + + # The merger can produce duplicate vowels. This is not allowed in orthography + delete_duplicates = pynini.string_map([("aa", "a"), ("oo", "o")]) # Removes vowels + delete_duplicates = pynini.cdrewrite(delete_duplicates, "", "", NEMO_SIGMA) + + remove_accents = pynini.cdrewrite( + accents, + pynini.union(NEMO_SPACE, pynini.accep("[BOS]")) + pynini.closure(NEMO_NOT_SPACE), + pynini.closure(NEMO_NOT_SPACE) + pynini.union("avo", "ava", "ésimo", "ésima"), + NEMO_SIGMA, + ) + merge_into_single_word = merge @ remove_accents @ delete_duplicates + + fraction_default = numerator + delete_space + insert_space + (denominator_plural @ merge_into_single_word) + + fraction_with_one = ( + numerator_one + delete_space + insert_space + (denominator_singular @ merge_into_single_word) + ) + + fraction_with_cardinal = strip_cardinal_apocope(numerator | numerator_one) + fraction_with_cardinal += ( + delete_space + pynutil.insert(" sobre ") + strip_cardinal_apocope(denominator_cardinal) + ) + + if not deterministic: + # There is an alternative rendering where ordinals act as adjectives for 'parte'. This requires use of the feminine + # Other rules will manage use of "un" at end, so just worry about endings + exceptions = pynini.string_map([("tercia", "tercera")]) + apply_exceptions = pynini.cdrewrite(exceptions, "", "", NEMO_SIGMA) + vowel_change = pynini.cdrewrite(pynini.cross("o", "a"), "", pynini.accep("[EOS]"), NEMO_SIGMA) + + denominator_singular_fem = shift_cardinal_gender(denominator_singular) @ vowel_change @ apply_exceptions + denominator_plural_fem = denominator_singular_fem + plural + + numerator_one_fem = shift_cardinal_gender(numerator_one) + numerator_fem = shift_cardinal_gender(numerator) + + fraction_with_cardinal |= ( + (numerator_one_fem | numerator_fem) + + delete_space + + pynutil.insert(" sobre ") + + shift_cardinal_gender(denominator_cardinal) + ) + + # Still need to manage stems + merge_stem = pynini.cdrewrite( + delete_space, "", pynini.union("avo", "ava", "avos", "avas"), NEMO_SIGMA + ) # For managing alternative spacing + merge_stem @= remove_accents @ delete_duplicates + + fraction_with_one_fem = numerator_one_fem + delete_space + insert_space + fraction_with_one_fem += pynini.union( + denominator_singular_fem @ merge_stem, + denominator_singular_fem @ merge_into_single_word, + ) # Both forms exists + fraction_with_one_fem += pynutil.insert(" parte") + fraction_with_one_fem @= pynini.cdrewrite( + pynini.cross("una media", "media"), "", "", NEMO_SIGMA + ) # "media" not "una media" + + fraction_default_fem = numerator_fem + delete_space + insert_space + fraction_default_fem += pynini.union( + denominator_plural_fem @ merge_stem, + denominator_plural_fem @ merge_into_single_word, + ) + fraction_default_fem += pynutil.insert(" partes") + + fraction_default |= ( + numerator + delete_space + insert_space + denominator_plural @ merge_stem + ) # Case of no merger + fraction_default |= fraction_default_fem + + fraction_with_one |= numerator_one + delete_space + insert_space + denominator_singular @ merge_stem + fraction_with_one |= fraction_with_one_fem + + fraction_with_one @= pynini.cdrewrite( + pynini.cross("un medio", "medio"), "", "", NEMO_SIGMA + ) # "medio" not "un medio" + + fraction = fraction_with_one | fraction_default | fraction_with_cardinal + graph_masc = optional_sign + pynini.closure(integer + delete_space + conjunction, 0, 1) + fraction + + # Manage cases of fem gender (only shows on integer except for "medio") + integer_fem = shift_cardinal_gender(integer) + fraction_default |= ( + shift_cardinal_gender(numerator) + + delete_space + + insert_space + + (denominator_plural @ pynini.cross("medios", "medias")) + ) + fraction_with_one |= ( + pynutil.delete(numerator_one) + delete_space + (denominator_singular @ pynini.cross("medio", "media")) + ) + + fraction_fem = fraction_with_one | fraction_default | fraction_with_cardinal + graph_fem = optional_sign + pynini.closure(integer_fem + delete_space + conjunction, 0, 1) + fraction_fem + + self.graph_masc = pynini.optimize(graph_masc) + self.graph_fem = pynini.optimize(graph_fem) + + self.graph = graph_masc | graph_fem + + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/measure.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..c7d80e61b3e59f6b0c1f419c65225ff6a64433f8 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/measure.py @@ -0,0 +1,115 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_preserve_order, + delete_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import ones +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +unit_plural_fem = pynini.string_file(get_abs_path("data/measures/measurements_plural_fem.tsv")) +unit_plural_masc = pynini.string_file(get_abs_path("data/measures/measurements_plural_masc.tsv")) + +unit_singular_fem = pynini.project(unit_plural_fem, "input") +unit_singular_masc = pynini.project(unit_plural_masc, "input") + +unit_plural_fem = pynini.project(unit_plural_fem, "output") +unit_plural_masc = pynini.project(unit_plural_masc, "output") + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { cardinal { integer: "dos" units: "gramos" } } -> "dos gramos" + measure { decimal { integer_part: "dos" quantity: "millones" units: "gramos" } } -> "dos millones de gramos" + + Args: + decimal: DecimalFst + cardinal: CardinalFst + fraction: FractionFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + + graph_decimal_masc = decimal.delete_tokens(decimal.graph_masc) + graph_decimal_fem = decimal.delete_tokens(decimal.graph_fem) + graph_cardinal_masc = cardinal.delete_tokens(cardinal.graph_masc) + graph_cardinal_fem = cardinal.delete_tokens(cardinal.graph_fem) + graph_fraction_fem = fraction.delete_tokens(fraction.graph_fem) + graph_fraction_masc = fraction.delete_tokens(fraction.graph_masc) + + unit_masc = (unit_plural_masc | unit_singular_masc) + pynini.closure( + NEMO_WHITE_SPACE + "por" + pynini.closure(NEMO_NOT_QUOTE, 1), 0, 1 + ) + unit_masc |= "por" + pynini.closure(NEMO_NOT_QUOTE, 1) + unit_masc = pynutil.delete("units: \"") + (pynini.closure(NEMO_NOT_QUOTE) @ unit_masc) + pynutil.delete("\"") + + unit_fem = (unit_plural_fem | unit_singular_fem) + pynini.closure( + NEMO_WHITE_SPACE + "por" + pynini.closure(NEMO_NOT_QUOTE, 1), 0, 1 + ) + unit_fem = pynutil.delete("units: \"") + (pynini.closure(NEMO_NOT_QUOTE) @ unit_fem) + pynutil.delete("\"") + + graph_masc = (graph_cardinal_masc | graph_decimal_masc) + NEMO_WHITE_SPACE + unit_masc + graph_masc |= graph_fraction_masc + NEMO_WHITE_SPACE + pynutil.insert("de ") + unit_masc + graph_masc |= pynutil.add_weight( + graph_fraction_masc @ (NEMO_SIGMA + pynini.union("medio", "medios")) + NEMO_WHITE_SPACE + unit_masc, -0.001 + ) # "medio litro" not "medio de litro" + + graph_fem = (graph_cardinal_fem | graph_decimal_fem) + NEMO_WHITE_SPACE + unit_fem + graph_fem |= graph_fraction_fem + NEMO_WHITE_SPACE + pynutil.insert("de ") + unit_fem + graph_fem |= pynutil.add_weight( + graph_fraction_fem @ (NEMO_SIGMA + pynini.union("media", "medias")) + NEMO_WHITE_SPACE + unit_fem, -0.001 + ) + + graph = graph_masc | graph_fem + + graph = ( + pynini.cdrewrite( + pynutil.insert(" de"), "quantity: \"" + pynini.closure(NEMO_NOT_QUOTE, 1), "\"", NEMO_SIGMA + ) + @ graph + ) # billones de xyz + + graph @= pynini.cdrewrite(pynini.cross(ones, "uno"), "", NEMO_WHITE_SPACE + "por", NEMO_SIGMA) + + # To manage alphanumeric combonations ("a-8, 5x"), we let them use a weighted default path. + alpha_num_unit = pynutil.delete("units: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + graph_alpha_num = pynini.union( + (graph_cardinal_masc | graph_decimal_masc) + NEMO_SPACE + alpha_num_unit, + alpha_num_unit + delete_extra_space + (graph_cardinal_masc | graph_decimal_masc), + ) + + math = pynutil.add_weight( + pynutil.delete("units: \"math\"") + delete_space + graph_cardinal_masc + delete_space, -1 + ) + + graph |= pynutil.add_weight(graph_alpha_num, 0.01) + graph |= math + + graph += delete_preserve_order + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/money.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..3ca61a17d9874e0c7857323c157b49581e5332b7 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/money.py @@ -0,0 +1,167 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_preserve_order, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import shift_cardinal_gender, strip_cardinal_apocope +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +fem = pynini.string_file((get_abs_path("data/money/currency_plural_fem.tsv"))) +masc = pynini.string_file((get_abs_path("data/money/currency_plural_masc.tsv"))) + +fem_singular = pynini.project(fem, "input") +masc_singular = pynini.project(masc, "input") + +fem_plural = pynini.project(fem, "output") +masc_plural = pynini.project(masc, "output") + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { currency_maj: "euro" integer_part: "un"} -> "un euro" + money { currency_maj: "euro" integer_part: "un" fractional_part: "cero cero un"} -> "uno coma cero cero uno euros" + money { integer_part: "un" currency_maj: "libra" fractional_part: "cuarenta" preserve_order: true} -> "una libra cuarenta" + money { integer_part: "un" currency_maj: "libra" fractional_part: "cuarenta" currency_min: "peniques" preserve_order: true} -> "una libra con cuarenta peniques" + money { fractional_part: "un" currency_min: "penique" preserve_order: true} -> "un penique" + + Args: + decimal: GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + maj_singular_masc = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_singular) + + pynutil.delete("\"") + ) + maj_singular_fem = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_singular) + + pynutil.delete("\"") + ) + + maj_plural_masc = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_plural) + + pynutil.delete("\"") + ) + maj_plural_fem = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_plural) + + pynutil.delete("\"") + ) + + maj_masc = maj_plural_masc | maj_singular_masc # Tagger kept quantity resolution stable + maj_fem = maj_plural_fem | maj_singular_fem + + min_singular_masc = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_singular) + + pynutil.delete("\"") + ) + min_singular_fem = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_singular) + + pynutil.delete("\"") + ) + + min_plural_masc = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_plural) + + pynutil.delete("\"") + ) + min_plural_fem = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_plural) + + pynutil.delete("\"") + ) + + min_masc = min_plural_masc | min_singular_masc + min_fem = min_plural_fem | min_singular_fem + + fractional_part = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + integer_part = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + optional_add_and = pynini.closure(pynutil.insert(pynini.union("con ", "y ")), 0, 1) + + # *** currency_maj + graph_integer_masc = integer_part + NEMO_SPACE + maj_masc + graph_integer_fem = shift_cardinal_gender(integer_part) + NEMO_SPACE + maj_fem + + graph_integer = graph_integer_fem | graph_integer_masc + + # *** currency_maj + (***) | ((con) *** current_min) + graph_integer_with_minor_masc = ( + graph_integer_masc + + NEMO_SPACE + + pynini.union( + optional_add_and + strip_cardinal_apocope(fractional_part), + (optional_add_and + fractional_part + NEMO_SPACE + min_masc), + (optional_add_and + shift_cardinal_gender(fractional_part) + NEMO_SPACE + min_fem), + ) # Could be minor currency that is different gender + + delete_preserve_order + ) + + graph_integer_with_minor_fem = ( + graph_integer_fem + + NEMO_SPACE + + pynini.union( + optional_add_and + shift_cardinal_gender(fractional_part), + (optional_add_and + fractional_part + NEMO_SPACE + min_masc), + (optional_add_and + shift_cardinal_gender(fractional_part) + NEMO_SPACE + min_fem), + ) # Could be minor currency that is different gender + + delete_preserve_order + ) + + graph_integer_with_minor = graph_integer_with_minor_fem | graph_integer_with_minor_masc + + ## *** coma *** currency_maj + graph_decimal_masc = decimal.graph_masc + NEMO_SPACE + maj_masc + + graph_decimal_fem = decimal.graph_fem + graph_decimal_fem |= decimal.numbers_only_quantity # can still have "x billions" with fem currency + graph_decimal_fem += NEMO_SPACE + maj_fem + + graph_decimal = graph_decimal_fem | graph_decimal_masc + graph_decimal = ( + pynini.cdrewrite( + pynutil.insert(" de"), "quantity: \"" + pynini.closure(NEMO_NOT_QUOTE, 1), "\"", NEMO_SIGMA + ) + @ graph_decimal + ) # formally it's millones/billones de *** + + # *** current_min + graph_minor_masc = fractional_part + NEMO_SPACE + min_masc + delete_preserve_order + graph_minor_fem = shift_cardinal_gender(fractional_part) + NEMO_SPACE + min_fem + delete_preserve_order + + graph_minor = graph_minor_fem | graph_minor_masc + + graph = graph_integer | graph_integer_with_minor | graph_decimal | graph_minor + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/ordinal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..90b129f3f9cd933ed0e1b543f7fb531ffcf8c298 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/ordinal.py @@ -0,0 +1,70 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, NEMO_SPACE, GraphFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import shift_number_gender + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinals + e.g. ordinal { integer: "tercer" } } -> "tercero" + -> "tercera" + -> "tercer" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + graph = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + # masculne gender we leave as is + graph_masc = graph + pynutil.delete(" morphosyntactic_features: \"gender_masc") + + # shift gender + graph_fem_ending = graph @ pynini.cdrewrite( + pynini.cross("o", "a"), "", NEMO_SPACE | pynini.accep("[EOS]"), NEMO_SIGMA + ) + graph_fem = shift_number_gender(graph_fem_ending) + pynutil.delete(" morphosyntactic_features: \"gender_fem") + + # Apocope just changes tercero and primero. May occur if someone wrote 11.er (uncommon) + graph_apocope = ( + pynini.cross("tercero", "tercer") + | pynini.cross("primero", "primer") + | pynini.cross("undécimo", "decimoprimer") + ) # In case someone wrote 11.er with deterministic + graph_apocope = (graph @ pynini.cdrewrite(graph_apocope, "", "", NEMO_SIGMA)) + pynutil.delete( + " morphosyntactic_features: \"apocope" + ) + + graph = graph_apocope | graph_masc | graph_fem + + if not deterministic: + # Plural graph + graph_plural = pynini.cdrewrite( + pynutil.insert("s"), pynini.union("o", "a"), NEMO_SPACE | pynini.accep("[EOS]"), NEMO_SIGMA + ) + + graph |= (graph @ graph_plural) + pynutil.delete("/plural") + + self.graph = graph + pynutil.delete("\"") + + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/telephone.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/telephone.py new file mode 100644 index 0000000000000000000000000000000000000000..6912a6bb30605a4c6b2c006b600013f9f84eca79 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/telephone.py @@ -0,0 +1,69 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing telephone, e.g. + telephone { number_part: "uno dos tres uno dos tres cinco seis siete ocho" } + -> uno dos tres uno dos tres cinco seis siete ocho + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="verbalize") + + optional_country_code = pynini.closure( + pynutil.delete("country_code: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + delete_space + + insert_space, + 0, + 1, + ) + + number_part = ( + pynutil.delete("number_part: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynini.closure(pynutil.add_weight(pynutil.delete(" "), -0.0001), 0, 1) + + pynutil.delete("\"") + ) + + optional_extension = pynini.closure( + delete_space + + insert_space + + pynutil.delete("extension: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\""), + 0, + 1, + ) + + graph = optional_country_code + number_part + optional_extension + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() + + +# number_part = pynutil.delete("number_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + +# delete_tokens = self.delete_tokens(number_part) +# self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/time.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..e0f0b15647f72c4f5ba547bfad67ff8e55543a3b --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/time.py @@ -0,0 +1,257 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.utils import get_abs_path + +alt_minutes = pynini.string_file(get_abs_path("data/time/alt_minutes.tsv")) + +morning_times = pynini.string_file(get_abs_path("data/time/morning_times.tsv")) +afternoon_times = pynini.string_file(get_abs_path("data/time/afternoon_times.tsv")) +evening_times = pynini.string_file(get_abs_path("data/time/evening_times.tsv")) + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g. + time { hours: "doce" minutes: "media" suffix: "a m" } -> doce y media de la noche + time { hours: "doce" } -> twelve o'clock + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + change_minutes = pynini.cdrewrite(alt_minutes, pynini.accep("[BOS]"), pynini.accep("[EOS]"), NEMO_SIGMA) + + morning_phrases = pynini.cross("am", "de la mañana") + afternoon_phrases = pynini.cross("pm", "de la tarde") + evening_phrases = pynini.cross("pm", "de la noche") + + # For the 12's + mid_times = pynini.accep("doce") + mid_phrases = ( + pynini.string_map([("pm", "del mediodía"), ("am", "de la noche")]) + if deterministic + else pynini.string_map( + [ + ("pm", "de la mañana"), + ("pm", "del día"), + ("pm", "del mediodía"), + ("am", "de la noche"), + ("am", "de la medianoche"), + ] + ) + ) + + hour = ( + pynutil.delete("hours:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + minute = ( + pynutil.delete("minutes:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + minute = (minute @ change_minutes) if deterministic else pynini.union(minute, minute @ change_minutes) + + suffix = ( + pynutil.delete("suffix:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + zone = ( + pynutil.delete("zone:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_zone = pynini.closure(delete_space + insert_space + zone, 0, 1) + second = ( + pynutil.delete("seconds:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + graph_hms = ( + hour + + pynutil.insert(" horas ") + + delete_space + + minute + + pynutil.insert(" minutos y ") + + delete_space + + second + + pynutil.insert(" segundos") + ) + + graph_hm = hour + delete_space + pynutil.insert(" y ") + minute + graph_hm |= pynini.union( + (hour @ morning_times) + + delete_space + + pynutil.insert(" y ") + + minute + + delete_space + + insert_space + + (suffix @ morning_phrases), + (hour @ afternoon_times) + + delete_space + + pynutil.insert(" y ") + + minute + + delete_space + + insert_space + + (suffix @ afternoon_phrases), + (hour @ evening_times) + + delete_space + + pynutil.insert(" y ") + + minute + + delete_space + + insert_space + + (suffix @ evening_phrases), + (hour @ mid_times) + + delete_space + + pynutil.insert(" y ") + + minute + + delete_space + + insert_space + + (suffix @ mid_phrases), + ) + + graph_h = pynini.union( + hour, + (hour @ morning_times) + delete_space + insert_space + (suffix @ morning_phrases), + (hour @ afternoon_times) + delete_space + insert_space + (suffix @ afternoon_phrases), + (hour @ evening_times) + delete_space + insert_space + (suffix @ evening_phrases), + (hour @ mid_times) + delete_space + insert_space + (suffix @ mid_phrases), + ) + + graph = (graph_hms | graph_hm | graph_h) + optional_zone + + if not deterministic: + graph_style_1 = pynutil.delete(" style: \"1\"") + graph_style_2 = pynutil.delete(" style: \"2\"") + + graph_menos = hour + delete_space + pynutil.insert(" menos ") + minute + graph_style_1 + graph_menos |= ( + (hour @ morning_times) + + delete_space + + pynutil.insert(" menos ") + + minute + + delete_space + + insert_space + + (suffix @ morning_phrases) + + graph_style_1 + ) + graph_menos |= ( + (hour @ afternoon_times) + + delete_space + + pynutil.insert(" menos ") + + minute + + delete_space + + insert_space + + (suffix @ afternoon_phrases) + + graph_style_1 + ) + graph_menos |= ( + (hour @ evening_times) + + delete_space + + pynutil.insert(" menos ") + + minute + + delete_space + + insert_space + + (suffix @ evening_phrases) + + graph_style_1 + ) + graph_menos |= ( + (hour @ mid_times) + + delete_space + + pynutil.insert(" menos ") + + minute + + delete_space + + insert_space + + (suffix @ mid_phrases) + + graph_style_1 + ) + graph_menos += optional_zone + + graph_para = minute + pynutil.insert(" para las ") + delete_space + hour + graph_style_2 + graph_para |= ( + minute + + pynutil.insert(" para las ") + + delete_space + + (hour @ morning_times) + + delete_space + + insert_space + + (suffix @ morning_phrases) + + graph_style_2 + ) + graph_para |= ( + minute + + pynutil.insert(" para las ") + + delete_space + + (hour @ afternoon_times) + + delete_space + + insert_space + + (suffix @ afternoon_phrases) + + graph_style_2 + ) + graph_para |= ( + minute + + pynutil.insert(" para las ") + + delete_space + + (hour @ evening_times) + + delete_space + + insert_space + + (suffix @ evening_phrases) + + graph_style_2 + ) + graph_para |= ( + minute + + pynutil.insert(" para las ") + + delete_space + + (hour @ mid_times) + + delete_space + + insert_space + + (suffix @ mid_phrases) + + graph_style_2 + ) + graph_para += optional_zone + graph_para @= pynini.cdrewrite( + pynini.cross(" las ", " la "), "para", "una", NEMO_SIGMA + ) # Need agreement with one + + graph |= graph_menos | graph_para + delete_tokens = self.delete_tokens(graph + delete_preserve_order) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/verbalize.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/verbalize.py new file mode 100644 index 0000000000000000000000000000000000000000..cc5e7694f244aabc3c8429e081428b4c7a41946b --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/verbalize.py @@ -0,0 +1,73 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.verbalizers.cardinal import CardinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.verbalizers.date import DateFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.verbalizers.decimals import DecimalFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.verbalizers.electronic import ElectronicFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.verbalizers.fraction import FractionFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.verbalizers.measure import MeasureFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.verbalizers.money import MoneyFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.verbalizers.ordinal import OrdinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.verbalizers.telephone import TelephoneFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.verbalizers.time import TimeFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + ordinal = OrdinalFst(deterministic=deterministic) + ordinal_graph = ordinal.fst + decimal = DecimalFst(deterministic=deterministic) + decimal_graph = decimal.fst + fraction = FractionFst(deterministic=deterministic) + fraction_graph = fraction.fst + date = DateFst(deterministic=deterministic) + date_graph = date.fst + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + measure_graph = measure.fst + electronic = ElectronicFst(deterministic=deterministic) + electronic_graph = electronic.fst + whitelist_graph = WhiteListFst(deterministic=deterministic).fst + money_graph = MoneyFst(decimal=decimal, deterministic=deterministic).fst + telephone_graph = TelephoneFst(deterministic=deterministic).fst + time_graph = TimeFst(deterministic=deterministic).fst + + graph = ( + cardinal_graph + | measure_graph + | decimal_graph + | ordinal_graph + | date_graph + | electronic_graph + | money_graph + | fraction_graph + | whitelist_graph + | telephone_graph + | time_graph + ) + self.fst = graph diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/verbalize_final.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/verbalize_final.py new file mode 100644 index 0000000000000000000000000000000000000000..5515c67ce11f7dd7faff932037f7990773d361df --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/es/verbalizers/verbalize_final.py @@ -0,0 +1,70 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.word import WordFst +from zonos3.vendor.nemo_text_processing.text_normalization.es.verbalizers.verbalize import VerbalizeFst +from zonos3.vendor.nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"es_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/dates/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/dates/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..341a77c5bc66dee5d2ba0edf888f91e5bf225e3c --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/dates/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/dates/eras.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/dates/eras.tsv new file mode 100644 index 0000000000000000000000000000000000000000..6127bea93f18d3233067a12d4b25580a82dba1b2 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/dates/eras.tsv @@ -0,0 +1,8 @@ +20s twenties +30s thirties +40s forties +50s fifties +60s sixties +70s seventies +80s eighties +90s nineties \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/dates/months.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/dates/months.tsv new file mode 100644 index 0000000000000000000000000000000000000000..98a4e7d5d5b6d6aceee049a7a315abb1e21f88ea --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/dates/months.tsv @@ -0,0 +1,12 @@ +1 janvier +2 février +3 mars +4 avril +5 mai +6 juin +7 juillet +8 août +9 septembre +10 octobre +11 novembre +12 décembre \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/fractions/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/fractions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/fractions/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/fractions/irregular_denominators.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/fractions/irregular_denominators.tsv new file mode 100644 index 0000000000000000000000000000000000000000..bb199174ca7990b19cf3b532b01367573e3d2a94 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/fractions/irregular_denominators.tsv @@ -0,0 +1,7 @@ +premième tout +deuxième demi +deuxièmes demi +troisième tiers +troisièmes tiers +quatrième quart +quatrièmes quarts \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/measures/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/measures/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/measures/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/measures/measurements.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/measures/measurements.tsv new file mode 100644 index 0000000000000000000000000000000000000000..55f0d46435462504b4a87f87c88c4db4c44ad3da --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/measures/measurements.tsv @@ -0,0 +1,17 @@ +m mètres +m² mètres carrés +m³ mètres cubes +s secondes +min minutes +h heures +° degrés +°C degrés celsius +g grammes +l litres +kg kilos +'' pouce +lb livres +% pour cent +‰ pour mille +km/h kilomètres heure +m/h mètres à l’heure \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/digit.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..4f5375b2940d04557fb2964776581151491930b0 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +1 un +2 deux +3 trois +4 quatre +5 cinq +6 six +7 sept +8 huit +9 neuf \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/eighty.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/eighty.tsv new file mode 100644 index 0000000000000000000000000000000000000000..bdb07292f323e72e48339646bfc3271dd18c1a56 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/eighty.tsv @@ -0,0 +1 @@ +8 quatre-vingt \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/ninety.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/ninety.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b00412f6d033c1597ef828ef52e4faa33f2dc08e --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/ninety.tsv @@ -0,0 +1 @@ +9 quatre-vingt \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/quantities.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/quantities.tsv new file mode 100644 index 0000000000000000000000000000000000000000..a17792e406a6f9feb5977fe25981542e64ddf1b4 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/quantities.tsv @@ -0,0 +1,10 @@ +million +millions +milliard +milliards +billion +billions +billiard +billiards +trillion +trillions \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/seventy.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/seventy.tsv new file mode 100644 index 0000000000000000000000000000000000000000..80c4116f7c7f93687b76bf5bb47d6160a565dc75 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/seventy.tsv @@ -0,0 +1 @@ +7 soixante \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/teens.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/teens.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c22d6879ea19c67cde0d44352550844880f728aa --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/teens.tsv @@ -0,0 +1,9 @@ +11 onze +12 douze +13 treize +14 quatorze +15 quinze +16 seize +17 dix-sept +18 dix-huit +19 dix-neuf \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/ten.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/ten.tsv new file mode 100644 index 0000000000000000000000000000000000000000..e7e79c1397e06c478ca09ec165efb337c69e06de --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/ten.tsv @@ -0,0 +1 @@ +10 dix \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/tens_simple.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/tens_simple.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b72cf3dd8e0a7a441148bdf4527c6e08eed4ff0a --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/tens_simple.tsv @@ -0,0 +1,5 @@ +2 vingt +3 trente +4 quarante +5 cinquante +6 soixante \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/zero.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/zero.tsv new file mode 100644 index 0000000000000000000000000000000000000000..2af43a7086f918593571d14ad147de530e9b15af --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/numbers/zero.tsv @@ -0,0 +1 @@ +0 zéro \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/irregular_numbers.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/irregular_numbers.tsv new file mode 100644 index 0000000000000000000000000000000000000000..2d21198f3ee1e88f8a50bca59705c537b3c02138 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/irregular_numbers.tsv @@ -0,0 +1,30 @@ +quatre quatrième +cinq cinquième +neuf neuvième +onze onzième +douze douzième +treize treizième +quatorze quatorzième +quinze quinzième +seize seizième +dix-neuf dix neuvième +trente trentième +quarante quarantième +cinquante cinquantième +soixante soixantième +soixante-onze soixante onzième +soixante-douze soixante douzième +soixante-treize soixante treizième +soixante-quatorze soixante quatorzième +soixante-quinze soixante quinzième +soixante-seize soixante seizième +soixante-dix-neuf soixante dix neuvième +quatre-vingt-onze quatre vingt onzième +quatre-vingt-douze quatre vingt douzième +quatre-vingt-treize quatre vingt treizième +quatre-vingt-quatorze quatre vingt quatorzième +quatre-vingt-quinze quatre vingt quinzième +quatre-vingt-seize quatre vingt seizième +quatre-vingt-dix-neuf quatre vingt dix neuvième +cents centième +mille millième \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/suffixes.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/suffixes.tsv new file mode 100644 index 0000000000000000000000000000000000000000..73adb8e043daca73a33a791255f8f64063806ace --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/ordinals/suffixes.tsv @@ -0,0 +1,6 @@ +e ième +ème ième +° ième +er ier +ère ière +es ièmes \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/whitelist.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/whitelist.tsv new file mode 100644 index 0000000000000000000000000000000000000000..dc563bdab6a3b86a2a41bc9e24be837a15df13d7 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/data/whitelist.tsv @@ -0,0 +1,13 @@ +Mᵐᵉ madame +Mᵐᵉˢ mesdames +Mˡˡᵉ mademoiselle +Mˡˡᵉˢ mademoiselles +Dʳ docteur +Dʳˢ docteurs +Dʳᵉ docteure +Dʳᵉˢ docteures +apr. J.-C. après jésus-christ +av. J.-C. avant Jésus-Christ +le hon. l’honorable +le très hon. le très hononrable +% pour cent \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/cardinal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..a1ba0d55cd188f6fe56968f03b424c92a285a4dd --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/cardinal.py @@ -0,0 +1,143 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst, insert_space +from zonos3.vendor.nemo_text_processing.text_normalization.fr.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + "1000" -> cardinal { integer: "mille" } + "2,000,000" -> cardinal { integer: "deux millions" } + This grammar covers from single digits to hundreds of billions ("milliards" in French). + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + # Single digits + zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + digits = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) # 1 to 9 + digits_no_one = (NEMO_DIGIT - "1") @ digits + one = "1" @ digits + + # Double digits + ten = pynini.string_file(get_abs_path("data/numbers/ten.tsv")) + teens = pynini.string_file(get_abs_path("data/numbers/teens.tsv")) # 11 to 19 + teens_no_one = (pynini.accep("1") + (NEMO_DIGIT - "1")) @ teens + + # Simple tens + tens = pynini.string_file(get_abs_path("data/numbers/tens_simple.tsv")) + ties_simple_unit = pynutil.insert("-") + digits_no_one + ties_simple_one = insert_space + pynini.cross("1", "et un") + ties_simple = tens + (pynutil.delete("0") | (ties_simple_unit | ties_simple_one)) # 20 to 69 + + # Complex tens + seventy = pynini.string_file(get_abs_path("data/numbers/seventy.tsv")) + seventy_unit = pynutil.insert("-") + ((pynutil.insert("1") + NEMO_DIGIT) @ teens_no_one) + seventy_one = insert_space + pynini.cross("1", "et onze") + seventies = seventy + (pynini.cross("0", "-dix") | (seventy_unit | seventy_one)) # 70 to 79 + + eighty = pynini.string_file(get_abs_path("data/numbers/eighty.tsv")) + eighties = eighty + (pynini.cross("0", "s") | (pynutil.insert("-") + digits)) # 80 to 89 + + ninety = pynini.string_file(get_abs_path("data/numbers/ninety.tsv")) + nineties_unit = pynutil.insert("-") + ((pynutil.insert("1") + NEMO_DIGIT) @ teens) + nineties = ninety + (pynini.cross("0", "-dix") | nineties_unit) # 90 to 99 + + all_double_digits = ten | teens | ties_simple | seventies | eighties | nineties + self.all_double_digits = all_double_digits + + one_to_all_tens = digits | all_double_digits + self.one_to_all_tens = one_to_all_tens.optimize() + + # Hundreds + hundreds_parts = (pynutil.delete("0") + insert_space + digits) | (insert_space + all_double_digits) + one_hundreds = pynini.cross("1", "cent") + (pynutil.delete("00") | hundreds_parts) + multiple_hundreds = (digits_no_one + insert_space + pynutil.insert("cent")) + ( + pynini.cross("00", "s") | hundreds_parts + ) + all_hundreds = one_hundreds | multiple_hundreds + self.all_hundreds = all_hundreds + + # Thousands + delete_separator = pynini.closure(pynutil.delete(" "), 0, 1) + one_thousand = pynini.cross("1", "mille") + delete_separator + other_thousands = ( + (digits_no_one | all_double_digits | all_hundreds) + + insert_space + + pynutil.insert("mille") + + delete_separator + ) + all_thousands = ( + ((one_thousand | other_thousands) + pynutil.delete("000")) + | (one_thousand + pynutil.delete("00") + insert_space + (pynini.cross("1", "et un") | digits_no_one)) + | (other_thousands + pynutil.delete("00") + insert_space + digits) + | ((one_thousand | other_thousands) + pynutil.delete("0") + insert_space + all_double_digits) + | ((one_thousand | other_thousands) + insert_space + all_hundreds) + ) + + # Millions + digits_to_hundreds_no_one = digits_no_one | all_double_digits | all_hundreds + digits_to_thousands_no_one = digits_no_one | all_double_digits | all_hundreds | all_thousands + millions_components = pynini.closure(delete_separator + pynini.closure(NEMO_DIGIT, 3), 2) + delete_zeros = pynini.closure(pynutil.delete("0"), 0, 6) + all_millions = ( + one + insert_space + pynutil.insert("million") + | (digits_to_hundreds_no_one + insert_space + pynutil.insert("millions")) + ) + ( + millions_components + @ ( + delete_zeros + + pynini.closure(insert_space + (digits_to_thousands_no_one | pynini.cross("1", "et un")), 0, 1) + ) + ) + + # Billions + digits_to_millions_no_one = digits_to_thousands_no_one | all_millions + billions_components = pynini.closure(delete_separator + pynini.closure(NEMO_DIGIT, 3), 3) + delete_zeros = pynini.closure(pynutil.delete("0"), 0, 9) + all_billions = ( + one + insert_space + pynutil.insert("milliard") + | (digits_to_hundreds_no_one + insert_space + pynutil.insert("milliards")) + ) + ( + billions_components + @ ( + delete_zeros + + pynini.closure(insert_space + (digits_to_millions_no_one | pynini.cross("1", "et un")), 0, 1) + ) + ) + + # All Numbers Union + final_graph_masc = zero | one_to_all_tens | all_hundreds | all_thousands | all_millions | all_billions + + # Adding adjustment for fem gender (choice of gender will be random) + final_graph_fem = final_graph_masc | ( + final_graph_masc @ (pynini.closure(NEMO_SIGMA) + pynini.cross("un", "une")) + ) + final_graph = final_graph_fem | final_graph_masc + + self.all_nums_no_tokens = final_graph + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + final_graph + pynutil.insert("\"") + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/date.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..bee6249d65a84e04b2848f8624652c9db2629774 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/date.py @@ -0,0 +1,97 @@ +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.utils import get_abs_path + +# TODO: add articles? 'le...' + +month_numbers = pynini.string_file(get_abs_path("data/dates/months.tsv")) +eras = pynini.string_file(get_abs_path("data/dates/eras.tsv")) +delete_leading_zero = ( + pynutil.delete("0") | (NEMO_DIGIT - "0") +) + NEMO_DIGIT # reminder, NEMO_DIGIT = filter on digits + + +class DateFst(GraphFst): + '''Finite state transducer for classyfing dates, e.g.: + '02.03.2003' -> date {day: 'deux' month: 'mai' year: 'deux mille trois' preserve order: true} + ''' + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="dates", kind="classify") + + cardinal_graph = cardinal.all_nums_no_tokens + + # 'le' -> 'le', 'les' -> 'les' + le_determiner = pynini.accep("le ") | pynini.accep("les ") + self.optional_le = pynini.closure(le_determiner, 0, 1) + + # '01' -> 'un' + optional_leading_zero = delete_leading_zero | NEMO_DIGIT + valid_day_number = pynini.union(*[str(x) for x in range(1, 32)]) + premier = pynini.string_map([("1", "premier")]) + day_number_to_word = premier | cardinal_graph + + digit_to_day = self.optional_le + optional_leading_zero @ valid_day_number @ day_number_to_word + self.day_graph = pynutil.insert("day: \"") + digit_to_day + pynutil.insert("\"") + + # '03' -> 'mars' + normalize_month_number = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 13)]) + number_to_month = month_numbers.optimize() + month_graph = normalize_month_number @ number_to_month + self.month_graph = pynutil.insert("month: \"") + month_graph + pynutil.insert("\"") + + # 2025 -> deux mille vingt cinq + accept_year_digits = (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 1, 3) + digits_to_year = accept_year_digits @ cardinal_graph + self.year_graph = pynutil.insert("year: \"") + digits_to_year + pynutil.insert("\"") + + # Putting it all together + self.fst = pynini.accep("") + + for separator in ["/", ".", "-"]: + self.fst |= ( + pynutil.insert("date { ") + + self.day_graph + + pynutil.delete(separator) + + pynutil.insert(" ") + + self.month_graph + + pynini.closure(pynutil.delete(separator) + pynutil.insert(" ") + self.year_graph, 0, 1) + + pynutil.insert(" preserve_order: true }") + ) + + # Accepts "janvier", "février", etc + month_name_graph = pynutil.insert("month: \"") + month_numbers.project("output") + pynutil.insert("\"") + + self.fst |= ( + pynutil.insert("date { ") + + self.day_graph + + pynini.accep(" ") + + month_name_graph + + pynini.closure(pynini.accep(" ") + self.year_graph, 0, 1) + + pynutil.insert(" preserve_order: true}") + ) + + # Accepts "70s", "80s", etc + self.fst |= pynutil.insert("date { year: \"") + eras + pynutil.insert("\" preserve_order: true }") + + # Accepts date ranges, "17-18-19 juin" -> date { day: "17" day: "18": day: "19"} + for separator in ["-", "/"]: + day_range_graph = ( + pynutil.insert("day: \"") + + pynini.closure(digit_to_day + pynutil.delete(separator) + pynutil.insert(" "), 1) + + digit_to_day + + pynutil.insert("\"") + ) + + self.fst |= ( + pynutil.insert("date { ") + + day_range_graph + + pynini.accep(" ") + + month_name_graph + + pynini.closure(pynini.accep(" ") + self.year_graph, 0, 1) + + pynutil.insert(" preserve_order: true }") + ) + + self.fst = self.fst.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/decimals.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/decimals.py new file mode 100644 index 0000000000000000000000000000000000000000..cdec2a41b89eba6118d7f5099af2b28fccc59651 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/decimals.py @@ -0,0 +1,116 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.fr.utils import get_abs_path + +quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv")) +digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) +zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + + +def get_quantity(decimal_graph: "pynini.FstLike", cardinal_graph: "pynini.FstLike") -> "pynini.FstLike": + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, + e.g. 2 millions -> integer_part: "deux" quantity: "millions" + e.g. 2,4 millions -> integer_part: "deux" fractional_part: "quatre" quantity: "millions" + Args: + decimal_graph: DecimalFST + cardinal_graph: CardinalFST + """ + delete_separator = pynini.closure(pynutil.delete(" "), 0, 1) + numbers = pynini.closure(NEMO_DIGIT, 1, 6) @ cardinal_graph + numbers = pynini.cdrewrite(pynutil.delete(delete_separator), "", "", NEMO_SIGMA) @ numbers + + res = ( + pynutil.insert('integer_part: "') + + numbers + + pynutil.insert('"') + + NEMO_SPACE + + pynutil.insert('quantity: "') + + quantities + + pynutil.insert('"') + ) + res |= decimal_graph + NEMO_SPACE + pynutil.insert('quantity: "') + quantities + pynutil.insert('"') + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + -11,406 millions -> decimal { negative: "true" integer_part: "onze" fractional_part: "quatre cent six" quantity: "millions" preserve_order: true } + 114 billions -> decimal { integer_part: "cent quatorze" quantity: "billions" preserve_order: true } + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + graph_digit = digit | zero + + if not deterministic: + graph = pynini.union(graph_digit, cardinal.all_hundreds, cardinal.one_to_all_tens) + graph += pynini.closure(insert_space + graph) + + else: + # General pattern is 1-3 digits: map as cardinal, default to tens followed by digits otherwise \ + graph = pynini.union( + pynutil.add_weight(graph_digit + pynini.closure(insert_space + zero), -0.00001), + pynutil.add_weight(cardinal.all_double_digits + pynini.closure(insert_space + zero), -0.00002), + pynutil.add_weight(cardinal.all_hundreds + pynini.closure(insert_space + zero), 0.00001), + pynutil.add_weight( + cardinal.all_double_digits + + pynini.closure(insert_space + cardinal.all_double_digits, 1) + + pynini.closure(insert_space + zero, 0, 1) + + (pynini.closure(insert_space + graph_digit, 0, 1) | pynini.closure(insert_space + zero, 0)), + -0.00002, + ), # Read out as tens and a possible trailing digit or zeroes + zero + + pynini.closure(insert_space + zero) + + pynini.closure(insert_space + graph_digit), # For cases such as "1,001" + ) + + graph = graph.optimize() + + delete_separator = pynutil.delete(",") + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + + graph_fractional = pynutil.insert('fractional_part: "') + graph + pynutil.insert('"') + + integers = cardinal.all_nums_no_tokens + graph_integer = pynutil.insert('integer_part: "') + integers + pynutil.insert('"') + final_graph_wo_sign = graph_integer + delete_separator + insert_space + graph_fractional + + final_graph_wo_negative = final_graph_wo_sign | get_quantity(final_graph_wo_sign, integers).optimize() + self.final_graph_wo_negative = final_graph_wo_negative + + final_graph = optional_graph_negative + final_graph_wo_negative + self.final_graph = final_graph + + final_graph += pynutil.insert(" preserve_order: true") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/fraction.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..47c391bf3519840e059236d2a42a567e599bf3e3 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/fraction.py @@ -0,0 +1,70 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fraction + "-4 1/3" -> + tokens { fraction { negative: "true" integer_part: "quatre" numerator: "un" denominator: "trois" morphosyntactic_features: "ième" } } + Args: + cardinal: CardinalFst + ordinal: OrdinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction will be generated (used for audio-based normalization) - TBD + """ + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + + cardinals = cardinal.all_nums_no_tokens + sing_numerator = pynini.accep("1") @ cardinals + pl_numerators = (pynini.closure(NEMO_DIGIT) - "1") @ cardinals + + add_denom_suffix = pynini.closure(NEMO_DIGIT) + pynutil.insert("e") + denominators = add_denom_suffix @ ordinal.graph + change_denom_label = pynini.cross("integer", "denominator") + pluralize_denom = pynini.closure(NEMO_SIGMA) + pynini.cross("\"ième\"", "\"ièmes\"") + + sing_fraction_graph = ( + pynutil.insert("numerator: \"") + + sing_numerator + + pynutil.insert("\" ") + + pynutil.delete("/") + + (denominators @ (change_denom_label + pynini.closure(NEMO_SIGMA))) + ) + + pl_fraction_graph = ( + pynutil.insert("numerator: \"") + + pl_numerators + + pynutil.insert("\" ") + + pynutil.delete("/") + + (denominators @ (change_denom_label + pluralize_denom)) + ) + + integer_part = pynutil.insert("integer_part: \"") + cardinals + pynutil.insert("\"") + optional_integer_part = pynini.closure(integer_part + pynini.accep(" "), 0, 1) + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + final_graph = optional_minus_graph + optional_integer_part + (sing_fraction_graph | pl_fraction_graph) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/ordinal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..713c96732f6c9314ab991e8f15f973bf852fb984 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/ordinal.py @@ -0,0 +1,48 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.utils import get_abs_path + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal + "2e" -> ordinal { integer: "deux" morphosyntactic_features: "ième" } + This grammar covers from single digits to hundreds of billions ("milliardième" in French). + This FST also records the ending of the ordinal (called "morphosyntactic_features"). + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify") + + numbers = cardinal.all_nums_no_tokens + numbers_graph = pynutil.insert("integer: \"") + numbers + pynutil.insert("\"") + + suffixes = pynini.string_file(get_abs_path("data/ordinals/suffixes.tsv")) + suffixes_graph = pynutil.insert("morphosyntactic_features: \"") + suffixes + pynutil.insert("\"") + + final_graph = numbers_graph + pynutil.insert(NEMO_SPACE) + suffixes_graph + self.graph = final_graph + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/tokenize_and_classify.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/tokenize_and_classify.py new file mode 100644 index 0000000000000000000000000000000000000000..541ee6b43b2aedbf577838818ab1b3e3d113eebf --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/tokenize_and_classify.py @@ -0,0 +1,126 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.taggers.cardinal import CardinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.taggers.date import DateFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.taggers.decimals import DecimalFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.taggers.fraction import FractionFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.taggers.ordinal import OrdinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.taggers.whitelist import WhiteListFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.taggers.word import WordFst +from zonos3.vendor.nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. + For deployment, this grammar will be compiled and exported to OpenFst Finate State aRchive (FAR) File. + More details to deployment at NeMo-text-processing/tools/text_processing_deployment. + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = False, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"_{input_case}_fr_tn_{deterministic}_deterministic{whitelist_file}.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating ClassifyFst grammars. This might take some time...") + + self.cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = self.cardinal.fst + + self.ordinal = OrdinalFst(cardinal=self.cardinal, deterministic=deterministic) + ordinal_graph = self.ordinal.fst + + self.decimal = DecimalFst(cardinal=self.cardinal, deterministic=deterministic) + decimal_graph = self.decimal.fst + + self.fraction = FractionFst( + cardinal=self.cardinal, + ordinal=self.ordinal, + deterministic=deterministic, + ) + fraction_graph = self.fraction.fst + word_graph = WordFst(deterministic=deterministic).fst + self.whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) + whitelist_graph = self.whitelist.fst + punct_graph = PunctuationFst(deterministic=deterministic).fst + + self.date = DateFst(self.cardinal, deterministic=deterministic) + date_graph = self.date.fst + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(date_graph, 1.1) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(fraction_graph, 1.09) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(word_graph, 200) + ) + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") + punct = pynini.closure( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct), + 1, + ) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure((delete_extra_space).ques + token_plus_punct) + graph = delete_space + graph + delete_space + graph |= punct + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logger.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/whitelist.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..2c80e5c06be4ed592f8c68a44b2f885423e8570d --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/whitelist.py @@ -0,0 +1,63 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space +from zonos3.vendor.nemo_text_processing.text_normalization.fr.utils import get_abs_path, load_labels + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist, e.g. + "sr." -> tokens { name: "bonjour" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + input_file: path to a file with whitelist replacements + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(input_case, file): + whitelist = load_labels(file) + if input_case == "lower_cased": + whitelist = [[x[0].lower()] + x[1:] for x in whitelist] + graph = pynini.string_map(whitelist) + return graph + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv")) + if not deterministic and input_case != "lower_cased": + graph |= pynutil.add_weight( + _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist.tsv")), weight=0.0001 + ) + + if input_file: + whitelist_provided = _get_whitelist_graph(input_case, input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + if not deterministic: + units_graph = _get_whitelist_graph(input_case, file=get_abs_path("data/measures/measurements.tsv")) + graph |= units_graph + + self.graph = graph + self.final_graph = convert_space(self.graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.final_graph + pynutil.insert("\"")).optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/word.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..67e162dbec7f3e0df3ec49c5b9dff94259513b7c --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/taggers/word.py @@ -0,0 +1,33 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying word. + e.g. bonjour -> tokens { name: "bonjour" } + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/utils.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7523e576221aa4246c3579a56c8f3094b846f7d7 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/utils.py @@ -0,0 +1,43 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + label_tsv = open(abs_path) + labels = list(csv.reader(label_tsv, delimiter="\t")) + label_tsv.close() + return labels diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cf281f0908d225f51d921e9ddbdf92f9b14d99 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/cardinal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..ca85b56ebf55b4a3101f0c83bd7d41e610535f1d --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/cardinal.py @@ -0,0 +1,36 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, insert_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinals + e.g. cardinal { negative: "true" integer: "un milliard et un" } -> "moins un milliard et un" + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\" ", "moins") + insert_space, 0, 1) + number = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + final_graph = optional_sign + number + + self.fst = self.delete_tokens(final_graph).optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/date.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..a6bf4b27771cdd3de0297a8675bfccbc72f13055 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/date.py @@ -0,0 +1,51 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SPACE, + GraphFst, + delete_preserve_order, +) + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + date {day: "deux" month: "mars" year: "deux mille trois" preserve_order: true} -> deux mars deux mille trois + Args: + ordinal: OrdinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + day = pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + month = pynutil.delete("month: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + year = pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + decade = pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + graph_dmy = day + NEMO_SPACE + month + pynini.closure(NEMO_SPACE + year, 0, 1) + delete_preserve_order + graph_my = month + NEMO_SPACE + year + delete_preserve_order + graph_decade = decade + delete_preserve_order + + self.graph = graph_dmy | graph_my | graph_decade + + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/decimals.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/decimals.py new file mode 100644 index 0000000000000000000000000000000000000000..3c63101352af538dd62a47ce8ef0addb291c6a3f --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/decimals.py @@ -0,0 +1,68 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + decimal { negative: "true" integer_part: "onze" fractional_part: "quatre cent six" quantity: "millions" preserve_order: true } -> moins onze virgule quatre cent six millions + decimal { integer_part: "cent quatorze" quantity: "billions" preserve_order: true } -> cent quatorze billions + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "moins ") + delete_space, 0, 1) + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + fractional_default = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + conjunction = pynutil.insert(" virgule ") + fractional = conjunction + fractional_default + + quantity = ( + delete_space + + insert_space + + pynutil.delete("quantity: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_quantity = pynini.closure(quantity, 0, 1) + + graph = optional_sign + pynini.union( + (integer + quantity), (integer + delete_space + fractional + optional_quantity) + ) + + self.numbers_only_quantity = ( + optional_sign + + pynini.union((integer + quantity), (integer + delete_space + fractional + quantity)).optimize() + ) + + graph += delete_preserve_order + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/fraction.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/fraction.py new file mode 100644 index 0000000000000000000000000000000000000000..14d236cefc77099b5071f5fa95315a0a538af43e --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/fraction.py @@ -0,0 +1,58 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.fr.utils import get_abs_path + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fraction + e.g. tokens { fraction { integer: "treinta y tres" numerator: "cuatro" denominator: "quinto" } } -> + treinta y tres y cuatro quintos + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + + numerator = pynutil.delete("numerator: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + denominator = ordinal.graph + irregular_denominators = pynini.string_file(get_abs_path("data/fractions/irregular_denominators.tsv")) + denominator = pynini.union(pynutil.add_weight(denominator @ irregular_denominators, -0.01), denominator) + denominator = (pynini.cross("denominator:", "integer:") + pynini.closure(NEMO_SIGMA)) @ denominator + + irregular_half = pynini.closure(NEMO_SIGMA) + pynini.cross("et un demi", "et demi") + + optional_integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + optional_integer = pynini.closure( + optional_integer + pynini.accep(NEMO_SPACE) + pynutil.insert("et") + insert_space, 0, 1 + ) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\" ", "moins") + insert_space, 0, 1) + + final_graph = optional_sign + optional_integer + numerator + pynini.accep(NEMO_SPACE) + denominator + final_graph = pynini.union(pynutil.add_weight(final_graph @ irregular_half, -0.01), final_graph) + self.fst = self.delete_tokens(final_graph).optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/ordinal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/ordinal.py new file mode 100644 index 0000000000000000000000000000000000000000..1e970fc279969e1b86927009abf821844410eeee --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/ordinal.py @@ -0,0 +1,69 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.utils import get_abs_path + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinals + e.g. ordinal { integer: "deux" morphosyntactic_features: "ième" } -> "deuxième" + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True, strip_dashes: bool = False): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + ones = pynini.cross("un", "prem") + + irregular_numbers = pynini.string_file(get_abs_path("data/ordinals/irregular_numbers.tsv")) + irregular_numbers = pynini.closure(pynini.closure(NEMO_NOT_QUOTE, 1) + NEMO_SPACE) + irregular_numbers + exceptions = pynini.project(irregular_numbers, "input") + exception_suffix = ( + pynutil.delete(" morphosyntactic_features: \"ième") + + pynini.closure(pynini.accep("s"), 0, 1) + + pynutil.delete("\"") + ) + irregular_numbers_graph = ( + pynutil.delete("integer: \"") + irregular_numbers + pynutil.delete("\"") + exception_suffix + ) + + numbers = pynini.closure(NEMO_NOT_QUOTE, 1) + numbers = pynini.difference(numbers, exceptions) + + if strip_dashes: + remove_dashes = pynini.closure(NEMO_ALPHA, 1) + pynini.cross("-", " ") + pynini.closure(NEMO_ALPHA, 1) + remove_dashes = pynini.closure(remove_dashes, 0) + numbers = pynini.union(numbers, pynutil.add_weight(numbers @ remove_dashes, -0.0001)) + + regular_ordinals = pynini.union(numbers, pynutil.add_weight(ones, -0.0001)) + regular_ordinals_graph = ( + pynutil.delete("integer: \"") + + regular_ordinals + + pynutil.delete("\"") + + pynutil.delete(" morphosyntactic_features: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + final_graph = pynini.union(regular_ordinals_graph, irregular_numbers_graph) + self.graph = final_graph + + self.fst = self.delete_tokens(final_graph).optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/verbalize.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/verbalize.py new file mode 100644 index 0000000000000000000000000000000000000000..ba1629dd6bea85cb6ab2c172649de7c8f964e2cd --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/verbalize.py @@ -0,0 +1,48 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.verbalizers.cardinal import CardinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.verbalizers.date import DateFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.verbalizers.decimals import DecimalFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.verbalizers.fraction import FractionFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.verbalizers.ordinal import OrdinalFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo-text-processing/tools/text_processing_deployment. + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + ordinal = OrdinalFst(deterministic=deterministic) + ordinal_graph = ordinal.fst + decimal = DecimalFst(deterministic=deterministic) + decimal_graph = decimal.fst + fraction = FractionFst(ordinal=ordinal, deterministic=deterministic) + fraction_graph = fraction.fst + whitelist_graph = WhiteListFst(deterministic=deterministic).fst + date = DateFst(deterministic=deterministic) + date_graph = date.fst + + graph = cardinal_graph | decimal_graph | ordinal_graph | fraction_graph | whitelist_graph | date_graph + self.fst = graph diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/verbalize_final.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/verbalize_final.py new file mode 100644 index 0000000000000000000000000000000000000000..7c638c700508f60d7fedb713e33301305e4d3b6a --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/fr/verbalizers/verbalize_final.py @@ -0,0 +1,70 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.word import WordFst +from zonos3.vendor.nemo_text_processing.text_normalization.fr.verbalizers.verbalize import VerbalizeFst +from zonos3.vendor.nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"fr_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) + logger.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2a9b2773c588d04f7eec36d300aa5816ccb1640e --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from zonos3.vendor.nemo_text_processing.text_normalization.it.taggers.tokenize_and_classify import ClassifyFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.verbalizers.verbalize import VerbalizeFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/electronic/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/electronic/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/electronic/domain.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/electronic/domain.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c5c26d736702cae7cdddbbf7a9d5e5704707e2ce --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/electronic/domain.tsv @@ -0,0 +1,14 @@ +.com punto com +.org punto org +.gov punto gov +.uk punto UK +.fr punto FR +.net punto net +.html punto html +.py punto python +.ru punto RU +.us punto US +.de punto DE +.it punto IT +.jpg punto jpeg +.edu punto edu \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/electronic/server_name.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/electronic/server_name.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9a0c7562948684993372a2c389c9f32297550e5f --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/electronic/server_name.tsv @@ -0,0 +1,11 @@ +gmail +nvidia +outlook +hotmail +yahoo +live +yandex +orange +wanadoo +web +comcast \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/electronic/symbols.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/electronic/symbols.tsv new file mode 100644 index 0000000000000000000000000000000000000000..56c9270c93111a58019ffa5f1141d30623c1fbd1 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/electronic/symbols.tsv @@ -0,0 +1,22 @@ +. punto +- trattino +_ trattino basso +! punto esclamativo +# cancelletto +$ dollaro +% percento +& e commerciale +' apostrofo +* asterisco ++ più +/ slash += uguale +? punto interrogativo +^ esponente +` accento +{ apri graffa +| barra verticale +} chiudi graffa +~ tilde +, virgola +: due punti diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/measure/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/measure/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/measure/measurements.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/measure/measurements.tsv new file mode 100644 index 0000000000000000000000000000000000000000..bb359e6a0bd8d46a68708aed1eb0ff782c089ec9 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/measure/measurements.tsv @@ -0,0 +1,64 @@ +d giorno +f fahrenheit +c celsius +°C grado celsius +°F grado fahrenheit +K kelvin +km chilometro +m metro +cm centimetro +mm millimetro +μm micrometro +nm nanometro +dm decimetro +pm picometro +hm ettometro +ha ettaro +mi miglio +m² metro quadrato -0.0001 +km² chilometro quadrato -0.0001 +m2 metro quadrato +km2 chilometro quadrato +m3 metro cubo +ft foot +g grammo +µg microgrammo +mg milligrammo +kg chilogrammo +lb libbra +h ora +s secondo +min minuto +ms millisecondo +μs microsecondo +hz hertz +kw kilowatt +ghz gigahertz +khz kilohertz +mhz megahertz +v volt +mc megacoulomb +mA milliampere +A ampere +tw terawatt +mv millivolt +mw megawatt +gw gigawatt +ω ohm +db decibel +gb gigabyte +kb kilobit +pb petabit +mb megabyte +kb kilobyte +tb terabyte +kv kilovolt +mv megavolt +kn kilonewton +ml millilitro +l litro +dl decilitro +bar bar +kcal chilocaloria +cal caloria +% percento \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/measure/suppletive.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/measure/suppletive.tsv new file mode 100644 index 0000000000000000000000000000000000000000..31d2e1975e93839929cfa9bf09902e88ae5286c2 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/measure/suppletive.tsv @@ -0,0 +1,13 @@ +grado celsius gradi celsius +metro metri +ora ore +grammo grammi +secondo secondi +minuto minuti +chilometro chilometri +bit bits +byte bytes +caloria calorie +miglio miglia +litro litri +giorno giorni \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/money/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/money/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/money/currency_major.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/money/currency_major.tsv new file mode 100644 index 0000000000000000000000000000000000000000..93a991e4344abe01d81b8c79c7a6397f2f4b2e15 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/money/currency_major.tsv @@ -0,0 +1,11 @@ +€ euro +$ dollaro +£ sterlina +usd dollaro statunitense +eur euro +cad dollaro canadese +chf franco svizzero +hkd dollaro di hong kong +¥ yen +rs rupia +₽ rublo diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/money/currency_minor.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/money/currency_minor.tsv new file mode 100644 index 0000000000000000000000000000000000000000..59c91fff9c2768071367089d643d2ef8597d0557 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/money/currency_minor.tsv @@ -0,0 +1,11 @@ +€ centesimo +$ centesimo +£ penny +usd centesimo +eur centesimo +cad centesimo +chf centesimo +hkd centesimo +¥ yen +rs paisa +₽ copeca diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/money/currency_plural_fem.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/money/currency_plural_fem.tsv new file mode 100644 index 0000000000000000000000000000000000000000..542c49d1dce1a405c630f71cbe45cfa6e0df8721 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/money/currency_plural_fem.tsv @@ -0,0 +1,2 @@ +sterlina sterline +rupia rupie diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/money/currency_plural_masc.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/money/currency_plural_masc.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b3905f693cf01d8949afa4e304f5cd5e36ae08b2 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/money/currency_plural_masc.tsv @@ -0,0 +1,10 @@ +centesimo centesimi +euro euro +dollaro dollari +penny penny +dollaro canadese dollari canadesi +franco svizzero franchi svizzeri +dollaro statunitense dollari statunitensi +rublo rubli +dollaro di hong hong dollari di hong kong +copeca copeche diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/digit.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..4f2bf34e405925de3c2c4d2c29e95068c49a03bf --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +uno 1 +due 2 +tre 3 +quattro 4 +cinque 5 +sei 6 +sette 7 +otto 8 +nove 9 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/hundreds.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/hundreds.tsv new file mode 100644 index 0000000000000000000000000000000000000000..20f4aa2d16907a70b8dad16a8111f27251d1bf12 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/hundreds.tsv @@ -0,0 +1,8 @@ +duecento 2 +trecento 3 +quattrocento 4 +cinquecento 5 +seicento 6 +settecento 7 +ottocento 8 +novecento 9 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/quantities.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/quantities.tsv new file mode 100644 index 0000000000000000000000000000000000000000..fac386746f3e73498eab9015b0997669f58f2425 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/quantities.tsv @@ -0,0 +1,2 @@ +milioni +miliardi \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/teen.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/teen.tsv new file mode 100644 index 0000000000000000000000000000000000000000..5c256ec6edb33d1002afb2c4796a06b9fb78aeac --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +dieci 10 +undici 11 +dodici 12 +tredici 13 +quattordici 14 +quindici 15 +sedici 16 +diciassette 17 +diciotto 18 +diciannove 19 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens.tsv new file mode 100644 index 0000000000000000000000000000000000000000..5637114c26793f0b65549322038bdf63c2748884 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens.tsv @@ -0,0 +1,8 @@ +venti 2 +trenta 3 +quaranta 4 +cinquanta 5 +sessanta 6 +settanta 7 +ottanta 8 +novanta 9 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens_eight.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens_eight.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1cd57016368caeab4f9453ce692c904d9c51747f --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens_eight.tsv @@ -0,0 +1,8 @@ +ventotto 28 +trentotto 38 +quarantotto 48 +cinquantotto 58 +sessantotto 68 +settantotto 78 +ottantotto 88 +novantotto 98 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens_one.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens_one.tsv new file mode 100644 index 0000000000000000000000000000000000000000..3ffbcec03d076193a169bf8539b1ae9464185840 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/tens_one.tsv @@ -0,0 +1,8 @@ +ventuno 21 +trentuno 31 +quarantuno 41 +cinquantuno 51 +sessantuno 61 +settantuno 71 +ottantuno 81 +novantuno 91 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/zero.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/zero.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c479272d40391d642dfa768cfa16be064e42ae5c --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/numbers/zero.tsv @@ -0,0 +1 @@ +zero 0 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/whitelist/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/whitelist/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/whitelist/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/whitelist/whitelist.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/whitelist/whitelist.tsv new file mode 100644 index 0000000000000000000000000000000000000000..15d65d81d2c081612e319f9a370fedf4bbe4bff5 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/data/whitelist/whitelist.tsv @@ -0,0 +1,65 @@ +dr dottor +dr. dottor +Dr. dottor +dott. dottor +Dott. dottor +dott.sa dottoressa +Dott.sa dottoressa +prof. professore +Prof. professore +proff. professori +Proff. professori +prof.ssa professoressa +Prof.ssa professoressa +Sig. signore +Sig.na signorina +sig. signore +sig.na signorina +sig signore +Sigg. signori +sigg. signori +Sig.ri signori +sig.ri signori +Sig.re signore +sig.re signore +Sig.ra signora +sig.ra signora +Sig.re signorine +sig.re signorine +Avv. avvocato +avv. avvocato +Arch. architetto +arch. architetto +Rag. ragioniere +rag. ragioniere +Geom. geometra +geom. geometra +Ing. ingegnere +ing. ingegnere +Ingg. ingegneri +ingg. ingegneri +Fam. famiglia +fam. famiglia +f.lli fratelli +F.lli fratelli +Rev. reverendo +On. onorevole +on. onorevole +Onn. onorevoli +onn. onorevoli +Min. ministro +Sen. senatore +c.v.d. come volevasi dimostrare +ecc. eccetera +ecc eccetera +v.le viale +p.za piazza +l.go largo +loc. località +ist. istituto +segr. segreteria +set. settore +uff. ufficio +dip. dipartimento +div. divisione ++ più diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/cardinal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..873afb703fb62f674555fa0af9a238a4d127a650 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/cardinal.py @@ -0,0 +1,183 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import cardinal_separator +from zonos3.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path + +zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) +digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) +teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv"))) +tens = pynini.invert(pynini.string_file(get_abs_path("data/numbers/tens.tsv"))) +tens_one = pynini.invert(pynini.string_file(get_abs_path("data/numbers/tens_one.tsv"))) +tens_eight = pynini.invert(pynini.string_file(get_abs_path("data/numbers/tens_eight.tsv"))) +hundreds = pynini.invert(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv"))) + + +def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Helper function for parsing number strings. Converts common cardinal strings (groups of three digits delineated by 'cardinal_separator' - see graph_utils) + and converts to a string of digits: + "1 000" -> "1000" + "1.000.000" -> "1000000" + Args: + fst: Any pynini.FstLike object. Function composes fst onto string parser fst + Returns: + fst: A pynini.FstLike object + """ + exactly_three_digits = NEMO_DIGIT**3 # for blocks of three + up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) # for start of string + + cardinal_string = pynini.closure( + NEMO_DIGIT, 1 + ) # For string w/o punctuation (used for page numbers, thousand series) + + cardinal_string |= ( + up_to_three_digits + + pynutil.delete(cardinal_separator) + + pynini.closure(exactly_three_digits + pynutil.delete(cardinal_separator)) + + exactly_three_digits + ) + + return cardinal_string @ fst + + +class CardinalFst(GraphFst): + ''' + Finite state transducer for classifying cardinals in Italian, e.g. + "1000" -> cardinal { integer: "mille" } + "2.000.000" -> cardinal { integer: "due milioni" } + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + ''' + + def __init__(self, deterministic: bool = True): + super().__init__(name='cardinal', kind='classify', deterministic=deterministic) + + # single digit + graph_digit = digit + digits_no_one = (NEMO_DIGIT - "1") @ graph_digit + + # double digit + graph_tens = teen + graph_tens |= tens + (pynutil.delete('0') | graph_digit) + graph_tens |= pynutil.add_weight(tens_one, -0.01) + graph_tens |= pynutil.add_weight(tens_eight, -0.01) + + self.tens = graph_tens.optimize() + + self.two_digit_no_zero = pynini.union( + graph_digit, graph_tens, (pynini.cross('0', NEMO_SPACE) + graph_digit) + ).optimize() + + # three digit + graph_hundreds = hundreds + pynini.union( + pynutil.delete('00'), insert_space + graph_tens, (pynini.cross("0", NEMO_SPACE) + graph_digit) + ) + graph_hundreds |= ( + pynini.cross('1', 'cento') + + insert_space + + pynini.union(graph_tens, pynutil.delete("0") + graph_digit, pynutil.delete("00")) + ) + + self.hundreds = graph_hundreds.optimize() + + # three digit leading zeros + graph_hundreds_component = pynini.union(graph_hundreds, pynutil.delete("0") + graph_tens) + + graph_hundreds_component_at_least_one_none_zero_digit = graph_hundreds_component | ( + pynutil.delete("00") + graph_digit + ) + + graph_hundreds_component_at_least_one_none_zero_digit_no_one = graph_hundreds_component | ( + pynutil.delete("00") + digits_no_one + ) + + # thousands + graph_thousands_component_at_least_one_none_zero_digit = pynini.union( + pynutil.delete("000") + graph_hundreds_component_at_least_one_none_zero_digit, + graph_hundreds_component_at_least_one_none_zero_digit_no_one + + pynutil.insert("mila") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + pynini.cross("001", "mille") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + ) + + graph_thousands_component_at_least_one_none_zero_digit_no_one = pynini.union( + pynutil.delete("000") + graph_hundreds_component_at_least_one_none_zero_digit_no_one, + graph_hundreds_component_at_least_one_none_zero_digit_no_one + + pynutil.insert("mila") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + pynini.cross("001", "mille") + + ((insert_space + graph_hundreds_component_at_least_one_none_zero_digit) | pynutil.delete("000")), + ) + + # higher + graph_million = pynutil.add_weight(pynini.cross("000001", "un milione"), -0.001) + graph_million |= graph_thousands_component_at_least_one_none_zero_digit_no_one + pynutil.insert(" milioni") + graph_million |= pynutil.delete("000000") + graph_million += insert_space + + graph_billion = pynutil.add_weight(pynini.cross("000001", "un miliardo"), -0.001) + graph_billion |= graph_thousands_component_at_least_one_none_zero_digit_no_one + pynutil.insert(" miliardi") + graph_billion |= pynutil.delete("000000") + graph_billion += insert_space + + graph_trillion = pynutil.add_weight(pynini.cross("000001", "un trilione"), -0.001) + graph_trillion |= graph_thousands_component_at_least_one_none_zero_digit_no_one + pynutil.insert(" trilioni") + graph_trillion |= pynutil.delete("000000") + graph_trillion += insert_space + + graph = ( + graph_trillion + + graph_billion + + graph_million + + (graph_thousands_component_at_least_one_none_zero_digit | pynutil.delete("000000")) + ) + + self.graph = ( + ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT**24 + @ graph + @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) + @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) + @ pynini.cdrewrite( + pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 2), NEMO_SPACE), NEMO_ALPHA, NEMO_ALPHA, NEMO_SIGMA + ) + ) + + self.graph |= zero + + self.graph = filter_punctuation(self.graph).optimize() + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/decimals.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/decimals.py new file mode 100644 index 0000000000000000000000000000000000000000..ec45d3820ecb01a5965c39d5522ced118f402f8e --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/decimals.py @@ -0,0 +1,121 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import ( + cardinal_separator, + decimal_separator, + strip_cardinal_apocope, +) +from zonos3.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path + +quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv")) +digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) +zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) + + +def get_quantity(decimal_graph: "pynini.FstLike", cardinal_graph: "pynini.FstLike") -> "pynini.FstLike": + """ """ + numbers = pynini.closure(NEMO_DIGIT, 1, 6) @ cardinal_graph + numbers = pynini.cdrewrite(pynutil.delete(cardinal_separator), "", "", NEMO_SIGMA) @ numbers + + res = ( + pynutil.insert('integer_part: "') + + numbers + + pynutil.insert('"') + + NEMO_SPACE + + pynutil.insert('quantity: "') + + quantities + + pynutil.insert('"') + ) + res |= decimal_graph + NEMO_SPACE + pynutil.insert('quantity: "') + quantities + pynutil.insert('"') + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + -2 milioni: decimal { negative: "true" integer_part: "due" quantity: "milioni" preserve_order: true } --> + meno due milioni + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + graph_digit = digit | zero + + if not deterministic: + graph = pynini.union(graph_digit, cardinal.hundreds, cardinal.tens) + graph += pynini.closure(insert_space + graph) + + else: + + graph = pynini.union( + pynutil.add_weight(graph_digit + pynini.closure(insert_space + zero), -0.00001), + pynutil.add_weight(cardinal.tens + pynini.closure(insert_space + zero), -0.00002), + pynutil.add_weight(cardinal.hundreds + pynini.closure(insert_space + zero), 0.00001), + pynutil.add_weight( + cardinal.tens + + pynini.closure(insert_space + cardinal.tens, 1) + + pynini.closure(insert_space + zero, 0, 1) + + (pynini.closure(insert_space + graph_digit, 0, 1) | pynini.closure(insert_space + zero, 0)), + -0.00002, + ), # Read out as tens and a possible trailing digit or zeroes + zero + + pynini.closure(insert_space + zero) + + pynini.closure(insert_space + graph_digit), # For cases such as "1,010" + ) + + # Technically decimals should be space delineated groups of three, e.g. (1,333 333). This removes any possible spaces + strip_formatting = pynini.cdrewrite(delete_space, "", "", NEMO_SIGMA) + graph = strip_formatting @ graph + + self.graph = graph.optimize() + + graph_separator = pynutil.delete(decimal_separator) + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + + self.graph_fractional = pynutil.insert('fractional_part: "') + self.graph + pynutil.insert('"') + + graph_integer = ( + strip_cardinal_apocope(cardinal.graph) + if deterministic + else pynini.union(cardinal.graph, strip_cardinal_apocope(cardinal.graph)) + ) + + self.graph_integer = pynutil.insert('integer_part: "') + graph_integer + pynutil.insert('"') + final_graph_wo_sign = self.graph_integer + graph_separator + insert_space + self.graph_fractional + + self.final_graph_wo_negative = ( + final_graph_wo_sign | get_quantity(final_graph_wo_sign, cardinal.graph).optimize() + ) + final_graph = optional_graph_negative + self.final_graph_wo_negative + + final_graph += pynutil.insert(" preserve_order: true") + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/electronic.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..48bfdc1486d0d7c58ece3e109f86891a4173d353 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/electronic.py @@ -0,0 +1,107 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SPACE, + GraphFst, + at, + colon, + domain_string, + double_quotes, + double_slash, + http, + https, + period, + protocol_string, + username_string, + www, +) +from zonos3.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path, load_labels + +# common_domains = [x[0] for x in load_labels(get_abs_path("data/electronic/domain.tsv"))] + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying electronic: email addresses + e.g. "abc.def@studente.università.it" -> electronic { username: "abc.def" domain: "studente.università.it" preserve_order: true } + e.g. "www.abc.com/123" -> electronic { protocol: "www." domain: "abc.com/123" preserve_order: true } + e.g. "https://github.com/NVIDIA/electronic.py" -> electronic { protocol: "https://" domain: "github.com/NVIDIA/electronic.py" preserve_order: true } + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + dot = pynini.accep(".") + + symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] + symbols = pynini.union(*symbols) + symbols_no_period = pynini.difference(symbols, dot) + accepted_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols_no_period), 1) + all_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols), 1) + + # domains + domain = dot + accepted_characters + domain_graph = ( + pynutil.insert(domain_string + colon + NEMO_SPACE + double_quotes) + + (accepted_characters + pynini.closure(domain, 1)) + + pynutil.insert(double_quotes) + ) + + # email + username = ( + pynutil.insert(username_string + colon + NEMO_SPACE + double_quotes) + + all_characters + + pynutil.insert(double_quotes) + + pynini.cross(at, NEMO_SPACE) + ) + email = username + domain_graph + + # social media tags + tag = ( + pynutil.delete(at) + + pynutil.insert(username_string + colon + NEMO_SPACE + double_quotes) + + (accepted_characters | (accepted_characters + pynini.closure(domain, 1))) + + pynutil.insert(double_quotes) + ) + + # url + protocol_start = pynini.accep(https + colon + double_slash) | pynini.accep(http + colon + double_slash) + protocol_end = ( + pynini.accep(www + period) + if deterministic + else pynini.accep(www + period) | pynini.cross((www + period), "vu vu vu.") + ) + protocol = protocol_start | protocol_end | (protocol_start + protocol_end) + protocol = ( + pynutil.insert(protocol_string + colon + NEMO_SPACE + double_quotes) + + protocol + + pynutil.insert(double_quotes) + ) + + url = protocol + pynutil.insert(NEMO_SPACE) + (domain_graph) + + graph = url | domain_graph | email | tag + self.graph = graph + + final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true")) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/measure.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..07b96ce39d4ece0e1f9a52ff17b51428285cd7d1 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/measure.py @@ -0,0 +1,173 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NON_BREAKING_SPACE, + NEMO_SIGMA, + GraphFst, + convert_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path + +unit_singular = pynini.string_file(get_abs_path("data/measure/measurements.tsv")) +suppletive = pynini.string_file(get_abs_path("data/measure/suppletive.tsv")) + + +def singular_to_plural(): + _o = NEMO_SIGMA + pynini.cross("o", "") + pynutil.insert("i") + _a = NEMO_SIGMA + pynini.cross("a", "") + pynutil.insert("e") + _e = NEMO_SIGMA + pynini.cross("e", "") + pynutil.insert("i") + + graph_plural = plurals._priority_union(suppletive, pynini.union(_o, _a, _e), NEMO_SIGMA).optimize() + + return graph_plural + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure, e.g. + "2,4 g" -> measure { cardinal { integer_part: "due" fractional_part: "quattro" units: "grammi" preserve_order: true } } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.graph + + graph_unit_singular = convert_space(unit_singular) + graph_unit_plural = convert_space(suppletive) + graph_unit_plural |= graph_unit_singular @ pynini.cdrewrite(convert_space(suppletive), "", "[EOS]", NEMO_SIGMA) + optional_graph_negative = pynini.closure("-", 0, 1) + + graph_unit_denominator = ( + pynini.cross("/", "per") + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_singular + ) + + optional_unit_denominator = pynini.closure( + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, + 0, + 1, + ) + + unit_plural = ( + pynutil.insert("units: \"") + + (graph_unit_plural + (optional_unit_denominator) | graph_unit_denominator) + + pynutil.insert("\"") + ) + + unit_singular_graph = ( + pynutil.insert("units: \"") + + ((graph_unit_singular + optional_unit_denominator) | graph_unit_denominator) + + pynutil.insert("\"") + ) + + subgraph_decimal = decimal.fst + insert_space + pynini.closure(pynutil.delete(" "), 0, 1) + unit_plural + + subgraph_cardinal = ( + (optional_graph_negative + (pynini.closure(NEMO_DIGIT) - "1")) @ cardinal.fst + + insert_space + + pynini.closure(pynutil.delete(" "), 0, 1) + + unit_plural + ) + + subgraph_cardinal |= ( + (optional_graph_negative + pynini.accep("1")) + @ cardinal.fst + @ pynini.cdrewrite(pynini.cross("uno", "un"), "", "", NEMO_SIGMA) + + insert_space + + pynini.closure(pynutil.delete(" "), 0, 1) + + unit_singular_graph + ) + + cardinal_dash_alpha = ( + pynutil.insert("cardinal { integer: \"") + + cardinal_graph + + pynutil.delete('-') + + pynutil.insert("\" } units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert("\"") + ) + + alpha_dash_cardinal = ( + pynutil.insert("units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.delete('-') + + pynutil.insert("\"") + + pynutil.insert(" cardinal { integer: \"") + + cardinal_graph + + pynutil.insert("\" }") + ) + + decimal_dash_alpha = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynutil.delete('-') + + pynutil.insert("\" } units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert("\"") + ) + + decimal_times = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert("\" } units: \"") + + pynini.union('x', 'X') + + pynutil.insert("\"") + ) + + cardinal_times = ( + pynutil.insert("cardinal { integer: \"") + + cardinal_graph + + pynutil.insert("\" } units: \"") + + pynini.union('x', 'X') + + pynutil.insert("\"") + ) + + alpha_dash_decimal = ( + pynutil.insert("units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.delete('-') + + pynutil.insert("\"") + + pynutil.insert(" decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(" }") + ) + + final_graph = ( + subgraph_decimal + | subgraph_cardinal + | cardinal_dash_alpha + | alpha_dash_cardinal + | decimal_dash_alpha + | decimal_times + | alpha_dash_decimal + | cardinal_times + ) + # final_graph += pynutil.insert(" preserve_order: true") + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/money.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..d04b60458bcc0884253f8e7d1b7662b32a3e2815 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/money.py @@ -0,0 +1,176 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import decimal_separator +from zonos3.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path, load_labels + +maj_singular_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) +maj_singular = pynini.string_file((get_abs_path("data/money/currency_major.tsv"))) +min_singular = pynini.string_file(get_abs_path("data/money/currency_minor.tsv")) +fem_plural = pynini.string_file((get_abs_path("data/money/currency_plural_fem.tsv"))) +masc_plural = pynini.string_file((get_abs_path("data/money/currency_plural_masc.tsv"))) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money, e.g. + "€1" -> money { currency_maj: "euro" integer_part: "un"} + "€1,000" -> money { currency_maj: "euro" integer_part: "un" } + "4,2 £" -> money { integer_part: "quattro" currency_maj: "sterline" fractional_part: "venti" currency_min: "penny" preserve_order: true } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + graph_decimal_final = decimal.final_graph_wo_negative + + maj_singular_graph = maj_singular + min_singular_graph = min_singular + maj_plural_graph = maj_singular @ (fem_plural | masc_plural) + min_plural_graph = min_singular @ (fem_plural | masc_plural) + + graph_maj_singular = pynutil.insert("currency_maj: \"") + maj_singular_graph + pynutil.insert("\"") + graph_maj_plural = pynutil.insert("currency_maj: \"") + maj_plural_graph + pynutil.insert("\"") + + graph_integer_one = pynutil.insert("integer_part: \"") + pynini.cross("1", "un") + pynutil.insert("\"") + + decimal_with_quantity = (NEMO_SIGMA + NEMO_ALPHA) @ graph_decimal_final + + graph_decimal_plural = pynini.union( + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + graph_decimal_final, # $1,05 + graph_decimal_final + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_plural, # 1,05 $ + ) + graph_decimal_plural = ((NEMO_SIGMA - "1") + decimal_separator + NEMO_SIGMA) @ graph_decimal_plural + + graph_decimal_singular = pynini.union( + graph_maj_singular + pynini.closure(delete_space, 0, 1) + insert_space + graph_decimal_final, # $1,05 + graph_decimal_final + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_singular, # 1,05 $ + ) + graph_decimal_singular = (pynini.accep("1") + decimal_separator + NEMO_SIGMA) @ graph_decimal_singular + + graph_decimal = pynini.union( + graph_decimal_singular, + graph_decimal_plural, + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + decimal_with_quantity, + ) + + graph_integer = ( + pynutil.insert("integer_part: \"") + ((NEMO_SIGMA - "1") @ cardinal_graph) + pynutil.insert("\"") + ) + + graph_integer_only = pynini.union( + graph_maj_singular + pynini.closure(delete_space, 0, 1) + insert_space + graph_integer_one, + graph_integer_one + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_singular, + ) + graph_integer_only |= pynini.union( + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + graph_integer, + graph_integer + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_plural, + ) + + graph = graph_integer_only | graph_decimal + + # remove trailing zeros of non zero number in the first 2 digits and fill up to 2 digits + # e.g. 2000 -> 20, 0200->02, 01 -> 01, 10 -> 10 + # not accepted: 002, 00, 0, + two_digits_fractional_part = ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(pynutil.delete("0")) + ) @ ( + (pynutil.delete("0") + (NEMO_DIGIT - "0")) + | ((NEMO_DIGIT - "0") + pynutil.insert("0")) + | ((NEMO_DIGIT - "0") + NEMO_DIGIT) + ) + + graph_min_singular = pynutil.insert("currency_min: \"") + min_singular_graph + pynutil.insert("\"") + graph_min_plural = pynutil.insert("currency_min: \"") + min_plural_graph + pynutil.insert("\"") + + # format ** euro ** centesimo + decimal_graph_with_minor = None + for curr_symbol, _ in maj_singular_labels: + preserve_order = pynutil.insert(" preserve_order: true") + + integer_plus_maj = pynini.union( + graph_integer + insert_space + pynutil.insert(curr_symbol) @ graph_maj_plural, + graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular, + ) + # non zero integer part + integer_plus_maj = (pynini.closure(NEMO_DIGIT) - "0") @ integer_plus_maj + + graph_fractional_one = ( + pynutil.insert("fractional_part: \"") + + two_digits_fractional_part @ pynini.cross("1", "un") + + pynutil.insert("\"") + ) + + graph_fractional = ( + two_digits_fractional_part @ (pynini.closure(NEMO_DIGIT, 1, 2) - "1") @ cardinal.two_digit_no_zero + ) + graph_fractional = pynutil.insert("fractional_part: \"") + graph_fractional + pynutil.insert("\"") + + fractional_plus_min = pynini.union( + graph_fractional + insert_space + pynutil.insert(curr_symbol) @ graph_min_plural, + graph_fractional_one + insert_space + pynutil.insert(curr_symbol) @ graph_min_singular, + ) + + decimal_graph_with_minor_curr = ( + integer_plus_maj + pynini.cross(decimal_separator, NEMO_SPACE) + fractional_plus_min + ) + decimal_graph_with_minor_curr |= pynutil.add_weight( + integer_plus_maj + + pynini.cross(decimal_separator, NEMO_SPACE) + + pynutil.insert("fractional_part: \"") + + two_digits_fractional_part @ cardinal.two_digit_no_zero + + pynutil.insert("\""), + weight=0.0001, + ) + + decimal_graph_with_minor_curr |= pynutil.delete("0,") + fractional_plus_min + decimal_graph_with_minor_curr = pynini.union( + pynutil.delete(curr_symbol) + + pynini.closure(delete_space, 0, 1) + + decimal_graph_with_minor_curr + + preserve_order, + decimal_graph_with_minor_curr + + preserve_order + + pynini.closure(delete_space, 0, 1) + + pynutil.delete(curr_symbol), + ) + + decimal_graph_with_minor = ( + decimal_graph_with_minor_curr + if decimal_graph_with_minor is None + else pynini.union(decimal_graph_with_minor, decimal_graph_with_minor_curr) + ) + + final_graph = graph | pynutil.add_weight(decimal_graph_with_minor, -0.001) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/time.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..46d782d6ef1b6b162c9da5c0bbd850a74ec7d286 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/time.py @@ -0,0 +1,80 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, insert_space + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + 15:30:30 tokens { time { hours: "15" minutes: "30" seconds: "30" preserve_order: true } } -> quindici e mezza trenta secondi + 12:15 tokens { time { hours: "12" minutes: "15" } } -> dodici e un quarto + 03:38 tokens { time { hours: "3" minutes: "38" } } -> tre e trentotto minuti + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + + labels_hour = [str(x) for x in range(0, 25)] + labels_minute_single = [str(x) for x in range(1, 10)] + labels_minute_double = [str(x) for x in range(10, 60)] + + delete_leading_zero_to_double_digit = (pynutil.delete("0") | (NEMO_DIGIT - "0")) + NEMO_DIGIT + + graph_hour = pynini.union(*labels_hour) + + graph_minute_single = pynini.union(*labels_minute_single) + graph_minute_double = pynini.union(*labels_minute_double) + + final_graph_hour_only = pynutil.insert("hours: \"") + graph_hour + pynutil.insert("\"") + final_graph_hour = ( + pynutil.insert("hours: \"") + delete_leading_zero_to_double_digit @ graph_hour + pynutil.insert("\"") + ) + final_graph_minute = ( + pynutil.insert("minutes: \"") + + (pynutil.delete("0") + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + final_graph_second = ( + pynutil.insert("seconds: \"") + + (pynutil.delete("0") + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + + graph_hm = ( + final_graph_hour + pynutil.delete(":") + (pynutil.delete("00") | (insert_space + final_graph_minute)) + ) + + graph_hms = ( + final_graph_hour + + pynutil.delete(":") + + (pynini.cross("00", " minutes: \"0\"") | (insert_space + final_graph_minute)) + + pynutil.delete(":") + + (pynini.cross("00", " seconds: \"0\"") | (insert_space + final_graph_second)) + + pynutil.insert(" preserve_order: true") + ) + + graph_h = final_graph_hour_only + + final_graph = (graph_hm | graph_h | graph_hms).optimize() + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/tokenize_and_classify.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/tokenize_and_classify.py new file mode 100644 index 0000000000000000000000000000000000000000..a2fe6340ce807a35f70b525edfbd2e4224b74d62 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/tokenize_and_classify.py @@ -0,0 +1,138 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.taggers.cardinal import CardinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.taggers.decimals import DecimalFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.taggers.electronic import ElectronicFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.taggers.measure import MeasureFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.taggers.money import MoneyFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.taggers.time import TimeFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.taggers.whitelist import WhiteListFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.taggers.word import WordFst +from zonos3.vendor.nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. + For deployment, this grammar will be compiled and exported to OpenFst Finite State aRchive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = False, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"_{input_case}_it_tn_{deterministic}_deterministic{whitelist_file}.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating ClassifyFst grammars. This might take some time...") + + self.cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = self.cardinal.fst + + self.decimal = DecimalFst(cardinal=self.cardinal, deterministic=deterministic) + decimal_graph = self.decimal.fst + + word_graph = WordFst(deterministic=deterministic).fst + + self.whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) + whitelist_graph = self.whitelist.fst + + self.electronic = ElectronicFst(deterministic=deterministic) + electronic_graph = self.electronic.fst + + self.measure = MeasureFst( + cardinal=self.cardinal, + decimal=self.decimal, + deterministic=deterministic, + ) + measure_graph = self.measure.fst + + self.money = MoneyFst( + cardinal=self.cardinal, + decimal=self.decimal, + deterministic=deterministic, + ) + money_graph = self.money.fst + + self.time = TimeFst(deterministic=deterministic) + time_graph = self.time.fst + + punct_graph = PunctuationFst(deterministic=deterministic).fst + + classify = ( + pynutil.add_weight(whitelist_graph, 0.0) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(electronic_graph, 1.1) + | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(word_graph, 100) + ) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") + + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure((delete_extra_space).ques + token_plus_punct) + + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() + no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) + self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/whitelist.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/whitelist.py new file mode 100644 index 0000000000000000000000000000000000000000..1104d6ff400d6d43cc79f7db52b2098f6c401b17 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/whitelist.py @@ -0,0 +1,65 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space +from zonos3.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path, load_labels + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist, e.g. + "dr." -> tokens { name: "dottor" } + "Avv." -> tokens { name: "avvocato" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + input_file: path to a file with whitelist replacements + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(input_case, file): + whitelist = load_labels(file) + if input_case == "lower_cased": + whitelist = [[x[0].lower()] + x[1:] for x in whitelist] + graph = pynini.string_map(whitelist) + return graph + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist/whitelist.tsv")) + if not deterministic and input_case != "lower_cased": + graph |= pynutil.add_weight( + _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist/whitelist.tsv")), weight=0.0001 + ) + + if input_file: + whitelist_provided = _get_whitelist_graph(input_case, input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + if not deterministic: + units_graph = _get_whitelist_graph(input_case, file=get_abs_path("data/measure/measurements.tsv")) + graph |= units_graph + + self.graph = graph + self.final_graph = convert_space(self.graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.final_graph + pynutil.insert("\"")).optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/word.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/word.py new file mode 100644 index 0000000000000000000000000000000000000000..5f0e019a8ad5746920d52e3c166c5be319a1c16f --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/taggers/word.py @@ -0,0 +1,34 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying word. + e.g. mangiare -> tokens { name: "mangiare" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/utils.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..be8bdb5addc0c542fe49d252e1a4240c5d61037f --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/utils.py @@ -0,0 +1,42 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebc808fad7ac87484dbabf02dacacdd7f826c14 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/cardinal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..3ef5246753cf3fe5a947cfc4afd711dea7aa8d6e --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/cardinal.py @@ -0,0 +1,43 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinals + e.g. cardinal { integer: "due" } -> "due" + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + optional_sign = pynini.closure(pynini.cross("negative: \"true\" ", "meno "), 0, 1) + self.optional_sign = optional_sign + integer = pynini.closure(NEMO_NOT_QUOTE, 1) + + self.integer = pynutil.delete(" \"") + integer + pynutil.delete("\"") + + integer = pynutil.delete("integer:") + self.integer + self.numbers = integer + graph = optional_sign + self.numbers + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/decimal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/decimal.py new file mode 100644 index 0000000000000000000000000000000000000000..e883487196ae68619d7437dc8a7b966a90e337b5 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/decimal.py @@ -0,0 +1,76 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + decimal { negative: "true" integer_part: "venti" fractional_part: "trentaquattro" quantity: "miliardi" } -> + meno venti virgola trentaquattro + decimal { integer_part: "un milione" fractional_part: "zero zero zero" quantity: "milioni" preserve_order: true } --> + un milione virgola zero zero zero + decimal { integer_part: "due" quantity: "milioni" preserve_order: true } --> + due milioni + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "meno ") + delete_space, 0, 1) + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + fractional_default = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + conjunction = pynutil.insert(" virgola ") + fractional = conjunction + fractional_default + + quantity = ( + delete_space + + insert_space + + pynutil.delete("quantity: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_quantity = pynini.closure(quantity, 0, 1) + + graph = optional_sign + pynini.union( + (integer + quantity), (integer + delete_space + fractional + optional_quantity) + ) + + self.numbers_only_quantity = ( + optional_sign + + pynini.union((integer + quantity), (integer + delete_space + fractional + quantity)).optimize() + ) + + self.graph = (graph + delete_preserve_order).optimize() + + graph += delete_preserve_order + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/electronic.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/electronic.py new file mode 100644 index 0000000000000000000000000000000000000000..4dc9ab2f6d2359e6cb773e7b03bcfff2f9b447b9 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/electronic.py @@ -0,0 +1,88 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( # Common string literals; expand as you see fit. + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + colon, + delete_preserve_order, + domain_string, + double_quotes, + protocol_string, + username_string, +) +from zonos3.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path + +digit_no_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) +zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) + +graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")) +server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv")) +domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv")) + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for verbalizing electronic + e.g. electronic { username: "abc.def2" domain: "studenti.università.it" } -> + "a b c punto d e f due chiocciola s t u d e n t i punto u n i v e r s i t à punto IT + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) + + graph_digit = digit_no_zero | zero + + def add_space_after_char(): + return pynini.closure(NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) + pynutil.insert(NEMO_SPACE)) + ( + NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) + ) + + verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", NEMO_SIGMA) + + user_name = ( + pynutil.delete(username_string + colon + NEMO_SPACE + double_quotes) + + add_space_after_char() + + pynutil.delete(double_quotes) + ) + user_name @= verbalize_characters + + convert_defaults = pynutil.add_weight(NEMO_NOT_QUOTE, weight=0.0001) | server_common | domain_common + domain = convert_defaults + pynini.closure(pynutil.insert(NEMO_SPACE) + convert_defaults) + domain @= verbalize_characters + + domain = ( + pynutil.delete(domain_string + colon + NEMO_SPACE + double_quotes) + domain + pynutil.delete(double_quotes) + ) + protocol = ( + pynutil.delete(protocol_string + colon + NEMO_SPACE + double_quotes) + + add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", NEMO_SIGMA) + + pynutil.delete(double_quotes) + ) + + self.graph = (pynini.closure(protocol + NEMO_SPACE, 0, 1) + domain) | ( + user_name + NEMO_SPACE + pynutil.insert("chiocciola ") + domain + | (pynutil.insert("chiocciola ") + user_name) + ) + + delete_tokens = self.delete_tokens(self.graph + delete_preserve_order) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/measure.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/measure.py new file mode 100644 index 0000000000000000000000000000000000000000..5fa4ec0ef109bfa7ab8a2f8d2ae447bc04bc5f41 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/measure.py @@ -0,0 +1,52 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_extra_space, + delete_preserve_order, +) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { cardinal { integer: "due" units: "grammi" } } -> "due grammi" + + Args: + decimal: decimal GraphFst + cardinal: cardinal GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst, deterministic: bool): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + + unit = pynutil.delete("units: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph_decimal = decimal.fst + graph_cardinal = cardinal.fst + + graph = (graph_cardinal | graph_decimal) + pynini.accep(" ") + unit + + graph |= unit + delete_extra_space + (graph_cardinal | graph_decimal) + graph += delete_preserve_order + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/money.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/money.py new file mode 100644 index 0000000000000000000000000000000000000000..ddada7e28604e72043fca15038eb53c29a53ce98 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/money.py @@ -0,0 +1,160 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_preserve_order, +) +from zonos3.vendor.nemo_text_processing.text_normalization.es.graph_utils import shift_cardinal_gender, strip_cardinal_apocope +from zonos3.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path + +fem = pynini.string_file((get_abs_path("data/money/currency_plural_fem.tsv"))) +masc = pynini.string_file((get_abs_path("data/money/currency_plural_masc.tsv"))) + +fem_singular = pynini.project(fem, "input") +masc_singular = pynini.project(masc, "input") + +fem_plural = pynini.project(fem, "output") +masc_plural = pynini.project(masc, "output") + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { currency_maj: "euro" integer_part: "un"} -> "un euro" + money { integer_part: "quattro" currency_maj: "sterline" fractional_part: "venti" currency_min: "penny" preserve_order: true } -> "quattro sterline venti penny" + + Args: + decimal: GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + maj_singular_masc = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_singular) + + pynutil.delete("\"") + ) + maj_singular_fem = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_singular) + + pynutil.delete("\"") + ) + + maj_plural_masc = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_plural) + + pynutil.delete("\"") + ) + maj_plural_fem = ( + pynutil.delete("currency_maj: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_plural) + + pynutil.delete("\"") + ) + + maj_masc = maj_plural_masc | maj_singular_masc + maj_fem = maj_plural_fem | maj_singular_fem + + min_singular_masc = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_singular) + + pynutil.delete("\"") + ) + min_singular_fem = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_singular) + + pynutil.delete("\"") + ) + + min_plural_masc = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ masc_plural) + + pynutil.delete("\"") + ) + min_plural_fem = ( + pynutil.delete("currency_min: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ fem_plural) + + pynutil.delete("\"") + ) + + min_masc = min_plural_masc | min_singular_masc + min_fem = min_plural_fem | min_singular_fem + + fractional_part = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + integer_part = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + optional_add_and = pynini.closure(pynutil.insert(pynini.union("con ", "y ")), 0, 1) + + graph_integer_masc = integer_part + NEMO_SPACE + maj_masc + graph_integer_fem = shift_cardinal_gender(integer_part) + NEMO_SPACE + maj_fem + + graph_integer = graph_integer_fem | graph_integer_masc + + graph_integer_with_minor_masc = ( + graph_integer_masc + + NEMO_SPACE + + pynini.union( + optional_add_and + strip_cardinal_apocope(fractional_part), + (optional_add_and + fractional_part + NEMO_SPACE + min_masc), + (optional_add_and + shift_cardinal_gender(fractional_part) + NEMO_SPACE + min_fem), + ) + + delete_preserve_order + ) + + graph_integer_with_minor_fem = ( + graph_integer_fem + + NEMO_SPACE + + pynini.union( + optional_add_and + shift_cardinal_gender(fractional_part), + (optional_add_and + fractional_part + NEMO_SPACE + min_masc), + (optional_add_and + shift_cardinal_gender(fractional_part) + NEMO_SPACE + min_fem), + ) + + delete_preserve_order + ) + + graph_integer_with_minor = graph_integer_with_minor_fem | graph_integer_with_minor_masc + + graph_decimal_masc = decimal.graph + NEMO_SPACE + maj_masc + + graph_decimal_fem = decimal.graph + graph_decimal_fem += NEMO_SPACE + maj_fem + + graph_decimal = graph_decimal_fem | graph_decimal_masc + graph_decimal = ( + pynini.cdrewrite( + pynutil.insert(" di"), "quantity: \"" + pynini.closure(NEMO_NOT_QUOTE, 1), "\"", NEMO_SIGMA + ) + @ graph_decimal + ) + + graph_minor_masc = fractional_part + NEMO_SPACE + min_masc + delete_preserve_order + graph_minor_fem = shift_cardinal_gender(fractional_part) + NEMO_SPACE + min_fem + delete_preserve_order + + graph_minor = graph_minor_fem | graph_minor_masc + + graph = graph_integer | graph_integer_with_minor | graph_decimal | graph_minor + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/time.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/time.py new file mode 100644 index 0000000000000000000000000000000000000000..3fe32466be2c984d9f073b8116b738dd5d5adf64 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/time.py @@ -0,0 +1,85 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, delete_preserve_order +from zonos3.vendor.nemo_text_processing.text_normalization.it.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g. + tokens { time { hours: "15" minutes: "30" seconds: "30" preserve_order: true } } -> quindici e mezza trenta secondi + tokens { time { hours: "12" minutes: "15" } } -> dodici e un quarto + tokens { time { hours: "3" minutes: "38" } } -> tre e trentotto minuti + Args: + cardinal_tagger: cardinal_tagger tagger GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal_tagger: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + graph_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))).optimize() + number_verbalization = graph_zero | cardinal_tagger.two_digit_no_zero + hour = pynutil.delete("hours: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + hour_verbalized = hour @ number_verbalization + minute = pynutil.delete("minutes: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + second = pynutil.delete("seconds: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + + graph_hms_15 = ( + hour_verbalized + + pynini.accep(" ") + + pynutil.insert("e ") + + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("15", "un quarto")) + + pynini.accep(" ") + + pynutil.insert("e ") + + second @ number_verbalization + + pynutil.insert(" secondi") + ) + + graph_hms_30 = ( + hour_verbalized + + pynini.accep(" ") + + pynutil.insert("e ") + + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("30", " mezza")) + + pynini.accep(" ") + + pynutil.insert("e ") + + second @ number_verbalization + + pynutil.insert(" secondi") + ) + + graph_hm_15 = ( + hour_verbalized + + pynini.accep(" ") + + pynutil.insert("e ") + + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("15", "un quarto")) + ) + + graph_hm_30 = ( + hour_verbalized + + pynini.accep(" ") + + pynutil.insert("e ") + + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("30", " mezza")) + ) + + graph_h = hour_verbalized + + self.graph = graph_hms_30 | graph_hms_30 | graph_hms_15 | graph_hm_30 | graph_hm_15 | graph_h + + delete_tokens = self.delete_tokens(self.graph + delete_preserve_order) + self.fst = delete_tokens.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/verbalize.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/verbalize.py new file mode 100644 index 0000000000000000000000000000000000000000..f37e9a248a78583bc6bf37f99c8b742414fc6e41 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/verbalize.py @@ -0,0 +1,63 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.taggers.cardinal import CardinalFst as CardinalTagger +from zonos3.vendor.nemo_text_processing.text_normalization.it.verbalizers.cardinal import CardinalFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.verbalizers.decimal import DecimalFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.verbalizers.electronic import ElectronicFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.verbalizers.measure import MeasureFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.verbalizers.money import MoneyFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.verbalizers.time import TimeFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + decimal = DecimalFst(deterministic=deterministic) + decimal_graph = decimal.fst + electronic = ElectronicFst(deterministic=deterministic) + electronic_graph = electronic.fst + whitelist_graph = WhiteListFst(deterministic=deterministic).fst + measure = MeasureFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic) + measure_graph = measure.fst + money = MoneyFst(decimal=decimal, deterministic=deterministic) + money_graph = money.fst + cardinal_tagger = CardinalTagger(deterministic=deterministic) + time = TimeFst(cardinal_tagger=cardinal_tagger, deterministic=deterministic) + time_graph = time.fst + + graph = ( + cardinal_graph + | decimal_graph + | electronic_graph + | whitelist_graph + | measure_graph + | money_graph + | time_graph + ) + + self.fst = graph diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/verbalize_final.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/verbalize_final.py new file mode 100644 index 0000000000000000000000000000000000000000..5ea847a820f0af89fa73a2ce08f2b699ad9cdcd6 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/it/verbalizers/verbalize_final.py @@ -0,0 +1,70 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from zonos3.vendor.nemo_text_processing.text_normalization.en.verbalizers.word import WordFst +from zonos3.vendor.nemo_text_processing.text_normalization.it.verbalizers.verbalize import VerbalizeFst +from zonos3.vendor.nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"it_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..571e528f9274e0e9012092509a1918c9e4cf5948 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from zonos3.vendor.nemo_text_processing.text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst +from zonos3.vendor.nemo_text_processing.text_normalization.ja.verbalizers.verbalize import VerbalizeFst +from zonos3.vendor.nemo_text_processing.text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9155f923f186a086a69de138e984e2fab2817ec --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9155f923f186a086a69de138e984e2fab2817ec --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/day.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/day.tsv new file mode 100644 index 0000000000000000000000000000000000000000..09258cb4c8d742834f6d7994580e16d863fe2783 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/day.tsv @@ -0,0 +1,31 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 +25 二十五 +26 二十六 +27 二十七 +28 二十八 +29 二十九 +30 三十 +31 三十一 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/era.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/era.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b932b7fb3fae11c5ef32e39ed90739d3b136ff86 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/era.tsv @@ -0,0 +1,12 @@ +令和 +平成 +昭和 +大正 +明治 +西暦 +和暦 +西洋暦 +グレゴリオ暦 +紀元前 +紀元 +紀元後 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/era_abbrev.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/era_abbrev.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1c95675c27ddf66508601e68834ccf48e463a3f9 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/era_abbrev.tsv @@ -0,0 +1,5 @@ +R. 令和 +H. 平成 +S. 昭和 +T. 大正 +M. 明治 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/month.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/month.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f992b4d28d2509cd2b4822ac28690259d64917f8 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/month.tsv @@ -0,0 +1,12 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/week.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/week.tsv new file mode 100644 index 0000000000000000000000000000000000000000..41556b4db2f74a429f6e571e46450f57dddbb868 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/date/week.tsv @@ -0,0 +1,15 @@ +月 月曜日 +火 火曜日 +水 水曜日 +木 木曜日 +金 金曜日 +土 土曜日 +日 日曜日 +祝 祝日 +月曜日 +火曜日 +水曜日 +木曜日 +金曜日 +土曜日 +日曜日 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/numbers/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/numbers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9155f923f186a086a69de138e984e2fab2817ec --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/numbers/digit.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/numbers/digit.tsv new file mode 100644 index 0000000000000000000000000000000000000000..ede6b97b73ae912917667f3a00a9efb71580bee3 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/numbers/teen.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/numbers/teen.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1585fe836701348ae4543e3001c2a10151d750f2 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/numbers/ties.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/numbers/ties.tsv new file mode 100644 index 0000000000000000000000000000000000000000..562e632656a6b6fd752296d2a30e419206052bf5 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/numbers/ties.tsv @@ -0,0 +1,8 @@ +2 二十 +3 三十 +4 四十 +5 五十 +6 六十 +7 七十 +8 八十 +9 九十 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7fab21718dad9ed8089512e9d513cab84074dfc2 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv @@ -0,0 +1 @@ +0 零 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/symbol.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/symbol.tsv new file mode 100644 index 0000000000000000000000000000000000000000..67ad785f0e2e9e4eec5c13063a884f10e8904160 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/symbol.tsv @@ -0,0 +1,23 @@ +& アンド +# ハッシュタグ +@ アット +§ セクション +™ トレードマーク +® 登録商標マーク +© 著作権 +_ アンダースコア +% パーセント +* 星印 ++ プラス +/ スラッシュ += エコール +^ 曲折アクセント記号 +| 縦棒 +~ ティルダ +$ ドール +£ ポンド +€ ユーロ +₩ ウォン +¥ 円 +° 度 +º 度 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/time/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/time/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9155f923f186a086a69de138e984e2fab2817ec --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/time/division.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/time/division.tsv new file mode 100644 index 0000000000000000000000000000000000000000..f6beedce80aec13a68e584811154d73bd47609e1 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/time/division.tsv @@ -0,0 +1,23 @@ +今朝 +今夜 +今晩 +午前 +午後 +夕方 +夜中 +夜 +夜半 +朝 +早朝 +明け方 +昼 +深夜 +毎朝 +毎夜 +毎晩 +毎日 +真夜中 +翌日 +未明 +正午 +真夜中の \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/time/hour.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/time/hour.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1d5c084253396e5d4183fa198f14241fcc175a95 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/time/hour.tsv @@ -0,0 +1,24 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/time/minute.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/time/minute.tsv new file mode 100644 index 0000000000000000000000000000000000000000..5e8276a00502b43948252b6a5af495e33d580e2f --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/time/minute.tsv @@ -0,0 +1,60 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 +25 二十五 +26 二十六 +27 二十七 +28 二十八 +29 二十九 +30 三十 +31 三十一 +32 三十二 +33 三十三 +34 三十四 +35 三十五 +36 三十六 +37 三十七 +38 三十八 +39 三十九 +40 四十 +41 四十一 +42 四十二 +43 四十三 +44 四十四 +45 四十五 +46 四十六 +47 四十七 +48 四十八 +49 四十九 +50 五十 +51 五十一 +52 五十二 +53 五十三 +54 五十四 +55 五十五 +56 五十六 +57 五十七 +58 五十八 +59 五十九 +60 六十 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/time/second.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/time/second.tsv new file mode 100644 index 0000000000000000000000000000000000000000..5e8276a00502b43948252b6a5af495e33d580e2f --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/time/second.tsv @@ -0,0 +1,60 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 +25 二十五 +26 二十六 +27 二十七 +28 二十八 +29 二十九 +30 三十 +31 三十一 +32 三十二 +33 三十三 +34 三十四 +35 三十五 +36 三十六 +37 三十七 +38 三十八 +39 三十九 +40 四十 +41 四十一 +42 四十二 +43 四十三 +44 四十四 +45 四十五 +46 四十六 +47 四十七 +48 四十八 +49 四十九 +50 五十 +51 五十一 +52 五十二 +53 五十三 +54 五十四 +55 五十五 +56 五十六 +57 五十七 +58 五十八 +59 五十九 +60 六十 \ No newline at end of file diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/whitelist.tsv b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/whitelist.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d0d7bef709e55ad4823f9b8b8d9ef9ababbc610d --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/data/whitelist.tsv @@ -0,0 +1,40 @@ +Dr. ドクター +dr. ドクター +Mr. ミスター +mr. ミスター +Ms. ミス +ms. ミス +Mrs. ミシーズ +mrs. ミシーず +st. ストリード +St. ストリード +mt. マウント +Mt. マウント +Prof. プロフェッサー +prof. プロフェッサー +sr. シニア +Sr. シニア +jr. ジュニア +Jr. ジュニア +rd. ロード +Rd. ロード +Ave. アベニュー +ave. アベニュー +no. ナンバー +No. ナンバー +( カッコ +) カッコ閉じる ++ プラス +- マイナス +Σ シグマ +η エタ +κ カッパ +ω オメガ +σ シグマ +α アルフ +ν ヌー +δ デルタ +ι イオタ +vs. バーサス +Ph.D. p h d +etc. エトセトラ diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/graph_utils.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7b14ce56150e237cd00a16a8e2b48dbe104f481c --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/graph_utils.py @@ -0,0 +1,297 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from zonos3.vendor.nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = u"\u00a0" +NEMO_NARROW_NON_BREAK_SPACE = u"\u202f" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_LOWER_NOT_A = pynini.union( + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", +).optimize() +NEMO_SPACES_AND_ALHPANUMERICS = pynini.closure( + NEMO_SIGMA + | pynini.closure(NEMO_SPACE) + | pynini.closure(NEMO_NON_BREAKING_SPACE) + | pynini.closure(NEMO_DIGIT) + | pynini.closure(NEMO_WHITE_SPACE) +).optimize() + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) +) + +suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv")) +# _v = pynini.union("a", "e", "i", "o", "u") +_c = pynini.union( + "b", "c", "d", "f", "g", "h", "j", "k", "l", "m", "n", "p", "q", "r", "s", "t", "v", "w", "x", "y", "z" +) +_ies = NEMO_SIGMA + _c + pynini.cross("y", "ies") +_es = NEMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es") +_s = NEMO_SIGMA + pynutil.insert("s") + +graph_plural = plurals._priority_union( + suppletive, plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), NEMO_SIGMA +).optimize() + +SINGULAR_TO_PLURAL = graph_plural +PLURAL_TO_SINGULAR = pynini.invert(graph_plural) +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) +TO_UPPER = pynini.invert(TO_LOWER) +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("minus", "Minus").optimize() + + +def capitalized_input_graph( + graph: 'pynini.FstLike', original_graph_weight: float = None, capitalized_graph_weight: float = None +) -> 'pynini.FstLike': + """ + Allow graph input to be capitalized, e.g. for ITN) + + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +def get_plurals(fst): + """ + Given singular returns plurals + + Args: + fst: Fst + + Returns plurals to given singular forms + """ + return SINGULAR_TO_PLURAL @ fst + + +def get_singulars(fst): + """ + Given plural returns singulars + + Args: + fst: Fst + + Returns singulars to given plural forms + """ + return PLURAL_TO_SINGULAR @ fst + + +def convert_space(fst) -> 'pynini.FstLike': + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [written_capitalized, spoken.capitalize()], # first letter capitalized + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], # # add pairs with the all letters capitalized + ] + ) + + spoken_no_space = spoken.replace(" ", "") + # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" + if len(spoken) == (2 * len(spoken_no_space) - 1): + logging.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [[written, spoken_no_space], [written_capitalized, spoken_no_space.upper()]] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/taggers/__init__.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/taggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9155f923f186a086a69de138e984e2fab2817ec --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/taggers/cardinal.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/taggers/cardinal.py new file mode 100644 index 0000000000000000000000000000000000000000..b531333b37018266b8723ce854a7930880ec1858 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/taggers/cardinal.py @@ -0,0 +1,191 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.ja.graph_utils import NEMO_DIGIT, GraphFst +from zonos3.vendor.nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. 23 -> cardinal { integer: "二十三" } + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + + no_one = pynini.difference(NEMO_DIGIT, "1") + no_zero_and_one = pynini.difference(no_one, "0") + + graph_digit_alt = no_zero_and_one @ graph_digit + graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) + graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) + + graph_all = (graph_ties + (graph_digit | pynutil.delete('0'))) | graph_teen | graph_digit + + hundreds = NEMO_DIGIT**3 + graph_hundred_component = (pynini.cross('1', '百') | (graph_digit_alt + pynutil.insert('百'))) + pynini.union( + pynini.closure(pynutil.delete('0')), (pynini.closure(pynutil.delete('0')) + graph_all) + ) + graph_hundred = hundreds @ graph_hundred_component + + thousands = NEMO_DIGIT**4 + graph_thousand_component = (pynini.cross('1', '千') | (graph_digit_alt + pynutil.insert('千'))) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_thousand_component_alt = (graph_digit + pynutil.insert('千')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + # this grammar is for larger number in later gramamr + graph_thousand = thousands @ graph_thousand_component + + ten_thousands = NEMO_DIGIT**5 + graph_ten_thousand_component = (graph_digit + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_ten_thousand = ten_thousands @ graph_ten_thousand_component + self.man = graph_ten_thousand.optimize() + + hundred_thousands = NEMO_DIGIT**6 + hundred_thousands_position = NEMO_DIGIT**2 + hundred_thousands_position = hundred_thousands_position @ graph_all + graph_hundred_thousand_component = (hundred_thousands_position + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_hundred_thousand = hundred_thousands @ graph_hundred_thousand_component + + millions = NEMO_DIGIT**7 + million_position = NEMO_DIGIT**3 + million_position = million_position @ graph_hundred_component + graph_million_component = (million_position + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_million = millions @ graph_million_component + + ten_millions = NEMO_DIGIT**8 + ten_million_position = NEMO_DIGIT**4 + ten_million_position = ten_million_position @ graph_thousand_component_alt + graph_ten_million_component = (ten_million_position + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_ten_million = ten_millions @ graph_ten_million_component + + hundred_millions = NEMO_DIGIT**9 + graph_hundred_million_component = (graph_digit + pynutil.insert('億')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_hundred_million = hundred_millions @ graph_hundred_million_component + + thousand_millions = NEMO_DIGIT**10 + thousand_millions_position = NEMO_DIGIT**2 + thousand_millions_position = thousand_millions_position @ graph_all + graph_thousand_million_component = (thousand_millions_position + pynutil.insert('億')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_thousand_million = thousand_millions @ graph_thousand_million_component + + ten_billions = NEMO_DIGIT**11 + ten_billions_position = NEMO_DIGIT**3 + ten_billions_position = ten_billions_position @ graph_hundred_component + graph_ten_billions_component = (ten_billions_position + pynutil.insert('億')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_ten_billions = ten_billions @ graph_ten_billions_component + + hundred_billions = NEMO_DIGIT**12 + hundred_billions_position = NEMO_DIGIT**4 + hundred_billions_position = hundred_billions_position @ graph_thousand_component_alt + graph_hundred_billions_component = (hundred_billions_position + pynutil.insert('億')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_hundred_billions = hundred_billions @ graph_hundred_billions_component + + graph = pynini.union( + graph_hundred_billions, + graph_ten_billions, + graph_thousand_million, + graph_hundred_million, + graph_ten_million, + graph_million, + graph_hundred_thousand, + graph_ten_thousand, + graph_thousand, + graph_hundred, + graph_all, + graph_zero, + ) + self.just_cardinals = graph.optimize() + + optional_sign = ( + pynutil.insert("negative: \"") + (pynini.accep("-") | pynini.cross("マイナス", "-")) + pynutil.insert("\"") + ) + + final_graph = ( + optional_sign + pynutil.insert(" ") + pynutil.insert("integer: \"") + graph + pynutil.insert("\"") + ) | (pynutil.insert("integer: \"") + graph + pynutil.insert("\"")) + + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/taggers/date.py b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/taggers/date.py new file mode 100644 index 0000000000000000000000000000000000000000..c01e523ec3b4f13137191d239cf3269c86e3ce05 --- /dev/null +++ b/vendor/zonos3/vendor/nemo_text_processing/text_normalization/ja/taggers/date.py @@ -0,0 +1,227 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from zonos3.vendor.nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_NARROW_NON_BREAK_SPACE, + NEMO_NON_BREAKING_SPACE, + GraphFst, +) +from zonos3.vendor.nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + 2024/01/30 -> date { year: "二千二十四" month: "一" day: "三十" } + 2024/1/30 -> date { year: "二千二十四" month: "一" day: "三十" } + 2024-01-30 -> date { year: "二千二十四" month: "一" day: "三十" } + 2024.01.30 -> date { year: "二千二十四" month: "一" day: "三十" } + H.6 -> date { era: "平成" yera: "六年" } + R.1 -> date { era: "令和" "year: "元年" } + S.5 -> date { era: "昭和" "year: "五年" } + T.5 -> date { era: "大正" "year: "五年" } + M.5 -> date { era: "明治" "year: "五年" } + 21日月曜日 -> tokens { date { day: "二十一日" weekday: "月曜日" } } + 70年代 -> date { year: "七十年代" } + 西暦794年 -> tokens { date { era: "西暦" year: "七百九十四年" } } + 1月1日(月)〜3日(水) + -> tokens { date { month: "一月" day: "一日" weekday: "月曜日" } } tokens { name: "から" } tokens { date { day: "三日" weekday: "水曜日" } } + 70〜80年代 + -> tokens { cardinal { integer: "七十" } } tokens { name: "から" } tokens { date { year: "八十年代" } } + 7月5〜9日(月〜金) + -> tokens { date { month: "七月" } } tokens { cardinal { integer: "五" } } tokens { name: "から" } tokens { date { day: "九日" weekday: "月曜日" } } tokens { name: "から" } tokens { date { weekday: "金曜日" } } + 7月初旬〜9月中旬 + -> tokens { date { month: "七月" } } tokens { name: "初" } tokens { name: "旬" } tokens { name: "から" } tokens { date { month: "九月" } } tokens { name: "中" } tokens { name: "旬" } + 3〜4月 + -> tokens { cardinal { integer: "三" } } tokens { name: "から" } tokens { date { month: "四月" } } + 2023年3月1日(水)〜6月12日(火) + -> tokens { date { year: "二千二十三年" month: "三月" day: "一日" weekday: "水曜日" } } tokens { name: "から" } tokens { date { month: "六月" day: "十二日" weekday: "火曜日" } } + 10月中旬〜11月上旬 + -> tokens { date { month: "十月" } } tokens { date { month: "中旬" } } tokens { name: "から" } tokens { date { month: "十一月" } } tokens { date { month: "上旬" } } + 1976年7月17日〜8月1日 + -> tokens { date { year: "千九百七十六年" month: "七月" day: "十七日" } } tokens { name: "から" } tokens { date { month: "八月" day: "一日" } } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + graph_cardinal = cardinal.just_cardinals + + month = pynutil.delete("0").ques + pynini.string_file(get_abs_path("data/date/month.tsv")) + day = pynutil.delete("0").ques + pynini.string_file(get_abs_path("data/date/day.tsv")) + week = pynini.string_file(get_abs_path("data/date/week.tsv")) + era = pynini.string_file(get_abs_path("data/date/era.tsv")) + era_abbrev = pynini.string_file(get_abs_path("data/date/era_abbrev.tsv")) + + signs = pynutil.delete("/") | pynutil.delete(".") | pynutil.delete("-") + delete_spaces = pynini.closure( + pynutil.delete(" ") | pynutil.delete(NEMO_NARROW_NON_BREAK_SPACE) | pynutil.delete(NEMO_NON_BREAKING_SPACE) + ) + + era_component = pynutil.insert("era: \"") + era + pynutil.insert("\"") + era_abbrev_component = pynutil.insert("era: \"") + era_abbrev + pynutil.insert("\"") + year_component = pynutil.insert("year: \"") + graph_cardinal + pynutil.insert("年") + pynutil.insert("\"") + month_component = pynutil.insert("month: \"") + month + pynutil.insert("月") + pynutil.insert("\"") + day_component = pynutil.insert("day: \"") + day + pynutil.insert("日") + pynutil.insert("\"") + + front_bracket = ( + ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete("(") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + | ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete("(") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + | ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete("(") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + ) + preceding_bracket = ( + ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete(")") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + | ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete(")") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + | ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete(")") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + ) + # this graph optionally accepts () around weekday to accomodate to inputs like (月〜金), thus being longer + + week_component = ( + (front_bracket + pynutil.insert("weekday: \"") + week + preceding_bracket + pynutil.insert("\"")) + | ( + front_bracket + + pynutil.insert("weekday: \"") + + week + + pynini.cross("〜", "から") + + week + + preceding_bracket + + pynutil.insert("\"") + ) + | ( + front_bracket + + pynutil.insert("weekday: \"") + + week + + pynutil.delete("・") + + week + + preceding_bracket + + pynutil.insert("\"") + ) + ) + + # era, year, month, date + graph_basic_date = ( # (R.|令和)2024/01/01, (R.|令和)2024/01/01/(水), (R.|令和)01/01, (R.|令和)01/01(水) + pynini.closure(era_component + pynutil.insert(" "), 0, 1) + + pynini.closure(year_component + signs + pynutil.insert(" "), 1) + + month_component + + signs + + pynutil.insert(" ") + + day_component + + pynini.closure(pynutil.insert(" ") + week_component, 0, 1) + ) + + # 2024年, 9月, 28日 + individual_year_component = ( + pynini.closure(era_component + pynutil.insert(" "), 0, 1) + + pynutil.insert("year: \"") + + graph_cardinal + + pynini.accep("年") + + pynutil.insert("\"") + ) + # this extra individual year component is to accomodate inputs R. 2024 with out "年" + # the inputs may or maynot include "年", thus below: + individual_year_component_2 = ( + pynini.closure(era_component + pynutil.insert(" "), 0, 1) + + pynutil.insert("year: \"") + + graph_cardinal + + (pynini.accep("世紀") | pynini.accep("")) + + pynutil.insert("\"") + ) | ( + era_abbrev_component + + pynutil.insert(" ") + + pynutil.insert("year: \"") + + graph_cardinal + + pynutil.insert("年") + + pynutil.insert("\"") + ) + + individual_month_component = ( + pynutil.insert("month: \"") + month + pynini.accep("月") + pynutil.insert("\"") + ) | ( + pynutil.insert("month: \"") + + (pynini.accep("中旬") | pynini.accep("下旬") | pynini.accep("上旬")) + + pynutil.insert("\"") + ) + individual_day_component = ( + pynutil.insert("day: \"") + graph_cardinal + pynini.accep("日") + pynutil.insert("\"") + ) + + graph_individual_component = ( + individual_year_component + | individual_month_component + | individual_day_component + | week_component + | individual_year_component_2 + ) + pynini.closure(pynutil.insert(" ") + week_component, 0, 1) + + # combined the above individual date components + graph_individual_component_combined = ( + (individual_year_component + pynutil.insert(" ") + individual_month_component) + | (individual_month_component + pynutil.insert(" ") + individual_day_component) + | ( + individual_year_component + + pynutil.insert(" ") + + individual_month_component + + pynutil.insert(" ") + + individual_day_component + ) + ) + pynini.closure(pynutil.insert(" ") + week_component, 0, 1) + + nendai = pynini.accep("年代") + era_nendai = ( + pynini.closure(era_component + pynutil.insert(" "), 0, 1) + + pynutil.insert("year: \"") + + graph_cardinal + + nendai + + pynutil.insert("\"") + ) + + graph_all_date = ( + graph_basic_date | graph_individual_component | graph_individual_component_combined | era_nendai + ) + + final_graph = graph_all_date + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize()