"""The NTv3 annotation benchmark: test species, released files and published scores.""" from dataclasses import dataclass DATASET_REPO = "InstaDeepAI/NTv3_annot_dataset" @dataclass(frozen=True) class TestSpecies: """One held-out species of the benchmark.""" name: str clade: str assembly: str released_gff: str gene_f1: float cds_f1: float @property def model_repo(self) -> str: """Hugging Face repo of the clade model, which also holds its predictions.""" return f"InstaDeepAI/NTv3_annot_{self.clade}" @property def dataset_dir(self) -> str: """Folder of the species in ``DATASET_REPO``.""" return f"{self.clade}/test_species/{self.assembly}" @property def reference_gff(self) -> str: """Benchmark reference in ``DATASET_REPO``.""" return f"{self.dataset_dir}/reference_longest_isoform.gff3.gz" @property def genome_fasta(self) -> str: """Soft-masked genome in ``DATASET_REPO`` (bgzipped, indexed).""" return f"{self.dataset_dir}/genome.fa.gz" def _species( name: str, clade: str, assembly: str, tag: str, gene_f1: float, cds_f1: float ) -> TestSpecies: return TestSpecies( name, clade, assembly, f"annot_results/ntv3_{name}_{tag}.gff3.gz", gene_f1, cds_f1, ) # Gene / CDS F1 (%) of the released NTv3 GFF3s, whole genome, as in the paper. TEST_SPECIES = { s.name: s for s in ( _species( "bos_taurus", "mammals", "GCF_002263795.3_ARS-UCD2.0", "GCF_002263795.3", 58.06, 90.60, ), _species( "delphinapterus_leucas", "mammals", "GCF_002288925.2_ASM228892v3", "GCF_002288925.2", 54.66, 89.68, ), _species( "brachypodium_distachyon", "plants", "GCF_000005505.3_Brachypodium_distachyon_v3.0", "GCF_000005505.3", 71.89, 89.68, ), _species( "eschscholzia_californica", "plants", "EcalifornicavarOrangeKingHAP1v1.1", "OrangeKingHAP1v1.1", 73.17, 91.07, ), _species( "freycinetia_multiflora", "plants", "FmultifloravarABG20040013HAP1v1.1", "ABG20040013HAP1v1.1", 68.36, 88.73, ), _species( "medicago_truncatula", "plants", "GCF_003473485.1_MtrunA17r5.0-ANR", "GCF_003473485.1", 72.63, 89.33, ), _species( "mimulus_guttatus", "plants", "MguttatusvarIM62v3.1", "IM62v3.1", 80.73, 93.54, ), _species( "bombyx_mori", "insects", "GCF_030269925.1_ASM3026992v2", "GCF_030269925.1", 56.53, 85.19, ), _species( "cataglyphis_hispanica", "insects", "GCF_021464435.1_ULB_Chis1_1.0", "GCF_021464435.1", 63.94, 87.85, ), _species( "colias_croceus", "insects", "GCF_905220415.1_ilColCroc2.1", "GCF_905220415.1", 62.65, 88.13, ), _species( "danaus_plexippus", "insects", "GCF_018135715.1_MEX_DaPlex", "GCF_018135715.1", 65.68, 89.38, ), _species( "leptidea_sinapis", "insects", "GCF_905404315.1_ilLepSina1.1", "GCF_905404315.1", 55.00, 81.99, ), _species( "nymphalis_io", "insects", "GCF_905147045.1_ilAglIoxx1.1", "GCF_905147045.1", 66.02, 89.19, ), _species( "osmia_bicornis", "insects", "GCF_907164935.1_iOsmBic2.1", "GCF_907164935.1", 58.75, 86.26, ), _species( "tribolium_castaneum", "insects", "GCF_031307605.1_icTriCast1.1", "GCF_031307605.1", 53.05, 80.60, ), _species( "vanessa_cardui", "insects", "GCF_905220365.1_ilVanCard2.1", "GCF_905220365.1", 64.37, 88.77, ), _species( "zerene_cesonia", "insects", "GCF_012273895.1_Zerene_cesonia_1.1", "GCF_012273895.1", 61.31, 85.14, ), ) }