""" Versioned UD treebank sizes and annotation provenance for the teaching app. Counts come from each official repository's r2.17/stats.xml. TrainingWords counts syntactic words, not surface tokens or whole-corpus words. """ TREEBANK_CATALOG = [ { "Language": "German", "Treebank": "HDT", "FullName": "Hamburg Dependency Treebank", "Tokens": "3,399,390", "TokensRaw": 3399390, "Sentences": "189,928", "SentencesRaw": 189928, "Genres": "News (Heise Online)", "Conformance": "Manual source annotations converted to UD; conversion has targeted manual corrections.", "BestFor": "Included to compare a much larger training set with GSD.", "ModelID": "german-hdt-ud-2.17-251125", "WordsRaw": 3455580, "Words": "3,455,580", "TrainingWordsRaw": 2799069, "TrainingWords": "2,799,069", "UDVersion": "2.17", "SourceURL": "https://universaldependencies.org/treebanks/de_hdt/index.html", "StatsURL": "https://github.com/UniversalDependencies/UD_German-HDT/blob/r2.17/stats.xml" }, { "Language": "German", "Treebank": "GSD", "FullName": "Google Universal Dependency Treebank", "Tokens": "287,708", "TokensRaw": 287708, "Sentences": "15,589", "SentencesRaw": 15589, "Genres": "News / Reviews / Web", "Conformance": "Manual source POS and syntax converted to UD; some other annotation layers are automatic.", "BestFor": "Included as a smaller German training set with a different annotation history.", "ModelID": "german-gsd-ud-2.17-251125", "WordsRaw": 292756, "Words": "292,756", "TrainingWordsRaw": 263777, "TrainingWords": "263,777", "UDVersion": "2.17", "SourceURL": "https://universaldependencies.org/treebanks/de_gsd/index.html", "StatsURL": "https://github.com/UniversalDependencies/UD_German-GSD/blob/r2.17/stats.xml" }, { "Language": "English", "Treebank": "EWT", "FullName": "English Web Treebank", "Tokens": "251,489", "TokensRaw": 251489, "Sentences": "16,622", "SentencesRaw": 16622, "Genres": "Blogs / Answers / Reviews / Email", "Conformance": "Converted dependencies manually corrected to UD; later consistency fixes. POS conversion also has corrections.", "BestFor": "Included as the English starting model and a comparison with GUM and LinES.", "ModelID": "english-ewt-ud-2.17-251125", "WordsRaw": 254818, "Words": "254,818", "TrainingWordsRaw": 204577, "TrainingWords": "204,577", "UDVersion": "2.17", "SourceURL": "https://universaldependencies.org/treebanks/en_ewt/index.html", "StatsURL": "https://github.com/UniversalDependencies/UD_English-EWT/blob/r2.17/stats.xml" }, { "Language": "English", "Treebank": "GUM", "FullName": "Georgetown University Multilayer", "Tokens": "229,851", "TokensRaw": 229851, "Sentences": "13,263", "SentencesRaw": 13263, "Genres": "Multiple written and spoken genres", "Conformance": "Earlier converted dependencies were manually corrected; newer texts are annotated directly in UD.", "BestFor": "Included to compare directly reviewed UD syntax with EWT and LinES.", "ModelID": "english-gum-ud-2.17-251125", "WordsRaw": 233926, "Words": "233,926", "TrainingWordsRaw": 177410, "TrainingWords": "177,410", "UDVersion": "2.17", "SourceURL": "https://universaldependencies.org/treebanks/en_gum/index.html", "StatsURL": "https://github.com/UniversalDependencies/UD_English-GUM/blob/r2.17/stats.xml" }, { "Language": "English", "Treebank": "LinES", "FullName": "Linköping English-Swedish Treebank", "Tokens": "105,137", "TokensRaw": 105137, "Sentences": "5,696", "SentencesRaw": 5696, "Genres": "Literature / Manuals / Europarl", "Conformance": "Manual source annotations converted to UD and partially reviewed.", "BestFor": "Included as the smallest selectable English training set and a contrast in conversion history.", "ModelID": "english-lines-ud-2.17-251125", "WordsRaw": 106305, "Words": "106,305", "TrainingWordsRaw": 64684, "TrainingWords": "64,684", "UDVersion": "2.17", "SourceURL": "https://universaldependencies.org/treebanks/en_lines/index.html", "StatsURL": "https://github.com/UniversalDependencies/UD_English-LinES/blob/r2.17/stats.xml" }, { "Language": "German", "Treebank": "LIT", "FullName": "German Literary Treebank", "Tokens": "40,340", "TokensRaw": 40340, "Sentences": "1,920", "SentencesRaw": 1920, "Genres": "Early Romantic philosophical fragments", "Conformance": "Dependencies manually annotated in UD; POS converted with some manual corrections.", "BestFor": "Exploration only: no training split or selectable model in this app.", "ModelID": None, "WordsRaw": 40450, "Words": "40,450", "TrainingWordsRaw": 0, "TrainingWords": "No training split", "UDVersion": "2.17", "SourceURL": "https://universaldependencies.org/treebanks/de_lit/index.html", "StatsURL": "https://github.com/UniversalDependencies/UD_German-LIT/blob/r2.17/stats.xml" }, { "Language": "German", "Treebank": "PUD", "FullName": "Parallel Universal Dependencies (German)", "Tokens": "21,001", "TokensRaw": 21001, "Sentences": "1,000", "SentencesRaw": 1000, "Genres": "Parallel News / Wikipedia", "Conformance": "Manual source annotations converted to UD; released as a test set.", "BestFor": "Exploration only: aligned sentences for cross-language comparison; no training split.", "ModelID": None, "WordsRaw": 21332, "Words": "21,332", "TrainingWordsRaw": 0, "TrainingWords": "No training split", "UDVersion": "2.17", "SourceURL": "https://universaldependencies.org/treebanks/de_pud/index.html", "StatsURL": "https://github.com/UniversalDependencies/UD_German-PUD/blob/r2.17/stats.xml" }, { "Language": "English", "Treebank": "PUD", "FullName": "Parallel Universal Dependencies (English)", "Tokens": "21,051", "TokensRaw": 21051, "Sentences": "1,000", "SentencesRaw": 1000, "Genres": "Parallel News / Wikipedia", "Conformance": "Manual source annotations converted to UD; released as a test set.", "BestFor": "Exploration only: aligned sentences for cross-language comparison; no training split.", "ModelID": None, "WordsRaw": 21180, "Words": "21,180", "TrainingWordsRaw": 0, "TrainingWords": "No training split", "UDVersion": "2.17", "SourceURL": "https://universaldependencies.org/treebanks/en_pud/index.html", "StatsURL": "https://github.com/UniversalDependencies/UD_English-PUD/blob/r2.17/stats.xml" } ] AVAILABLE_MODELS = { "English": { "UDPipe · english-ewt-ud-2.17-251125": "english-ewt-ud-2.17-251125", "UDPipe · english-gum-ud-2.17-251125": "english-gum-ud-2.17-251125", "UDPipe · english-lines-ud-2.17-251125": "english-lines-ud-2.17-251125" }, "German": { "UDPipe · german-hdt-ud-2.17-251125": "german-hdt-ud-2.17-251125", "UDPipe · german-gsd-ud-2.17-251125": "german-gsd-ud-2.17-251125" } } # Keep corpus exploration independent of the selected parser model. GREW_UD_VERSION = "2.18" _EXPLORATION_PROMPTS = { ("German", "HDT"): "Explore subject position in news sentences: find nsubj relations and compare subjects before and after the verb.", ("German", "GSD"): "Search for relative clauses (acl:relcl). How do their attachment and word order compare with the examples you parsed?", ("English", "EWT"): "Find coordination (conj) in web texts. Which word is the head, and where does the coordinating conjunction attach?", ("English", "GUM"): "Explore relative clauses (acl:relcl) across genres. Find three contrasting examples and explain their structures.", ("English", "LinES"): "Find prepositional noun modifiers (nmod) in literary text. Compare their case dependents with those in your own sentences.", ("German", "LIT"): "Explore subjects and objects in literary sentences. Which word-order patterns differ from the news examples in HDT?", ("German", "PUD"): "Choose a sentence and inspect its subjects, objects, and modifiers. Compare its English counterpart using the same sent_id.", ("English", "PUD"): "Choose a sentence and inspect its subjects, objects, and modifiers. Compare its German counterpart using the same sent_id.", } for _treebank in TREEBANK_CATALOG: _key = (_treebank["Language"], _treebank["Treebank"]) _treebank["GrewURL"] = ( "https://universal.grew.fr/?corpus=UD_" + _treebank["Language"] + "-" + _treebank["Treebank"] + "@" + GREW_UD_VERSION ) _treebank["ExplorePrompt"] = _EXPLORATION_PROMPTS[_key]