Spaces:
Sleeping
Sleeping
Elias Entrup commited on
Commit ·
e29f910
1
Parent(s): 24309d4
Add fractional counting
Browse files- api.py +44 -1
- dashboard.py +25 -11
- data_processing.py +162 -114
- data_visualization.py +37 -30
api.py
CHANGED
|
@@ -33,7 +33,6 @@ def ojs_articles_by_year() -> pl.DataFrame:
|
|
| 33 |
|
| 34 |
def ojs_article_for_institution(ror: str) -> pl.LazyFrame:
|
| 35 |
"""Fetch all OJS articles for the institution with pub year > min_year."""
|
| 36 |
-
|
| 37 |
query = (
|
| 38 |
Works()
|
| 39 |
.select(
|
|
@@ -62,6 +61,49 @@ def ojs_article_for_institution(ror: str) -> pl.LazyFrame:
|
|
| 62 |
with ARTICLE_CACHE_PATH.open('a') as f:
|
| 63 |
f.write(json.dumps(article) + '\n')
|
| 64 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
return pl.scan_ndjson(ARTICLE_CACHE_PATH)
|
| 66 |
|
| 67 |
|
|
@@ -105,6 +147,7 @@ def article_disciplines(ror: str) -> pl.LazyFrame:
|
|
| 105 |
|
| 106 |
def get_ror_suggestions(text: str) -> list[dict]:
|
| 107 |
"""Retrieve 50 best matches for ROR ID + name for a given input text."""
|
|
|
|
| 108 |
r = requests.get(
|
| 109 |
'https://api.ror.org/organizations?query=' + text,
|
| 110 |
headers={'Client-Id': ROR_API_KEY},
|
|
|
|
| 33 |
|
| 34 |
def ojs_article_for_institution(ror: str) -> pl.LazyFrame:
|
| 35 |
"""Fetch all OJS articles for the institution with pub year > min_year."""
|
|
|
|
| 36 |
query = (
|
| 37 |
Works()
|
| 38 |
.select(
|
|
|
|
| 61 |
with ARTICLE_CACHE_PATH.open('a') as f:
|
| 62 |
f.write(json.dumps(article) + '\n')
|
| 63 |
|
| 64 |
+
logger.info('Converting.')
|
| 65 |
+
(
|
| 66 |
+
pl.scan_ndjson(ARTICLE_CACHE_PATH)
|
| 67 |
+
.rename({'id': 'work_id'})
|
| 68 |
+
.explode('authorships')
|
| 69 |
+
.unnest('authorships')
|
| 70 |
+
.unnest('author')
|
| 71 |
+
.rename({'id': 'author_id', 'display_name': 'author_name'})
|
| 72 |
+
.filter(pl.col('institutions').list.len() > 0)
|
| 73 |
+
.with_columns(pl.col('institutions').list.get(0).alias('institution'))
|
| 74 |
+
.with_columns(
|
| 75 |
+
(1 / pl.col('work_id').count().over('work_id')).alias('author_weight')
|
| 76 |
+
)
|
| 77 |
+
.unnest('institution', separator=':')
|
| 78 |
+
.unnest('primary_location')
|
| 79 |
+
.unnest('source', separator=':')
|
| 80 |
+
.unnest('primary_topic', separator=':')
|
| 81 |
+
.unnest('primary_topic:field', separator=':')
|
| 82 |
+
.rename(
|
| 83 |
+
{
|
| 84 |
+
'institution:ror': 'ror',
|
| 85 |
+
'source:issn': 'issn',
|
| 86 |
+
'primary_topic:field:display_name': 'field',
|
| 87 |
+
'institution:country_code': 'country_code',
|
| 88 |
+
}
|
| 89 |
+
)
|
| 90 |
+
.filter(
|
| 91 |
+
pl.col('ror').is_not_null(),
|
| 92 |
+
pl.col('author_id').is_not_null(),
|
| 93 |
+
)
|
| 94 |
+
.select(
|
| 95 |
+
'ror',
|
| 96 |
+
'author_id',
|
| 97 |
+
'work_id',
|
| 98 |
+
'publication_year',
|
| 99 |
+
'issn',
|
| 100 |
+
'field',
|
| 101 |
+
'country_code',
|
| 102 |
+
'author_weight',
|
| 103 |
+
)
|
| 104 |
+
.sink_parquet('cache.parquet')
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
return pl.scan_ndjson(ARTICLE_CACHE_PATH)
|
| 108 |
|
| 109 |
|
|
|
|
| 147 |
|
| 148 |
def get_ror_suggestions(text: str) -> list[dict]:
|
| 149 |
"""Retrieve 50 best matches for ROR ID + name for a given input text."""
|
| 150 |
+
logger.info('Getting ROR institutions.')
|
| 151 |
r = requests.get(
|
| 152 |
'https://api.ror.org/organizations?query=' + text,
|
| 153 |
headers={'Client-Id': ROR_API_KEY},
|
dashboard.py
CHANGED
|
@@ -57,16 +57,30 @@ if len(institution_input) > 2:
|
|
| 57 |
)
|
| 58 |
ror_id = option['ror']
|
| 59 |
|
|
|
|
|
|
|
| 60 |
if ror_id is not None and st.button('Analyze'):
|
| 61 |
articles = ojs_article_for_institution(ror_id)
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
)
|
| 58 |
ror_id = option['ror']
|
| 59 |
|
| 60 |
+
fractional = st.toggle('Use Fractional Counting')
|
| 61 |
+
|
| 62 |
if ror_id is not None and st.button('Analyze'):
|
| 63 |
articles = ojs_article_for_institution(ror_id)
|
| 64 |
+
if fractional:
|
| 65 |
+
st.plotly_chart(get_ojs_article_count_line(articles, ror_id, fractional=True))
|
| 66 |
+
st.plotly_chart(get_discipline_bar(articles, ror_id, fractional=True))
|
| 67 |
+
st.warning(
|
| 68 |
+
'Small number of articles below due to testing '
|
| 69 |
+
'with DOAJ and then matching with OJS journals'
|
| 70 |
+
)
|
| 71 |
+
st.plotly_chart(
|
| 72 |
+
get_ojs_journal_locations_bar(articles, ror_id, fractional=True)
|
| 73 |
+
)
|
| 74 |
+
st.plotly_chart(get_country_collab_net(articles, fractional=True))
|
| 75 |
+
st.plotly_chart(get_institution_collab_map(articles, fractional=True))
|
| 76 |
+
else:
|
| 77 |
+
st.plotly_chart(get_share_of_ojs_articles_pie(ror_id))
|
| 78 |
+
st.plotly_chart(get_ojs_article_count_line(articles))
|
| 79 |
+
st.plotly_chart(get_discipline_bar(articles, ror_id))
|
| 80 |
+
st.warning(
|
| 81 |
+
'Small number of articles below due to testing '
|
| 82 |
+
'with DOAJ and then matching with OJS journals'
|
| 83 |
+
)
|
| 84 |
+
st.plotly_chart(get_ojs_journal_locations_bar(articles, ror_id))
|
| 85 |
+
st.plotly_chart(get_country_collab_net(articles))
|
| 86 |
+
st.plotly_chart(get_institution_collab_map(articles))
|
data_processing.py
CHANGED
|
@@ -1,5 +1,4 @@
|
|
| 1 |
import logging
|
| 2 |
-
from itertools import combinations
|
| 3 |
|
| 4 |
import country_converter as coco
|
| 5 |
import polars as pl
|
|
@@ -21,38 +20,67 @@ def get_journals() -> pl.LazyFrame:
|
|
| 21 |
return pl.scan_csv(BEACON_PATH)
|
| 22 |
|
| 23 |
|
| 24 |
-
def articles_to_publication_year_count(
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
|
| 33 |
|
| 34 |
-
def articles_to_disciplines_count(
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
|
| 54 |
|
| 55 |
-
def articles_to_ojs_locations(
|
|
|
|
|
|
|
|
|
|
| 56 |
journals = (
|
| 57 |
get_journals()
|
| 58 |
.select('issn', 'country_consolidated')
|
|
@@ -60,18 +88,25 @@ def articles_to_ojs_locations(df: pl.LazyFrame, mark_country_code: str) -> pl.Da
|
|
| 60 |
.filter(pl.col('issns').is_not_null(), pl.col('issns').list.len() > 0)
|
| 61 |
)
|
| 62 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
return (
|
| 64 |
-
df.
|
| 65 |
-
.unnest('primary_location:source', separator=':')
|
| 66 |
-
.explode('primary_location:source:issn')
|
| 67 |
-
.join(
|
| 68 |
-
journals.explode('issns'),
|
| 69 |
-
left_on='primary_location:source:issn',
|
| 70 |
-
right_on='issns',
|
| 71 |
-
)
|
| 72 |
-
.group_by('country_consolidated')
|
| 73 |
-
.agg(pl.col('id').unique().len())
|
| 74 |
-
.with_columns(
|
| 75 |
pl.col('country_consolidated')
|
| 76 |
.alias('country_name')
|
| 77 |
.map_batches(
|
|
@@ -83,68 +118,76 @@ def articles_to_ojs_locations(df: pl.LazyFrame, mark_country_code: str) -> pl.Da
|
|
| 83 |
.otherwise(pl.lit('Different country'))
|
| 84 |
.alias('color'),
|
| 85 |
)
|
| 86 |
-
.rename({'
|
| 87 |
.sort('Count', descending=True)
|
| 88 |
-
.collect()
|
| 89 |
)
|
| 90 |
|
| 91 |
|
| 92 |
def get_country_code_for_ror(articles: pl.LazyFrame, ror: str) -> str:
|
| 93 |
-
|
| 94 |
-
return next(
|
| 95 |
-
institution['country_code']
|
| 96 |
-
for author in df['authorships'][0] # type: ignore
|
| 97 |
-
for institution in author['institutions']
|
| 98 |
-
if institution['ror'] == ror
|
| 99 |
-
)
|
| 100 |
|
| 101 |
|
| 102 |
-
def articles_to_country_collab_count(
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
.
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
)
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
lambda codes: list(combinations(sorted(codes), 2)),
|
| 121 |
-
return_dtype=pl.List(pl.List(pl.String)),
|
| 122 |
)
|
| 123 |
-
.alias('pairs')
|
| 124 |
)
|
| 125 |
-
|
| 126 |
-
.group_by('
|
| 127 |
-
.
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
pl.col('
|
| 131 |
-
|
| 132 |
)
|
| 133 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
.with_columns(
|
| 135 |
-
pl.col('
|
| 136 |
.alias('country_a')
|
| 137 |
.map_batches(lambda x: pl.Series(coco.convert(x, to='name_short'))),
|
| 138 |
-
pl.col('
|
| 139 |
.alias('country_b')
|
| 140 |
.map_batches(lambda x: pl.Series(coco.convert(x, to='name_short'))),
|
| 141 |
)
|
| 142 |
-
.
|
| 143 |
-
.collect()
|
| 144 |
)
|
|
|
|
| 145 |
|
| 146 |
|
| 147 |
-
def articles_to_institution_collab_count(
|
|
|
|
|
|
|
| 148 |
if not ROR_PATH.exists():
|
| 149 |
download_ror_dataset()
|
| 150 |
|
|
@@ -165,41 +208,46 @@ def articles_to_institution_collab_count(articles: pl.LazyFrame) -> pl.DataFrame
|
|
| 165 |
)
|
| 166 |
)
|
| 167 |
|
| 168 |
-
|
| 169 |
-
articles.
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
.
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
.agg(pl.col('ror').unique().drop_nulls())
|
| 181 |
-
.filter(pl.col('ror').list.len() >= 2)
|
| 182 |
-
.with_columns(
|
| 183 |
-
pl.col('ror')
|
| 184 |
-
.map_elements(
|
| 185 |
-
lambda codes: list(combinations(sorted(codes), 2)),
|
| 186 |
-
return_dtype=pl.List(pl.List(pl.String)),
|
| 187 |
)
|
| 188 |
-
.alias('pairs')
|
| 189 |
)
|
| 190 |
-
|
| 191 |
-
.group_by('
|
| 192 |
-
.
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
pl.col('
|
|
|
|
| 196 |
)
|
| 197 |
-
|
| 198 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 199 |
.rename({'name': 'Institution 1', 'lat': 'lat1', 'lng': 'lng1'})
|
| 200 |
-
.join(ror_df, left_on='
|
| 201 |
.rename({'name': 'Institution 2', 'lat': 'lat2', 'lng': 'lng2'})
|
| 202 |
-
.
|
|
|
|
|
|
|
| 203 |
.sort('count', descending=True)
|
| 204 |
-
.collect()
|
| 205 |
)
|
|
|
|
|
|
| 1 |
import logging
|
|
|
|
| 2 |
|
| 3 |
import country_converter as coco
|
| 4 |
import polars as pl
|
|
|
|
| 20 |
return pl.scan_csv(BEACON_PATH)
|
| 21 |
|
| 22 |
|
| 23 |
+
def articles_to_publication_year_count(
|
| 24 |
+
df: pl.LazyFrame, fractional: bool, ror: str | None = None
|
| 25 |
+
) -> pl.DataFrame:
|
| 26 |
+
if fractional and ror is not None:
|
| 27 |
+
return (
|
| 28 |
+
df.filter(pl.col('ror') == ror)
|
| 29 |
+
.group_by('publication_year')
|
| 30 |
+
.agg(pl.col('author_weight').sum())
|
| 31 |
+
.sort('publication_year')
|
| 32 |
+
.rename({'publication_year': 'Publication Year', 'author_weight': 'Count'})
|
| 33 |
+
.collect() # type: ignore
|
| 34 |
+
)
|
| 35 |
+
elif fractional and ror is None:
|
| 36 |
+
raise Exception('Invalid function arguments')
|
| 37 |
+
else:
|
| 38 |
+
return (
|
| 39 |
+
df.group_by('publication_year')
|
| 40 |
+
.agg(pl.col('work_id').unique().len())
|
| 41 |
+
.sort('publication_year')
|
| 42 |
+
.rename({'publication_year': 'Publication Year', 'work_id': 'Count'})
|
| 43 |
+
.collect() # type: ignore
|
| 44 |
+
)
|
| 45 |
|
| 46 |
|
| 47 |
+
def articles_to_disciplines_count(
|
| 48 |
+
df: pl.LazyFrame, ror: str, fractional: bool
|
| 49 |
+
) -> pl.DataFrame:
|
| 50 |
+
if fractional:
|
| 51 |
+
return (
|
| 52 |
+
df.filter(pl.col('ror') == ror)
|
| 53 |
+
.group_by('field')
|
| 54 |
+
.agg(pl.col('author_weight').sum().alias('Count'))
|
| 55 |
+
.rename({'field': 'Field'})
|
| 56 |
+
.sort('Count', descending=True)
|
| 57 |
+
.with_columns(pl.lit('In OJS Journal').alias('Type'))
|
| 58 |
+
.select('Field', 'Count', 'Type')
|
| 59 |
+
.collect() # type: ignore
|
| 60 |
+
)
|
| 61 |
+
else:
|
| 62 |
+
all_articles = (
|
| 63 |
+
article_disciplines(ror)
|
| 64 |
+
.rename({'key_display_name': 'Field'})
|
| 65 |
+
.select('Field', 'count')
|
| 66 |
+
.rename({'count': 'Count'})
|
| 67 |
+
.with_columns(pl.lit('All articles').alias('Type'))
|
| 68 |
+
)
|
| 69 |
+
df = (
|
| 70 |
+
df.group_by('field')
|
| 71 |
+
.agg(pl.col('work_id').unique().len())
|
| 72 |
+
.rename({'field': 'Field', 'work_id': 'Count'})
|
| 73 |
+
.sort('Count', descending=True)
|
| 74 |
+
.with_columns(pl.lit('In OJS Journal').alias('Type'))
|
| 75 |
+
.select('Field', 'Count', 'Type')
|
| 76 |
+
)
|
| 77 |
+
return pl.concat([df, all_articles]).collect() # type: ignore
|
| 78 |
|
| 79 |
|
| 80 |
+
def articles_to_ojs_locations(
|
| 81 |
+
df: pl.LazyFrame, fractional: bool, ror: str
|
| 82 |
+
) -> pl.DataFrame:
|
| 83 |
+
mark_country_code = get_country_code_for_ror(df, ror)
|
| 84 |
journals = (
|
| 85 |
get_journals()
|
| 86 |
.select('issn', 'country_consolidated')
|
|
|
|
| 88 |
.filter(pl.col('issns').is_not_null(), pl.col('issns').list.len() > 0)
|
| 89 |
)
|
| 90 |
|
| 91 |
+
df = (
|
| 92 |
+
df.filter(pl.col('ror') == ror)
|
| 93 |
+
.explode('issn')
|
| 94 |
+
.join(journals.explode('issns'), left_on='issn', right_on='issns')
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
if fractional and ror is not None:
|
| 98 |
+
df = df.group_by('country_consolidated').agg(
|
| 99 |
+
pl.col('author_weight').sum().alias('Count')
|
| 100 |
+
)
|
| 101 |
+
elif fractional and ror is None:
|
| 102 |
+
raise Exception('Invalid function arguments')
|
| 103 |
+
else:
|
| 104 |
+
df = df.group_by('country_consolidated').agg(
|
| 105 |
+
pl.col('work_id').unique().len().alias('Count')
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
return (
|
| 109 |
+
df.with_columns(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
pl.col('country_consolidated')
|
| 111 |
.alias('country_name')
|
| 112 |
.map_batches(
|
|
|
|
| 118 |
.otherwise(pl.lit('Different country'))
|
| 119 |
.alias('color'),
|
| 120 |
)
|
| 121 |
+
.rename({'country_name': 'Country Name'})
|
| 122 |
.sort('Count', descending=True)
|
| 123 |
+
.collect()
|
| 124 |
)
|
| 125 |
|
| 126 |
|
| 127 |
def get_country_code_for_ror(articles: pl.LazyFrame, ror: str) -> str:
|
| 128 |
+
return articles.filter(pl.col('ror') == ror).head(1).collect()['country_code'][0] # type:ignore
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
|
| 130 |
|
| 131 |
+
def articles_to_country_collab_count(
|
| 132 |
+
articles: pl.LazyFrame, fractional: bool
|
| 133 |
+
) -> dict[str, pl.DataFrame]:
|
| 134 |
+
if fractional:
|
| 135 |
+
# TODO: how to measure edge for fractional counting
|
| 136 |
+
nodes = articles.group_by('country_code').agg(
|
| 137 |
+
pl.col('author_weight').sum().alias('weight')
|
| 138 |
+
)
|
| 139 |
+
df = articles.group_by(['work_id', 'country_code']).agg(
|
| 140 |
+
pl.col('author_weight').sum().alias('country_weight')
|
| 141 |
+
)
|
| 142 |
+
df = (
|
| 143 |
+
df.join(df, on='work_id', how='inner', suffix='_b')
|
| 144 |
+
.filter(pl.col('country_code') < pl.col('country_code_b'))
|
| 145 |
+
.with_columns(
|
| 146 |
+
(pl.col('country_weight') + pl.col('country_weight_b')).alias(
|
| 147 |
+
'pair_weight'
|
| 148 |
+
)
|
|
|
|
|
|
|
| 149 |
)
|
|
|
|
| 150 |
)
|
| 151 |
+
else:
|
| 152 |
+
nodes = articles.group_by('country_code').len().rename({'len': 'weight'})
|
| 153 |
+
df = articles.select('work_id', 'country_code').unique()
|
| 154 |
+
df = (
|
| 155 |
+
df.join(df, on='work_id', how='inner', suffix='_b')
|
| 156 |
+
.filter(pl.col('country_code') < pl.col('country_code_b'))
|
| 157 |
+
.with_columns(pl.lit(1).alias('pair_weight'))
|
| 158 |
)
|
| 159 |
+
|
| 160 |
+
nodes = (
|
| 161 |
+
nodes.with_columns(
|
| 162 |
+
pl.col('country_code')
|
| 163 |
+
.alias('country')
|
| 164 |
+
.map_batches(lambda x: pl.Series(coco.convert(x, to='name_short'))),
|
| 165 |
+
)
|
| 166 |
+
.filter(pl.col('country') != 'not found')
|
| 167 |
+
.select('weight', 'country')
|
| 168 |
+
.collect()
|
| 169 |
+
)
|
| 170 |
+
edges = (
|
| 171 |
+
df.group_by(['country_code', 'country_code_b'])
|
| 172 |
+
.agg(pl.col('pair_weight').sum().alias('count'))
|
| 173 |
+
.sort('count', descending=True)
|
| 174 |
.with_columns(
|
| 175 |
+
pl.col('country_code')
|
| 176 |
.alias('country_a')
|
| 177 |
.map_batches(lambda x: pl.Series(coco.convert(x, to='name_short'))),
|
| 178 |
+
pl.col('country_code_b')
|
| 179 |
.alias('country_b')
|
| 180 |
.map_batches(lambda x: pl.Series(coco.convert(x, to='name_short'))),
|
| 181 |
)
|
| 182 |
+
.select('country_a', 'country_b', 'count')
|
| 183 |
+
.collect()
|
| 184 |
)
|
| 185 |
+
return {'edges': edges, 'nodes': nodes} # type: ignore
|
| 186 |
|
| 187 |
|
| 188 |
+
def articles_to_institution_collab_count(
|
| 189 |
+
articles: pl.LazyFrame, fractional: bool
|
| 190 |
+
) -> dict[str, pl.DataFrame]:
|
| 191 |
if not ROR_PATH.exists():
|
| 192 |
download_ror_dataset()
|
| 193 |
|
|
|
|
| 208 |
)
|
| 209 |
)
|
| 210 |
|
| 211 |
+
if fractional:
|
| 212 |
+
nodes = articles.group_by('ror').agg(
|
| 213 |
+
pl.col('author_weight').sum().alias('weight')
|
| 214 |
+
)
|
| 215 |
+
df = articles.group_by(['work_id', 'ror']).agg(
|
| 216 |
+
pl.col('author_weight').sum().alias('ror_weight')
|
| 217 |
+
)
|
| 218 |
+
df = (
|
| 219 |
+
df.join(df, on='work_id', how='inner', suffix='_b')
|
| 220 |
+
.filter(pl.col('ror') < pl.col('ror_b'))
|
| 221 |
+
.with_columns(
|
| 222 |
+
(pl.col('ror_weight') + pl.col('ror_weight_b')).alias('pair_weight')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
)
|
|
|
|
| 224 |
)
|
| 225 |
+
else:
|
| 226 |
+
nodes = articles.group_by('ror').len().rename({'len': 'weight'})
|
| 227 |
+
df = articles.select('work_id', 'ror').unique()
|
| 228 |
+
df = (
|
| 229 |
+
df.join(df, on='work_id', how='inner', suffix='_b')
|
| 230 |
+
.filter(pl.col('ror') < pl.col('ror_b'))
|
| 231 |
+
.with_columns(pl.lit(1).alias('pair_weight'))
|
| 232 |
)
|
| 233 |
+
|
| 234 |
+
nodes = (
|
| 235 |
+
nodes.join(ror_df, left_on='ror', right_on='id')
|
| 236 |
+
.select('weight', 'name', 'lat', 'lng')
|
| 237 |
+
.rename({'name': 'Institution'})
|
| 238 |
+
.collect()
|
| 239 |
+
)
|
| 240 |
+
edges = (
|
| 241 |
+
df.group_by(['ror', 'ror_b'])
|
| 242 |
+
.agg(pl.col('pair_weight').sum().alias('count'))
|
| 243 |
+
.join(ror_df, left_on='ror', right_on='id')
|
| 244 |
.rename({'name': 'Institution 1', 'lat': 'lat1', 'lng': 'lng1'})
|
| 245 |
+
.join(ror_df, left_on='ror_b', right_on='id')
|
| 246 |
.rename({'name': 'Institution 2', 'lat': 'lat2', 'lng': 'lng2'})
|
| 247 |
+
.select(
|
| 248 |
+
'Institution 1', 'lat1', 'lng1', 'Institution 2', 'lat2', 'lng2', 'count'
|
| 249 |
+
)
|
| 250 |
.sort('count', descending=True)
|
| 251 |
+
.collect()
|
| 252 |
)
|
| 253 |
+
return {'edges': edges, 'nodes': nodes} # type: ignore
|
data_visualization.py
CHANGED
|
@@ -10,7 +10,6 @@ from data_processing import (
|
|
| 10 |
articles_to_institution_collab_count,
|
| 11 |
articles_to_ojs_locations,
|
| 12 |
articles_to_publication_year_count,
|
| 13 |
-
get_country_code_for_ror,
|
| 14 |
)
|
| 15 |
|
| 16 |
|
|
@@ -23,18 +22,22 @@ def get_share_of_ojs_articles_pie(ror: str) -> go.Figure:
|
|
| 23 |
)
|
| 24 |
|
| 25 |
|
| 26 |
-
def get_ojs_article_count_line(
|
|
|
|
|
|
|
| 27 |
return px.line(
|
| 28 |
-
articles_to_publication_year_count(articles),
|
| 29 |
x='Publication Year',
|
| 30 |
y='Count',
|
| 31 |
title='OJS Article per Year',
|
| 32 |
)
|
| 33 |
|
| 34 |
|
| 35 |
-
def get_discipline_bar(
|
|
|
|
|
|
|
| 36 |
return px.bar(
|
| 37 |
-
articles_to_disciplines_count(articles, ror),
|
| 38 |
x='Field',
|
| 39 |
y='Count',
|
| 40 |
color='Type',
|
|
@@ -44,10 +47,11 @@ def get_discipline_bar(articles: pl.LazyFrame, ror: str) -> go.Figure:
|
|
| 44 |
)
|
| 45 |
|
| 46 |
|
| 47 |
-
def get_ojs_journal_locations_bar(
|
| 48 |
-
|
|
|
|
| 49 |
return px.bar(
|
| 50 |
-
articles_to_ojs_locations(articles,
|
| 51 |
x='Country Name',
|
| 52 |
y='Count',
|
| 53 |
color='color',
|
|
@@ -75,14 +79,14 @@ def __create_network_chart(graph: nx.Graph) -> go.Figure:
|
|
| 75 |
)
|
| 76 |
)
|
| 77 |
|
|
|
|
| 78 |
node_x, node_y, node_text, node_size = [], [], [], []
|
| 79 |
-
for node in graph.nodes():
|
| 80 |
x, y = pos[node]
|
| 81 |
node_x.append(x)
|
| 82 |
node_y.append(y)
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
node_size.append(5 + (2 * degree / max_weight))
|
| 86 |
|
| 87 |
node_trace = go.Scatter(
|
| 88 |
x=node_x,
|
|
@@ -117,11 +121,15 @@ def __create_network_chart(graph: nx.Graph) -> go.Figure:
|
|
| 117 |
return fig
|
| 118 |
|
| 119 |
|
| 120 |
-
def get_country_collab_net(
|
| 121 |
-
|
|
|
|
|
|
|
| 122 |
G = nx.Graph()
|
| 123 |
-
for row in collabs.iter_rows(named=True):
|
| 124 |
G.add_edge(row['country_a'], row['country_b'], weight=row['count'])
|
|
|
|
|
|
|
| 125 |
|
| 126 |
fig = __create_network_chart(G)
|
| 127 |
fig.layout.title = 'Network of Country Collaborations'
|
|
@@ -129,35 +137,34 @@ def get_country_collab_net(articles: pl.LazyFrame) -> go.Figure:
|
|
| 129 |
return fig
|
| 130 |
|
| 131 |
|
| 132 |
-
def get_institution_collab_map(
|
|
|
|
|
|
|
| 133 |
"""Get world map with co-authoring institutes highlighted."""
|
| 134 |
threshold = 1_000
|
| 135 |
-
|
|
|
|
| 136 |
|
| 137 |
-
nodes =
|
| 138 |
-
[
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
{'Institution 2': 'Institution 1', 'lat2': 'lat1', 'lng2': 'lng1'}
|
| 142 |
-
),
|
| 143 |
-
]
|
| 144 |
-
).unique()
|
| 145 |
|
| 146 |
fig = go.Figure()
|
| 147 |
fig.add_trace(
|
| 148 |
go.Scattergeo(
|
| 149 |
-
lon=nodes['
|
| 150 |
-
lat=nodes['
|
| 151 |
-
text=nodes['Institution
|
| 152 |
mode='markers',
|
| 153 |
marker={'size': 4, 'color': 'rgba(0, 100, 125)', 'opacity': 0.9},
|
| 154 |
hoverinfo='text',
|
| 155 |
)
|
| 156 |
)
|
| 157 |
|
| 158 |
-
max_collab =
|
| 159 |
|
| 160 |
-
for row in
|
| 161 |
fig.add_trace(
|
| 162 |
go.Scattergeo(
|
| 163 |
lon=[row['lng1'], row['lng2']],
|
|
|
|
| 10 |
articles_to_institution_collab_count,
|
| 11 |
articles_to_ojs_locations,
|
| 12 |
articles_to_publication_year_count,
|
|
|
|
| 13 |
)
|
| 14 |
|
| 15 |
|
|
|
|
| 22 |
)
|
| 23 |
|
| 24 |
|
| 25 |
+
def get_ojs_article_count_line(
|
| 26 |
+
articles: pl.LazyFrame, ror: str | None = None, fractional: bool = False
|
| 27 |
+
) -> go.Figure:
|
| 28 |
return px.line(
|
| 29 |
+
articles_to_publication_year_count(articles, fractional, ror),
|
| 30 |
x='Publication Year',
|
| 31 |
y='Count',
|
| 32 |
title='OJS Article per Year',
|
| 33 |
)
|
| 34 |
|
| 35 |
|
| 36 |
+
def get_discipline_bar(
|
| 37 |
+
articles: pl.LazyFrame, ror: str, fractional: bool = False
|
| 38 |
+
) -> go.Figure:
|
| 39 |
return px.bar(
|
| 40 |
+
articles_to_disciplines_count(articles, ror, fractional),
|
| 41 |
x='Field',
|
| 42 |
y='Count',
|
| 43 |
color='Type',
|
|
|
|
| 47 |
)
|
| 48 |
|
| 49 |
|
| 50 |
+
def get_ojs_journal_locations_bar(
|
| 51 |
+
articles: pl.LazyFrame, ror: str, fractional: bool = False
|
| 52 |
+
) -> go.Figure:
|
| 53 |
return px.bar(
|
| 54 |
+
articles_to_ojs_locations(articles, fractional, ror),
|
| 55 |
x='Country Name',
|
| 56 |
y='Count',
|
| 57 |
color='color',
|
|
|
|
| 79 |
)
|
| 80 |
)
|
| 81 |
|
| 82 |
+
max_weight = max(d['weight'] for _, d in graph.nodes(data=True))
|
| 83 |
node_x, node_y, node_text, node_size = [], [], [], []
|
| 84 |
+
for node, data in graph.nodes(data=True):
|
| 85 |
x, y = pos[node]
|
| 86 |
node_x.append(x)
|
| 87 |
node_y.append(y)
|
| 88 |
+
node_text.append(f'{node}<br>Total collabs: {data["weight"]}')
|
| 89 |
+
node_size.append(5 + (2 * data['weight'] / max_weight))
|
|
|
|
| 90 |
|
| 91 |
node_trace = go.Scatter(
|
| 92 |
x=node_x,
|
|
|
|
| 121 |
return fig
|
| 122 |
|
| 123 |
|
| 124 |
+
def get_country_collab_net(
|
| 125 |
+
articles: pl.LazyFrame, fractional: bool = False
|
| 126 |
+
) -> go.Figure:
|
| 127 |
+
collabs = articles_to_country_collab_count(articles, fractional)
|
| 128 |
G = nx.Graph()
|
| 129 |
+
for row in collabs['edges'].iter_rows(named=True):
|
| 130 |
G.add_edge(row['country_a'], row['country_b'], weight=row['count'])
|
| 131 |
+
for row in collabs['nodes'].iter_rows(named=True):
|
| 132 |
+
G.add_node(row['country'], weight=row['weight'])
|
| 133 |
|
| 134 |
fig = __create_network_chart(G)
|
| 135 |
fig.layout.title = 'Network of Country Collaborations'
|
|
|
|
| 137 |
return fig
|
| 138 |
|
| 139 |
|
| 140 |
+
def get_institution_collab_map(
|
| 141 |
+
articles: pl.LazyFrame, fractional: bool = False
|
| 142 |
+
) -> go.Figure:
|
| 143 |
"""Get world map with co-authoring institutes highlighted."""
|
| 144 |
threshold = 1_000
|
| 145 |
+
collabs = articles_to_institution_collab_count(articles, fractional)
|
| 146 |
+
edges = collabs['edges'][:threshold]
|
| 147 |
|
| 148 |
+
nodes = collabs['nodes'].filter(
|
| 149 |
+
pl.col('Institution').is_in(edges['Institution 1'].implode())
|
| 150 |
+
| pl.col('Institution').is_in(edges['Institution 2'].implode()),
|
| 151 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 152 |
|
| 153 |
fig = go.Figure()
|
| 154 |
fig.add_trace(
|
| 155 |
go.Scattergeo(
|
| 156 |
+
lon=nodes['lng'],
|
| 157 |
+
lat=nodes['lat'],
|
| 158 |
+
text=nodes['Institution'],
|
| 159 |
mode='markers',
|
| 160 |
marker={'size': 4, 'color': 'rgba(0, 100, 125)', 'opacity': 0.9},
|
| 161 |
hoverinfo='text',
|
| 162 |
)
|
| 163 |
)
|
| 164 |
|
| 165 |
+
max_collab = edges['count'].max()
|
| 166 |
|
| 167 |
+
for row in edges.iter_rows(named=True):
|
| 168 |
fig.add_trace(
|
| 169 |
go.Scattergeo(
|
| 170 |
lon=[row['lng1'], row['lng2']],
|