Delete Climate_site

Climate_site/pages/2_Scientific_publications_and_Authors.py

@@ -1,211 +0,0 @@
-
-import streamlit as st
-
-st.set_page_config(page_title="Scientific publications and authors", page_icon="📖")
-
-
-from streamlit_folium import folium_static
-import folium
-import pandas as pd
-#from python_scripts import functions
-from python_scripts import paper_functions
-from python_scripts import others
-
-@st.cache
-def convert_df(df):
-    # IMPORTANT: Cache the conversion to prevent computation on every rerun
-    return df.to_csv().encode('utf-8')
-
-
-header = st.container()
-
-paper_finder = st.container()
-
-
-with paper_finder:
-    
-  
-  st.markdown("# Scientific publications and authors related to IEA technologies")
-  st.sidebar.header("Scientific publications and authors")
-  st.write(
-        """You can find the most related scientific publications and authors for each climate related IEA technology. Enjoy!"""
-    )
-
-  st.header('Which papers are related to the technology?')
-
-
-  st.subheader("Select the technology you want to look at")
-  dic_technologies, dic_categories, list_categories_tech, list_technologies = paper_functions.finder()
-
-  list_technologies = [ ( list_categories_tech[i] , i ) for i in range(len(list_categories_tech))  ] 
-  key0 = lambda t: t[0]
-    
-    
-   # Choose the category
-  tech_category = st.selectbox('Select a category',list_technologies, format_func = key0)
-
-    
-  # After a selection is made the choices will update
-  # Choose the technology and number
-  technologies = dic_technologies[tech_category[0]]
-
-  technology = st.selectbox('Select a technology',technologies,format_func = key0)
-  
-
-  # Select whether or not to use keywords
-  list_keyword_option = [ ("No pre-select them for me" , False) , ("Yes" , True )]
-  list_keyword_bool = st.selectbox('Would you like to enter your own keywords?',list_keyword_option,format_func = key0)
-
-  if list_keyword_bool[1] == True:
-    tech_category_keyword = st.text_input("Keywords", 'Input your text here')
-  else:
-    tech_categories = dic_categories[technology[1]]
-    tech_category_keyword = st.radio(
-      "Select a keyword to search on: ",
-      tech_categories)
-
-    
-  # Select type of papers (related to climate or not)
-  list_climate = [ ("Any related papers" , False ) , ("Climate related papers" , True)]
-  patent_type_bool = st.selectbox('What type of papers do you want?',list_climate,format_func = key0)
-
-  size = st.slider('How many papers do you want?', 0, 100, 5)
-
-  st.subheader("Look at the related scientific publications and authors")
-  #try:
-  #if st.button('Get more information about the technology'):
-  reference_text, text1 , text2  = paper_functions.extract_quantitative_data_technology(
-  technologies = technology[0],
-  number_technology = technology[1],
-  )
-  
-  st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Technology details:</strong> {reference_text}' , unsafe_allow_html = True)
-  st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Deployment target and Announced development target:</strong> {text1}' , unsafe_allow_html = True)
-  st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Announced cost reduction targets:</strong> {text2}' , unsafe_allow_html = True)
-
-  #except:
-  #  st.write("No information about the technology")
-    
-
-
-  #try:
-  if st.button('Get related papers'):
-     paper_rank_df = paper_functions.get_ranking_related_papers(
-     technologies = tech_category[1],
-     number_technology = technology[1],
-     research_words = tech_category_keyword,
-     carbon_related = patent_type_bool[1],
-     size = size
-     )
-     st.dataframe(paper_rank_df)
-     #print(paper_rank_df)
-  #except:
-  #  st.write("No paper found, try with other key words")
-        
-  #try:  
-  if st.button('Get the main authors'):
-     authors = paper_functions.main_authors(
-     technologies = tech_category[1],
-     number_technology = technology[1],
-     research_words = tech_category_keyword,
-     carbon_related = patent_type_bool[1],
-     size = size
-     )
-     st.dataframe(authors)
-
-
-     csv = convert_df(authors)
-
-     st.download_button(
-     label="Download table as CSV",
-     data=csv,
-     file_name='authors.csv',
-     )
-     #print(patent_rank_df)
-#      
- # except:
- #   st.write("No author found, try with other key words")
-  
-  #      
-  #try:
-    
-  if st.button('Get the map of the main authors'):
-    map_author = paper_functions.map_authors(
-    technologies = tech_category[1],
-    number_technology = technology[1],
-    research_words = tech_category_keyword,
-    carbon_related = patent_type_bool[1],
-    size = size)
-    #st.map(map_author)
-    m = folium.Map(zoom_start = 1)
-    dic_geo = {}
-    for elem in map_author.index:
-
-        if map_author["institution"][elem] != "Max Planck Institute for Plasma Physics - Greifswald":
-          longitude = map_author["longitude"][elem]
-          latitude = map_author["latitude"][elem]
-          if str(longitude) + " " + str(latitude) not in dic_geo:
-            dic_geo[str(longitude) + " " + str(latitude)] = {}
-            dic_geo[str(longitude) + " " + str(latitude)]["authors"] = set()
-            dic_geo[str(longitude) + " " + str(latitude)]["institutions"] = set()
-            dic_geo[str(longitude) + " " + str(latitude)]["dates"] = set()
-          dic_geo[str(longitude) + " " + str(latitude)]["authors"].add(map_author["author"][elem])
-          dic_geo[str(longitude) + " " + str(latitude)]["institutions"].add(map_author["institution"][elem])
-          dic_geo[str(longitude) + " " + str(latitude)]["dates"].add(map_author["date"][elem])
-
-    for elem in dic_geo:
-      folium.Marker([float(elem.split()[1]), float(elem.split()[0])], popup="Author: " + ", ".join(list(dic_geo[elem]["authors"])) + ", Institution: " + ", ".join(list(dic_geo[elem]["institutions"])) + ", Date: " + ", ".join(list(dic_geo[elem]["dates"])), tooltip=", ".join(list(dic_geo[elem]["authors"]))).add_to(m)
-    folium_static(m)
-         
-  #except:
-  #  st.write("No author found, try with other key words")
-  #      
-  #      
-  #
-  
-  
-  #      
-  #      
-  #      
-  if st.button('Get more information about the related_projects'):
-    try:
-      reference_text,key_initiative,location_entities,papers = paper_functions.related_projects(
-      technologies = technology[0], 
-      number_technology = technology[1], 
-      carbon_related= patent_type_bool[1], 
-      size = size)
-      
-      st.write(f'<p style="color:Blue"> From IEA Website:' , unsafe_allow_html = True)
-      st.write(f'<strong>Technology details:</strong>  {reference_text}' ,  unsafe_allow_html = True)
-      st.write(f'<strong>Technology key initiative:</strong> {key_initiative}' ,  unsafe_allow_html = True)
-      st.write(f'<strong>Extracted projects and organization name:</strong> {location_entities}' ,  unsafe_allow_html = True)
-      st.write(f'<p style="color:Red"> From OpenAlex Website:' , unsafe_allow_html = True)
-      st.dataframe(papers)
-    except:
-       try:
-          st.write(f'<p style="color:Blue"> From IEA Website:' , unsafe_allow_html = True)
-          st.write(f'<strong>Technology details:</strong>  {reference_text}' , unsafe_allow_html = True)
-          st.write(f'<strong>Technology key initiatives:</strong> {key_initiative}' ,  unsafe_allow_html = True)
-          st.write(f'<strong>Extracted projects and organization name:</strong> {location_entities}' ,  unsafe_allow_html = True)
-          st.write(f'<p style="color:Red"> From OpenAlex Website:' , unsafe_allow_html = True)
-          st.write(f'No related paper found')
-       except:
-          st.write("No key initatives found")
-          
-          
-  try:
-    paper_id = st.text_input("More data on a paper?", 'Enter an OpenAlex paper id (ex: W100000)')
-    url , title , abstract , date , authors , institutions = paper_functions.extract_quantitative_data_paper(
-    work_id = paper_id)
-    
-    st.write(f'<p style="color:Red"> From OpenAlex Website:' , unsafe_allow_html = True)
-    st.write(f'<p style="color:Black"> <strong> Paper link:</strong> <a href="url">{url}' , unsafe_allow_html = True)
-    st.write(f'<strong> Paper title:</strong> {title}' , unsafe_allow_html = True)
-    st.write(f'<strong> Paper abstract:</strong> {abstract}' , unsafe_allow_html = True)
-    st.write(f'<strong> Paper date:</strong> {date}' , unsafe_allow_html = True)
-    st.write(f'<strong> Paper co-inventors:</strong> {authors}' , unsafe_allow_html = True)
-    st.write(f'<strong> Paper assignees:</strong> {institutions}' , unsafe_allow_html = True)
-  #      
-  except:
-    st.write("Enter a valid work id")
-  

Climate_site/pages/3_Related_Companies.py

@@ -1,88 +0,0 @@
-import streamlit as st
-
-st.set_page_config(page_title="Related Companies", page_icon="🏛")
-
-
-import pandas as pd
-from python_scripts import functions
-from python_scripts import functions_companies
-
-@st.cache
-def convert_df(df):
-    # IMPORTANT: Cache the conversion to prevent computation on every rerun
-    return df.to_csv()
-
-
-header = st.container()
-
-company_finder = st.container()
-
-
-with company_finder:
-    
-  
-  st.markdown("# Related companies")
-  st.sidebar.header("Related companies")
-  st.write(
-        """You can find the most related companies for each climate related IEA technology. Enjoy!"""
-    )
-
-
-  st.subheader("Select the technology you want to look at")
-  dic_technologies, dic_categories, list_categories_tech, list_technologies = functions_companies.finder()
-
-  list_technologies = [ ( list_categories_tech[i] , i ) for i in range(len(list_categories_tech))  ] 
-  key0 = lambda t: t[0]
-    
-    
-   # Choose the category
-  tech_category = st.selectbox('Select a category',list_technologies, format_func = key0)
-
-    
-  # After a selection is made the choices will update
-  # Choose the technology and number
-  technologies = dic_technologies[tech_category[0]]
-
-  technology = st.selectbox('Select a technology',technologies,format_func = key0)
-  
-  size = st.slider('How many companies do you want?', 0, 50, 10)
-  
-
-  st.subheader("Look at the related companies")
-    
-    
-  try:
-     reference_text, text1 , text2 = functions_companies.extract_quantitative_data_technology(
-     technologies = technology[0],
-     number_technology = technology[1],
-     )
-  
-     st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Technology details:</strong> {reference_text}' , unsafe_allow_html = True)
-     st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Deployment target and Announced development target:</strong> {text1}' , unsafe_allow_html = True)
-     st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Announced cost reduction targets:</strong> {text2}' , unsafe_allow_html = True)
-
-  except:
-    st.write("No information about the technology")
-    
-
-
-  #try:
-  if st.button('Get related companies'):
-    companies_ranked = functions_companies.related_VC_deals(
-    category = tech_category[1],
-    number_technology = technology[1],
-    size = size
-    )
-    st.dataframe(companies_ranked.style.format({"year_established": '{:.0f}'}))
-    csv = convert_df(companies_ranked)
-
-    st.download_button(
-         label="Download table as CSV",
-         data=csv,
-        file_name='companies.csv',
-            
-        )
-  #except:
-  #  st.write("No company found, try with other key words")
-        
-  

Climate_site/pages/4_University_contests.py

@@ -1,69 +0,0 @@
-import streamlit as st
-
-st.set_page_config(page_title="Related university projects", page_icon="🏛")
-
-
-import pandas as pd
-from python_scripts import functions
-from python_scripts import functions_contest
-
-
-
-header = st.container()
-
-company_finder = st.container()
-
-
-with company_finder:
-    
-  
-  st.markdown("# Related university projects")
-  st.sidebar.header("Related university projects")
-  st.write(
-        """You can find the most related projects which participated to the Big Ideas contest: [Big Ideas website](https://bigideascontest.org). Enjoy!"""
-  )
-
-
-  st.subheader("Select the technology you want to look at")
-  dic_technologies, dic_categories, list_categories_tech, list_technologies = functions_contest.finder()
-
-  list_technologies = [ ( list_categories_tech[i] , i ) for i in range(len(list_categories_tech))  ] 
-  key0 = lambda t: t[0]
-    
-    
-   # Choose the category
-  tech_category = st.selectbox('Select a category',list_technologies, format_func = key0)
-
-    
-  # After a selection is made the choices will update
-  # Choose the technology and number
-  technologies = dic_technologies[tech_category[0]]
-
-  technology = st.selectbox('Select a technology',technologies,format_func = key0)
-    
-
-  st.subheader("Look at the related projects")
-    
-    
-  try:
-     reference_text, text1 , text2 = functions_contest.extract_quantitative_data_technology(
-     technologies = technology[0],
-     number_technology = technology[1],
-     )
-  
-     st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Technology details:</strong> {reference_text}' , unsafe_allow_html = True)
-     st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Deployment target and Announced development target:</strong> {text1}' , unsafe_allow_html = True)
-     st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Announced cost reduction targets:</strong> {text2}' , unsafe_allow_html = True)
-
-  except:
-    st.write("No information about the technology")
-    
-
-
-  try:
-      if st.button('Get related projects'):
-        companies_ranked = functions_contest.big_ideas_encoding(
-        number_technology = technology[1])
-        st.dataframe(companies_ranked)
-  except:
-    st.write("No project found.")

Climate_site/pages/5_Your_own_research:_Patents.py

@@ -1,109 +0,0 @@
-import streamlit as st
-
-st.set_page_config(page_title="Search patents related to your own technology", page_icon="🙋‍♀️")
-
-import pandas as pd
-from python_scripts import patent_functions_own_details
-from streamlit_folium import folium_static
-import folium
-
-
-
-header = st.container()
-
-own_paper_finder = st.container()
-
-
-with own_paper_finder:
-    
-  
-  st.markdown("# Your own research: Patents and Inventors ")
-  st.sidebar.header("Patents and Inventors")
-  st.write(
-        """You can find the most related patents and inventors to your technology. Enjoy!"""
-    )
-
-
-  st.subheader("We are searching for the patents which contain (in the title or abstract) the key words you can provide below:")
- 
-  key_words = st.text_input("Keywords", 'Input your text here')
-  
-  st.subheader("The patents are ranked according to the similary between their abstract and the description you can provide below:")
-
-  details = st.text_input("Details", 'Input your text here')
-  
-  size = st.slider('How many patents do you want?', 0, 200, 20)
-
-
-    
-    
-    
-  try:
-    if st.button('Get related patents'):
-       patent_rank_df = patent_functions_own_details.get_ranking_own_research(research_key_words = key_words,  details = details  , size=size)
- 
-       st.dataframe(patent_rank_df)
-    
-
-  except:
-    st.write("No patent found, try with other key words")
-    
-    
-    
-        
-  try:  
-    if st.button('Get the main inventors'):
-       inventors = patent_functions_own_details.main_inventors(research_key_words = key_words,  details = details   , size=size)
-       st.dataframe(inventors)
-  
-  except:
-    st.write("No inventor found, try with other key words")
-
-    
-    
-
-  try:
-      
-    if st.button('Get the map of the main inventors patents'):
-      map_inventor = patent_functions_own_details.map_inventors(
-      research_key_words = key_words,  details = details   , size=size)
-
-      m = folium.Map(zoom_start = 1)
-      dic_geo = {}
-      for elem in map_inventor.index:
-  
-        longitude = map_inventor["longitude"][elem]
-        latitude = map_inventor["latitude"][elem]
-        if str(longitude) + " " + str(latitude) not in dic_geo:
-          dic_geo[str(longitude) + " " + str(latitude)] = {}
-          dic_geo[str(longitude) + " " + str(latitude)]["inventors"] = set()
-          dic_geo[str(longitude) + " " + str(latitude)]["dates"] = set()
-       
-        dic_geo[str(longitude) + " " + str(latitude)]["inventors"].add(map_inventor["inventor_name"][elem])
-        dic_geo[str(longitude) + " " + str(latitude)]["dates"].add(map_inventor["patent_date"][elem])
-  
-      for elem in dic_geo:
-        folium.Marker([float(elem.split()[1]), float(elem.split()[0])], popup="Inventors: " + ", ".join(list(dic_geo[elem]["inventors"])) + ", Date: " + ", ".join(list(dic_geo[elem]["dates"])), tooltip=", ".join(list(dic_geo[elem]["inventors"]))).add_to(m)
-      folium_static(m)
-           
-  except:
-    st.write("No inventor found, try with other key words")
-    
-    
-  try:
-      patent_id = st.text_input("More data on a patent?", 'Enter a US patent (ex: US-100000)')
-      url , title , abstract , date , co_inventors , assignees = patent_functions_own_details.extract_quantitative_data_patent(
-      patent_id = patent_id)
-
-      st.write(f'<p style="color:Black"> <strong> Patent link:</strong> <a href="url">{url}' , unsafe_allow_html = True)
-      st.write(f'<p style="color:Blue"> From PatentsView Website:' , unsafe_allow_html = True)
-      st.write(f'<strong> Patent title:</strong> {title}' , unsafe_allow_html = True)
-      st.write(f'<strong> Patent abstract:</strong> {abstract}' , unsafe_allow_html = True)
-      st.write(f'<strong> Patent date:</strong> {date}' , unsafe_allow_html = True)
-      st.write(f'<strong> Patent co-inventors:</strong> {co_inventors}' , unsafe_allow_html = True)
-      st.write(f'<strong> Patent assignees:</strong> {assignees}' , unsafe_allow_html = True)
-        
-  except:
-    st.write("Enter a valid patent id")
-
-    

Climate_site/pages/6_Your_own_research:_Scientific_Publications.py

@@ -1,105 +0,0 @@
-import streamlit as st
-
-st.set_page_config(page_title="Search papers with your own technology", page_icon="🙋‍♀️")
-
-import pandas as pd
-from python_scripts import paper_functions_own_details
-from streamlit_folium import folium_static
-import folium
-
-
-
-header = st.container()
-
-own_paper_finder = st.container()
-
-
-with own_paper_finder:
-    
-  
-  st.markdown("# Your own research: Scientific publications and authors ")
-  st.sidebar.header("Scientific publications and authors")
-  st.write(
-        """You can find the most related scientific publications and authors to your technology. Enjoy!"""
-    )
-
-
-  st.subheader("We are searching for the scientific publications which contain (in the title, abstract or text) the key words you can provide below:")
- 
-  key_words = st.text_input("Keywords", 'Input your text here')
-  
-  st.subheader("The scientific publications are ranked according to the similary between their abstract and the description you can provide below:")
-
-  details = st.text_input("Details", 'Input your text here')
-  
-  size = st.slider('How many papers do you want?', 0, 200, 20)
-
-
-  try:
-    if st.button('Get related papers'):
-       paper_rank_df = paper_functions_own_details.get_ranking_own_research(research_key_words = key_words,  details = details  , display= False , size=size)
- 
-       st.dataframe(paper_rank_df)
-        #print(paper_rank_df)
-  except:
-    st.write("No paper found, try with other key words")
-    
-    
-    
-    
-  #try:  
-  if st.button('Get the main authors'):
-     authors = paper_functions_own_details.main_authors(research_key_words = key_words,  details = details   , size=size)
-     st.dataframe(authors)
-  
-  #except:
-  #  st.write("No author found, try with other key words")
-
-
-
-
-  try:
-   
-    if st.button('Get the map of the main authors'):
-      map_author = paper_functions_own_details.map_authors(research_key_words = key_words,  details = details   , size=size)
-      m = folium.Map(zoom_start = 1)
-      dic_geo = {}
-      for elem in map_author.index:
-      
-            longitude = map_author["longitude"][elem]
-            latitude = map_author["latitude"][elem]
-            if str(longitude) + " " + str(latitude) not in dic_geo:
-              dic_geo[str(longitude) + " " + str(latitude)] = {}
-              dic_geo[str(longitude) + " " + str(latitude)]["authors"] = set()
-              dic_geo[str(longitude) + " " + str(latitude)]["institutions"] = set()
-              dic_geo[str(longitude) + " " + str(latitude)]["dates"] = set()
-            dic_geo[str(longitude) + " " + str(latitude)]["authors"].add(map_author["author"][elem])
-            dic_geo[str(longitude) + " " + str(latitude)]["institutions"].add(map_author["institution"][elem])
-            dic_geo[str(longitude) + " " + str(latitude)]["dates"].add(map_author["date"][elem])
-      
-      for elem in dic_geo:
-        folium.Marker([float(elem.split()[1]), float(elem.split()[0])], popup="Author: " + ", ".join(list(dic_geo[elem]["authors"])) + ", Institution: " + ", ".join(list(dic_geo[elem]["institutions"])) + ", Date: " + ", ".join(list(dic_geo[elem]["dates"])), tooltip=", ".join(list(dic_geo[elem]["authors"]))).add_to(m)
-      folium_static(m)
-            
-  except:
-    st.write("No author found, try with other key words")
-
-    
-    
-    
-    
-  try:
-    paper_id = st.text_input("More data on a paper?", 'Enter an OpenAlex paper id (ex: W100000)')
-    url , title , abstract , date , authors , institutions = paper_functions_own_details.extract_quantitative_data_paper(
-    work_id = paper_id)
-    
-    st.write(f'<p style="color:Red"> From OpenAlex Website:' , unsafe_allow_html = True)
-    st.write(f'<p style="color:Black"> <strong> Paper link:</strong> <a href="url">{url}' , unsafe_allow_html = True)
-    st.write(f'<strong> Paper title:</strong> {title}' , unsafe_allow_html = True)
-    st.write(f'<strong> Paper abstract:</strong> {abstract}' , unsafe_allow_html = True)
-    st.write(f'<strong> Paper date:</strong> {date}' , unsafe_allow_html = True)
-    st.write(f'<strong> Paper co-inventors:</strong> {authors}' , unsafe_allow_html = True)
-    st.write(f'<strong> Paper assignees:</strong> {institutions}' , unsafe_allow_html = True)
-  #      
-  except:
-    st.write("Enter a valid work id")

Climate_site/pages/7_Your_own_research:_Related_Scientist.py

@@ -1,107 +0,0 @@
-import streamlit as st
-
-st.set_page_config(page_title="Search related scientist", page_icon="🙋‍♀️")
-
-import pandas as pd
-from python_scripts import related_scientist
-from streamlit_folium import folium_static
-import folium
-
-
-
-header = st.container()
-
-own_paper_finder = st.container()
-
-
-with own_paper_finder:
-    
-  
-  st.markdown("# Searching the scientists related to somebody")
-  st.sidebar.header("Related scientists")
-
-
-
-  st.subheader("We are looking for the scientists who cite the scientist of your choice.")
- 
-  key_name = st.text_input("Enter the name of a scientist", 'Scientist name')
-    
-  key0 = lambda t: t[0]
-  key1 = lambda t: t[1]
-  
-
-  #try:
-  if key_name == 'Scientist name':
-    st.write("Enter a scientist name")
-    
-  else:
-
-    pandas_scientific_ids , scientist_ids = related_scientist.author_id_from_name(name = key_name)
-
-    st.dataframe(pandas_scientific_ids)
-
-
-
-    #st.write('Select the identifier(s) which correspond to the scientist you are looking for:')
-
-
-    author_ids = st.multiselect("Select the identifier(s) which correspond to the scientist you are looking for: ", scientist_ids, format_func = key1 )
-
-    size = st.slider('How many scientists do you want?', 0, 100, 10)
-
-    year = st.selectbox("I want to look at the citations since: ", [ k for k in range(1980, 2023) ] , index = 38)
-
-
-    list_option = [ ("No, I want to look at all scientist you can find." , 0) , ("Yes. I want to search by continent." , 1 ), ("Yes. I want to search by country." , 2 )]
-    list_keyword_bool = st.selectbox('Would you like to look at the scientist from a specific country?',list_option,format_func = key0)
-
-    if list_keyword_bool[1] == 2:
-      country_code = st.text_input("Enter a country code", 'US')
-    elif list_keyword_bool[1] == 1:
-      country_code = st.selectbox("Select a continent", [ "Europe" , "North America" , "Asia" , "South America", "Oceania" , "Africa" ] , index = 0)
-    else:
-      country_code = False
-    
-    if len(author_ids) == 0:
-        st.write("Select the identifier(s) which correspond to the scientist you are looking for")
-        
-    else:
-
-
-
-        if st.button('Get the related scientist (it can take a few minutes, please, be patient 😊)'):
-
-            scientists = related_scientist.from_author_id(
-            main_author_ids = [ author_id[0] for author_id in author_ids] ,
-            year = year,
-            country_code = country_code,
-            size = size)
-
-            st.dataframe(scientists[0])
-
-            map_author = scientists[1]
-            m = folium.Map(zoom_start = 1)
-            dic_geo = {}
-            for elem in map_author.index:
-                  if map_author["institution"][elem] != "Max Planck Institute for Plasma Physics - Greifswald":
-
-                      longitude = map_author["longitude"][elem]
-                      latitude = map_author["latitude"][elem]
-                      if str(longitude) + " " + str(latitude) not in dic_geo:
-                        dic_geo[str(longitude) + " " + str(latitude)] = {}
-                        dic_geo[str(longitude) + " " + str(latitude)]["authors"] = set()
-                        dic_geo[str(longitude) + " " + str(latitude)]["institutions"] = set()
-                      dic_geo[str(longitude) + " " + str(latitude)]["authors"].add(map_author["author_name"][elem])
-                      dic_geo[str(longitude) + " " + str(latitude)]["institutions"].add(map_author["institution"][elem])
-
-            for elem in dic_geo:
-              folium.Marker([float(elem.split()[1]), float(elem.split()[0])], popup="Author: " + ", ".join(list(dic_geo[elem]["authors"])) + ", Institution: " + ", ".join(list(dic_geo[elem]["institutions"])), tooltip=", ".join(list(dic_geo[elem]["authors"]))).add_to(m)
-            folium_static(m)
-
-
-
-
-  #except:
-  #  st.write("No scientist corresponding to this name was found.")
-    
-  

Climate_site/python_scripts/__init__.py

@@ -1 +0,0 @@
-__all__ = ["functions","paper_functions","patent_functions", "others"]

Climate_site/python_scripts/functions.py

@@ -1,125 +0,0 @@
-##packages code
-import pandas as pd
-import numpy as np
-import json, requests 
-import pandas as pd
-
-#import spacy
-#!python -m spacy download en_core_web_lg
-#spacy_nlp = spacy.load("en_core_web_lg")
-
-#import en_core_web_lg
-#spacy_nlp = en_core_web_lg.load()
-
-#################### Test Scripts #############################
-
-def URL(base_URL , entity_type , filters):
-    url = base_URL + entity_type + filters 
-    return url
-
-
-def get_data(url):
-    url = requests.get(url)
-    text = url.text
-    import json
-    data = json.loads(text)
-    return data
-
-## encoding the abstract
-
-
-def reconstruction_abstract(abstract_inverted_index):
-    # return the abstract is the abstract exists in the database, else, return None
-    
-    if abstract_inverted_index != None:
-        
-        list_values = list(abstract_inverted_index.values())
-        list_keys = list(abstract_inverted_index.keys())
-        #from the words in the abstract (keys of abstract_inverted_index) and their position in the text (values of abstract_inverted_index), reconstruct the abstract
-        
-        size_abstract = max([ max(elem) for elem in abstract_inverted_index.values() ] )
-        
-        abstract = [""]*(size_abstract +1)
-        
-        for i in range(len(list_values)):
-            for pos in list_values[i]:
-                abstract[pos] = list_keys[i]
-        
-        return " ".join(list(abstract))
-    
-    else:
-        return None
-            
-    
-## calculate efficiently the dot product between two vectors
-
-def norm(vector):
-    return np.sqrt(sum(x * x for x in vector))    
-
-def cosine_similarity2(vec_a, vec_b):
-        norm_a = norm(vec_a)
-        norm_b = norm(vec_b)
-        dot = sum(a * b for a, b in zip(vec_a, vec_b))
-        return dot / (norm_a * norm_b)
-
-################################### Extracted texts ###############################################################
-
-def extract_quantitative_data_paper( work_id):
-    
-
-    try:
-        url = "https://api.openalex.org/works/" + str(work_id)
-        data = get_data(url)
-        date = data["publication_date"]
-        title = data["title"]
-        abstract = reconstruction_abstract(data["abstract_inverted_index"])
-        concepts = ", ".join( [elem["display_name"] for elem in data["concepts"]] )
-        authors = ", ".join( [elem["author"]["display_name"] for elem in data["authorships"]] )
-        institutions = ", ".join( set([elem["institutions"][0]["display_name"] for elem in data["authorships"]]) )
-        print("\033[96mFROM OpenAlex: ")
-        print("\033[92mPaper link: "  + url)
-        print(" ")
-        print("\033[92mTitle: \x1b[0m"  + title)
-        print(" ")
-        print("\033[92mConcepts: \x1b[0m"  + concepts)
-        print(" ")
-        print("\033[92mDate: \x1b[0m"  + date)
-        print(" ")
-        print("\033[92mAuthors: \x1b[0m"  + authors)
-        print(" ")
-        print("\033[92mInstitutions: \x1b[0m"  + institutions)
-        print(" ")
-        # sentences = extract_sentences_with_numbers(abstract , "\033[92mAbstract\x1b[0m")
-        return [url, date, title, abstract, concepts, authors, institutions]
-    except:
-        print("Enter a valid work id from OpenAlex")
-        
-   
-def extract_quantitative_data_patent( patent_id):
-
-    try: 
-        url = "https://api.patentsview.org/patents/query?q={%22patent_id%22:%22" + str(patent_id) + "%22}&f=[%22patent_number%22,%22patent_title%22,%22patent_abstract%22,%22patent_date%22,%22inventor_last_name%22,%22inventor_first_name%22,%22assignee_organization%22]"
-        data = get_data(url)["patents"][0]
-        title = data["patent_title"]
-        abstract = data["patent_abstract"]
-        print("\033[96mFROM PATENTSVIEW:")
-        print("\033[92mPatent link: \x1b[0m"  + url)
-        print(" ")
-        print("\033[92mTitle: \x1b[0m"  + title)
-        print(" ")
-        print("\033[92mAbstract: \x1b[0m"  + abstract)
-        print(" ")
-        print("\033[92mDate: \x1b[0m"  + data["patent_date"])
-        print(" ")
-        print("\033[92mInventors: \x1b[0m"   ,  ", ".join([ data["inventors"][i]["inventor_first_name"] + " " + data["inventors"][i]["inventor_last_name"] for i in range(len(data["inventors"])) ]) )
-        print(" ")
-        print("\033[92mAssignee: \x1b[0m" , ", ".join( [ str(data["assignees"][i]["assignee_organization"]) for i in range(len(data["assignees"])) ] ) )
-        assignees = ', '.join( [ str(data["assignees"][i]["assignee_organization"]) for i in range(len(data["assignees"])) ] )
-        inventors = ", ".join([ data["inventors"][i]["inventor_first_name"] + " " + data["inventors"][i]["inventor_last_name"] for i in range(len(data["inventors"])) ])
-    except:
-        print("Enter a valid patent_id from PatentsView")
-
-    return data, url, assignees, inventors
-    
-
-#print("Do you want the quantitative information corresponding to a paper?")

Climate_site/python_scripts/functions_companies.py

@@ -1,447 +0,0 @@
-##packages code
-
-import streamlit as st
-from shapely.geometry import Point
-import pandas as pd
-import re
-import json
-
-def load_dic():
-    f = open("Synapse_project/Climate_site/python_scripts/related_companies.json","r")
-    dic_companies = json.load(f)
-    return dic_companies
-    
-
-def load_data():
-    url = "Synapse_project/Climate_site/python_scripts/preqin_companies_IEA.tsv"
-    table = pd.read_csv(url, delimiter = "\t" , index_col = 0)
-    table = table.astype({'portfolio_company_id': 'str'})
-
-    return table
-
-table_companies = load_data()
-dic_companies = load_dic()
-
-
-#################### General Functions #############################
-
-
-
-def print_extracted_text(name_file):
-
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-    count = 0
-    for index, line in enumerate(lines):
-        read_line = line.strip()
-        print(read_line)
-
-    file.close()
-    
-    
-    iea.txt
-
-def details(name_file , display):
-    
-    file = open("Synapse_project/Climate_site/python_scripts/iea.txt", "r")
-    lines = file.readlines()
-
-    mark = 0 
-    dic_details = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-
-
-        if mark == 1 and line != "" and line[0] == "*":
-            
-            if display == True:
-                print(count)
-                print(text)
-                print(" ")
-            dic_details[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line.split(" 	")[-1] == "Details" or line.split(" 	")[-1] == "Hide":
-            mark = 1
-            text = ""
-            
-    return dic_details
-
-
-
-
-
-
-
-def key_initiatives(name_file , display ):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-
-      
-    mark = 0 
-    dic_key_initiatives = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-
-            count += 1
-
-
-
-        if mark == 1 and line != "" and ( (line[0].isnumeric() and  ">" in line and " 	" in line) or line == "*Deployment targets:*" or line == "*Announced development targets:*"):
-            if display == True: 
-                print(count)
-                print(text)
-                print(" ")
-            
-            dic_key_initiatives[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line == "*Key initiatives:*":
-
-
-            mark = 1
-            text = ""
-            
-    return dic_key_initiatives
-
-
-
-
-def deployment_target(name_file , display):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-
-
-    mark = 0 
-    dic_target = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-
-
-
-        if mark == 1 and line != "" and  ((line[0].isnumeric() and ">" in line and " 	" in line)  or line == "*Announced cost reduction targets:*" or line == "*Announced development targets:*"):
-            
-            if display == True:
-                print(count)
-                print(text)
-                print(" ")
-                
-            dic_target[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line == "*Deployment targets:*" or line == "*Announced development targets:*":
-
-            mark = 1
-            text = ""
-            
-    return dic_target
-
-
- 
-    
-def cost_reduction_target(name_file , display):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-    
-    mark = 0 
-    dic_cost = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-
-
-            count += 1
-
-
-
-        if mark == 1 and line != ""  and (line[0].isnumeric() and ">" in line and " 	" in line) :
-            
-            if display == True: 
-                print(count)
-                print(text)
-                print(" ")
-                
-            dic_cost[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line == "*Announced cost reduction targets:*":
-
-            mark = 1
-            text = ""
-            
-    return dic_cost
-
-
-
-def key_words(name_file, display ):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    
-    lines = file.readlines()
-    
-    list_categories = []
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-            
-            if display == True:
-                print("Technologies"  , count+1 , ":")
-
-        if line != "":
-
-            if line[0].isnumeric() and ">" in line and " 	" in line:
-                i = 0
-                try:
-                    line = line.split(" 	")[2]
-                except:
-                    print(line)
-                    break
-                
-                if "Details" not in lines[index] and "Moderate" not in lines[index]:
-                
-                    while "	" not in line:
-                        i += 1
-                        if "Details"==lines[index + i][:7] or "End-use"==lines[index + i][:7]:
-                            break
-                        else:
-                            line = line + "  " +  lines[index + i]
-
-                #if " Production" in line:
-                    #line = line.replace(" Production" , "")
-
-                line = line.replace("\n" , " ")
-                line = line.replace("/" , " ")
-                line = line.replace("-" , " ")
-                line = line.split(" 	")[0]
-
-                if "  " in line:
-                    line = line.replace("  ", " ")
-                line = line.split(">")
-
-
-                if "(" in line[-1]:
-                    line[-1] = line[-1].split("(")[0] 
-
-
-                for i in range(len(line)):
-
-                    # remove multiple spaces
-                    line[i] = re.sub(' +', ' ', line[i])
-                    # remove trailing spaces
-                    line[i] = line[i].strip()
-
-    
-
-                if display == True:
-                    print(line)
-                    print(" ")
-                    
-                if '' in line:
-                    line.remove('')
-
-                list_categories.append([count , line])
-                
-    return list_categories
-
-
-
-def technology(name_file, display ):
-    # Filepath too specific, need to change to relative path
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-    
-    list_categories = []
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-            
-            if display == True:
-                print("Technologies"  , count+1 , ":")
-
-        if line != "":
-
-            if line[0].isnumeric() and ">" in line and " 	" in line:
-                i = 0
-                try:
-                    line = line.split(" 	")[1]
-                except:
-                    print(line)
-                    break
-                
-
-                line = line.replace("\n" , " ")
-                line = line.replace("/" , " ")
-                line = line.replace("-" , " ")
-                line = line.strip()
-                line = re.sub(' +', ' ', line)
-                line = line.split(" 	")[0]
-                line = line.split(">")
-
-
-                if "(" in line[-1]:
-                    line[-1] = line[-1].split("(")[0] 
-
-
-                for i in range(len(line)):
-
-                    # remove multiple spaces
-                    line[i] = re.sub(' +', ' ', line[i])
-                    # remove trailing spaces
-                    line[i] = line[i].strip()
-
-    
-
-                if display == True:
-                    print(line)
-                    print(" ")
-                
-
-                list_categories.append([count , line])
-                
-    return list_categories
-
-
-#################### Companies Functions #############################
-
-
-def extract_quantitative_data_technology(technologies, number_technology):
-    
-      
-    name_file = "iea"
-    dic_target = deployment_target(name_file , False)
-    dic_cost = cost_reduction_target(name_file , False)
-    dic_details = details(name_file , False)
-    cost_target_text = 'No information'
-    cost_text = 'No information'
-    
-    if number_technology in dic_details:
-        reference_text = dic_details[number_technology]
-            
-
-    if number_technology in dic_target:
-        cost_target_text = dic_target[number_technology]
-
-    if number_technology in dic_cost:
-        cost_text = dic_cost[number_technology]
-
-    return reference_text, cost_target_text, cost_text
-
-
-def related_VC_deals(category , number_technology , size):
-    
-    number_technology = str(number_technology)
-    
-   
-    list_results = list(dic_companies[number_technology].keys()) 
-    res = table_companies[table_companies["portfolio_company_id"].isin(list_results)].set_index("portfolio_company_id")
-    
-    table = res.copy()
-    for elem in table.index:
-        table.loc[ elem , "score" ] = dic_companies[number_technology][str(elem)]
-        
-    table = table.sort_values("score" , ascending = False).head(size)
-    table = table[['portfolio_company_name', 'year_established','portfolio_company_website','firm_about',
-       'portfolio_company_country',  'portfolio_company_state',
-        'firm_othernames', 'industry_classification',
-       'primary_industry', 'sub_industries', 'score']]
-
-
-    table["year_established"] = table["year_established"].replace(",", "", regex=True).astype(int, errors='ignore')
-
-
-
-    return table
-
-
-
-
-################################### Extracted texts ###############################################################
-
-#@title Which patents are related to the technology?
-
-
-def finder():
-    name_file = 'iea'
-    res = technology("iea", False )
-    list_categories_tech = []
-    list_categories = key_words("iea" , False)
-    list_technologies = [ ( ", ".join(list_categories[i][1]) , i ) for i in range(len(list_categories))  ] 
-    dic_technologies = {}
-    for i in range(len(res)):
-        names = res[i][1]
-        if ", ".join(names) not in list_categories_tech:
-            list_categories_tech.append(", ".join(names))
-            dic_technologies[", ".join(names)] = []
-        dic_technologies[", ".join(names)].append( (", ".join(list_categories[i][1]) , i ))
-        
-
-    list_climate = [ ("Any related papers" , False ) , ("Climate related papers" , True)]
-
-    dic_categories = {}
-    for elem in list_technologies:
-        list_words = elem[0].split(",")[-3:]
-        for i in range(len(list_words)):
-            if "CCUS" in list_words[i]:
-                list_words[i] = list_words[i].replace("CCUS" , "carbon capture storage")
-        dic_categories[elem[1]]  = [  ", ".join([ " ".join(words.split()[:3]) for words in list_words ] )  ,  ", ".join([ " ".join(words.split()[:3]) for words in list_words[:-1] ]) , ", ".join([ " ".join(words.split()[:3]) for words in list_words[1:] ] ) ]  
-    
-    return dic_technologies, dic_categories, list_categories_tech, list_technologies

Climate_site/python_scripts/functions_contest.py

@@ -1,478 +0,0 @@
-##packages code
-
-import streamlit as st
-from shapely.geometry import Point
-from math import radians, cos, sin, asin, sqrt
-import pandas as pd
-import re
-import json
-
-from sentence_transformers import SentenceTransformer, util
-
-@st.cache_resource
-def model_nlp():
-    model = SentenceTransformer('all-mpnet-base-v2')
-    return model
-
-
-def load_data():
-    url = "Synapse_project/Climate_site/python_scripts/big_ideas_contest.tsv"
-    dic = pd.read_csv(url, delimiter = "\t" , index_col = 0).to_dict('index')
-    return dic
-
-
-
-#################### General Functions #############################
-
-def clean_encoding(encoded_text):
-    if encoded_text == None:
-        return None
-    else:
-        if "\n" in encoded_text:
-            encoded_text = encoded_text.replace("\n" , "")
-        encoded_text = encoded_text[1:-1]
-        encoded_text = list(map(float , encoded_text.split(", ")))
-        return encoded_text
-    
-
-def norm(vector):
-    return sqrt(sum(x * x for x in vector))    
-
-def cosine_similarity2(vec_a, vec_b):
-        norm_a = norm(vec_a)
-        norm_b = norm(vec_b)
-        dot = sum(a * b for a, b in zip(vec_a, vec_b))
-        return dot / (norm_a * norm_b)
-
-def print_extracted_text(name_file):
-
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-    count = 0
-    for index, line in enumerate(lines):
-        read_line = line.strip()
-        print(read_line)
-
-    file.close()
-    
-    
-    iea.txt
-
-def details(name_file , display):
-    
-    file = open("Synapse_project/Climate_site/python_scripts/iea.txt", "r")
-    lines = file.readlines()
-
-    mark = 0 
-    dic_details = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-
-
-        if mark == 1 and line != "" and line[0] == "*":
-            
-            if display == True:
-                print(count)
-                print(text)
-                print(" ")
-            dic_details[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line.split(" 	")[-1] == "Details" or line.split(" 	")[-1] == "Hide":
-            mark = 1
-            text = ""
-            
-    return dic_details
-
-
-
-
-
-
-
-def key_initiatives(name_file , display ):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-
-      
-    mark = 0 
-    dic_key_initiatives = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-
-            count += 1
-
-
-
-        if mark == 1 and line != "" and ( (line[0].isnumeric() and  ">" in line and " 	" in line) or line == "*Deployment targets:*" or line == "*Announced development targets:*"):
-            if display == True: 
-                print(count)
-                print(text)
-                print(" ")
-            
-            dic_key_initiatives[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line == "*Key initiatives:*":
-
-
-            mark = 1
-            text = ""
-            
-    return dic_key_initiatives
-
-
-
-
-def deployment_target(name_file , display):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-
-
-    mark = 0 
-    dic_target = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-
-
-
-        if mark == 1 and line != "" and  ((line[0].isnumeric() and ">" in line and " 	" in line)  or line == "*Announced cost reduction targets:*" or line == "*Announced development targets:*"):
-            
-            if display == True:
-                print(count)
-                print(text)
-                print(" ")
-                
-            dic_target[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line == "*Deployment targets:*" or line == "*Announced development targets:*":
-
-            mark = 1
-            text = ""
-            
-    return dic_target
-
-
- 
-    
-def cost_reduction_target(name_file , display):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-    
-    mark = 0 
-    dic_cost = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-
-
-            count += 1
-
-
-
-        if mark == 1 and line != ""  and (line[0].isnumeric() and ">" in line and " 	" in line) :
-            
-            if display == True: 
-                print(count)
-                print(text)
-                print(" ")
-                
-            dic_cost[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line == "*Announced cost reduction targets:*":
-
-            mark = 1
-            text = ""
-            
-    return dic_cost
-
-
-
-def key_words(name_file, display ):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    
-    lines = file.readlines()
-    
-    list_categories = []
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-            
-            if display == True:
-                print("Technologies"  , count+1 , ":")
-
-        if line != "":
-
-            if line[0].isnumeric() and ">" in line and " 	" in line:
-                i = 0
-                try:
-                    line = line.split(" 	")[2]
-                except:
-                    print(line)
-                    break
-                
-                if "Details" not in lines[index] and "Moderate" not in lines[index]:
-                
-                    while "	" not in line:
-                        i += 1
-                        if "Details"==lines[index + i][:7] or "End-use"==lines[index + i][:7]:
-                            break
-                        else:
-                            line = line + "  " +  lines[index + i]
-
-                #if " Production" in line:
-                    #line = line.replace(" Production" , "")
-
-                line = line.replace("\n" , " ")
-                line = line.replace("/" , " ")
-                line = line.replace("-" , " ")
-                line = line.split(" 	")[0]
-
-                if "  " in line:
-                    line = line.replace("  ", " ")
-                line = line.split(">")
-
-
-                if "(" in line[-1]:
-                    line[-1] = line[-1].split("(")[0] 
-
-
-                for i in range(len(line)):
-
-                    # remove multiple spaces
-                    line[i] = re.sub(' +', ' ', line[i])
-                    # remove trailing spaces
-                    line[i] = line[i].strip()
-
-    
-
-                if display == True:
-                    print(line)
-                    print(" ")
-                    
-                if '' in line:
-                    line.remove('')
-
-                list_categories.append([count , line])
-                
-    return list_categories
-
-
-
-def technology(name_file, display ):
-    # Filepath too specific, need to change to relative path
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-    
-    list_categories = []
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-            
-            if display == True:
-                print("Technologies"  , count+1 , ":")
-
-        if line != "":
-
-            if line[0].isnumeric() and ">" in line and " 	" in line:
-                i = 0
-                try:
-                    line = line.split(" 	")[1]
-                except:
-                    print(line)
-                    break
-                
-
-                line = line.replace("\n" , " ")
-                line = line.replace("/" , " ")
-                line = line.replace("-" , " ")
-                line = line.strip()
-                line = re.sub(' +', ' ', line)
-                line = line.split(" 	")[0]
-                line = line.split(">")
-
-
-                if "(" in line[-1]:
-                    line[-1] = line[-1].split("(")[0] 
-
-
-                for i in range(len(line)):
-
-                    # remove multiple spaces
-                    line[i] = re.sub(' +', ' ', line[i])
-                    # remove trailing spaces
-                    line[i] = line[i].strip()
-
-    
-
-                if display == True:
-                    print(line)
-                    print(" ")
-                
-
-                list_categories.append([count , line])
-                
-    return list_categories
-
-          
-
-
-#################### Companies Functions #############################
-
-
-def extract_quantitative_data_technology(technologies, number_technology):
-    
-      
-    name_file = "iea"
-    dic_target = deployment_target(name_file , False)
-    dic_cost = cost_reduction_target(name_file , False)
-    dic_details = details(name_file , False)
-    cost_target_text = 'No information'
-    cost_text = 'No information'
-    
-    if number_technology in dic_details:
-        reference_text = dic_details[number_technology]
-            
-
-    if number_technology in dic_target:
-        cost_target_text = dic_target[number_technology]
-
-    if number_technology in dic_cost:
-        cost_text = dic_cost[number_technology]
-
-    return reference_text, cost_target_text, cost_text
-
-
-
-def big_ideas_encoding(number_technology):
-    
-    model = model_nlp()
-    
-    dic_big_ideas = load_data()
-    
-    dic_matches = {}
-    
-
-    name_file = 'iea'
-    dic_details = details(name_file , False)
-    k = 0 
-        
-        
-        
-    IEA_encoding = model.encode(dic_details[number_technology]  , convert_to_tensor=False , show_progress_bar = False).tolist()
-    for count_contest in dic_big_ideas:
-        if pd.isna(dic_big_ideas[count_contest]['encoded_description']) != True:
-            score = cosine_similarity2(IEA_encoding, clean_encoding(dic_big_ideas[count_contest]['encoded_description']))
-            if score > 0.45:
-                dic_matches[k] = dic_big_ideas[count_contest]
-                dic_matches[k]["score"] = score
-                k += 1
-                
-    
-                
-    if len(dic_matches) > 0:
-        return pd.DataFrame(dic_matches).T.sort_values("score" , ascending = False)[["Project_name" ,  "Description" ,"Project_date" , "University" , "Field" , "Team_members", "score" ]]  
-    else:
-        return "No related project found"  
-    
-    
-     
-
-
-
-
-
-
-################################### Extracted texts ###############################################################
-
-#@title Which patents are related to the technology?
-
-
-def finder():
-    name_file = 'iea'
-    res = technology("iea", False )
-    list_categories_tech = []
-    list_categories = key_words("iea" , False)
-    list_technologies = [ ( ", ".join(list_categories[i][1]) , i ) for i in range(len(list_categories))  ] 
-    dic_technologies = {}
-    for i in range(len(res)):
-        names = res[i][1]
-        if ", ".join(names) not in list_categories_tech:
-            list_categories_tech.append(", ".join(names))
-            dic_technologies[", ".join(names)] = []
-        dic_technologies[", ".join(names)].append( (", ".join(list_categories[i][1]) , i ))
-        
-
-    list_climate = [ ("Any related papers" , False ) , ("Climate related papers" , True)]
-
-    dic_categories = {}
-    for elem in list_technologies:
-        list_words = elem[0].split(",")[-3:]
-        for i in range(len(list_words)):
-            if "CCUS" in list_words[i]:
-                list_words[i] = list_words[i].replace("CCUS" , "carbon capture storage")
-        dic_categories[elem[1]]  = [  ", ".join([ " ".join(words.split()[:3]) for words in list_words ] )  ,  ", ".join([ " ".join(words.split()[:3]) for words in list_words[:-1] ]) , ", ".join([ " ".join(words.split()[:3]) for words in list_words[1:] ] ) ]  
-    
-    return dic_technologies, dic_categories, list_categories_tech, list_technologies

Climate_site/python_scripts/others.py

@@ -1,7 +0,0 @@
-import numpy as np
-
-def other(list):
-    return np.sum(list)
-
-def add(x,y):
-    return x+y

Climate_site/python_scripts/paper_functions.py

@@ -1,1220 +0,0 @@
-##packages code
-import streamlit as st
-from shapely.geometry import Point
-import pandas as pd
-from tqdm import tqdm
-
-import numpy as np
-import json, requests 
-import pandas as pd
-#from pandas.io.json import json_normalize
-
-import matplotlib.pyplot as plt
-import seaborn as sns
-from math import radians, cos, sin, asin, sqrt
-
-#import spacy
-#spacy_nlp = spacy.load("en_core_web_lg")
-
-#@st.cache_resource
-#def spacy_func():
-#    spacy_nlp = spacy.load("en_core_web_lg")
-#    return spacy_nlp
-
-from sentence_transformers import SentenceTransformer, util
-
-@st.cache_resource
-def model_nlp():
-    model = SentenceTransformer('all-MiniLM-L6-v2')
-    return model
-
-
-@st.cache_data  # 👈 Add the caching decorator
-def load_data():
-    url = "Synapse_project/Climate_site/python_scripts/institutions.tsv"
-    dic = pd.read_csv(url, delimiter = "\t" , index_col = 1).to_dict('index')
-    return dic
-
-dic_institutions = load_data()
-
-
-import unicodedata
-
-from metaphone import doublemetaphone
-from fuzzywuzzy import fuzz
-from difflib import SequenceMatcher
-import re
-
-import geopandas as gpd
-from geopandas import GeoDataFrame
-#################### General Functions #############################
-
-def URL(base_URL , entity_type , filters):
-    url = base_URL + entity_type + filters 
-    return url
-
-
-def get_data(url):
-    url = requests.get(url)
-    text = url.text
-    import json
-    data = json.loads(text)
-    return data
-
-## encoding the abstract
-
-
-def reconstruction_abstract(abstract_inverted_index):
-    # return the abstract is the abstract exists in the database, else, return None
-    
-    if abstract_inverted_index != None:
-        
-        list_values = list(abstract_inverted_index.values())
-        list_keys = list(abstract_inverted_index.keys())
-        #from the words in the abstract (keys of abstract_inverted_index) and their position in the text (values of abstract_inverted_index), reconstruct the abstract
-        
-        size_abstract = max([ max(elem) for elem in abstract_inverted_index.values() ] )
-        
-        abstract = [""]*(size_abstract +1)
-        
-        for i in range(len(list_values)):
-            for pos in list_values[i]:
-                abstract[pos] = list_keys[i]
-        
-        return " ".join(list(abstract))
-    
-    else:
-        return None
-            
-    
-## calculate efficiently the dot product between two vectors
-
-def norm(vector):
-    return np.sqrt(sum(x * x for x in vector))    
-
-def cosine_similarity2(vec_a, vec_b):
-        norm_a = norm(vec_a)
-        norm_b = norm(vec_b)
-        dot = sum(a * b for a, b in zip(vec_a, vec_b))
-        return dot / (norm_a * norm_b)
-
-## Extracted texts
-
-def print_extracted_text(name_file):
-
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-    count = 0
-    for index, line in enumerate(lines):
-        read_line = line.strip()
-        print(read_line)
-
-    file.close()
-    
-    
-    iea.txt
-
-def details(name_file , display):
-    
-    file = open("Synapse_project/Climate_site/python_scripts/iea.txt", "r")
-    lines = file.readlines()
-
-    mark = 0 
-    dic_details = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-
-
-        if mark == 1 and line != "" and line[0] == "*":
-            
-            if display == True:
-                print(count)
-                print(text)
-                print(" ")
-            dic_details[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line.split(" 	")[-1] == "Details" or line.split(" 	")[-1] == "Hide":
-            mark = 1
-            text = ""
-            
-    return dic_details
-
-
-
-
-
-
-
-def key_initiatives(name_file , display ):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-
-      
-    mark = 0 
-    dic_key_initiatives = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-
-            count += 1
-
-
-
-        if mark == 1 and line != "" and ( (line[0].isnumeric() and  ">" in line and " 	" in line) or line == "*Deployment targets:*" or line == "*Announced development targets:*"):
-            if display == True: 
-                print(count)
-                print(text)
-                print(" ")
-            
-            dic_key_initiatives[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line == "*Key initiatives:*":
-
-
-            mark = 1
-            text = ""
-            
-    return dic_key_initiatives
-
-
-
-
-def deployment_target(name_file , display):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-
-
-    mark = 0 
-    dic_target = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-
-
-
-        if mark == 1 and line != "" and  ((line[0].isnumeric() and ">" in line and " 	" in line)  or line == "*Announced cost reduction targets:*" or line == "*Announced development targets:*"):
-            
-            if display == True:
-                print(count)
-                print(text)
-                print(" ")
-                
-            dic_target[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line == "*Deployment targets:*" or line == "*Announced development targets:*":
-
-            mark = 1
-            text = ""
-            
-    return dic_target
-
-
- 
-    
-def cost_reduction_target(name_file , display):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-    
-    mark = 0 
-    dic_cost = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-
-
-            count += 1
-
-
-
-        if mark == 1 and line != ""  and (line[0].isnumeric() and ">" in line and " 	" in line) :
-            
-            if display == True: 
-                print(count)
-                print(text)
-                print(" ")
-                
-            dic_cost[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line == "*Announced cost reduction targets:*":
-
-            mark = 1
-            text = ""
-            
-    return dic_cost
-
-
-
-def key_words(name_file, display ):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    
-    lines = file.readlines()
-    
-    list_categories = []
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-            
-            if display == True:
-                print("Technologies"  , count+1 , ":")
-
-        if line != "":
-
-            if line[0].isnumeric() and ">" in line and " 	" in line:
-                i = 0
-                try:
-                    line = line.split(" 	")[2]
-                except:
-                    print(line)
-                    break
-                
-                if "Details" not in lines[index] and "Moderate" not in lines[index]:
-                
-                    while "	" not in line:
-                        i += 1
-                        if "Details"==lines[index + i][:7] or "End-use"==lines[index + i][:7]:
-                            break
-                        else:
-                            line = line + "  " +  lines[index + i]
-
-                #if " Production" in line:
-                    #line = line.replace(" Production" , "")
-
-                line = line.replace("\n" , " ")
-                line = line.replace("/" , " ")
-                line = line.replace("-" , " ")
-                line = line.split(" 	")[0]
-
-                if "  " in line:
-                    line = line.replace("  ", " ")
-                line = line.split(">")
-
-
-                if "(" in line[-1]:
-                    line[-1] = line[-1].split("(")[0] 
-
-
-                for i in range(len(line)):
-
-                    # remove multiple spaces
-                    line[i] = re.sub(' +', ' ', line[i])
-                    # remove trailing spaces
-                    line[i] = line[i].strip()
-
-    
-
-                if display == True:
-                    print(line)
-                    print(" ")
-                    
-                if '' in line:
-                    line.remove('')
-
-                list_categories.append([count , line])
-                
-    return list_categories
-
-
-
-def technology(name_file, display ):
-    # Filepath too specific, need to change to relative path
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-    
-    list_categories = []
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-            
-            if display == True:
-                print("Technologies"  , count+1 , ":")
-
-        if line != "":
-
-            if line[0].isnumeric() and ">" in line and " 	" in line:
-                i = 0
-                try:
-                    line = line.split(" 	")[1]
-                except:
-                    print(line)
-                    break
-                
-
-                line = line.replace("\n" , " ")
-                line = line.replace("/" , " ")
-                line = line.replace("-" , " ")
-                line = line.strip()
-                line = re.sub(' +', ' ', line)
-                line = line.split(" 	")[0]
-                line = line.split(">")
-
-
-                if "(" in line[-1]:
-                    line[-1] = line[-1].split("(")[0] 
-
-
-                for i in range(len(line)):
-
-                    # remove multiple spaces
-                    line[i] = re.sub(' +', ' ', line[i])
-                    # remove trailing spaces
-                    line[i] = line[i].strip()
-
-    
-
-                if display == True:
-                    print(line)
-                    print(" ")
-                
-
-                list_categories.append([count , line])
-                
-    return list_categories
-
-
-#################### Paper Functions #############################
-
-def related_papers(number_technology  , carbon_related , display , key_words  ):
-    """Returns a dictionary of data related to the papers the user chooses
-    
-    This is done through generating URLs to connect with OpenAlex API and parsing through the json file
-    
-    Parameters
-    ----------
-    number_technology : int, the index of the technology with respect to the 23 IEA technologies
-    list_categories : str, an array of all the technologies related to the 23 IEA technologies
-    carbon_related: bool, a setting to select papers related to carbon capture
-    display:
-    key_words: str, an array of keywords selected based off technology/ user input
-
-    """   
-    dic = {}
-
-    max_count = 0
-    base_URL_OA = f'https://api.openalex.org/'
-    filter_works = f'works?'
-        
-
-    filter_openalex = f"search=" + key_words
-
-
-    if carbon_related == True:
-        filter_openalex += "&filter=concepts.id:https://openalex.org/C132651083|https://openalex.org/C115343472|https://openalex.org/C530467964&per_page=100&mailto=emma_scharfmann@berkeley.edu"
-    
-    else:
-        filter_openalex += "&per_page=100&mailto=emma_scharfmann@berkeley.edu"
-        
-        
-    filter_openalex = filter_openalex.replace(" " , "%20")
-    
-    url = URL(base_URL_OA , filter_works, filter_openalex) 
-    data = get_data(url)
-    count = data["meta"]["count"] 
-
-
-
-    if display == True:
-        print(  data["meta"]["count"] , [elem[i] for i in range(1,len(elem))]  )
-        print(url)
-    
-    for i in range(len(data["results"])):
-        dic[ data["results"][i]["id"]] = {}
-        dic[ data["results"][i]["id"]]["title"] = data["results"][i]["title"]
-        dic[ data["results"][i]["id"]]["abstract"] = reconstruction_abstract(data["results"][i]["abstract_inverted_index"])
-        dic[ data["results"][i]["id"]]["concepts"] = data["results"][i]["concepts"]
-        dic[ data["results"][i]["id"]]["date"] = data["results"][i]["publication_date"]
-        dic[ data["results"][i]["id"]]["authorships"] = data["results"][i]["authorships"]
-        dic[ data["results"][i]["id"]]["cited_by_count"] = data["results"][i]["cited_by_count"]
-
-        if len(data["results"][i]["authorships"]) > 0:
-            if data["results"][i]["authorships"][0]["institutions"] != []:
-                dic[ data["results"][i]["id"]]["countries"] = data["results"][i]["authorships"][0]["institutions"][0]["country_code"]
-                dic[ data["results"][i]["id"]]["institutions"] = data["results"][i]["authorships"][0]["institutions"][0]["display_name"]
-            else:
-                dic[ data["results"][i]["id"]]["countries"] = ""
-                dic[ data["results"][i]["id"]]["institutions"] = ""
-            dic[ data["results"][i]["id"]]["authors"] = data["results"][i]["authorships"][0]["author"]["display_name"]
-
-            for j in range(1 , len(data["results"][i]["authorships"])):
-                if data["results"][i]["authorships"][j]["institutions"] != []:
-                    dic[ data["results"][i]["id"]]["institutions"] += ", " + data["results"][i]["authorships"][j]["institutions"][0]["display_name"]
-                    dic[ data["results"][i]["id"]]["countries"] = data["results"][i]["authorships"][j]["institutions"][0]["country_code"]
-                dic[ data["results"][i]["id"]]["authors"] += ", " + data["results"][i]["authorships"][j]["author"]["display_name"]
-
-    if display == True: 
-        print(" ")
-        
-    return dic
-        
-
-## Ranking Papers
-
-def ranking_papers(number_technology,  dic_details   , display , size , carbon_related , search_words ):
-    """Returns a dictionary of data related to the papers the user chooses
-    
-    This is done through generating URLs to connect with OpenAlex API and parsing through the json file
-    
-    Parameters
-    ----------
-    number_technology : int, the index of the technology with respect to the 500+ IEA technologies
-    dic_details : str, dictionary where key is index of technology with respect to 500+ IEA technolgies and values are the details related to the technologies
-    
-    dic: bool, a setting to select papers related to carbon capture
-    list_categories: str, an array of keywords selected based off technology/ user input
-    display: bool, chooses to display information or not
-    size: str, an array of keywords selected based off technology/ user input
-
-    """   
-    model = model_nlp()
-    dic = related_papers(number_technology  , carbon_related , display , search_words  )
-    
-    
-    dic_scores_papers = {}
-    
-    
-
-    if display == True:
-        print("Technology " + str(number_technology))
-        print("Key words: " , list_categories[number_technology][1])
-            
-    if number_technology in dic_details:
-        reference_text = dic_details[number_technology]
-            
-        if display== True:
-            print("Technology details: " , reference_text)
-            print(" ")
-        encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist()
-        
-        
-        if len(dic) == 0:
-            return "No paper found, select other key words" 
-        
-        
-        for ids in list(dic.keys()):
-
-            dic_scores_papers[ids] = {}
-            
-            if dic[ids]["title"] != None:
-                encoded_title = model.encode(dic[ids]["title"], convert_to_tensor=False).tolist()
-                score_title = cosine_similarity2(encoded_title, encoded_text)
-            else:
-                score_title = None
-
-        
-
-            if dic[ids]["abstract"] != None:
-                encoded_abstract = model.encode(dic[ids]["abstract"], convert_to_tensor=False).tolist()
-                score_abstract = cosine_similarity2(encoded_abstract, encoded_text)
-            else:
-                score_abstract = None
-
-
-            #concepts = ""
-            #for elem in dic[ids]["concepts"]:
-            #    concepts += elem["display_name"] + " "
-            #encoded_concepts = model.encode(concepts, convert_to_tensor=False).tolist()
-            #score_concepts = cosine_similarity2(encoded_concepts, encoded_text)
-
-            dic_scores_papers[ids]["title comparison"] = score_title
-            dic_scores_papers[ids]["abstract comparison"] = score_abstract
-            #dic_scores_papers[ids]["concepts comparison"] = score_concepts
-            
-            dic_scores_papers[ids]["title"] = dic[ids]["title"]
-            dic_scores_papers[ids]["citations"] = dic[ids]["cited_by_count"]
-            
-
-            dic_scores_papers[ids]["date"] = dic[ids]["date"][:4]
-            if "institutions" in dic[ids]:
-                dic_scores_papers[ids]["institutions"] = dic[ids]["institutions"]
-            #    dic_scores_papers[ids]["countries"] = dic[ids]["countries"]
-                
-            else:
-                dic_scores_papers[ids]["institutions"] = None
-            #    dic_scores_papers[ids]["countries"] = None
-            
-            if "authors" in dic[ids] and dic[ids]["authors"] != None:
-                dic_scores_papers[ids]["number of co-authors"] = len(dic[ids]["authors"].split(","))
-                dic_scores_papers[ids]["authors"] = dic[ids]["authors"]
-            else:
-                dic_scores_papers[ids]["number of co-authors"] = None
-                dic_scores_papers[ids]["authors"] = None
-            
-
-        return dic , dic_scores_papers
-        
-
-
-
-
-def get_ranking_related_papers( technologies , number_technology , carbon_related , size , research_words):
-    name_file = "iea"
-    dic_details = details(name_file , False)
-    dic , dic_scores_papers = ranking_papers(number_technology,  dic_details   , False , size , carbon_related , research_words )
-    
-    if dic_scores_papers == {}:
-        return "No paper found"
-    
-    elif type(dic_scores_papers) == str:
-        return dic_scores_papers
-    
-    else:
-        return pd.DataFrame(dic_scores_papers).T.sort_values(by="abstract comparison" , ascending = False).head(size)
-     
-
-## Extract quantitative data
-
-def extract_sentences_with_numbers(text , text_name):
-    if text != None:
-        text = text.replace("CO 2" , "CO2")
-        text = text.replace("CO 3" , "CO3")
-        text = text.replace("CO(2)" , "CO2")
-        text = text.replace("CO(3)" , "CO3")
-        
-        print(text_name + ": " , text)
-        print(" ")
-
-
-
-        list_text = list(text)
-        for i in range(1,len(list_text)-1):
-            if list_text[i] == " " and list_text[i-1] == "." and list_text[i+1].isupper():
-                list_text[i] = "~"
-
-        text = "".join(list_text)
-        text = text.split("~")
-        for sentence in text:                
-
-            if any(char.isdigit() for char in sentence):
-                
-                
-                
-                if "CO2" in sentence:
-                    print("\x1b[31mCARBON RELATED:\x1b[0m", sentence)
-                    print(" ")
-
-
-                    
-                if "GJ" in sentence or "MJ" in sentence:
-                    print("\x1b[31mENERGY:\x1b[0m" , sentence)
-                    print(" ")
-
-
-                    
-                ##price
-                if "€"  in sentence or "$" in sentence or "EUR" in sentence or "dollars" in sentence.lower():
-                    print("\x1b[31mPRICE:\x1b[0m" , sentence)
-                    print(" ")
-
-
-                    
-                ##dates
-                digits = []
-                for word in sentence.replace("," , "").replace("%" , "").replace("." , " ").split():
-                    if word.isdigit() and 1850 < int(word) < 2200 :
-                        digits.append(word)
-                if digits != []:
-                    print("\x1b[31mDATE:\x1b[0m" , sentence)
-                    print(" ")
-
-
-                    
-                ##CO quantity
-                if "Mt" in sentence or "tC" in sentence or "t-C" in sentence:
-                    print("\x1b[31mCARBON QUANTITY:\x1b[0m" , sentence)
-                    print(" ")
- 
-    print(" ")
-          
-    
-def extract_quantitative_data_technology(technologies, number_technology):
-    
-    model = model_nlp()
-    count = 0
-      
-    name_file = "iea"
-    dic_target = deployment_target(name_file , False)
-    dic_cost = cost_reduction_target(name_file , False)
-    dic_details = details(name_file , False)
-    cost_target_text = 'No information'
-    cost_text = 'No information'
-    
-    if number_technology in dic_details:
-        reference_text = dic_details[number_technology]
-            
-        #print("\033[96mFROM IEA website: ")
-        #print("\033[92mTechnology details: \x1b[0m" , reference_text)
-        #print(" ")
-        #encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist()
-
-    if number_technology in dic_target:
-        cost_target_text = dic_target[number_technology]
-        #print("\033[96mFROM IEA website: ")
-        #sentences = extract_sentences_with_numbers(cost_target_text , "\033[92mDeployment target and Announced development target\x1b[0m")
-
-    if number_technology in dic_cost:
-        cost_text = dic_cost[number_technology]
-        #print("\033[96mFROM IEA website: ")
-        #sentences = extract_sentences_with_numbers(cost_text , "\033[92mAnnounced cost reduction targets\x1b[0m")
-    
-    return reference_text, cost_target_text, cost_text
-
-
-def extract_quantitative_data_paper(work_id):
-
-    url = "https://api.openalex.org/works/" + str(work_id)
-    url_google = "https://explore.openalex.org/works/" + str(work_id)
-    
-    data = get_data(url)
-    date = data["publication_date"]
-    title = data["title"]
-    abstract = reconstruction_abstract(data["abstract_inverted_index"])
-    concepts = ", ".join( [elem["display_name"] for elem in data["concepts"]] )
-    authors = ", ".join( [elem["author"]["display_name"] for elem in data["authorships"]] )
-    institutions = ", ".join( set([elem["institutions"][0]["display_name"] for elem in data["authorships"] if len(elem["institutions"]) > 0]) )
-    return url_google , title , abstract , date , authors , institutions
-            
-
-    
-
-## Get papers related to cited projects
-
-def related_projects(technologies, number_technology , carbon_related , size):
-    
-    spacy_nlp = spacy_func()
-    model = model_nlp()
-
-    name_file = "iea"
-    dic_details = details(name_file , False)
-    dic_key_initiatives = key_initiatives(name_file , False )
-    
-    
-    dic2 = {}
-
-    reference_text = dic_details[number_technology]
-    print("\033[92mTechnology details: \x1b[0m" , reference_text)
-    print(" ")
-    encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist()
-     # reference_text key_initiative location_entities
-    if number_technology in dic_key_initiatives:
-        key_initiative = dic_key_initiatives[number_technology]
-        print("\033[92mTechnology key initiatives: \x1b[0m" ,  key_initiative)
-
-        location_entities = set()
-        doc = spacy_nlp(key_initiative.strip())
-        for j in doc.ents:
-            entry = str(j.lemma_).lower()
-            key_initiative = key_initiative.replace(str(j).lower(), "")
-            if j.label_ in ["ORG"]:
-                location_entities.add(j)
-        print(" ")
-        print("\033[92mExtracted projects and organizations names: \x1b[0m", location_entities)
-
-
-        base_URL_OA = f'https://api.openalex.org/'
-        filter_works = f'works?'
-        for element in location_entities:
-
-            filter_openalex = f"search=" 
-            filter_openalex += str(element) 
-            if carbon_related == True:
-                filter_openalex += "&filter=concepts.id:https://openalex.org/C2780021121&per_page=50&mailto=emma_scharfmann@berkeley.edu"
-            else:
-                filter_openalex += "&per_page=50&mailto=emma_scharfmann@berkeley.edu"
-
-            filter_openalex = filter_openalex.replace(" " , "%20")
-
-            url = URL(base_URL_OA , filter_works, filter_openalex) 
-
-            data = get_data(url)
-            count = len(data["results"])
-            if count < 40:
-
-                for k in range(count):
-                    
-                    if data["results"][k]["abstract_inverted_index"] != None:
-                        encoded_abstract = model.encode(reconstruction_abstract(data["results"][k]["abstract_inverted_index"]) ,  convert_to_tensor=False).tolist()
-                        abstract_similarity = cosine_similarity2(encoded_abstract, encoded_text)
-                    else:
-                        abstract_similarity = None
-                        
-                    if data["results"][k]["title"] != None:
-                        encoded_title = model.encode(data["results"][k]["title"] ,  convert_to_tensor=False).tolist()
-                        title_similarity = cosine_similarity2(encoded_title, encoded_text)
-                    else:
-                        title_similarity = None
-                        
-                    work_id = data["results"][k]["id"]
-                    dic2[work_id] = {}
-                    dic2[work_id]["project/institution name"] = str(element)
-                    dic2[work_id]["title comparison"] = title_similarity
-                    dic2[work_id]["abstract comparison"] = abstract_similarity
-                    dic2[work_id]["title"] = data["results"][k]["title"]
-                    #dic2[work_id]["abstract"] = reconstruction_abstract(data["results"][k]["abstract_inverted_index"])
-                    dic2[work_id]["date"] = data["results"][k]["publication_date"]
-
-            else:
-                list_categories = key_words( name_file, False)
-                key_word = list_categories[number_technology]
-                elem = key_word[1]
-
-                string = ""
-                for j in range(1,len(elem)):
-                    if elem[j] == "CCUS":
-                        elem[j] = "Carbon capture and storage"
-                    string += elem[j]  + " " 
-
-
-                string += str(element)
-                filter_openalex = f"search="  + string 
-
-                if carbon_related == True:
-                    filter_openalex += "&filter=concepts.id:https://openalex.org/C2780021121&per_page=50&mailto=emma_scharfmann@berkeley.edu"
-                else:
-                    filter_openalex += "&per_page=50&mailto=emma_scharfmann@berkeley.edu"
-                filter_openalex = filter_openalex.replace(" " , "%20")
-                url = URL(base_URL_OA , filter_works, filter_openalex) 
-                data = get_data(url)
-                count = len(data["results"])
-                for k in range(count):
-                    
-                    if data["results"][k]["abstract_inverted_index"] != None:
-                        encoded_abstract = model.encode(reconstruction_abstract(data["results"][k]["abstract_inverted_index"]) ,  convert_to_tensor=False).tolist()
-                        abstract_similarity = cosine_similarity2(encoded_abstract, encoded_text)
-                    else:
-                        abstract_similarity = None
-                        
-                    if data["results"][k]["title"] != None:
-                        encoded_title = model.encode(data["results"][k]["title"] ,  convert_to_tensor=False).tolist()
-                        title_similarity = cosine_similarity2(encoded_title, encoded_text)
-                    else:
-                        title_similarity = None
-                        
-                    work_id = data["results"][k]["id"]
-                    dic2[work_id] = {}
-                    dic2[work_id]["project/institution name"] = str(element)
-                    dic2[work_id]["title comparison"] = title_similarity
-                    dic2[work_id]["abstract comparison"] = abstract_similarity
-                    dic2[work_id]["title"] = data["results"][k]["title"]
-                    #dic2[work_id]["abstract"] = reconstruction_abstract(data["results"][k]["abstract_inverted_index"])
-                    dic2[work_id]["date"] = data["results"][k]["publication_date"]
-                    
-                    
-
-
-        if dic2 != {}:
-            return reference_text,key_initiative,location_entities,pd.DataFrame(dic2).T.sort_values(by="abstract comparison" , ascending = False).head(size)
-        else:
-            return reference_text,key_initiative,location_entities, "No paper found"
-
-
-## Mapping the authors
-
-#merge the nobiliary particles with the last name
-#ln_suff file can be modified if more or less nobiliary particles want to be suppressed
-
-ln_suff = ['oster',
- 'nordre',
- 'vaster',
- 'aust',
- 'vesle',
- 'da',
- 'van t',
- 'af',
- 'al',
- 'setya',
- 'zu',
- 'la',
- 'na',
- 'mic',
- 'ofver',
- 'el',
- 'vetle',
- 'van het',
- 'dos',
- 'ui',
- 'vest',
- 'ab',
- 'vste',
- 'nord',
- 'van der',
- 'bin',
- 'ibn',
- 'war',
- 'fitz',
- 'alam',
- 'di',
- 'erch',
- 'fetch',
- 'nga',
- 'ka',
- 'soder',
- 'lille',
- 'upp',
- 'ua',
- 'te',
- 'ni',
- 'bint',
- 'von und zu',
- 'vast',
- 'vestre',
- 'over',
- 'syd',
- 'mac',
- 'nin',
- 'nic',
- 'putri',
- 'bet',
- 'verch',
- 'norr',
- 'bath',
- 'della',
- 'van',
- 'ben',
- 'du',
- 'stor',
- 'das',
- 'neder',
- 'abu',
- 'degli',
- 'vre',
- 'ait',
- 'ny',
- 'opp',
- 'pour',
- 'kil',
- 'der',
- 'oz',
- 'von',
- 'at',
- 'nedre',
- 'van den',
- 'setia',
- 'ap',
- 'gil',
- 'myljom',
- 'van de',
- 'stre',
- 'dele',
- 'mck',
- 'de',
- 'mellom',
- 'mhic',
- 'binti',
- 'ath',
- 'binte',
- 'snder',
- 'sre',
- 'ned',
- 'ter',
- 'bar',
- 'le',
- 'mala',
- 'ost',
- 'syndre',
- 'sr',
- 'bat',
- 'sndre',
- 'austre',
- 'putra',
- 'putera',
- 'av',
- 'lu',
- 'vetch',
- 'ver',
- 'puteri',
- 'mc',
- 'tre',
- 'st']
-
-#suppress all the unwanted suffixes from a string. 
-#name_del file can be modified if more or less suffixes want to be suppressed 
-
-name_del = ['2nd', '3rd', 'Jr', 'Jr.', 'Junior', 'Sr', 'Sr.', 'Senior']
-
-
-def name_delete(string):
-    for elmt in name_del:
-        if f" {elmt}" in string:
-            return string.replace(f" {elmt}","")
-    return string
-
-def ln_suff_merge(string):
-    for suff in ln_suff:
-        if f" {suff} " in string or string.startswith(f"{suff} "):
-            return string.replace(f"{suff} ",suff.replace(" ",""))
-    return string
-
-#normalize a string dat that represents often a name. 
-
-def normalize(data):
-    normal = unicodedata.normalize('NFKD', data).encode('ASCII', 'ignore')
-    val = normal.decode("utf-8")
-    # delete unwanted elmt
-    val = name_delete(val)
-    # lower full name in upper
-    val = re.sub(r"[A-Z]{3,}", lambda x: x.group().lower(), val)
-    # add space in front of upper case
-    val = re.sub(r"(\w)([A-Z])", r"\1 \2", val)
-    # Lower case
-    val = val.lower()
-    # remove special characters
-    val = re.sub('[^A-Za-z0-9 ]+', ' ', val)
-    # remove multiple spaces
-    val = re.sub(' +', ' ', val)
-    # remove trailing spaces
-    val = val.strip()
-    # suffix merge
-    val = ln_suff_merge(val)
-
-    return val
-
-
-def main_authors(technologies,number_technology , carbon_related , size , research_words):
-    
-    name_file = "iea"
-    list_categories = key_words( name_file, False)
-    dic_details = details(name_file , False)
-    dic_papers , dic_papers_ranked = ranking_papers(number_technology,  dic_details   , False , size , carbon_related , research_words )
-    
-    dic_papers_co_authors = {}
-        
-
-
-    for paper in list(dic_papers_ranked.keys())[:size]:
-
-        for k in range(len(dic_papers[paper]["authorships"])):
-            coauthor_id = dic_papers[paper]["authorships"][k]["author"]["id"]
-            
-            
-            author_name = dic_papers[paper]["authorships"][k]["author"]["display_name"]
-            author_name_norm = normalize(dic_papers[paper]["authorships"][k]["author"]["display_name"]).split()
-
-            if len(author_name_norm ) > 0:
-                author_name_norm = author_name_norm[0] + " " + author_name_norm[-1]
-
-                if author_name_norm not in dic_papers_co_authors:
-                    dic_papers_co_authors[author_name_norm] = {}
-                    dic_papers_co_authors[author_name_norm]["Author's name(s)"] = author_name
-                    if coauthor_id != None :
-                        dic_papers_co_authors[author_name_norm]["Author's id(s)"] = coauthor_id[21:]
-                    else:
-                        dic_papers_co_authors[author_name_norm]["Author's id(s)"] = ""
-                    dic_papers_co_authors[author_name_norm]["Number of occurence within the " + str(size) + " most related papers"] = 1
-                    dic_papers_co_authors[author_name_norm]["Number of related citations"] = dic_papers[paper]["cited_by_count"]
-                else:
-                    dic_papers_co_authors[author_name_norm]["Number of occurence within the " + str(size) + " most related papers"] += 1
-                    if author_name not in dic_papers_co_authors[author_name_norm]["Author's name(s)"]:
-                        dic_papers_co_authors[author_name_norm]["Author's name(s)"] += ", " + author_name[0]
-                    if coauthor_id != None and coauthor_id[21:] not in dic_papers_co_authors[author_name_norm]["Author's id(s)"]:                                                              
-                        dic_papers_co_authors[author_name_norm]["Author's id(s)"] += ", " + coauthor_id[21:]
-                        dic_papers_co_authors[author_name_norm]["Number of related citations"] += dic_papers[paper]["cited_by_count"]
-
-
-
-    if dic_papers_co_authors != {}:
-        dic_papers_co_authors = {k: v for k, v in sorted(dic_papers_co_authors.items(), key=lambda item: item[1]["Number of occurence within the " + str(size) + " most related papers"] , reverse = True)}
-
-        for author_name_norm in list(dic_papers_co_authors.keys())[:size]:
-            list_ids = dic_papers_co_authors[author_name_norm]["Author's id(s)"].split(", ")
-            work_count = 0
-            cited_by_count = 0
-            institutions = ''
-            for elem in list_ids:
-                if len(elem) > 3:
-                    url = 'https://api.openalex.org/authors?filter=openalex_id:' + elem + "&page=1&per_page=200&mailto=emma_scharfmann@berkeley.edu"
-                    try:
-                      data = get_data(url)["results"][0]
-                    except:
-                      continue
-      
-                    work_count += data["works_count"]
-                    cited_by_count += data["cited_by_count"]
-                    if data["last_known_institution"] != None and len(data["last_known_institution"]) > 0 and data["last_known_institution"]["display_name"] != None:
-                        institutions += data["last_known_institution"]["display_name"]
-                    
-            dic_papers_co_authors[author_name_norm]["Last Known Institution"] = institutions
-            dic_papers_co_authors[author_name_norm]["Number of works"] = work_count
-            dic_papers_co_authors[author_name_norm]["Number of citations"] = cited_by_count
-
-            
-        return pd.DataFrame(dic_papers_co_authors, index = [ "Author's name(s)" ,  "Number of occurence within the " + str(size) + " most related papers" , "Last Known Institution" , "Number of works" , "Number of citations" ,  "Number of related citations", "Author's id(s)"]).T
-
-    else:
-        return ("Select another category")
-
-
-def map_authors(technologies, number_technology , carbon_related , size , research_words):
-
-    name_file = "iea"
-    dic_details = details(name_file , False)
-    dic_papers , dic_papers_ranked = ranking_papers(number_technology,  dic_details   , False , size , carbon_related , research_words )
-            
-    dic_papers_co_authors = {}
-    count = 0 
-
-    for paper in list(dic_papers_ranked.keys())[:size]:
-        
-
-        for k in range(len(dic_papers[paper]["authorships"])):
-            
-            if dic_papers[paper]["authorships"][k]["institutions"] != [] and "id" in dic_papers[paper]["authorships"][k]["institutions"][0] and dic_papers[paper]["authorships"][k]["institutions"][0]["id"] != None:
-                institution_id = dic_papers[paper]["authorships"][k]["institutions"][0]["id"][21:]
-                if institution_id in dic_institutions:
-                    data = dic_institutions[institution_id]
-
-                    dic_papers_co_authors[count] = {}
-                    dic_papers_co_authors[count]["longitude"] = data["longitude"]
-                    dic_papers_co_authors[count]["latitude"] = data["latitude"]
-                    dic_papers_co_authors[count]["abstract comparison"] = dic_papers_ranked[paper]["abstract comparison"]
-                    dic_papers_co_authors[count]["author"] = dic_papers[paper]["authorships"][k]["author"]["display_name"]
-                    dic_papers_co_authors[count]["institution"] = dic_papers[paper]["authorships"][k]["institutions"][0]["display_name"]
-                    dic_papers_co_authors[count]["date"] = dic_papers[paper]["date"]
-                    
-                    count += 1
-    
-                        
-    if dic_papers_co_authors == {}:
-        return "No papers"
-    
-
-    
-    map_df = pd.DataFrame(dic_papers_co_authors).T
-    map_df["longitude"]=map_df['longitude'].astype(float)
-    map_df['latitude']=map_df['latitude'].astype(float)
-    map_df = map_df[map_df["latitude"].notnull()]
-
-    return map_df
-       
-
-################################### Extracted texts ###############################################################
-
-#@title Which patents are related to the technology?
-def finder():
-    name_file = 'iea'
-    res = technology("iea", False )
-    list_categories_tech = []
-    list_categories = key_words("iea" , False)
-    list_technologies = [ ( ", ".join(list_categories[i][1]) , i ) for i in range(len(list_categories))  ] 
-    dic_technologies = {}
-    for i in range(len(res)):
-        names = res[i][1]
-        if ", ".join(names) not in list_categories_tech:
-            list_categories_tech.append(", ".join(names))
-            dic_technologies[", ".join(names)] = []
-        dic_technologies[", ".join(names)].append( (", ".join(list_categories[i][1]) , i ))
-        
-
-    list_climate = [ ("Any related papers" , False ) , ("Climate related papers" , True)]
-
-    dic_categories = {}
-    for elem in list_technologies:
-        list_words = elem[0].split(",")[-3:]
-        for i in range(len(list_words)):
-            if "CCUS" in list_words[i]:
-                list_words[i] = list_words[i].replace("CCUS" , "carbon capture storage")
-        dic_categories[elem[1]]  = [  ", ".join([ " ".join(words.split()[:3]) for words in list_words ] )  ,  ", ".join([ " ".join(words.split()[:3]) for words in list_words[:-1] ]) , ", ".join([ " ".join(words.split()[:3]) for words in list_words[1:] ] ) ]  
-    
-    return dic_technologies, dic_categories, list_categories_tech, list_technologies
-                                       
-# technologies_widget = widgets.Dropdown(options=list_categories_tech, 
-#                                             description="Choose the category:" , 
-#                                             style = {'description_width':'initial' } , 
-#                                             layout=Layout(width='500px'));
-
-# category_widget = widgets.RadioButtons( values = 1,
-#                                         description='Patents where the abstract contains:' , 
-#                                         style = {'description_width':'initial' } , 
-#                                         layout=Layout(width='1000px'));
-                                       
-# number_technology_widget = widgets.Dropdown(options=list_technologies, 
-#                                             description="Choose the technology:" , 
-#                                             style = {'description_width':'initial' } , 
-#                                             layout=Layout(width='500px'));
-
-# def update_category(*args):
-#     category_widget.options = dic_categories[number_technology_widget.value]
-    
-# def update_technology(*args):
-#     number_technology_widget.options = dic_technologies[technologies_widget.value]
-    
-                                       
-# number_technology_widget.observe(update_category)
-# technologies_widget.observe(update_technology)
-
-# print("Which patents are related to the technology?")
-# interact_manual(get_ranking_patents, 
-#                 technologies=technologies_widget,
-#                 number_technology= number_technology_widget ,    
-#                 carbon_related=widgets.Dropdown(options=list_climate, 
-#                                                 description="Select the type of patents:" , 
-#                                                 style = {'description_width':'initial' }   , 
-#                                                 layout=Layout(width='500px')  ) , 
-#                 size = widgets.IntSlider(min=3, 
-#                                          max=100, 
-#                                          value=10, 
-#                                          step=1, 
-#                                          description="Select the number of patents:" ,  
-#                                          style = {'description_width':'initial' } , 
-#                                          layout=Layout(width='500px')),
-#                category = category_widget);

Climate_site/python_scripts/paper_functions_own_details.py

@@ -1,542 +0,0 @@
-##packages code
-from shapely.geometry import Point
-import pandas as pd
-from tqdm import tqdm
-import streamlit as st
-import numpy as np
-import json, requests 
-import pandas as pd
-#from pandas.io.json import json_normalize
-
-import matplotlib.pyplot as plt
-import seaborn as sns
-from math import radians, cos, sin, asin, sqrt
-
-import spacy
-#!python -m spacy download en_core_web_lg
-
-
-from sentence_transformers import SentenceTransformer, util
-
-@st.cache_resource
-def model_nlp():
-    model = SentenceTransformer('all-MiniLM-L6-v2')
-    return model
-
-import unicodedata
-
-from metaphone import doublemetaphone
-from fuzzywuzzy import fuzz
-from difflib import SequenceMatcher
-import re
-
-import geopandas as gpd
-from geopandas import GeoDataFrame
-
-
-@st.cache_data  # 👈 Add the caching decorator
-def load_data():
-    url = "Synapse_project/Climate_site/python_scripts/institutions.tsv"
-    dic = pd.read_csv(url, delimiter = "\t" , index_col = 1).to_dict('index')
-    return dic
-
-dic_institutions = load_data()
-
-
-#################### General Functions #############################
-
-def URL(base_URL , entity_type , filters):
-    url = base_URL + entity_type + filters 
-    return url
-
-
-def get_data(url):
-    url = requests.get(url)
-    text = url.text
-    import json
-    data = json.loads(text)
-    return data
-
-## encoding the abstract
-
-
-def reconstruction_abstract(abstract_inverted_index):
-    # return the abstract is the abstract exists in the database, else, return None
-    
-    if abstract_inverted_index != None:
-        
-        list_values = list(abstract_inverted_index.values())
-        list_keys = list(abstract_inverted_index.keys())
-        #from the words in the abstract (keys of abstract_inverted_index) and their position in the text (values of abstract_inverted_index), reconstruct the abstract
-        
-        size_abstract = max([ max(elem) for elem in abstract_inverted_index.values() ] )
-        
-        abstract = [""]*(size_abstract +1)
-        
-        for i in range(len(list_values)):
-            for pos in list_values[i]:
-                abstract[pos] = list_keys[i]
-        
-        return " ".join(list(abstract))
-    
-    else:
-        return None
-            
-    
-## calculate efficiently the dot product between two vectors
-
-def norm(vector):
-    return np.sqrt(sum(x * x for x in vector))    
-
-def cosine_similarity2(vec_a, vec_b):
-        norm_a = norm(vec_a)
-        norm_b = norm(vec_b)
-        dot = sum(a * b for a, b in zip(vec_a, vec_b))
-        return dot / (norm_a * norm_b)
-
-
-
-
-#################### Paper Functions #############################
-
-def related_papers_own_research(  research_key_words , display  ):
-
-    dic = {}
-
-    max_count = 0
-    base_URL_OA = f'https://api.openalex.org/'
-    filter_works = f'works?'
-    
-        
-
-    filter_openalex = f"search=" + research_key_words + "&per_page=200&mailto=emma_scharfmann@berkeley.edu"
-        
-        
-    filter_openalex = filter_openalex.replace(" " , "%20")
-    
-    url = URL(base_URL_OA , filter_works, filter_openalex) 
-    data = get_data(url)
-    count = data["meta"]["count"] 
-
-
-
-    if display == True:
-        print(  data["meta"]["count"] , [elem[i] for i in range(1,len(elem))]  )
-        print(url)
-    
-    for i in range(len(data["results"])):
-        dic[ data["results"][i]["id"]] = {}
-        dic[ data["results"][i]["id"]]["title"] = data["results"][i]["title"]
-        dic[ data["results"][i]["id"]]["abstract"] = reconstruction_abstract(data["results"][i]["abstract_inverted_index"])
-        dic[ data["results"][i]["id"]]["concepts"] = data["results"][i]["concepts"]
-        dic[ data["results"][i]["id"]]["date"] = data["results"][i]["publication_date"]
-        dic[ data["results"][i]["id"]]["authorships"] = data["results"][i]["authorships"]
-        dic[ data["results"][i]["id"]]["cited_by_count"] = data["results"][i]["cited_by_count"]
-
-        if len(data["results"][i]["authorships"]) > 0:
-            if data["results"][i]["authorships"][0]["institutions"] != []:
-                dic[ data["results"][i]["id"]]["countries"] = data["results"][i]["authorships"][0]["institutions"][0]["country_code"]
-                dic[ data["results"][i]["id"]]["institutions"] = data["results"][i]["authorships"][0]["institutions"][0]["display_name"]
-            else:
-                dic[ data["results"][i]["id"]]["countries"] = ""
-                dic[ data["results"][i]["id"]]["institutions"] = ""
-            dic[ data["results"][i]["id"]]["authors"] = data["results"][i]["authorships"][0]["author"]["display_name"]
-
-            for j in range(1 , len(data["results"][i]["authorships"])):
-                if data["results"][i]["authorships"][j]["institutions"] != []:
-                    dic[ data["results"][i]["id"]]["institutions"] += ", " + data["results"][i]["authorships"][j]["institutions"][0]["display_name"]
-                    dic[ data["results"][i]["id"]]["countries"] = data["results"][i]["authorships"][j]["institutions"][0]["country_code"]
-                dic[ data["results"][i]["id"]]["authors"] += ", " + data["results"][i]["authorships"][j]["author"]["display_name"]
-
-    if display == True: 
-        print(" ")
-        
-    return dic
-        
-    
-        
-
-## Ranking Papers
-
-def ranking_own_research(research_key_words,  details  , display):
-    
-    model = model_nlp()
-    
-    dic_scores_papers = {}
-    dic = related_papers_own_research( research_key_words , False)
-    
-    reference_text = details
-        
-    if display== True:
-        print("Technology details: " , reference_text)
-        print(" ")
-    encoded_text = model.encode(reference_text, convert_to_tensor=False , show_progress_bar = False).tolist()
-    
-
-    
-    for ids in list(dic.keys()):
-
-        dic_scores_papers[ids] = {}
-        
-        if dic[ids]["title"] != None:
-            encoded_title = model.encode(dic[ids]["title"], convert_to_tensor=False , show_progress_bar = False).tolist()
-            score_title = cosine_similarity2(encoded_title, encoded_text)
-        else:
-            score_title = None
-
-    
-
-        if dic[ids]["abstract"] != None:
-            encoded_abstract = model.encode(dic[ids]["abstract"], convert_to_tensor=False , show_progress_bar = False).tolist()
-            score_abstract = cosine_similarity2(encoded_abstract, encoded_text)
-        else:
-            score_abstract = None
-
-
-        #concepts = ""
-        #for elem in dic[ids]["concepts"]:
-        #    concepts += elem["display_name"] + " "
-        #encoded_concepts = model.encode(concepts, convert_to_tensor=False).tolist()
-        #score_concepts = cosine_similarity2(encoded_concepts, encoded_text)
-
-        dic_scores_papers[ids]["title comparison"] = score_title
-        dic_scores_papers[ids]["abstract comparison"] = score_abstract
-        #dic_scores_papers[ids]["concepts comparision"] = score_concepts
-        
-        dic_scores_papers[ids]["title"] = dic[ids]["title"]
-        dic_scores_papers[ids]["citations"] = dic[ids]["cited_by_count"]
-        
-
-        dic_scores_papers[ids]["date"] = dic[ids]["date"][:4]
-        if "institutions" in dic[ids]:
-            dic_scores_papers[ids]["institutions"] = dic[ids]["institutions"]
-        #    dic_scores_papers[ids]["countries"] = dic[ids]["countries"]
-            
-        else:
-            dic_scores_papers[ids]["institutions"] = None
-        #    dic_scores_papers[ids]["countries"] = None
-        
-        if "authors" in dic[ids]:
-            dic_scores_papers[ids]["number of co-authors"] = len(dic[ids]["authors"].split(","))
-            dic_scores_papers[ids]["authors"] = dic[ids]["authors"]
-
-    return dic , dic_scores_papers
-    
-
-
-
-    
-    
-    
-def get_ranking_own_research(research_key_words,  details  , display , size):
-    
-    dic , dic_scores_papers = ranking_own_research(research_key_words,  details  , display)
-    
-    if dic_scores_papers == {}:
-        return "No paper found"
-    
-    elif type(dic_scores_papers) == str:
-        return dic_scores_papers
-    
-    else:
-        return pd.DataFrame(dic_scores_papers).T.sort_values(by="abstract comparison" , ascending = False).head(size)
-     
-      
- 
-          
-    
-
-
-def extract_quantitative_data_paper(work_id):
-
-    url = "https://api.openalex.org/works/" + str(work_id)
-    url_google = "https://explore.openalex.org/works/" + str(work_id)
-    
-    data = get_data(url)
-    date = data["publication_date"]
-    title = data["title"]
-    abstract = reconstruction_abstract(data["abstract_inverted_index"])
-    concepts = ", ".join( [elem["display_name"] for elem in data["concepts"]] )
-    authors = ", ".join( [elem["author"]["display_name"] for elem in data["authorships"]] )
-    institutions = ", ".join( set([elem["institutions"][0]["display_name"] for elem in data["authorships"] if len(elem["institutions"]) > 0]) )
-    return url_google , title , abstract , date , authors , institutions
-            
-
-    
-
-
-## Mapping the authors
-
-#merge the nobiliary particles with the last name
-#ln_suff file can be modified if more or less nobiliary particles want to be suppressed
-
-ln_suff = ['oster',
- 'nordre',
- 'vaster',
- 'aust',
- 'vesle',
- 'da',
- 'van t',
- 'af',
- 'al',
- 'setya',
- 'zu',
- 'la',
- 'na',
- 'mic',
- 'ofver',
- 'el',
- 'vetle',
- 'van het',
- 'dos',
- 'ui',
- 'vest',
- 'ab',
- 'vste',
- 'nord',
- 'van der',
- 'bin',
- 'ibn',
- 'war',
- 'fitz',
- 'alam',
- 'di',
- 'erch',
- 'fetch',
- 'nga',
- 'ka',
- 'soder',
- 'lille',
- 'upp',
- 'ua',
- 'te',
- 'ni',
- 'bint',
- 'von und zu',
- 'vast',
- 'vestre',
- 'over',
- 'syd',
- 'mac',
- 'nin',
- 'nic',
- 'putri',
- 'bet',
- 'verch',
- 'norr',
- 'bath',
- 'della',
- 'van',
- 'ben',
- 'du',
- 'stor',
- 'das',
- 'neder',
- 'abu',
- 'degli',
- 'vre',
- 'ait',
- 'ny',
- 'opp',
- 'pour',
- 'kil',
- 'der',
- 'oz',
- 'von',
- 'at',
- 'nedre',
- 'van den',
- 'setia',
- 'ap',
- 'gil',
- 'myljom',
- 'van de',
- 'stre',
- 'dele',
- 'mck',
- 'de',
- 'mellom',
- 'mhic',
- 'binti',
- 'ath',
- 'binte',
- 'snder',
- 'sre',
- 'ned',
- 'ter',
- 'bar',
- 'le',
- 'mala',
- 'ost',
- 'syndre',
- 'sr',
- 'bat',
- 'sndre',
- 'austre',
- 'putra',
- 'putera',
- 'av',
- 'lu',
- 'vetch',
- 'ver',
- 'puteri',
- 'mc',
- 'tre',
- 'st']
-
-#suppress all the unwanted suffixes from a string. 
-#name_del file can be modified if more or less suffixes want to be suppressed 
-
-name_del = ['2nd', '3rd', 'Jr', 'Jr.', 'Junior', 'Sr', 'Sr.', 'Senior']
-
-
-def name_delete(string):
-    for elmt in name_del:
-        if f" {elmt}" in string:
-            return string.replace(f" {elmt}","")
-    return string
-
-def ln_suff_merge(string):
-    for suff in ln_suff:
-        if f" {suff} " in string or string.startswith(f"{suff} "):
-            return string.replace(f"{suff} ",suff.replace(" ",""))
-    return string
-
-#normalize a string dat that represents often a name. 
-
-def normalize(data):
-    normal = unicodedata.normalize('NFKD', data).encode('ASCII', 'ignore')
-    val = normal.decode("utf-8")
-    # delete unwanted elmt
-    val = name_delete(val)
-    # lower full name in upper
-    val = re.sub(r"[A-Z]{3,}", lambda x: x.group().lower(), val)
-    # add space in front of upper case
-    val = re.sub(r"(\w)([A-Z])", r"\1 \2", val)
-    # Lower case
-    val = val.lower()
-    # remove special characters
-    val = re.sub('[^A-Za-z0-9 ]+', ' ', val)
-    # remove multiple spaces
-    val = re.sub(' +', ' ', val)
-    # remove trailing spaces
-    val = val.strip()
-    # suffix merge
-    val = ln_suff_merge(val)
-
-    return val
-
-
-def main_authors( research_key_words  , details ,   size):
-        
-    dic_papers , dic_papers_ranked = ranking_own_research(  research_key_words  , details ,  False  )
-    
-    dic_papers_co_authors = {}
-        
-    for paper in list(dic_papers_ranked.keys())[:size]:
-
-        for k in range(len(dic_papers[paper]["authorships"])):
-            coauthor_id = dic_papers[paper]["authorships"][k]["author"]["id"]
-            
-            
-            author_name = dic_papers[paper]["authorships"][k]["author"]["display_name"]
-            author_name_norm = normalize(dic_papers[paper]["authorships"][k]["author"]["display_name"]).split()
-            if len (author_name_norm) > 0:
-                
-                author_name_norm = author_name_norm[0] + " " + author_name_norm[-1]
-
-                if author_name_norm not in dic_papers_co_authors:
-                    dic_papers_co_authors[author_name_norm] = {}
-                    dic_papers_co_authors[author_name_norm]["Author's name(s)"] = author_name
-                    if coauthor_id != None:
-                        dic_papers_co_authors[author_name_norm]["Author's id(s)"] = coauthor_id[21:]
-                    else:
-                        dic_papers_co_authors[author_name_norm]["Author's id(s)"] = ""
-                    dic_papers_co_authors[author_name_norm]["Number of occurence within the " + str(size) + " most related papers"] = 1
-                    dic_papers_co_authors[author_name_norm]["Number of related citations"] = dic_papers[paper]["cited_by_count"]
-                else:
-                    dic_papers_co_authors[author_name_norm]["Number of occurence within the " + str(size) + " most related papers"] += 1
-                    if author_name not in dic_papers_co_authors[author_name_norm]["Author's name(s)"]:
-                        dic_papers_co_authors[author_name_norm]["Author's name(s)"] += ", " + author_name
-                    if coauthor_id != None and coauthor_id[21:] not in dic_papers_co_authors[author_name_norm]["Author's id(s)"]:                                                              
-                        dic_papers_co_authors[author_name_norm]["Author's id(s)"] += ", " + coauthor_id[21:]
-                        dic_papers_co_authors[author_name_norm]["Number of related citations"] += dic_papers[paper]["cited_by_count"]
-
-
-
-    if dic_papers_co_authors != {}:
-        dic_papers_co_authors = {k: v for k, v in sorted(dic_papers_co_authors.items(), key=lambda item: item[1]["Number of occurence within the " + str(size) + " most related papers"] , reverse = True)}
-
-        for author_name_norm in list(dic_papers_co_authors.keys())[:size]:
-            list_ids = dic_papers_co_authors[author_name_norm]["Author's id(s)"].split(", ")
-            work_count = 0
-            cited_by_count = 0
-            institutions = ''
-            for elem in list_ids:
-                if len(elem) > 3:
-                    
-                    data = get_data("https://api.openalex.org/people/" + elem)
-                    work_count += data["works_count"]
-                    cited_by_count += data["cited_by_count"]
-                    if data["last_known_institution"] != None and data["last_known_institution"]["display_name"] != None:
-                        if institutions == '':
-                            institutions += data["last_known_institution"]["display_name"]
-                        else:
-                            institutions += ", " + data["last_known_institution"]["display_name"]
-                    
-            dic_papers_co_authors[author_name_norm]["Last Known Institution"] = institutions
-            dic_papers_co_authors[author_name_norm]["Number of works"] = work_count
-            dic_papers_co_authors[author_name_norm]["Number of citations"] = cited_by_count
-
-            
-        return pd.DataFrame(dic_papers_co_authors, index = [ "Author's name(s)" , "Author's id(s)" ,  "Number of occurence within the " + str(size) + " most related papers" , "Last Known Institution" , "Number of works" , "Number of citations" ,  "Number of related citations"]).T.style.hide(axis="index")
-
-    else:
-        return ("Select another category")
-
-
-def map_authors(research_key_words , details , size):
-
-    dic_papers , dic_papers_ranked = ranking_own_research(  research_key_words  , details ,  False  )
-
-    
-    dic_papers_co_authors = {}
-    count = 0
-    
-
-    for paper in list(dic_papers_ranked.keys())[:size]:
-        
-        for k in range(len(dic_papers[paper]["authorships"])):
-            
-            if dic_papers[paper]["authorships"][k]["institutions"] != [] and "id" in dic_papers[paper]["authorships"][k]["institutions"][0] and dic_papers[paper]["authorships"][k]["institutions"][0]["id"] != None:
-                institution_id = dic_papers[paper]["authorships"][k]["institutions"][0]["id"][21:]
-                if institution_id in dic_institutions:
-                    data = dic_institutions[institution_id]
-
-                    dic_papers_co_authors[count] = {}
-                    dic_papers_co_authors[count]["longitude"] = data["longitude"]
-                    dic_papers_co_authors[count]["latitude"] = data["latitude"]
-                    dic_papers_co_authors[count]["abstract comparison"] = dic_papers_ranked[paper]["abstract comparison"]
-                    dic_papers_co_authors[count]["author"] = dic_papers[paper]["authorships"][k]["author"]["display_name"]
-                    dic_papers_co_authors[count]["institution"] = dic_papers[paper]["authorships"][k]["institutions"][0]["display_name"]
-                    dic_papers_co_authors[count]["date"] = dic_papers[paper]["date"]
-                    
-                    count += 1
-                    
-    map_df = pd.DataFrame(dic_papers_co_authors).T
-                    
-    if dic_papers_co_authors == {}:
-        return map_df
-    
-
-    
-    
-    map_df["longitude"]=map_df['longitude'].astype(float)
-    map_df['latitude']=map_df['latitude'].astype(float)
-    map_df = map_df[map_df["latitude"].notnull()]
-
-    return map_df
-       
-
-
-

Climate_site/python_scripts/patent_functions.py

@@ -1,892 +0,0 @@
-##packages code
-import streamlit as st
-from shapely.geometry import Point
-import pandas as pd
-from tqdm import tqdm
-
-import numpy as np
-import json, requests 
-import pandas as pd
-#from pandas.io.json import json_normalize
-
-import matplotlib.pyplot as plt
-import seaborn as sns
-from math import radians, cos, sin, asin, sqrt
-
-
-from sentence_transformers import SentenceTransformer, util
-
-@st.cache_resource
-def model_nlp():
-    model = SentenceTransformer('all-MiniLM-L6-v2')
-    return model
-
-
-@st.cache_data  # 👈 Add the caching decorator
-def load_data():
-    url = "Synapse_project/Climate_site/python_scripts/institutions.tsv"
-    dic = pd.read_csv(url, delimiter = "\t" , index_col = 0).to_dict('index')
-    return dic
-
-dic_institutions = load_data()
-
-
-import unicodedata
-
-from metaphone import doublemetaphone
-from fuzzywuzzy import fuzz
-from difflib import SequenceMatcher
-import re
-
-import geopandas as gpd
-from geopandas import GeoDataFrame
-#################### General Functions #############################
-
-def URL(base_URL , entity_type , filters):
-    url = base_URL + entity_type + filters 
-    return url
-
-
-def get_data(url):
-    url = requests.get(url)
-    text = url.text
-    import json
-    data = json.loads(text)
-    return data
-
-## encoding the abstract
-
-
-def reconstruction_abstract(abstract_inverted_index):
-    # return the abstract is the abstract exists in the database, else, return None
-    
-    if abstract_inverted_index != None:
-        
-        list_values = list(abstract_inverted_index.values())
-        list_keys = list(abstract_inverted_index.keys())
-        #from the words in the abstract (keys of abstract_inverted_index) and their position in the text (values of abstract_inverted_index), reconstruct the abstract
-        
-        size_abstract = max([ max(elem) for elem in abstract_inverted_index.values() ] )
-        
-        abstract = [""]*(size_abstract +1)
-        
-        for i in range(len(list_values)):
-            for pos in list_values[i]:
-                abstract[pos] = list_keys[i]
-        
-        return " ".join(list(abstract))
-    
-    else:
-        return None
-            
-    
-## calculate efficiently the dot product between two vectors
-
-def norm(vector):
-    return np.sqrt(sum(x * x for x in vector))    
-
-def cosine_similarity2(vec_a, vec_b):
-        norm_a = norm(vec_a)
-        norm_b = norm(vec_b)
-        dot = sum(a * b for a, b in zip(vec_a, vec_b))
-        return dot / (norm_a * norm_b)
-
-## Extracted texts
-
-def print_extracted_text(name_file):
-
-    file = open('iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-    count = 0
-    for index, line in enumerate(lines):
-        read_line = line.strip()
-        print(read_line)
-
-    file.close()
-    
-    
-    iea.txt
-
-
-def details(name_file , display):
-    
-    file = open("Synapse_project/Climate_site/python_scripts/iea.txt", "r")
-    lines = file.readlines()
-
-    mark = 0 
-    dic_details = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-
-
-        if mark == 1 and line != "" and line[0] == "*":
-            
-            if display == True:
-                print(count)
-                print(text)
-                print(" ")
-            dic_details[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line.split(" 	")[-1] == "Details" or line.split(" 	")[-1] == "Hide":
-            mark = 1
-            text = ""
-            
-    return dic_details
-
-
-
-
-
-def key_initiatives(name_file , display ):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-
-      
-    mark = 0 
-    dic_key_initiatives = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-
-            count += 1
-
-
-
-        if mark == 1 and line != "" and ( (line[0].isnumeric() and  ">" in line and " 	" in line) or line == "*Deployment targets:*" or line == "*Announced development targets:*"):
-            if display == True: 
-                print(count)
-                print(text)
-                print(" ")
-            
-            dic_key_initiatives[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line == "*Key initiatives:*":
-
-
-            mark = 1
-            text = ""
-            
-    return dic_key_initiatives
-
-
-
-
-def deployment_target(name_file , display):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-
-
-    mark = 0 
-    dic_target = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-
-
-
-        if mark == 1 and line != "" and  ((line[0].isnumeric() and ">" in line and " 	" in line)  or line == "*Announced cost reduction targets:*" or line == "*Announced development targets:*"):
-            
-            if display == True:
-                print(count)
-                print(text)
-                print(" ")
-                
-            dic_target[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line == "*Deployment targets:*" or line == "*Announced development targets:*":
-
-            mark = 1
-            text = ""
-            
-    return dic_target
-
-
- 
-    
-def cost_reduction_target(name_file , display):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-    
-    mark = 0 
-    dic_cost = {}
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-
-
-            count += 1
-
-
-
-        if mark == 1 and line != ""  and (line[0].isnumeric() and ">" in line and " 	" in line) :
-            
-            if display == True: 
-                print(count)
-                print(text)
-                print(" ")
-                
-            dic_cost[count] = text
-            mark = 0
-
-
-        if mark == 1:
-            text = text + line + " "
-
-        if line == "*Announced cost reduction targets:*":
-
-            mark = 1
-            text = ""
-            
-    return dic_cost
-
-
-
-def key_words(name_file, display ):
-    
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    
-    lines = file.readlines()
-    
-    list_categories = []
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-            
-            if display == True:
-                print("Technologies"  , count+1 , ":")
-
-        if line != "":
-
-            if line[0].isnumeric() and ">" in line and " 	" in line:
-                i = 0
-                try:
-                    line = line.split(" 	")[2]
-                except:
-                    print(line)
-                    break
-                
-                if "Details" not in lines[index] and "Moderate" not in lines[index]:
-                
-                    while "	" not in line:
-                        i += 1
-                        if "Details"==lines[index + i][:7] or "End-use"==lines[index + i][:7]:
-                            break
-                        else:
-                            line = line + "  " +  lines[index + i]
-
-                #if " Production" in line:
-                    #line = line.replace(" Production" , "")
-
-                line = line.replace("\n" , " ")
-                line = line.replace("/" , " ")
-                line = line.replace("-" , " ")
-                line = line.split(" 	")[0]
-
-                if "  " in line:
-                    line = line.replace("  ", " ")
-                line = line.split(">")
-
-
-                if "(" in line[-1]:
-                    line[-1] = line[-1].split("(")[0] 
-
-
-                for i in range(len(line)):
-
-                    # remove multiple spaces
-                    line[i] = re.sub(' +', ' ', line[i])
-                    # remove trailing spaces
-                    line[i] = line[i].strip()
-
-    
-
-                if display == True:
-                    print(line)
-                    print(" ")
-                    
-                if '' in line:
-                    line.remove('')
-
-                list_categories.append([count , line])
-                
-    return list_categories
-
-
-
-def technology(name_file, display ):
-    # Filepath too specific, need to change to relative path
-    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
-    lines = file.readlines()
-    
-    list_categories = []
-    count = -1
-    for index, line in enumerate(lines):
-
-        line = line.strip()
-        
-        if line == "Close explanation":
-            break
-
-        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
-            count += 1
-            
-
-            if display == True:
-                print("Technologies"  , count+1 , ":")
-
-        if line != "":
-
-            if line[0].isnumeric() and ">" in line and " 	" in line:
-                i = 0
-                try:
-                    line = line.split(" 	")[1]
-                except:
-                    print(line)
-                    break
-                
-
-                line = line.replace("\n" , " ")
-                line = line.replace("/" , " ")
-                line = line.replace("-" , " ")
-                line = line.strip()
-                line = re.sub(' +', ' ', line)
-                line = line.split(" 	")[0]
-                line = line.split(">")
-
-
-                if "(" in line[-1]:
-                    line[-1] = line[-1].split("(")[0] 
-
-
-                for i in range(len(line)):
-
-                    # remove multiple spaces
-                    line[i] = re.sub(' +', ' ', line[i])
-                    # remove trailing spaces
-                    line[i] = line[i].strip()
-
-    
-
-                if display == True:
-                    print(line)
-                    print(" ")
-                
-
-                list_categories.append([count , line])
-                
-    return list_categories
-
-
-#################### Patent Functions #############################
-
-def related_patents(number_technology , research_words, carbon_related , display):
-    
-    name_file = "iea"
-    list_categories = key_words( name_file, False)
-
-    dic_patents = {}
-
-    max_count = 0
-    base_URL_PV = "https://api.patentsview.org/"
-    filter_works = "patents/query?"
-    filter_PV = "q={%22_and%22:[{%22_text_any%22:{%22patent_abstract%22:%22" 
-    filter_PV += research_words.replace("," , "")
-    
-    if carbon_related == True:
-        filter_PV += "%22}},{%22_eq%22:{%22cpc_group_id%22:%22Y02E%22}}]}&f=[%22patent_number%22,%22patent_title%22,%22assignee_country%22,%22patent_date%22,%22inventor_id%22,%22assignee_organization%22,%22inventor_longitude%22,%22inventor_latitude%22,%22inventor_last_name%22,%22inventor_first_name%22,%22cpc_subsection_title%22,%22assignee_city%22,%22patent_abstract%22,%22patent_kind%22,%22cpc_group_id%22,%22assignee_organization%22,%22citedby_patent_number%22]"
-    
-    else:
-        filter_PV += "%22}}]}&f=[%22patent_number%22,%22patent_title%22,%22assignee_country%22,%22patent_date%22,%22assignee_organization%22,%22inventor_longitude%22,%22inventor_latitude%22,%22inventor_last_name%22,%22inventor_id%22,%22inventor_first_name%22,%22cpc_subsection_title%22,%22assignee_city%22,%22patent_abstract%22,%22patent_kind%22,%22cpc_group_id%22,%22assignee_organization%22,%22citedby_patent_number%22]"
-
-        
-    filter_PV = filter_PV.replace(" " , "%20")
-
-        
-
-    url = URL(base_URL_PV , filter_works, filter_PV) 
-    data = get_data(url)
-
-
-    if display == True:
-        print(  data["total_patent_count"] , elem[-1] )
-        print(url)
-
-
-
-    
-    for i in range(data["count"]):
-        dic_patents[ "US-" + data["patents"][i]["patent_number"]] = {}
-        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["title"] = data["patents"][i]["patent_title"]
-        dic_patents["US-" + data["patents"][i]["patent_number"]]["abstract"] = data["patents"][i]["patent_abstract"]
-
-        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["assignee"] = str(data["patents"][i]["assignees"][0]["assignee_organization"])
-        dic_patents["US-" + data["patents"][i]["patent_number"]]["assignee_city"] = str(data["patents"][i]["assignees"][0]["assignee_city"])
-        dic_patents["US-" + data["patents"][i]["patent_number"]]["assignee_country"] = str(data["patents"][i]["assignees"][0]["assignee_country"])
-        for j in range(1, len(data["patents"][i]["assignees"])):
-            dic_patents[ "US-" + data["patents"][i]["patent_number"]]["assignee"] +=  ", " + str(data["patents"][i]["assignees"][j]["assignee_organization"])
-            dic_patents[ "US-" + data["patents"][i]["patent_number"]]["assignee_city"] +=  ", " + str(data["patents"][i]["assignees"][j]["assignee_city"])
-            dic_patents["US-" + data["patents"][i]["patent_number"]]["assignee_country"] +=  ", " + str(data["patents"][i]["assignees"][j]["assignee_country"])
-
-        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["list_inventors"] = data["patents"][i]["inventors"]
-
-        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["inventors"] = str(data["patents"][i]["inventors"][0]["inventor_first_name"]) + " " + str(data["patents"][i]["inventors"][0]["inventor_last_name"])
-        for j in range(1, len(data["patents"][i]["inventors"])):
-            dic_patents[ "US-" + data["patents"][i]["patent_number"]]["inventors"] +=  ", " + str(data["patents"][i]["inventors"][j]["inventor_first_name"]) + " " + str(data["patents"][i]["inventors"][j]["inventor_last_name"])
-
-
-        dic_patents["US-" + data["patents"][i]["patent_number"]]["date"] = data["patents"][i]["patent_date"]
-        dic_patents["US-" + data["patents"][i]["patent_number"]]["number_citations"] = len(data["patents"][i]["citedby_patents"])
-
-
-    if display == True:
-        print(" ")
-            
-            
-    return dic_patents
-
-
-
-## Ranking Patents
-
-def ranking_patents(number_technology , research_words, carbon_related , display):
-    
-    model = model_nlp()
-    
-    
-    name_file = "iea"
-    list_categories = key_words( name_file, False)
-    dic_details = details(name_file , False)
-    dic_patents = related_patents(number_technology , research_words, carbon_related , display)
-    
-
-    dic_scores = {}
-
-    if display == True:
-        print("Key words: " , list_categories[number_technology][1])
-        
-    if number_technology in dic_details:
-        reference_text = dic_details[number_technology]
-        if display == True:
-            print("Technology details: " , reference_text)
-            print(" ")
-        encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist()
-        
-        
-        if len(dic_patents ) == 0:
-            return "Select other key words"
-        
-        
-        else:
-            for ids in list(dic_patents.keys()):
-
-                dic_scores[ids] = {}
-
-                encoded_title = model.encode(dic_patents[ids]["title"], convert_to_tensor=False).tolist()
-                score_title = cosine_similarity2(encoded_title, encoded_text)
-
-
-                if dic_patents[ids]["abstract"] != None:
-                    encoded_abstract = model.encode(dic_patents[ids]["abstract"], convert_to_tensor=False).tolist()
-                    score_abstract = cosine_similarity2(encoded_abstract, encoded_text)
-                else:
-                    score_abstract = None
-
-                dic_scores[ids]["title comparision"] = score_title
-                dic_scores[ids]["abstract comparison"] = score_abstract
-                dic_scores[ids]["title"] = dic_patents[ids]["title"]
-                dic_scores[ids]["citations"] = dic_patents[ids]["number_citations"]
-                dic_scores[ids]["date"] = dic_patents[ids]["date"][:4]
-                dic_scores[ids]["assignee"] = dic_patents[ids]["assignee"]
-                #dic_scores[ids]["assignee_city"] = dic_patents[ids]["assignee_city"]
-                #dic_scores[ids]["assignee_country"] = dic_patents[ids]["assignee_country"]
-                dic_scores[ids]["inventors"] = dic_patents[ids]["inventors"]
-                dic_scores[ids]["number of co-inventors"] = len(dic_patents[ids]["inventors"].split(","))
-    return dic_patents , dic_scores
-
-
-def get_ranking_patents(technologies, number_technology ,category ,  carbon_related , size):
-    dic , dic_patents = ranking_patents(number_technology , category, carbon_related , False)
-    if dic_patents == {}:
-        return "No patent found, select other key words"
-
-    elif dic_patents == "Select other key words":
-        return dic_patents
-    
-    else:
-        return pd.DataFrame(dic_patents).T.sort_values(by="abstract comparison" , ascending = False)[:size]
-    
-    
-
-
-
-#normalize a string dat that represents often a name. 
-
-def normalize(data):
-    normal = unicodedata.normalize('NFKD', data).encode('ASCII', 'ignore')
-    val = normal.decode("utf-8")
-
-    # lower full name in upper
-    val = re.sub(r"[A-Z]{3,}", lambda x: x.group().lower(), val)
-    # add space in front of upper case
-    val = re.sub(r"(\w)([A-Z])", r"\1 \2", val)
-    # Lower case
-    val = val.lower()
-    # remove special characters
-    val = re.sub('[^A-Za-z0-9 ]+', ' ', val)
-    # remove multiple spaces
-    val = re.sub(' +', ' ', val)
-    # remove trailing spaces
-    val = val.strip()
-
-
-    return val
-
-
-def main_inventors(technologies, number_technology , carbon_related , category , size ):
-    display = False
-    dic_patents , dic_ranked = ranking_patents(number_technology , category, carbon_related , display)
-    dic_patents_co_inventors = {}
-    
-
-    for patent in list(dic_ranked.keys())[:size]:
-        for k in range(len(dic_patents[patent]["list_inventors"])):
-            
-            inventor_id = dic_patents[patent]["list_inventors"][k]["inventor_id"]
-            inventor_name = dic_patents[patent]["list_inventors"][k]["inventor_first_name"] + " " + dic_patents[patent]["list_inventors"][k]["inventor_last_name"]
-            inventor_name_norm = normalize(inventor_name).split()
-            inventor_name_norm = inventor_name_norm[0] + " " + inventor_name_norm[-1]
-
-            if inventor_name_norm not in dic_patents_co_inventors:
-                dic_patents_co_inventors[inventor_name_norm] = {}
-                dic_patents_co_inventors[inventor_name_norm]["Inventor's name"] = inventor_name 
-                dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"] =  inventor_id 
-                dic_patents_co_inventors[inventor_name_norm]["Number of occurence"] = 1
-                dic_patents_co_inventors[inventor_name_norm]["Number of related citations"] = dic_patents[patent]["number_citations"]
-
-            else:
-                if inventor_id not in dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"] :
-                    dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"] += ", " + inventor_id
-                if inventor_name not in dic_patents_co_inventors[inventor_name_norm]["Inventor's name"] :
-                    dic_patents_co_inventors[inventor_name_norm]["Inventor's name"] += ", " + inventor_name
-                dic_patents_co_inventors[inventor_name_norm]["Number of occurence"] += 1
-                dic_patents_co_inventors[inventor_name_norm]["Number of related citations"] += dic_patents[patent]["number_citations"]
-
-
-        dic_patents_co_inventors = {k: v for k, v in sorted(dic_patents_co_inventors.items(), key=lambda item: item[1]["Number of occurence"] , reverse = True)}
-    
-    if dic_patents_co_inventors == {}:
-    
-        
-        return "No patent, select other key words"
-    else:
-        
-        for inventor_name_norm in list(dic_patents_co_inventors.keys()):
-            list_inventors = dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"].split(", ")
-            work_count = 0
-            cited_by_count = 0
-            
-            for elem in list_inventors:
-                url = "https://api.patentsview.org/inventors/query?q={%22inventor_id%22:[%22" + elem + "%22]}&f=[%22inventor_total_num_patents%22,%22patent_num_cited_by_us_patents%22]"
-                data = get_data(url)["inventors"][0]
-                work_count += int(data["inventor_total_num_patents"])
-                for k in range(len(data["patents"])):
-                    cited_by_count += int(data["patents"][k]["patent_num_cited_by_us_patents"])
-                
-            dic_patents_co_inventors[inventor_name_norm]["Number of patents"] = work_count
-            dic_patents_co_inventors[inventor_name_norm]["Number of US patents citations"] = cited_by_count
-            
-        
-                
-                
-        return pd.DataFrame(dic_patents_co_inventors , index = ["Inventor's name", "PatentsView inventor's id", "Number of occurence"  , "Number of patents" ,"Number of US patents citations" ,  "Number of related citations"]).T
-
-
-    
-    
-def map_inventors(technologies, number_technology , carbon_related , category , size):
-    display = False
-    dic_patents , dic_ranked = ranking_patents(number_technology , category, carbon_related , display)
-    dic_patents_co_inventors = {}
-    count = 0
-    
-    for patent in list(dic_ranked.keys())[:size]:
-        for k in range(len(dic_patents[patent]["list_inventors"])):
-            
-            dic_patents_co_inventors[count] = {}
-            
-            dic_patents_co_inventors[count]["latitude"] = dic_patents[patent]["list_inventors"][k]["inventor_latitude"]
-            dic_patents_co_inventors[count]["longitude"] = dic_patents[patent]["list_inventors"][k]["inventor_longitude"]
-            dic_patents_co_inventors[count]["longitude"] = dic_patents[patent]["list_inventors"][k]["inventor_longitude"]
-            dic_patents_co_inventors[count]["inventor_name"]  = str(dic_patents[patent]["list_inventors"][k]["inventor_first_name"]) + " " + str(dic_patents[patent]["list_inventors"][k]["inventor_last_name"])
-            dic_patents_co_inventors[count]["patent_date"]  = dic_patents[patent]["date"]
-            count += 1
-                
-    if dic_patents_co_inventors == {}:
-        return "No patent, select other key words"
-    
-    
-    map_df = pd.DataFrame(dic_patents_co_inventors).T
-    map_df["longitude"]=map_df['longitude'].astype(float)
-    map_df['latitude']=map_df['latitude'].astype(float)
-    map_df = map_df[map_df["latitude"].notnull()]
-
-    return map_df
-
-    #geometry = [Point(xy) for xy in zip(map_df['Longitude'], map_df['Latitude'])]
-    #gdf = GeoDataFrame(map_df, geometry=geometry)   
-
-    #this is a simple map that goes with geopandas
-    #world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))
-    #gdf.plot(ax=world.plot( color='white', edgecolor='black' ), marker='o', color='red', markersize=15 , zorder = 1);
-    #plt.xlim([-180, 180])
-    #plt.ylim([-90, 90])
-
-
-    #plt.title("Main inventors: geographic location")
-    #plt.xlabel("Longitude")
-    #plt.ylabel("Latitude")
-    #plt.show()
-
-    
-## Extract quantitative data
-
-def extract_sentences_with_numbers(text , text_name):
-    
-    if text != None:
-        text = text.replace("CO 2" , "CO2")
-        text = text.replace("CO 3" , "CO3")
-        text = text.replace("CO(2)" , "CO2")
-        text = text.replace("CO(3)" , "CO3")
-        
-        print(text_name + ": " , text)
-        print(" ")
-
-
-
-        list_text = list(text)
-        for i in range(1,len(list_text)-1):
-            if list_text[i] == " " and list_text[i-1] == "." and list_text[i+1].isupper():
-                list_text[i] = "~"
-
-        text = "".join(list_text)
-        text = text.split("~")
-        for sentence in text:                
-
-            if any(char.isdigit() for char in sentence):
-                
-                
-                
-                if "CO2" in sentence:
-                    print("\x1b[31mCARBON RELATED:\x1b[0m", sentence)
-                    print(" ")
-
-
-                    
-                if "GJ" in sentence or "MJ" in sentence:
-                    print("\x1b[31mENERGY:\x1b[0m" , sentence)
-                    print(" ")
-
-
-                    
-                ##price
-                if "€"  in sentence or "$" in sentence or "EUR" in sentence or "dollars" in sentence.lower():
-                    print("\x1b[31mPRICE:\x1b[0m" , sentence)
-                    print(" ")
-
-
-                    
-                ##dates
-                digits = []
-                for word in sentence.replace("," , "").replace("%" , "").replace("." , " ").split():
-                    if word.isdigit() and 1850 < int(word) < 2200 :
-                        digits.append(word)
-                if digits != []:
-                    print("\x1b[31mDATE:\x1b[0m" , sentence)
-                    print(" ")
-
-
-                    
-                ##CO quantity
-                if "Mt" in sentence or "tC" in sentence or "t-C" in sentence:
-                    print("\x1b[31mCARBON QUANTITY:\x1b[0m" , sentence)
-                    print(" ")
- 
-    print(" ")
-          
-    
-def extract_quantitative_data_technology(technologies, number_technology):
-    count = 0
-      
-    name_file = "iea"
-    dic_target = deployment_target(name_file , False)
-    dic_cost = cost_reduction_target(name_file , False)
-    dic_details = details(name_file , False)
-    cost_target_text = 'No information'
-    cost_text = 'No information'
-    
-    if number_technology in dic_details:
-        reference_text = dic_details[number_technology]
-            
-        #print("\033[96mFROM IEA website: ")
-        #print("\033[92mTechnology details: \x1b[0m" , reference_text)
-        #print(" ")
-        #encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist()
-
-    if number_technology in dic_target:
-        cost_target_text = dic_target[number_technology]
-        #print("\033[96mFROM IEA website: ")
-        #sentences = extract_sentences_with_numbers(cost_target_text , "\033[92mDeployment target and Announced development target\x1b[0m")
-
-    if number_technology in dic_cost:
-        cost_text = dic_cost[number_technology]
-        #print("\033[96mFROM IEA website: ")
-        #sentences = extract_sentences_with_numbers(cost_text , "\033[92mAnnounced cost reduction targets\x1b[0m")
-    
-    return reference_text, cost_target_text , cost_text
-
-
-        
-def extract_quantitative_data_patent(patent_id):
-    
-
-    patent_id = patent_id[3:]
-        
-    url = "https://api.patentsview.org/patents/query?q={%22patent_id%22:%22" + str(patent_id) + "%22}&f=[%22patent_number%22,%22patent_title%22,%22patent_abstract%22,%22patent_date%22,%22inventor_last_name%22,%22inventor_first_name%22,%22assignee_organization%22]"
-    url_google = "https://patents.google.com/patent/US" + str(patent_id)
-    
-    data = get_data(url)["patents"][0]
-    title = data["patent_title"]
-    abstract = data["patent_abstract"]
-    co_inventors = ", ".join([ data["inventors"][i]["inventor_first_name"] + " " + data["inventors"][i]["inventor_last_name"] for i in range(len(data["inventors"])) ]) 
-    assignees =  ", ".join([ str(data["assignees"][i]["assignee_organization"]) for i in range(len(data["assignees"])) ] )
-
-    return url_google , title , abstract , data["patent_date"] , co_inventors , assignees
-
-
-
-
-
-    
-        
-    
-
-################################### Extracted texts ###############################################################
-
-#@title Which patents are related to the technology?
-def finder():
-    name_file = 'iea'
-
-    res = technology("iea", False )
-    list_categories_tech = []
-    list_categories = key_words(name_file , False)
-    list_technologies = [ ( ", ".join(list_categories[i][1]) , i ) for i in range(len(list_categories))  ] 
-
-    dic_technologies = {}
-    for i in range(len(res)):
-        names = res[i][1]
-        if ", ".join(names) not in list_categories_tech:
-            list_categories_tech.append(", ".join(names))
-            dic_technologies[", ".join(names)] = []
-        dic_technologies[", ".join(names)].append( (", ".join(list_categories[i][1]) , i ))
-        
-
-    list_climate = [ ("Any related patents" , False ) , ("Climate related patents" , True)]
-
-    dic_categories = {}
-    for elem in list_technologies:
-        list_words = elem[0].split(",")[-3:]
-
-        dic_categories[elem[1]]  = [  " ".join(list_words[-1].split()[:3]) ,  " ".join(list_words[-2].split()[:3]) if len(list_words) > 1 else "" ,  " ".join(list_words[-3].split()[:3]) if len(list_words) > 2 else ""  ]  
-    
-    return dic_technologies, dic_categories, list_categories_tech, list_technologies
-                                       
-# technologies_widget = widgets.Dropdown(options=list_categories_tech, 
-#                                             description="Choose the category:" , 
-#                                             style = {'description_width':'initial' } , 
-#                                             layout=Layout(width='500px'));
-
-# category_widget = widgets.RadioButtons( values = 1,
-#                                         description='Patents where the abstract contains:' , 
-#                                         style = {'description_width':'initial' } , 
-#                                         layout=Layout(width='1000px'));
-                                       
-# number_technology_widget = widgets.Dropdown(options=list_technologies, 
-#                                             description="Choose the technology:" , 
-#                                             style = {'description_width':'initial' } , 
-#                                             layout=Layout(width='500px'));
-
-# def update_category(*args):
-#     category_widget.options = dic_categories[number_technology_widget.value]
-    
-# def update_technology(*args):
-#     number_technology_widget.options = dic_technologies[technologies_widget.value]
-    
-                                       
-# number_technology_widget.observe(update_category)
-# technologies_widget.observe(update_technology)
-
-# print("Which patents are related to the technology?")
-# interact_manual(get_ranking_patents, 
-#                 technologies=technologies_widget,
-#                 number_technology= number_technology_widget ,    
-#                 carbon_related=widgets.Dropdown(options=list_climate, 
-#                                                 description="Select the type of patents:" , 
-#                                                 style = {'description_width':'initial' }   , 
-#                                                 layout=Layout(width='500px')  ) , 
-#                 size = widgets.IntSlider(min=3, 
-#                                          max=100, 
-#                                          value=10, 
-#                                          step=1, 
-#                                          description="Select the number of patents:" ,  
-#                                          style = {'description_width':'initial' } , 
-#                                          layout=Layout(width='500px')),
-#                category = category_widget);

Climate_site/python_scripts/patent_functions_own_details.py

@@ -1,475 +0,0 @@
-##packages code
-from shapely.geometry import Point
-import pandas as pd
-from tqdm import tqdm
-import streamlit as st
-import numpy as np
-import json, requests 
-import pandas as pd
-#from pandas.io.json import json_normalize
-
-import matplotlib.pyplot as plt
-import seaborn as sns
-from math import radians, cos, sin, asin, sqrt
-
-import spacy
-#!python -m spacy download en_core_web_lg
-
-
-from sentence_transformers import SentenceTransformer, util
-
-@st.cache_resource
-def model_nlp():
-    model = SentenceTransformer('all-MiniLM-L6-v2')
-    return model
-
-import unicodedata
-
-from metaphone import doublemetaphone
-from fuzzywuzzy import fuzz
-from difflib import SequenceMatcher
-import re
-
-import geopandas as gpd
-from geopandas import GeoDataFrame
-
-
-@st.cache_data  # 👈 Add the caching decorator
-def load_data():
-    url = "Synapse_project/Climate_site/python_scripts/institutions.tsv"
-    dic = pd.read_csv(url, delimiter = "\t" , index_col = 0).to_dict('index')
-    return dic
-
-dic_institutions = load_data()
-
-
-#################### General Functions #############################
-
-def URL(base_URL , entity_type , filters):
-    url = base_URL + entity_type + filters 
-    return url
-
-
-def get_data(url):
-    url = requests.get(url)
-    text = url.text
-    import json
-    data = json.loads(text)
-    return data
-
-    
-## calculate efficiently the dot product between two vectors
-
-def norm(vector):
-    return np.sqrt(sum(x * x for x in vector))    
-
-def cosine_similarity2(vec_a, vec_b):
-        norm_a = norm(vec_a)
-        norm_b = norm(vec_b)
-        dot = sum(a * b for a, b in zip(vec_a, vec_b))
-        return dot / (norm_a * norm_b)
-
-
-
-
-#################### Patent Functions #############################
-
-
-def related_patents(research_words , display):
-    
-
-    dic_patents = {}
-
-    max_count = 0
-    base_URL_PV = "https://api.patentsview.org/"
-    filter_works = "patents/query?"
-    filter_PV = "q={%22_and%22:[{%22_text_all%22:{%22patent_abstract%22:%22" 
-    filter_PV += research_words
-
-    filter_PV += "%22}}]}&f=[%22patent_number%22,%22patent_title%22,%22assignee_country%22,%22patent_date%22,%22assignee_organization%22,%22inventor_longitude%22,%22inventor_latitude%22,%22inventor_last_name%22,%22inventor_id%22,%22inventor_first_name%22,%22cpc_subsection_title%22,%22assignee_city%22,%22patent_abstract%22,%22patent_kind%22,%22cpc_group_id%22,%22assignee_organization%22,%22citedby_patent_number%22]"
-
-    filter_PV = filter_PV.replace(" " , "%20")
-
-        
-
-    url = URL(base_URL_PV , filter_works, filter_PV) 
-    data = get_data(url)
-
-
-    if display == True:
-        print(  data["total_patent_count"] , elem[-1] )
-        print(url)
-
-
-
-    
-    for i in range(data["count"]):
-        dic_patents[ "US-" + data["patents"][i]["patent_number"]] = {}
-        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["title"] = data["patents"][i]["patent_title"]
-        dic_patents["US-" + data["patents"][i]["patent_number"]]["abstract"] = data["patents"][i]["patent_abstract"]
-
-        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["assignee"] = str(data["patents"][i]["assignees"][0]["assignee_organization"])
-        dic_patents["US-" + data["patents"][i]["patent_number"]]["assignee_city"] = str(data["patents"][i]["assignees"][0]["assignee_city"])
-        dic_patents["US-" + data["patents"][i]["patent_number"]]["assignee_country"] = str(data["patents"][i]["assignees"][0]["assignee_country"])
-        for j in range(1, len(data["patents"][i]["assignees"])):
-            dic_patents[ "US-" + data["patents"][i]["patent_number"]]["assignee"] +=  ", " + str(data["patents"][i]["assignees"][j]["assignee_organization"])
-            dic_patents[ "US-" + data["patents"][i]["patent_number"]]["assignee_city"] +=  ", " + str(data["patents"][i]["assignees"][j]["assignee_city"])
-            dic_patents["US-" + data["patents"][i]["patent_number"]]["assignee_country"] +=  ", " + str(data["patents"][i]["assignees"][j]["assignee_country"])
-
-        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["list_inventors"] = data["patents"][i]["inventors"]
-
-        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["inventors"] = str(data["patents"][i]["inventors"][0]["inventor_first_name"]) + " " + str(data["patents"][i]["inventors"][0]["inventor_last_name"])
-        for j in range(1, len(data["patents"][i]["inventors"])):
-            dic_patents[ "US-" + data["patents"][i]["patent_number"]]["inventors"] +=  ", " + str(data["patents"][i]["inventors"][j]["inventor_first_name"]) + " " + str(data["patents"][i]["inventors"][j]["inventor_last_name"])
-
-
-        dic_patents["US-" + data["patents"][i]["patent_number"]]["date"] = data["patents"][i]["patent_date"]
-        dic_patents["US-" + data["patents"][i]["patent_number"]]["number_citations"] = len(data["patents"][i]["citedby_patents"])
-
-
-    if display == True:
-        print(" ")
-            
-            
-    return dic_patents
-  
-  
-  
-  
-  
-def ranking_patents( research_words, details):
-    
-    model = model_nlp()
-    
-    dic_patents = related_patents( research_words , False)
-    
-
-    dic_scores = {}
-
-
-    
-    reference_text = details
-
-    encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist()
-        
-        
-    if len(dic_patents ) == 0:
-        return "Select other key words"
-        
-    else:
-        for ids in list(dic_patents.keys()):
-
-            dic_scores[ids] = {}
-
-            encoded_title = model.encode(dic_patents[ids]["title"], convert_to_tensor=False).tolist()
-            score_title = cosine_similarity2(encoded_title, encoded_text)
-
-
-            if dic_patents[ids]["abstract"] != None:
-                encoded_abstract = model.encode(dic_patents[ids]["abstract"], convert_to_tensor=False).tolist()
-                score_abstract = cosine_similarity2(encoded_abstract, encoded_text)
-            else:
-                score_abstract = None
-
-            dic_scores[ids]["title comparision"] = score_title
-            dic_scores[ids]["abstract comparison"] = score_abstract
-            dic_scores[ids]["title"] = dic_patents[ids]["title"]
-            dic_scores[ids]["citations"] = dic_patents[ids]["number_citations"]
-            dic_scores[ids]["date"] = dic_patents[ids]["date"][:4]
-            dic_scores[ids]["assignee"] = dic_patents[ids]["assignee"]
-            dic_scores[ids]["inventors"] = dic_patents[ids]["inventors"]
-            dic_scores[ids]["number of co-inventors"] = len(dic_patents[ids]["inventors"].split(","))
-            
-    return dic_patents , dic_scores
-
-
-
-
-def get_ranking_own_research(research_key_words,  details  , size):
-    
-    dic , dic_scores_papers = ranking_patents(research_key_words,  details )
-    
-    if dic_scores_papers == {}:
-        return "No paper found"
-    
-    elif type(dic_scores_papers) == str:
-        return dic_scores_papers
-    
-    else:
-        return pd.DataFrame(dic_scores_papers).T.sort_values(by="abstract comparison" , ascending = False).head(size)
-     
- 
-
-def extract_quantitative_data_patent(patent_id):
-    
-
-    patent_id = patent_id[3:]
-        
-    url = "https://api.patentsview.org/patents/query?q={%22patent_id%22:%22" + str(patent_id) + "%22}&f=[%22patent_number%22,%22patent_title%22,%22patent_abstract%22,%22patent_date%22,%22inventor_last_name%22,%22inventor_first_name%22,%22assignee_organization%22]"
-    url_google = "https://patents.google.com/patent/US" + str(patent_id)
-    
-    data = get_data(url)["patents"][0]
-    title = data["patent_title"]
-    abstract = data["patent_abstract"]
-    co_inventors = ", ".join([ data["inventors"][i]["inventor_first_name"] + " " + data["inventors"][i]["inventor_last_name"] for i in range(len(data["inventors"])) ]) 
-    assignees =  ", ".join([ str(data["assignees"][i]["assignee_organization"]) for i in range(len(data["assignees"])) ] )
-
-    return url_google , title , abstract , data["patent_date"] , co_inventors , assignees
-
-
-
-
-    
-
-
-## Mapping the authors
-
-#merge the nobiliary particles with the last name
-#ln_suff file can be modified if more or less nobiliary particles want to be suppressed
-
-ln_suff = ['oster',
- 'nordre',
- 'vaster',
- 'aust',
- 'vesle',
- 'da',
- 'van t',
- 'af',
- 'al',
- 'setya',
- 'zu',
- 'la',
- 'na',
- 'mic',
- 'ofver',
- 'el',
- 'vetle',
- 'van het',
- 'dos',
- 'ui',
- 'vest',
- 'ab',
- 'vste',
- 'nord',
- 'van der',
- 'bin',
- 'ibn',
- 'war',
- 'fitz',
- 'alam',
- 'di',
- 'erch',
- 'fetch',
- 'nga',
- 'ka',
- 'soder',
- 'lille',
- 'upp',
- 'ua',
- 'te',
- 'ni',
- 'bint',
- 'von und zu',
- 'vast',
- 'vestre',
- 'over',
- 'syd',
- 'mac',
- 'nin',
- 'nic',
- 'putri',
- 'bet',
- 'verch',
- 'norr',
- 'bath',
- 'della',
- 'van',
- 'ben',
- 'du',
- 'stor',
- 'das',
- 'neder',
- 'abu',
- 'degli',
- 'vre',
- 'ait',
- 'ny',
- 'opp',
- 'pour',
- 'kil',
- 'der',
- 'oz',
- 'von',
- 'at',
- 'nedre',
- 'van den',
- 'setia',
- 'ap',
- 'gil',
- 'myljom',
- 'van de',
- 'stre',
- 'dele',
- 'mck',
- 'de',
- 'mellom',
- 'mhic',
- 'binti',
- 'ath',
- 'binte',
- 'snder',
- 'sre',
- 'ned',
- 'ter',
- 'bar',
- 'le',
- 'mala',
- 'ost',
- 'syndre',
- 'sr',
- 'bat',
- 'sndre',
- 'austre',
- 'putra',
- 'putera',
- 'av',
- 'lu',
- 'vetch',
- 'ver',
- 'puteri',
- 'mc',
- 'tre',
- 'st']
-
-#suppress all the unwanted suffixes from a string. 
-#name_del file can be modified if more or less suffixes want to be suppressed 
-
-name_del = ['2nd', '3rd', 'Jr', 'Jr.', 'Junior', 'Sr', 'Sr.', 'Senior']
-
-
-def name_delete(string):
-    for elmt in name_del:
-        if f" {elmt}" in string:
-            return string.replace(f" {elmt}","")
-    return string
-
-def ln_suff_merge(string):
-    for suff in ln_suff:
-        if f" {suff} " in string or string.startswith(f"{suff} "):
-            return string.replace(f"{suff} ",suff.replace(" ",""))
-    return string
-
-#normalize a string dat that represents often a name. 
-
-def normalize(data):
-    normal = unicodedata.normalize('NFKD', data).encode('ASCII', 'ignore')
-    val = normal.decode("utf-8")
-    # delete unwanted elmt
-    val = name_delete(val)
-    # lower full name in upper
-    val = re.sub(r"[A-Z]{3,}", lambda x: x.group().lower(), val)
-    # add space in front of upper case
-    val = re.sub(r"(\w)([A-Z])", r"\1 \2", val)
-    # Lower case
-    val = val.lower()
-    # remove special characters
-    val = re.sub('[^A-Za-z0-9 ]+', ' ', val)
-    # remove multiple spaces
-    val = re.sub(' +', ' ', val)
-    # remove trailing spaces
-    val = val.strip()
-    # suffix merge
-    val = ln_suff_merge(val)
-
-    return val
-
-
-def main_inventors( research_key_words , details , size):
-    
-    dic_patents , dic_ranked = ranking_patents(research_key_words , details)
-    dic_patents_co_inventors = {}
-    
-
-    for patent in list(dic_ranked.keys())[:size]:
-        for k in range(len(dic_patents[patent]["list_inventors"])):
-            
-            inventor_id = dic_patents[patent]["list_inventors"][k]["inventor_id"]
-            inventor_name = dic_patents[patent]["list_inventors"][k]["inventor_first_name"] + " " + dic_patents[patent]["list_inventors"][k]["inventor_last_name"]
-            inventor_name_norm = normalize(inventor_name).split()
-            inventor_name_norm = inventor_name_norm[0] + " " + inventor_name_norm[-1]
-
-            if inventor_name_norm not in dic_patents_co_inventors:
-                dic_patents_co_inventors[inventor_name_norm] = {}
-                dic_patents_co_inventors[inventor_name_norm]["Inventor's name"] = inventor_name 
-                dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"] =  inventor_id 
-                dic_patents_co_inventors[inventor_name_norm]["Number of occurence"] = 1
-                dic_patents_co_inventors[inventor_name_norm]["Number of related citations"] = dic_patents[patent]["number_citations"]
-
-            else:
-                if inventor_id not in dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"] :
-                    dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"] += ", " + inventor_id
-                if inventor_name not in dic_patents_co_inventors[inventor_name_norm]["Inventor's name"] :
-                    dic_patents_co_inventors[inventor_name_norm]["Inventor's name"] += ", " + inventor_name
-                dic_patents_co_inventors[inventor_name_norm]["Number of occurence"] += 1
-                dic_patents_co_inventors[inventor_name_norm]["Number of related citations"] += dic_patents[patent]["number_citations"]
-
-
-        dic_patents_co_inventors = {k: v for k, v in sorted(dic_patents_co_inventors.items(), key=lambda item: item[1]["Number of occurence"] , reverse = True)}
-    
-    if dic_patents_co_inventors == {}:
-    
-        
-        return "No patent, select other key words"
-    else:
-        
-        for inventor_name_norm in list(dic_patents_co_inventors.keys()):
-            list_inventors = dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"].split(", ")
-            work_count = 0
-            cited_by_count = 0
-            
-            for elem in list_inventors:
-                url = "https://api.patentsview.org/inventors/query?q={%22inventor_id%22:[%22" + elem + "%22]}&f=[%22inventor_total_num_patents%22,%22patent_num_cited_by_us_patents%22]"
-                data = get_data(url)["inventors"][0]
-                work_count += int(data["inventor_total_num_patents"])
-                for k in range(len(data["patents"])):
-                    cited_by_count += int(data["patents"][k]["patent_num_cited_by_us_patents"])
-                
-            dic_patents_co_inventors[inventor_name_norm]["Number of patents"] = work_count
-            dic_patents_co_inventors[inventor_name_norm]["Number of US patents citations"] = cited_by_count
-            
-        
-                
-                
-        return pd.DataFrame(dic_patents_co_inventors , index = ["Inventor's name", "PatentsView inventor's id", "Number of occurence"  , "Number of patents" ,"Number of US patents citations" ,  "Number of related citations"]).T.style.hide(axis="index")
-
-
-    
-def map_inventors(research_key_words , details , size):
-    display = False
-    dic_patents , dic_ranked = ranking_patents(research_key_words , details)
-    dic_patents_co_inventors = {}
-    count = 0
-    
-    for patent in list(dic_ranked.keys())[:size]:
-        for k in range(len(dic_patents[patent]["list_inventors"])):
-            
-            dic_patents_co_inventors[count] = {}
-            
-            dic_patents_co_inventors[count]["latitude"] = dic_patents[patent]["list_inventors"][k]["inventor_latitude"]
-            dic_patents_co_inventors[count]["longitude"] = dic_patents[patent]["list_inventors"][k]["inventor_longitude"]
-            dic_patents_co_inventors[count]["longitude"] = dic_patents[patent]["list_inventors"][k]["inventor_longitude"]
-            dic_patents_co_inventors[count]["inventor_name"]  = str(dic_patents[patent]["list_inventors"][k]["inventor_first_name"]) + " " + str(dic_patents[patent]["list_inventors"][k]["inventor_last_name"])
-            dic_patents_co_inventors[count]["patent_date"]  = dic_patents[patent]["date"]
-            count += 1
-                
-    if dic_patents_co_inventors == {}:
-        return "No patent, select other key words"
-    
-    
-    map_df = pd.DataFrame(dic_patents_co_inventors).T
-    map_df["longitude"]=map_df['longitude'].astype(float)
-    map_df['latitude']=map_df['latitude'].astype(float)
-    map_df = map_df[map_df["latitude"].notnull()]
-
-    return map_df
-

Climate_site/python_scripts/read_me.txt

@@ -1,13 +0,0 @@
----
-title: SynapseProject
-emoji: 🌍
-colorFrom: yellow
-colorTo: blue
-sdk: streamlit
-sdk_version: 1.25.0
-app_file: app.py
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

Climate_site/python_scripts/related_scientist.py

@@ -1,366 +0,0 @@
-##packages code
-
-
-import pandas as pd
-import streamlit as st
-import numpy as np
-import json, requests 
-#from pandas.io.json import json_normalize
-from tqdm import tqdm
-
-
-@st.cache_data  # 👈 Add the caching decorator
-def load_data():
-    url = "Synapse_project/Climate_site/python_scripts/institutions.tsv"
-    dic = pd.read_csv(url, delimiter = "\t" , index_col = 1).to_dict('index')
-    return dic
-
-dic_institutions = load_data()
-
-
-dic_country_codes = {'Europe': ['AD',
-  'AL',
-  'AT',
-  'BE',
-  'BG',
-  'BY',
-  'CZ',
-  'DE',
-  'DK',
-  'EE',
-  'FI',
-  'FR',
-  'GR',
-  'HU',
-  'IE',
-  'IS',
-  'IT',
-  'LI',
-  'LT',
-  'LU',
-  'LV',
-  'MK',
-  'MT',
-  'NL',
-  'NO',
-  'PL',
-  'PT',
-  'RO',
-  'RU',
-  'SE',
-  'SI',
-  'SK',
-  'SM',
-  'UA',
-  'VA',
-  'BA',
-  'HR',
-  'MD',
-  'MC',
-  'ME',
-  'RS',
-  'ES',
-  'CH',
-  'GB'],
- 'Asia': ['AF',
-  'AM',
-  'AZ',
-  'BD',
-  'BH',
-  'BN',
-  'BT',
-  'CN',
-  'CY',
-  'GE',
-  'ID',
-  'IL',
-  'IN',
-  'IQ',
-  'IR',
-  'JO',
-  'JP',
-  'KG',
-  'KP',
-  'KR',
-  'KW',
-  'LB',
-  'MM',
-  'MN',
-  'MV',
-  'MY',
-  'NP',
-  'OM',
-  'PH',
-  'PK',
-  'QA',
-  'SA',
-  'SG',
-  'SY',
-  'TH',
-  'TJ',
-  'TM',
-  'TR',
-  'UZ',
-  'VN',
-  'YE',
-  'KH',
-  'TL',
-  'KZ',
-  'LA',
-  'LK',
-  'AE'],
- 'North America': ['AG',
-  'BB',
-  'BS',
-  'BZ',
-  'CA',
-  'CR',
-  'CU',
-  'DM',
-  'DO',
-  'GT',
-  'GT',
-  'HN',
-  'JM',
-  'MX',
-  'NI',
-  'PA',
-  'TT',
-  'US',
-  'SV',
-  'GD',
-  'KN',
-  'LC',
-  'VC'],
- 'Africa': ['AO',
-  'BF',
-  'BI',
-  'BJ',
-  'BW',
-  'CD',
-  'CG',
-  'CI',
-  'CM',
-  'CV',
-  'DJ',
-  'EG',
-  'ER',
-  'ET',
-  'GA',
-  'GH',
-  'GM',
-  'GN',
-  'GW',
-  'KE',
-  'LR',
-  'LS',
-  'LY',
-  'MG',
-  'ML',
-  'MR',
-  'MU',
-  'MW',
-  'MZ',
-  'NA',
-  'NE',
-  'NG',
-  'RW',
-  'SC',
-  'SD',
-  'SL',
-  'SN',
-  'SO',
-  'ST',
-  'TG',
-  'TN',
-  'TZ',
-  'UG',
-  'ZM',
-  'ZW',
-  'DZ',
-  'CF',
-  'TD',
-  'KM',
-  'GQ',
-  'MA',
-  'ZA',
-  'SZ'],
- 'South America': ['AR',
-  'BO',
-  'BR',
-  'CL',
-  'CO',
-  'EC',
-  'GY',
-  'PE',
-  'PY',
-  'SR',
-  'UY',
-  'VE'],
- 'Oceania': ['AU',
-  'FJ',
-  'KI',
-  'MH',
-  'NR',
-  'NZ',
-  'PG',
-  'PW',
-  'SB',
-  'TO',
-  'TV',
-  'VU',
-  'FM',
-  'WS']}
-
-
-#################### General Functions #############################
-
-def URL(base_URL , entity_type , filters):
-    url = base_URL + entity_type + filters 
-    return url
-
-
-def get_data(url):
-    url = requests.get(url)
-    text = url.text
-    import json
-    data = json.loads(text)
-    return data
-
-## selecting the ids we want
-
-
-def author_id_from_name(name):
-    
-    dic_names = {}
-    
-    name = name.title()
-    
-    url = "https://api.openalex.org/authors?search=" + name + "&per_page=200&mailto=emma_scharfmann@berkeley.edu"
-    
-    data = get_data(url)["results"]
-
-    for k in range(len(data)):
-        author_id = data[k]["id"][21:]
-        dic_names[author_id] = {}
-        dic_names[author_id]["author_name"] = data[k]["display_name"]
-        
-        dic_names[author_id]["number_of_works"] = data[k]["works_count"]
-        dic_names[author_id]["number_of_citations"] = data[k]["cited_by_count"]
-        
-        dic_names[author_id]["field_of_study"] = ", ".join([ data[k]["x_concepts"][j]["display_name"] for j in range(min(5,len(data[k]["x_concepts"])))])
-
-        dic_names[author_id]["last_known_institution"] = None
-        dic_names[author_id]["country_code"] = None
-        
-        if data[k]["last_known_institution"] != None:
-            dic_names[author_id]["last_known_institution"] = data[k]["last_known_institution"]["display_name"]
-            dic_names[author_id]["country_code"] = data[k]["last_known_institution"]["country_code"]
-            
-        dic_names[author_id]["orcid"] = data[k]["orcid"]
-        
-            
-    return pd.DataFrame(dic_names).T , [ ( k , ", ".join([ elem["author_name"] , str(elem["last_known_institution"]) , str(elem["field_of_study"].split(", ")[0]) , "Number of works: " + str(elem["number_of_works"])  ] ) ) for k , elem in  dic_names.items()] 
-        
-        
-    
- 
-        
-def from_author_id(main_author_ids, year, country_code , size):
-    
-    dic_main_workers = {} 
-
-    
-    for main_author_id in main_author_ids:
-    
-        url = "https://api.openalex.org/works?filter=author.id:" + main_author_id + ",publication_year:>" + str(year) + "&per_page=200&mailto=emma_scharfmann@berkeley.edu"
-
-        data = get_data(url)["results"]
-
-        for k in tqdm(range(len(data))):
-            work_id = data[k]["id"][21:]
-            for j in range(len(data[k]["authorships"])):
-                author_id = data[k]["authorships"][j]["author"]["id"][21:]
-
-                if author_id != main_author_id:
-                    if author_id not in dic_main_workers:
-                        dic_main_workers[author_id] = {}
-                        dic_main_workers[author_id]["author_name"] = data[k]["authorships"][j]["author"]["display_name"]
-                        dic_main_workers[author_id]["co_authors"] = 0
-                        dic_main_workers[author_id]["citations"] = 0
-                        dic_main_workers[author_id]["institution"] = None
-                        dic_main_workers[author_id]["country_code"] = None
-                        dic_main_workers[author_id]["id"] = None
-                        dic_main_workers[author_id]["longitude"] = None
-                        dic_main_workers[author_id]["latitude"] = None
-                        
-                    if dic_main_workers[author_id]["institution"] == None and data[k]["authorships"][j]["institutions"] != [] and "display_name" in data[k]["authorships"][j]["institutions"][0]:
-                        dic_main_workers[author_id]["institution"] = data[k]["authorships"][j]["institutions"][0]["display_name"]
-                    if dic_main_workers[author_id]["country_code"] == None and data[k]["authorships"][j]["institutions"] != [] and "country_code" in data[k]["authorships"][j]["institutions"][0]:
-                        dic_main_workers[author_id]["country_code"] = data[k]["authorships"][j]["institutions"][0]["country_code"]
-                    if dic_main_workers[author_id]["id"] == None and data[k]["authorships"][j]["institutions"] != [] and "id" in data[k]["authorships"][j]["institutions"][0] and data[k]["authorships"][j]["institutions"][0]["id"] != None:
-                        dic_main_workers[author_id]["id"] = data[k]["authorships"][j]["institutions"][0]["id"][21:]
-
-
-
-                    dic_main_workers[author_id]["co_authors"] += 1
-
-            url = "https://api.openalex.org/works?filter=referenced_works:" + work_id + ",publication_year:>" + str(year) + "&per_page=200&mailto=emma_scharfmann@berkeley.edu"
-            
-
-                
-            citations_data = get_data(url)["results"]
-            for i in range(len(citations_data)):
-                citing_work_id = citations_data[i]["id"][21:]
-                for j in range(len(citations_data[i]["authorships"])):
-                    citing_author_id = citations_data[i]["authorships"][j]["author"]["id"][21:]
-
-                    if citing_author_id != main_author_id:
-                        if citing_author_id not in dic_main_workers:
-                            dic_main_workers[citing_author_id] = {}
-                            dic_main_workers[citing_author_id]["author_name"] = citations_data[i]["authorships"][j]["author"]["display_name"]
-                            dic_main_workers[citing_author_id]["co_authors"] = 0
-                            dic_main_workers[citing_author_id]["citations"] = 0
-                            dic_main_workers[citing_author_id]["institution"] = None
-                            dic_main_workers[citing_author_id]["country_code"] = None
-                            dic_main_workers[citing_author_id]["id"] = None
-                            dic_main_workers[citing_author_id]["longitude"] = None
-                            dic_main_workers[citing_author_id]["latitude"] = None
-                            
-                        if dic_main_workers[citing_author_id]["institution"] == None and (citations_data[i]["authorships"][j]["institutions"] != [] and "display_name" in citations_data[i]["authorships"][j]["institutions"][0]):
-                                dic_main_workers[citing_author_id]["institution"] = citations_data[i]["authorships"][j]["institutions"][0]["display_name"]
-                        if dic_main_workers[citing_author_id]["country_code"] == None and (citations_data[i]["authorships"][j]["institutions"] != [] and "country_code" in citations_data[i]["authorships"][j]["institutions"][0] ):
-                            dic_main_workers[citing_author_id]["country_code"] = citations_data[i]["authorships"][j]["institutions"][0]["country_code"]
-                        if dic_main_workers[citing_author_id]["id"] == None and ( citations_data[i]["authorships"][j]["institutions"] != [] and "id" in citations_data[i]["authorships"][j]["institutions"][0] and citations_data[i]["authorships"][j]["institutions"][0]["id"] != None):
-                            dic_main_workers[citing_author_id]["id"] = citations_data[i]["authorships"][j]["institutions"][0]["id"][21:]
-                                
-                        
-
-                        dic_main_workers[citing_author_id]["citations"] += 1
-                        
-                        
-    for author_id in dic_main_workers:
-        if dic_main_workers[author_id]["id"] != None:
-            institution_id = dic_main_workers[author_id]["id"] 
-            if institution_id in dic_institutions:
-                geo_data = dic_institutions[institution_id]
-                dic_main_workers[author_id]["longitude"] = geo_data["longitude"]
-                dic_main_workers[author_id]["latitude"] = geo_data["latitude"]
-
-
-    res = pd.DataFrame(dic_main_workers).T.sort_values("citations" , ascending = False)[['author_name', 'co_authors', 'citations', 'institution', 'country_code']]
-    
-    res_geo = pd.DataFrame(dic_main_workers).T.sort_values("citations" , ascending = False)[['author_name', 'co_authors', 'citations', 'institution', 'country_code','longitude', 'latitude']]
-    res_geo = res_geo[res_geo["longitude"].notnull()]
-    if country_code != False:
-        
-        if country_code in [ "Europe" , "North America" , "Asia" , "South America", "Oceania" , "Africa" ]:
-            
-            return res[res["country_code"].isin(dic_country_codes[country_code])].head(size) , res_geo[res_geo["country_code"].isin(dic_country_codes[country_code])].head(size)
-            
-        else:
-            return res[res["country_code"] == country_code].head(size) , res_geo[res_geo["country_code"] == country_code].head(size)
-    else: 
-        return res.head(size), res_geo.head(size)
-
-