Upload 35 files

Climate_site/pages/1_Patents_and_Inventors.py

@@ -0,0 +1,166 @@
+import streamlit as st
+
+st.set_page_config(page_title="Patents and Inventors", page_icon="📖")
+
+import pandas as pd
+from python_scripts import patent_functions
+from streamlit_folium import folium_static
+import folium
+
+header = st.container()
+patent_finder = st.container()
+
+
+with patent_finder:
+
+  st.markdown("# Patents and Inventors related to IEA technologies")
+  st.sidebar.header("Patents and Inventors")
+  st.write(
+        """You can find the most related patents and inventors for each climate related IEA technology. Enjoy!"""
+    )
+
+    
+
+  st.subheader("Select the technology you want to look at")
+  dic_technologies, dic_categories, list_categories_tech, list_technologies = patent_functions.finder()
+    
+  list_technologies = [ ( list_categories_tech[i] , i ) for i in range(len(list_categories_tech))  ] 
+  key0 = lambda t: t[0]
+  
+  
+    
+    
+  # Select the category  
+  tech_category = st.selectbox('Select a category',list_technologies,format_func = key0)
+
+    
+  # After a selection is made the choices will update
+  # Choose the technology and number
+  technologies = dic_technologies[tech_category[0]]
+  technology = st.selectbox('Select a technology',technologies,format_func = key0)
+  
+
+  # Select whether or not to use keywords
+  list_keyword_option = [ ("No pre-select them for me" , False) , ("Yes" , True )]
+  list_keyword_bool = st.selectbox('Would you like to enter your own keywords?',list_keyword_option,format_func = key0)
+
+  if list_keyword_bool[1] == True:
+    tech_category_keyword = st.text_input("Keywords", 'Input your text here')
+  else:
+    tech_categories = dic_categories[technology[1]]
+    tech_category_keyword = st.radio(
+      "Select a keyword to search on: ",
+      tech_categories)
+
+    
+  # Select type of patent (related to climate or not)
+  list_climate = [ ("Any related patents" , False ) , ("Climate related patents" , True)]
+  patent_type_bool = st.selectbox('What type of patent do you want?',list_climate,format_func = key0)
+  
+  size = st.slider('How many patents do you want?', 0, 100, 5)
+
+  
+  #try:
+
+  st.subheader("Look at the related patents and inventors")
+    
+  reference_text, text1 , text2 = patent_functions.extract_quantitative_data_technology(
+  technologies = technology[0],
+  number_technology = technology[1],
+  )
+    
+  st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Technology details:</strong> {reference_text}' , unsafe_allow_html = True)
+  st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Deployment target and Announced development target:</strong> {text1}' , unsafe_allow_html = True)
+  st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Announced cost reduction targets:</strong> {text2}' , unsafe_allow_html = True)
+
+    
+  #try:
+  if st.button('Get related patents'):
+        patent_rank_df = patent_functions.get_ranking_patents(
+        technologies = tech_category[1],
+        number_technology = technology[1],
+        category = tech_category_keyword,
+        carbon_related = patent_type_bool[1],
+        size = size
+      )
+        st.dataframe(patent_rank_df)
+        print(patent_rank_df)
+ # except:
+  #  st.write("No patent found, try with other key words")
+        
+  try:  
+    if st.button('Get the main inventors patents'):
+        patent_rank_df = patent_functions.main_inventors(
+        technologies = tech_category[1],
+        number_technology = technology[1],
+        category = tech_category_keyword,
+        carbon_related = patent_type_bool[1],
+        size = size
+      )
+        st.dataframe(patent_rank_df)
+        #print(patent_rank_df)
+          
+  except:
+    st.write("No inventor found, try with other key words")
+
+        
+  try:
+      
+    if st.button('Get the map of the main inventors patents'):
+      map_inventor = patent_functions.map_inventors(
+      technologies = tech_category[1],
+      number_technology = technology[1],
+      category = tech_category_keyword,
+      carbon_related = patent_type_bool[1],
+      size = size)
+      #st.map(map_inventor)
+      #print(map_inventor)
+      m = folium.Map(zoom_start = 1)
+      dic_geo = {}
+      for elem in map_inventor.index:
+  
+        longitude = map_inventor["longitude"][elem]
+        latitude = map_inventor["latitude"][elem]
+        if str(longitude) + " " + str(latitude) not in dic_geo:
+          dic_geo[str(longitude) + " " + str(latitude)] = {}
+          dic_geo[str(longitude) + " " + str(latitude)]["inventors"] = set()
+          dic_geo[str(longitude) + " " + str(latitude)]["dates"] = set()
+       
+        dic_geo[str(longitude) + " " + str(latitude)]["inventors"].add(map_inventor["inventor_name"][elem])
+        dic_geo[str(longitude) + " " + str(latitude)]["dates"].add(map_inventor["patent_date"][elem])
+  
+      for elem in dic_geo:
+        folium.Marker([float(elem.split()[1]), float(elem.split()[0])], popup="Inventors: " + ", ".join(list(dic_geo[elem]["inventors"])) + ", Date: " + ", ".join(list(dic_geo[elem]["dates"])), tooltip=", ".join(list(dic_geo[elem]["inventors"]))).add_to(m)
+      folium_static(m)
+           
+  except:
+    st.write("No inventor found, try with other key words")
+        
+        
+  
+
+  try:
+      patent_id = st.text_input("More data on a patent?", 'Enter a US patent (ex: US-100000)')
+      url , title , abstract , date , co_inventors , assignees = patent_functions.extract_quantitative_data_patent(
+      patent_id = patent_id)
+
+      st.write(f'<p style="color:Black"> <strong> Patent link:</strong> <a href="url">{url}' , unsafe_allow_html = True)
+      st.write(f'<p style="color:Blue"> From PatentsView Website:' , unsafe_allow_html = True)
+      st.write(f'<strong> Patent title:</strong> {title}' , unsafe_allow_html = True)
+      st.write(f'<strong> Patent abstract:</strong> {abstract}' , unsafe_allow_html = True)
+      st.write(f'<strong> Patent date:</strong> {date}' , unsafe_allow_html = True)
+      st.write(f'<strong> Patent co-inventors:</strong> {co_inventors}' , unsafe_allow_html = True)
+      st.write(f'<strong> Patent assignees:</strong> {assignees}' , unsafe_allow_html = True)
+        
+  except:
+    st.write("Enter a valid patent id")
+        
+        
+        
+   ## Get info about technology
+
+  #except NameError:
+  #  st.write(f'No patents found')
+  # st.write(f'list_categories_tech is {list_categories_tech}')
+  # st.write(f'list_technologies is {list_technologies}')
+  # st.write(f'dic_categories is {dic_categories}')

Climate_site/pages/2_Scientific_publications_and_Authors.py

@@ -0,0 +1,211 @@
+
+import streamlit as st
+
+st.set_page_config(page_title="Scientific publications and authors", page_icon="📖")
+
+
+from streamlit_folium import folium_static
+import folium
+import pandas as pd
+#from python_scripts import functions
+from python_scripts import paper_functions
+from python_scripts import others
+
+@st.cache
+def convert_df(df):
+    # IMPORTANT: Cache the conversion to prevent computation on every rerun
+    return df.to_csv().encode('utf-8')
+
+
+header = st.container()
+
+paper_finder = st.container()
+
+
+with paper_finder:
+    
+  
+  st.markdown("# Scientific publications and authors related to IEA technologies")
+  st.sidebar.header("Scientific publications and authors")
+  st.write(
+        """You can find the most related scientific publications and authors for each climate related IEA technology. Enjoy!"""
+    )
+
+  st.header('Which papers are related to the technology?')
+
+
+  st.subheader("Select the technology you want to look at")
+  dic_technologies, dic_categories, list_categories_tech, list_technologies = paper_functions.finder()
+
+  list_technologies = [ ( list_categories_tech[i] , i ) for i in range(len(list_categories_tech))  ] 
+  key0 = lambda t: t[0]
+    
+    
+   # Choose the category
+  tech_category = st.selectbox('Select a category',list_technologies, format_func = key0)
+
+    
+  # After a selection is made the choices will update
+  # Choose the technology and number
+  technologies = dic_technologies[tech_category[0]]
+
+  technology = st.selectbox('Select a technology',technologies,format_func = key0)
+  
+
+  # Select whether or not to use keywords
+  list_keyword_option = [ ("No pre-select them for me" , False) , ("Yes" , True )]
+  list_keyword_bool = st.selectbox('Would you like to enter your own keywords?',list_keyword_option,format_func = key0)
+
+  if list_keyword_bool[1] == True:
+    tech_category_keyword = st.text_input("Keywords", 'Input your text here')
+  else:
+    tech_categories = dic_categories[technology[1]]
+    tech_category_keyword = st.radio(
+      "Select a keyword to search on: ",
+      tech_categories)
+
+    
+  # Select type of papers (related to climate or not)
+  list_climate = [ ("Any related papers" , False ) , ("Climate related papers" , True)]
+  patent_type_bool = st.selectbox('What type of papers do you want?',list_climate,format_func = key0)
+
+  size = st.slider('How many papers do you want?', 0, 100, 5)
+
+  st.subheader("Look at the related scientific publications and authors")
+  #try:
+  #if st.button('Get more information about the technology'):
+  reference_text, text1 , text2  = paper_functions.extract_quantitative_data_technology(
+  technologies = technology[0],
+  number_technology = technology[1],
+  )
+  
+  st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Technology details:</strong> {reference_text}' , unsafe_allow_html = True)
+  st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Deployment target and Announced development target:</strong> {text1}' , unsafe_allow_html = True)
+  st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Announced cost reduction targets:</strong> {text2}' , unsafe_allow_html = True)
+
+  #except:
+  #  st.write("No information about the technology")
+    
+
+
+  #try:
+  if st.button('Get related papers'):
+     paper_rank_df = paper_functions.get_ranking_related_papers(
+     technologies = tech_category[1],
+     number_technology = technology[1],
+     research_words = tech_category_keyword,
+     carbon_related = patent_type_bool[1],
+     size = size
+     )
+     st.dataframe(paper_rank_df)
+     #print(paper_rank_df)
+  #except:
+  #  st.write("No paper found, try with other key words")
+        
+  #try:  
+  if st.button('Get the main authors'):
+     authors = paper_functions.main_authors(
+     technologies = tech_category[1],
+     number_technology = technology[1],
+     research_words = tech_category_keyword,
+     carbon_related = patent_type_bool[1],
+     size = size
+     )
+     st.dataframe(authors)
+
+
+     csv = convert_df(authors)
+
+     st.download_button(
+     label="Download table as CSV",
+     data=csv,
+     file_name='authors.csv',
+     )
+     #print(patent_rank_df)
+#      
+ # except:
+ #   st.write("No author found, try with other key words")
+  
+  #      
+  #try:
+    
+  if st.button('Get the map of the main authors'):
+    map_author = paper_functions.map_authors(
+    technologies = tech_category[1],
+    number_technology = technology[1],
+    research_words = tech_category_keyword,
+    carbon_related = patent_type_bool[1],
+    size = size)
+    #st.map(map_author)
+    m = folium.Map(zoom_start = 1)
+    dic_geo = {}
+    for elem in map_author.index:
+
+        if map_author["institution"][elem] != "Max Planck Institute for Plasma Physics - Greifswald":
+          longitude = map_author["longitude"][elem]
+          latitude = map_author["latitude"][elem]
+          if str(longitude) + " " + str(latitude) not in dic_geo:
+            dic_geo[str(longitude) + " " + str(latitude)] = {}
+            dic_geo[str(longitude) + " " + str(latitude)]["authors"] = set()
+            dic_geo[str(longitude) + " " + str(latitude)]["institutions"] = set()
+            dic_geo[str(longitude) + " " + str(latitude)]["dates"] = set()
+          dic_geo[str(longitude) + " " + str(latitude)]["authors"].add(map_author["author"][elem])
+          dic_geo[str(longitude) + " " + str(latitude)]["institutions"].add(map_author["institution"][elem])
+          dic_geo[str(longitude) + " " + str(latitude)]["dates"].add(map_author["date"][elem])
+
+    for elem in dic_geo:
+      folium.Marker([float(elem.split()[1]), float(elem.split()[0])], popup="Author: " + ", ".join(list(dic_geo[elem]["authors"])) + ", Institution: " + ", ".join(list(dic_geo[elem]["institutions"])) + ", Date: " + ", ".join(list(dic_geo[elem]["dates"])), tooltip=", ".join(list(dic_geo[elem]["authors"]))).add_to(m)
+    folium_static(m)
+         
+  #except:
+  #  st.write("No author found, try with other key words")
+  #      
+  #      
+  #
+  
+  
+  #      
+  #      
+  #      
+  if st.button('Get more information about the related_projects'):
+    try:
+      reference_text,key_initiative,location_entities,papers = paper_functions.related_projects(
+      technologies = technology[0], 
+      number_technology = technology[1], 
+      carbon_related= patent_type_bool[1], 
+      size = size)
+      
+      st.write(f'<p style="color:Blue"> From IEA Website:' , unsafe_allow_html = True)
+      st.write(f'<strong>Technology details:</strong>  {reference_text}' ,  unsafe_allow_html = True)
+      st.write(f'<strong>Technology key initiative:</strong> {key_initiative}' ,  unsafe_allow_html = True)
+      st.write(f'<strong>Extracted projects and organization name:</strong> {location_entities}' ,  unsafe_allow_html = True)
+      st.write(f'<p style="color:Red"> From OpenAlex Website:' , unsafe_allow_html = True)
+      st.dataframe(papers)
+    except:
+       try:
+          st.write(f'<p style="color:Blue"> From IEA Website:' , unsafe_allow_html = True)
+          st.write(f'<strong>Technology details:</strong>  {reference_text}' , unsafe_allow_html = True)
+          st.write(f'<strong>Technology key initiatives:</strong> {key_initiative}' ,  unsafe_allow_html = True)
+          st.write(f'<strong>Extracted projects and organization name:</strong> {location_entities}' ,  unsafe_allow_html = True)
+          st.write(f'<p style="color:Red"> From OpenAlex Website:' , unsafe_allow_html = True)
+          st.write(f'No related paper found')
+       except:
+          st.write("No key initatives found")
+          
+          
+  try:
+    paper_id = st.text_input("More data on a paper?", 'Enter an OpenAlex paper id (ex: W100000)')
+    url , title , abstract , date , authors , institutions = paper_functions.extract_quantitative_data_paper(
+    work_id = paper_id)
+    
+    st.write(f'<p style="color:Red"> From OpenAlex Website:' , unsafe_allow_html = True)
+    st.write(f'<p style="color:Black"> <strong> Paper link:</strong> <a href="url">{url}' , unsafe_allow_html = True)
+    st.write(f'<strong> Paper title:</strong> {title}' , unsafe_allow_html = True)
+    st.write(f'<strong> Paper abstract:</strong> {abstract}' , unsafe_allow_html = True)
+    st.write(f'<strong> Paper date:</strong> {date}' , unsafe_allow_html = True)
+    st.write(f'<strong> Paper co-inventors:</strong> {authors}' , unsafe_allow_html = True)
+    st.write(f'<strong> Paper assignees:</strong> {institutions}' , unsafe_allow_html = True)
+  #      
+  except:
+    st.write("Enter a valid work id")
+  

Climate_site/pages/3_Related_Companies.py

@@ -0,0 +1,88 @@
+import streamlit as st
+
+st.set_page_config(page_title="Related Companies", page_icon="🏛")
+
+
+import pandas as pd
+from python_scripts import functions
+from python_scripts import functions_companies
+
+@st.cache
+def convert_df(df):
+    # IMPORTANT: Cache the conversion to prevent computation on every rerun
+    return df.to_csv()
+
+
+header = st.container()
+
+company_finder = st.container()
+
+
+with company_finder:
+    
+  
+  st.markdown("# Related companies")
+  st.sidebar.header("Related companies")
+  st.write(
+        """You can find the most related companies for each climate related IEA technology. Enjoy!"""
+    )
+
+
+  st.subheader("Select the technology you want to look at")
+  dic_technologies, dic_categories, list_categories_tech, list_technologies = functions_companies.finder()
+
+  list_technologies = [ ( list_categories_tech[i] , i ) for i in range(len(list_categories_tech))  ] 
+  key0 = lambda t: t[0]
+    
+    
+   # Choose the category
+  tech_category = st.selectbox('Select a category',list_technologies, format_func = key0)
+
+    
+  # After a selection is made the choices will update
+  # Choose the technology and number
+  technologies = dic_technologies[tech_category[0]]
+
+  technology = st.selectbox('Select a technology',technologies,format_func = key0)
+  
+  size = st.slider('How many companies do you want?', 0, 50, 10)
+  
+
+  st.subheader("Look at the related companies")
+    
+    
+  try:
+     reference_text, text1 , text2 = functions_companies.extract_quantitative_data_technology(
+     technologies = technology[0],
+     number_technology = technology[1],
+     )
+  
+     st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Technology details:</strong> {reference_text}' , unsafe_allow_html = True)
+     st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Deployment target and Announced development target:</strong> {text1}' , unsafe_allow_html = True)
+     st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Announced cost reduction targets:</strong> {text2}' , unsafe_allow_html = True)
+
+  except:
+    st.write("No information about the technology")
+    
+
+
+  #try:
+  if st.button('Get related companies'):
+    companies_ranked = functions_companies.related_VC_deals(
+    category = tech_category[1],
+    number_technology = technology[1],
+    size = size
+    )
+    st.dataframe(companies_ranked.style.format({"year_established": '{:.0f}'}))
+    csv = convert_df(companies_ranked)
+
+    st.download_button(
+         label="Download table as CSV",
+         data=csv,
+        file_name='companies.csv',
+            
+        )
+  #except:
+  #  st.write("No company found, try with other key words")
+        
+  

Climate_site/pages/4_University_contests.py

@@ -0,0 +1,69 @@
+import streamlit as st
+
+st.set_page_config(page_title="Related university projects", page_icon="🏛")
+
+
+import pandas as pd
+from python_scripts import functions
+from python_scripts import functions_contest
+
+
+
+header = st.container()
+
+company_finder = st.container()
+
+
+with company_finder:
+    
+  
+  st.markdown("# Related university projects")
+  st.sidebar.header("Related university projects")
+  st.write(
+        """You can find the most related projects which participated to the Big Ideas contest: [Big Ideas website](https://bigideascontest.org). Enjoy!"""
+  )
+
+
+  st.subheader("Select the technology you want to look at")
+  dic_technologies, dic_categories, list_categories_tech, list_technologies = functions_contest.finder()
+
+  list_technologies = [ ( list_categories_tech[i] , i ) for i in range(len(list_categories_tech))  ] 
+  key0 = lambda t: t[0]
+    
+    
+   # Choose the category
+  tech_category = st.selectbox('Select a category',list_technologies, format_func = key0)
+
+    
+  # After a selection is made the choices will update
+  # Choose the technology and number
+  technologies = dic_technologies[tech_category[0]]
+
+  technology = st.selectbox('Select a technology',technologies,format_func = key0)
+    
+
+  st.subheader("Look at the related projects")
+    
+    
+  try:
+     reference_text, text1 , text2 = functions_contest.extract_quantitative_data_technology(
+     technologies = technology[0],
+     number_technology = technology[1],
+     )
+  
+     st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Technology details:</strong> {reference_text}' , unsafe_allow_html = True)
+     st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Deployment target and Announced development target:</strong> {text1}' , unsafe_allow_html = True)
+     st.write(f'<p style="color:Blue"> From IEA Website: <p style="color:Black"> <strong>Announced cost reduction targets:</strong> {text2}' , unsafe_allow_html = True)
+
+  except:
+    st.write("No information about the technology")
+    
+
+
+  try:
+      if st.button('Get related projects'):
+        companies_ranked = functions_contest.big_ideas_encoding(
+        number_technology = technology[1])
+        st.dataframe(companies_ranked)
+  except:
+    st.write("No project found.")

Climate_site/pages/5_Your_own_research:_Patents.py

@@ -0,0 +1,109 @@
+import streamlit as st
+
+st.set_page_config(page_title="Search patents related to your own technology", page_icon="🙋‍♀️")
+
+import pandas as pd
+from python_scripts import patent_functions_own_details
+from streamlit_folium import folium_static
+import folium
+
+
+
+header = st.container()
+
+own_paper_finder = st.container()
+
+
+with own_paper_finder:
+    
+  
+  st.markdown("# Your own research: Patents and Inventors ")
+  st.sidebar.header("Patents and Inventors")
+  st.write(
+        """You can find the most related patents and inventors to your technology. Enjoy!"""
+    )
+
+
+  st.subheader("We are searching for the patents which contain (in the title or abstract) the key words you can provide below:")
+ 
+  key_words = st.text_input("Keywords", 'Input your text here')
+  
+  st.subheader("The patents are ranked according to the similary between their abstract and the description you can provide below:")
+
+  details = st.text_input("Details", 'Input your text here')
+  
+  size = st.slider('How many patents do you want?', 0, 200, 20)
+
+
+    
+    
+    
+  try:
+    if st.button('Get related patents'):
+       patent_rank_df = patent_functions_own_details.get_ranking_own_research(research_key_words = key_words,  details = details  , size=size)
+ 
+       st.dataframe(patent_rank_df)
+    
+
+  except:
+    st.write("No patent found, try with other key words")
+    
+    
+    
+        
+  try:  
+    if st.button('Get the main inventors'):
+       inventors = patent_functions_own_details.main_inventors(research_key_words = key_words,  details = details   , size=size)
+       st.dataframe(inventors)
+  
+  except:
+    st.write("No inventor found, try with other key words")
+
+    
+    
+
+  try:
+      
+    if st.button('Get the map of the main inventors patents'):
+      map_inventor = patent_functions_own_details.map_inventors(
+      research_key_words = key_words,  details = details   , size=size)
+
+      m = folium.Map(zoom_start = 1)
+      dic_geo = {}
+      for elem in map_inventor.index:
+  
+        longitude = map_inventor["longitude"][elem]
+        latitude = map_inventor["latitude"][elem]
+        if str(longitude) + " " + str(latitude) not in dic_geo:
+          dic_geo[str(longitude) + " " + str(latitude)] = {}
+          dic_geo[str(longitude) + " " + str(latitude)]["inventors"] = set()
+          dic_geo[str(longitude) + " " + str(latitude)]["dates"] = set()
+       
+        dic_geo[str(longitude) + " " + str(latitude)]["inventors"].add(map_inventor["inventor_name"][elem])
+        dic_geo[str(longitude) + " " + str(latitude)]["dates"].add(map_inventor["patent_date"][elem])
+  
+      for elem in dic_geo:
+        folium.Marker([float(elem.split()[1]), float(elem.split()[0])], popup="Inventors: " + ", ".join(list(dic_geo[elem]["inventors"])) + ", Date: " + ", ".join(list(dic_geo[elem]["dates"])), tooltip=", ".join(list(dic_geo[elem]["inventors"]))).add_to(m)
+      folium_static(m)
+           
+  except:
+    st.write("No inventor found, try with other key words")
+    
+    
+  try:
+      patent_id = st.text_input("More data on a patent?", 'Enter a US patent (ex: US-100000)')
+      url , title , abstract , date , co_inventors , assignees = patent_functions_own_details.extract_quantitative_data_patent(
+      patent_id = patent_id)
+
+      st.write(f'<p style="color:Black"> <strong> Patent link:</strong> <a href="url">{url}' , unsafe_allow_html = True)
+      st.write(f'<p style="color:Blue"> From PatentsView Website:' , unsafe_allow_html = True)
+      st.write(f'<strong> Patent title:</strong> {title}' , unsafe_allow_html = True)
+      st.write(f'<strong> Patent abstract:</strong> {abstract}' , unsafe_allow_html = True)
+      st.write(f'<strong> Patent date:</strong> {date}' , unsafe_allow_html = True)
+      st.write(f'<strong> Patent co-inventors:</strong> {co_inventors}' , unsafe_allow_html = True)
+      st.write(f'<strong> Patent assignees:</strong> {assignees}' , unsafe_allow_html = True)
+        
+  except:
+    st.write("Enter a valid patent id")
+
+    

Climate_site/pages/6_Your_own_research:_Scientific_Publications.py

@@ -0,0 +1,105 @@
+import streamlit as st
+
+st.set_page_config(page_title="Search papers with your own technology", page_icon="🙋‍♀️")
+
+import pandas as pd
+from python_scripts import paper_functions_own_details
+from streamlit_folium import folium_static
+import folium
+
+
+
+header = st.container()
+
+own_paper_finder = st.container()
+
+
+with own_paper_finder:
+    
+  
+  st.markdown("# Your own research: Scientific publications and authors ")
+  st.sidebar.header("Scientific publications and authors")
+  st.write(
+        """You can find the most related scientific publications and authors to your technology. Enjoy!"""
+    )
+
+
+  st.subheader("We are searching for the scientific publications which contain (in the title, abstract or text) the key words you can provide below:")
+ 
+  key_words = st.text_input("Keywords", 'Input your text here')
+  
+  st.subheader("The scientific publications are ranked according to the similary between their abstract and the description you can provide below:")
+
+  details = st.text_input("Details", 'Input your text here')
+  
+  size = st.slider('How many papers do you want?', 0, 200, 20)
+
+
+  try:
+    if st.button('Get related papers'):
+       paper_rank_df = paper_functions_own_details.get_ranking_own_research(research_key_words = key_words,  details = details  , display= False , size=size)
+ 
+       st.dataframe(paper_rank_df)
+        #print(paper_rank_df)
+  except:
+    st.write("No paper found, try with other key words")
+    
+    
+    
+    
+  #try:  
+  if st.button('Get the main authors'):
+     authors = paper_functions_own_details.main_authors(research_key_words = key_words,  details = details   , size=size)
+     st.dataframe(authors)
+  
+  #except:
+  #  st.write("No author found, try with other key words")
+
+
+
+
+  try:
+   
+    if st.button('Get the map of the main authors'):
+      map_author = paper_functions_own_details.map_authors(research_key_words = key_words,  details = details   , size=size)
+      m = folium.Map(zoom_start = 1)
+      dic_geo = {}
+      for elem in map_author.index:
+      
+            longitude = map_author["longitude"][elem]
+            latitude = map_author["latitude"][elem]
+            if str(longitude) + " " + str(latitude) not in dic_geo:
+              dic_geo[str(longitude) + " " + str(latitude)] = {}
+              dic_geo[str(longitude) + " " + str(latitude)]["authors"] = set()
+              dic_geo[str(longitude) + " " + str(latitude)]["institutions"] = set()
+              dic_geo[str(longitude) + " " + str(latitude)]["dates"] = set()
+            dic_geo[str(longitude) + " " + str(latitude)]["authors"].add(map_author["author"][elem])
+            dic_geo[str(longitude) + " " + str(latitude)]["institutions"].add(map_author["institution"][elem])
+            dic_geo[str(longitude) + " " + str(latitude)]["dates"].add(map_author["date"][elem])
+      
+      for elem in dic_geo:
+        folium.Marker([float(elem.split()[1]), float(elem.split()[0])], popup="Author: " + ", ".join(list(dic_geo[elem]["authors"])) + ", Institution: " + ", ".join(list(dic_geo[elem]["institutions"])) + ", Date: " + ", ".join(list(dic_geo[elem]["dates"])), tooltip=", ".join(list(dic_geo[elem]["authors"]))).add_to(m)
+      folium_static(m)
+            
+  except:
+    st.write("No author found, try with other key words")
+
+    
+    
+    
+    
+  try:
+    paper_id = st.text_input("More data on a paper?", 'Enter an OpenAlex paper id (ex: W100000)')
+    url , title , abstract , date , authors , institutions = paper_functions_own_details.extract_quantitative_data_paper(
+    work_id = paper_id)
+    
+    st.write(f'<p style="color:Red"> From OpenAlex Website:' , unsafe_allow_html = True)
+    st.write(f'<p style="color:Black"> <strong> Paper link:</strong> <a href="url">{url}' , unsafe_allow_html = True)
+    st.write(f'<strong> Paper title:</strong> {title}' , unsafe_allow_html = True)
+    st.write(f'<strong> Paper abstract:</strong> {abstract}' , unsafe_allow_html = True)
+    st.write(f'<strong> Paper date:</strong> {date}' , unsafe_allow_html = True)
+    st.write(f'<strong> Paper co-inventors:</strong> {authors}' , unsafe_allow_html = True)
+    st.write(f'<strong> Paper assignees:</strong> {institutions}' , unsafe_allow_html = True)
+  #      
+  except:
+    st.write("Enter a valid work id")

Climate_site/pages/7_Your_own_research:_Related_Scientist.py

@@ -0,0 +1,107 @@
+import streamlit as st
+
+st.set_page_config(page_title="Search related scientist", page_icon="🙋‍♀️")
+
+import pandas as pd
+from python_scripts import related_scientist
+from streamlit_folium import folium_static
+import folium
+
+
+
+header = st.container()
+
+own_paper_finder = st.container()
+
+
+with own_paper_finder:
+    
+  
+  st.markdown("# Searching the scientists related to somebody")
+  st.sidebar.header("Related scientists")
+
+
+
+  st.subheader("We are looking for the scientists who cite the scientist of your choice.")
+ 
+  key_name = st.text_input("Enter the name of a scientist", 'Scientist name')
+    
+  key0 = lambda t: t[0]
+  key1 = lambda t: t[1]
+  
+
+  #try:
+  if key_name == 'Scientist name':
+    st.write("Enter a scientist name")
+    
+  else:
+
+    pandas_scientific_ids , scientist_ids = related_scientist.author_id_from_name(name = key_name)
+
+    st.dataframe(pandas_scientific_ids)
+
+
+
+    #st.write('Select the identifier(s) which correspond to the scientist you are looking for:')
+
+
+    author_ids = st.multiselect("Select the identifier(s) which correspond to the scientist you are looking for: ", scientist_ids, format_func = key1 )
+
+    size = st.slider('How many scientists do you want?', 0, 100, 10)
+
+    year = st.selectbox("I want to look at the citations since: ", [ k for k in range(1980, 2023) ] , index = 38)
+
+
+    list_option = [ ("No, I want to look at all scientist you can find." , 0) , ("Yes. I want to search by continent." , 1 ), ("Yes. I want to search by country." , 2 )]
+    list_keyword_bool = st.selectbox('Would you like to look at the scientist from a specific country?',list_option,format_func = key0)
+
+    if list_keyword_bool[1] == 2:
+      country_code = st.text_input("Enter a country code", 'US')
+    elif list_keyword_bool[1] == 1:
+      country_code = st.selectbox("Select a continent", [ "Europe" , "North America" , "Asia" , "South America", "Oceania" , "Africa" ] , index = 0)
+    else:
+      country_code = False
+    
+    if len(author_ids) == 0:
+        st.write("Select the identifier(s) which correspond to the scientist you are looking for")
+        
+    else:
+
+
+
+        if st.button('Get the related scientist (it can take a few minutes, please, be patient 😊)'):
+
+            scientists = related_scientist.from_author_id(
+            main_author_ids = [ author_id[0] for author_id in author_ids] ,
+            year = year,
+            country_code = country_code,
+            size = size)
+
+            st.dataframe(scientists[0])
+
+            map_author = scientists[1]
+            m = folium.Map(zoom_start = 1)
+            dic_geo = {}
+            for elem in map_author.index:
+                  if map_author["institution"][elem] != "Max Planck Institute for Plasma Physics - Greifswald":
+
+                      longitude = map_author["longitude"][elem]
+                      latitude = map_author["latitude"][elem]
+                      if str(longitude) + " " + str(latitude) not in dic_geo:
+                        dic_geo[str(longitude) + " " + str(latitude)] = {}
+                        dic_geo[str(longitude) + " " + str(latitude)]["authors"] = set()
+                        dic_geo[str(longitude) + " " + str(latitude)]["institutions"] = set()
+                      dic_geo[str(longitude) + " " + str(latitude)]["authors"].add(map_author["author_name"][elem])
+                      dic_geo[str(longitude) + " " + str(latitude)]["institutions"].add(map_author["institution"][elem])
+
+            for elem in dic_geo:
+              folium.Marker([float(elem.split()[1]), float(elem.split()[0])], popup="Author: " + ", ".join(list(dic_geo[elem]["authors"])) + ", Institution: " + ", ".join(list(dic_geo[elem]["institutions"])), tooltip=", ".join(list(dic_geo[elem]["authors"]))).add_to(m)
+            folium_static(m)
+
+
+
+
+  #except:
+  #  st.write("No scientist corresponding to this name was found.")
+    
+  

Climate_site/python_scripts/__init__.py

@@ -0,0 +1 @@
+__all__ = ["functions","paper_functions","patent_functions", "others"]

Climate_site/python_scripts/functions.py

@@ -0,0 +1,125 @@
+##packages code
+import pandas as pd
+import numpy as np
+import json, requests 
+import pandas as pd
+
+#import spacy
+#!python -m spacy download en_core_web_lg
+#spacy_nlp = spacy.load("en_core_web_lg")
+
+#import en_core_web_lg
+#spacy_nlp = en_core_web_lg.load()
+
+#################### Test Scripts #############################
+
+def URL(base_URL , entity_type , filters):
+    url = base_URL + entity_type + filters 
+    return url
+
+
+def get_data(url):
+    url = requests.get(url)
+    text = url.text
+    import json
+    data = json.loads(text)
+    return data
+
+## encoding the abstract
+
+
+def reconstruction_abstract(abstract_inverted_index):
+    # return the abstract is the abstract exists in the database, else, return None
+    
+    if abstract_inverted_index != None:
+        
+        list_values = list(abstract_inverted_index.values())
+        list_keys = list(abstract_inverted_index.keys())
+        #from the words in the abstract (keys of abstract_inverted_index) and their position in the text (values of abstract_inverted_index), reconstruct the abstract
+        
+        size_abstract = max([ max(elem) for elem in abstract_inverted_index.values() ] )
+        
+        abstract = [""]*(size_abstract +1)
+        
+        for i in range(len(list_values)):
+            for pos in list_values[i]:
+                abstract[pos] = list_keys[i]
+        
+        return " ".join(list(abstract))
+    
+    else:
+        return None
+            
+    
+## calculate efficiently the dot product between two vectors
+
+def norm(vector):
+    return np.sqrt(sum(x * x for x in vector))    
+
+def cosine_similarity2(vec_a, vec_b):
+        norm_a = norm(vec_a)
+        norm_b = norm(vec_b)
+        dot = sum(a * b for a, b in zip(vec_a, vec_b))
+        return dot / (norm_a * norm_b)
+
+################################### Extracted texts ###############################################################
+
+def extract_quantitative_data_paper( work_id):
+    
+
+    try:
+        url = "https://api.openalex.org/works/" + str(work_id)
+        data = get_data(url)
+        date = data["publication_date"]
+        title = data["title"]
+        abstract = reconstruction_abstract(data["abstract_inverted_index"])
+        concepts = ", ".join( [elem["display_name"] for elem in data["concepts"]] )
+        authors = ", ".join( [elem["author"]["display_name"] for elem in data["authorships"]] )
+        institutions = ", ".join( set([elem["institutions"][0]["display_name"] for elem in data["authorships"]]) )
+        print("\033[96mFROM OpenAlex: ")
+        print("\033[92mPaper link: "  + url)
+        print(" ")
+        print("\033[92mTitle: \x1b[0m"  + title)
+        print(" ")
+        print("\033[92mConcepts: \x1b[0m"  + concepts)
+        print(" ")
+        print("\033[92mDate: \x1b[0m"  + date)
+        print(" ")
+        print("\033[92mAuthors: \x1b[0m"  + authors)
+        print(" ")
+        print("\033[92mInstitutions: \x1b[0m"  + institutions)
+        print(" ")
+        # sentences = extract_sentences_with_numbers(abstract , "\033[92mAbstract\x1b[0m")
+        return [url, date, title, abstract, concepts, authors, institutions]
+    except:
+        print("Enter a valid work id from OpenAlex")
+        
+   
+def extract_quantitative_data_patent( patent_id):
+
+    try: 
+        url = "https://api.patentsview.org/patents/query?q={%22patent_id%22:%22" + str(patent_id) + "%22}&f=[%22patent_number%22,%22patent_title%22,%22patent_abstract%22,%22patent_date%22,%22inventor_last_name%22,%22inventor_first_name%22,%22assignee_organization%22]"
+        data = get_data(url)["patents"][0]
+        title = data["patent_title"]
+        abstract = data["patent_abstract"]
+        print("\033[96mFROM PATENTSVIEW:")
+        print("\033[92mPatent link: \x1b[0m"  + url)
+        print(" ")
+        print("\033[92mTitle: \x1b[0m"  + title)
+        print(" ")
+        print("\033[92mAbstract: \x1b[0m"  + abstract)
+        print(" ")
+        print("\033[92mDate: \x1b[0m"  + data["patent_date"])
+        print(" ")
+        print("\033[92mInventors: \x1b[0m"   ,  ", ".join([ data["inventors"][i]["inventor_first_name"] + " " + data["inventors"][i]["inventor_last_name"] for i in range(len(data["inventors"])) ]) )
+        print(" ")
+        print("\033[92mAssignee: \x1b[0m" , ", ".join( [ str(data["assignees"][i]["assignee_organization"]) for i in range(len(data["assignees"])) ] ) )
+        assignees = ', '.join( [ str(data["assignees"][i]["assignee_organization"]) for i in range(len(data["assignees"])) ] )
+        inventors = ", ".join([ data["inventors"][i]["inventor_first_name"] + " " + data["inventors"][i]["inventor_last_name"] for i in range(len(data["inventors"])) ])
+    except:
+        print("Enter a valid patent_id from PatentsView")
+
+    return data, url, assignees, inventors
+    
+
+#print("Do you want the quantitative information corresponding to a paper?")

Climate_site/python_scripts/functions_companies.py

@@ -0,0 +1,447 @@
+##packages code
+
+import streamlit as st
+from shapely.geometry import Point
+import pandas as pd
+import re
+import json
+
+def load_dic():
+    f = open("Synapse_project/Climate_site/python_scripts/related_companies.json","r")
+    dic_companies = json.load(f)
+    return dic_companies
+    
+
+def load_data():
+    url = "Synapse_project/Climate_site/python_scripts/preqin_companies_IEA.tsv"
+    table = pd.read_csv(url, delimiter = "\t" , index_col = 0)
+    table = table.astype({'portfolio_company_id': 'str'})
+
+    return table
+
+table_companies = load_data()
+dic_companies = load_dic()
+
+
+#################### General Functions #############################
+
+
+
+def print_extracted_text(name_file):
+
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+    count = 0
+    for index, line in enumerate(lines):
+        read_line = line.strip()
+        print(read_line)
+
+    file.close()
+    
+    
+    iea.txt
+
+def details(name_file , display):
+    
+    file = open("Synapse_project/Climate_site/python_scripts/iea.txt", "r")
+    lines = file.readlines()
+
+    mark = 0 
+    dic_details = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+
+
+        if mark == 1 and line != "" and line[0] == "*":
+            
+            if display == True:
+                print(count)
+                print(text)
+                print(" ")
+            dic_details[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line.split(" 	")[-1] == "Details" or line.split(" 	")[-1] == "Hide":
+            mark = 1
+            text = ""
+            
+    return dic_details
+
+
+
+
+
+
+
+def key_initiatives(name_file , display ):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+
+      
+    mark = 0 
+    dic_key_initiatives = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+
+            count += 1
+
+
+
+        if mark == 1 and line != "" and ( (line[0].isnumeric() and  ">" in line and " 	" in line) or line == "*Deployment targets:*" or line == "*Announced development targets:*"):
+            if display == True: 
+                print(count)
+                print(text)
+                print(" ")
+            
+            dic_key_initiatives[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line == "*Key initiatives:*":
+
+
+            mark = 1
+            text = ""
+            
+    return dic_key_initiatives
+
+
+
+
+def deployment_target(name_file , display):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+
+
+    mark = 0 
+    dic_target = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+
+
+
+        if mark == 1 and line != "" and  ((line[0].isnumeric() and ">" in line and " 	" in line)  or line == "*Announced cost reduction targets:*" or line == "*Announced development targets:*"):
+            
+            if display == True:
+                print(count)
+                print(text)
+                print(" ")
+                
+            dic_target[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line == "*Deployment targets:*" or line == "*Announced development targets:*":
+
+            mark = 1
+            text = ""
+            
+    return dic_target
+
+
+ 
+    
+def cost_reduction_target(name_file , display):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+    
+    mark = 0 
+    dic_cost = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+
+
+            count += 1
+
+
+
+        if mark == 1 and line != ""  and (line[0].isnumeric() and ">" in line and " 	" in line) :
+            
+            if display == True: 
+                print(count)
+                print(text)
+                print(" ")
+                
+            dic_cost[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line == "*Announced cost reduction targets:*":
+
+            mark = 1
+            text = ""
+            
+    return dic_cost
+
+
+
+def key_words(name_file, display ):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    
+    lines = file.readlines()
+    
+    list_categories = []
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+            
+            if display == True:
+                print("Technologies"  , count+1 , ":")
+
+        if line != "":
+
+            if line[0].isnumeric() and ">" in line and " 	" in line:
+                i = 0
+                try:
+                    line = line.split(" 	")[2]
+                except:
+                    print(line)
+                    break
+                
+                if "Details" not in lines[index] and "Moderate" not in lines[index]:
+                
+                    while "	" not in line:
+                        i += 1
+                        if "Details"==lines[index + i][:7] or "End-use"==lines[index + i][:7]:
+                            break
+                        else:
+                            line = line + "  " +  lines[index + i]
+
+                #if " Production" in line:
+                    #line = line.replace(" Production" , "")
+
+                line = line.replace("\n" , " ")
+                line = line.replace("/" , " ")
+                line = line.replace("-" , " ")
+                line = line.split(" 	")[0]
+
+                if "  " in line:
+                    line = line.replace("  ", " ")
+                line = line.split(">")
+
+
+                if "(" in line[-1]:
+                    line[-1] = line[-1].split("(")[0] 
+
+
+                for i in range(len(line)):
+
+                    # remove multiple spaces
+                    line[i] = re.sub(' +', ' ', line[i])
+                    # remove trailing spaces
+                    line[i] = line[i].strip()
+
+    
+
+                if display == True:
+                    print(line)
+                    print(" ")
+                    
+                if '' in line:
+                    line.remove('')
+
+                list_categories.append([count , line])
+                
+    return list_categories
+
+
+
+def technology(name_file, display ):
+    # Filepath too specific, need to change to relative path
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+    
+    list_categories = []
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+            
+            if display == True:
+                print("Technologies"  , count+1 , ":")
+
+        if line != "":
+
+            if line[0].isnumeric() and ">" in line and " 	" in line:
+                i = 0
+                try:
+                    line = line.split(" 	")[1]
+                except:
+                    print(line)
+                    break
+                
+
+                line = line.replace("\n" , " ")
+                line = line.replace("/" , " ")
+                line = line.replace("-" , " ")
+                line = line.strip()
+                line = re.sub(' +', ' ', line)
+                line = line.split(" 	")[0]
+                line = line.split(">")
+
+
+                if "(" in line[-1]:
+                    line[-1] = line[-1].split("(")[0] 
+
+
+                for i in range(len(line)):
+
+                    # remove multiple spaces
+                    line[i] = re.sub(' +', ' ', line[i])
+                    # remove trailing spaces
+                    line[i] = line[i].strip()
+
+    
+
+                if display == True:
+                    print(line)
+                    print(" ")
+                
+
+                list_categories.append([count , line])
+                
+    return list_categories
+
+
+#################### Companies Functions #############################
+
+
+def extract_quantitative_data_technology(technologies, number_technology):
+    
+      
+    name_file = "iea"
+    dic_target = deployment_target(name_file , False)
+    dic_cost = cost_reduction_target(name_file , False)
+    dic_details = details(name_file , False)
+    cost_target_text = 'No information'
+    cost_text = 'No information'
+    
+    if number_technology in dic_details:
+        reference_text = dic_details[number_technology]
+            
+
+    if number_technology in dic_target:
+        cost_target_text = dic_target[number_technology]
+
+    if number_technology in dic_cost:
+        cost_text = dic_cost[number_technology]
+
+    return reference_text, cost_target_text, cost_text
+
+
+def related_VC_deals(category , number_technology , size):
+    
+    number_technology = str(number_technology)
+    
+   
+    list_results = list(dic_companies[number_technology].keys()) 
+    res = table_companies[table_companies["portfolio_company_id"].isin(list_results)].set_index("portfolio_company_id")
+    
+    table = res.copy()
+    for elem in table.index:
+        table.loc[ elem , "score" ] = dic_companies[number_technology][str(elem)]
+        
+    table = table.sort_values("score" , ascending = False).head(size)
+    table = table[['portfolio_company_name', 'year_established','portfolio_company_website','firm_about',
+       'portfolio_company_country',  'portfolio_company_state',
+        'firm_othernames', 'industry_classification',
+       'primary_industry', 'sub_industries', 'score']]
+
+
+    table["year_established"] = table["year_established"].replace(",", "", regex=True).astype(int, errors='ignore')
+
+
+
+    return table
+
+
+
+
+################################### Extracted texts ###############################################################
+
+#@title Which patents are related to the technology?
+
+
+def finder():
+    name_file = 'iea'
+    res = technology("iea", False )
+    list_categories_tech = []
+    list_categories = key_words("iea" , False)
+    list_technologies = [ ( ", ".join(list_categories[i][1]) , i ) for i in range(len(list_categories))  ] 
+    dic_technologies = {}
+    for i in range(len(res)):
+        names = res[i][1]
+        if ", ".join(names) not in list_categories_tech:
+            list_categories_tech.append(", ".join(names))
+            dic_technologies[", ".join(names)] = []
+        dic_technologies[", ".join(names)].append( (", ".join(list_categories[i][1]) , i ))
+        
+
+    list_climate = [ ("Any related papers" , False ) , ("Climate related papers" , True)]
+
+    dic_categories = {}
+    for elem in list_technologies:
+        list_words = elem[0].split(",")[-3:]
+        for i in range(len(list_words)):
+            if "CCUS" in list_words[i]:
+                list_words[i] = list_words[i].replace("CCUS" , "carbon capture storage")
+        dic_categories[elem[1]]  = [  ", ".join([ " ".join(words.split()[:3]) for words in list_words ] )  ,  ", ".join([ " ".join(words.split()[:3]) for words in list_words[:-1] ]) , ", ".join([ " ".join(words.split()[:3]) for words in list_words[1:] ] ) ]  
+    
+    return dic_technologies, dic_categories, list_categories_tech, list_technologies

Climate_site/python_scripts/functions_contest.py

@@ -0,0 +1,478 @@
+##packages code
+
+import streamlit as st
+from shapely.geometry import Point
+from math import radians, cos, sin, asin, sqrt
+import pandas as pd
+import re
+import json
+
+from sentence_transformers import SentenceTransformer, util
+
+@st.cache_resource
+def model_nlp():
+    model = SentenceTransformer('all-mpnet-base-v2')
+    return model
+
+
+def load_data():
+    url = "Synapse_project/Climate_site/python_scripts/big_ideas_contest.tsv"
+    dic = pd.read_csv(url, delimiter = "\t" , index_col = 0).to_dict('index')
+    return dic
+
+
+
+#################### General Functions #############################
+
+def clean_encoding(encoded_text):
+    if encoded_text == None:
+        return None
+    else:
+        if "\n" in encoded_text:
+            encoded_text = encoded_text.replace("\n" , "")
+        encoded_text = encoded_text[1:-1]
+        encoded_text = list(map(float , encoded_text.split(", ")))
+        return encoded_text
+    
+
+def norm(vector):
+    return sqrt(sum(x * x for x in vector))    
+
+def cosine_similarity2(vec_a, vec_b):
+        norm_a = norm(vec_a)
+        norm_b = norm(vec_b)
+        dot = sum(a * b for a, b in zip(vec_a, vec_b))
+        return dot / (norm_a * norm_b)
+
+def print_extracted_text(name_file):
+
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+    count = 0
+    for index, line in enumerate(lines):
+        read_line = line.strip()
+        print(read_line)
+
+    file.close()
+    
+    
+    iea.txt
+
+def details(name_file , display):
+    
+    file = open("Synapse_project/Climate_site/python_scripts/iea.txt", "r")
+    lines = file.readlines()
+
+    mark = 0 
+    dic_details = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+
+
+        if mark == 1 and line != "" and line[0] == "*":
+            
+            if display == True:
+                print(count)
+                print(text)
+                print(" ")
+            dic_details[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line.split(" 	")[-1] == "Details" or line.split(" 	")[-1] == "Hide":
+            mark = 1
+            text = ""
+            
+    return dic_details
+
+
+
+
+
+
+
+def key_initiatives(name_file , display ):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+
+      
+    mark = 0 
+    dic_key_initiatives = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+
+            count += 1
+
+
+
+        if mark == 1 and line != "" and ( (line[0].isnumeric() and  ">" in line and " 	" in line) or line == "*Deployment targets:*" or line == "*Announced development targets:*"):
+            if display == True: 
+                print(count)
+                print(text)
+                print(" ")
+            
+            dic_key_initiatives[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line == "*Key initiatives:*":
+
+
+            mark = 1
+            text = ""
+            
+    return dic_key_initiatives
+
+
+
+
+def deployment_target(name_file , display):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+
+
+    mark = 0 
+    dic_target = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+
+
+
+        if mark == 1 and line != "" and  ((line[0].isnumeric() and ">" in line and " 	" in line)  or line == "*Announced cost reduction targets:*" or line == "*Announced development targets:*"):
+            
+            if display == True:
+                print(count)
+                print(text)
+                print(" ")
+                
+            dic_target[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line == "*Deployment targets:*" or line == "*Announced development targets:*":
+
+            mark = 1
+            text = ""
+            
+    return dic_target
+
+
+ 
+    
+def cost_reduction_target(name_file , display):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+    
+    mark = 0 
+    dic_cost = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+
+
+            count += 1
+
+
+
+        if mark == 1 and line != ""  and (line[0].isnumeric() and ">" in line and " 	" in line) :
+            
+            if display == True: 
+                print(count)
+                print(text)
+                print(" ")
+                
+            dic_cost[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line == "*Announced cost reduction targets:*":
+
+            mark = 1
+            text = ""
+            
+    return dic_cost
+
+
+
+def key_words(name_file, display ):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    
+    lines = file.readlines()
+    
+    list_categories = []
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+            
+            if display == True:
+                print("Technologies"  , count+1 , ":")
+
+        if line != "":
+
+            if line[0].isnumeric() and ">" in line and " 	" in line:
+                i = 0
+                try:
+                    line = line.split(" 	")[2]
+                except:
+                    print(line)
+                    break
+                
+                if "Details" not in lines[index] and "Moderate" not in lines[index]:
+                
+                    while "	" not in line:
+                        i += 1
+                        if "Details"==lines[index + i][:7] or "End-use"==lines[index + i][:7]:
+                            break
+                        else:
+                            line = line + "  " +  lines[index + i]
+
+                #if " Production" in line:
+                    #line = line.replace(" Production" , "")
+
+                line = line.replace("\n" , " ")
+                line = line.replace("/" , " ")
+                line = line.replace("-" , " ")
+                line = line.split(" 	")[0]
+
+                if "  " in line:
+                    line = line.replace("  ", " ")
+                line = line.split(">")
+
+
+                if "(" in line[-1]:
+                    line[-1] = line[-1].split("(")[0] 
+
+
+                for i in range(len(line)):
+
+                    # remove multiple spaces
+                    line[i] = re.sub(' +', ' ', line[i])
+                    # remove trailing spaces
+                    line[i] = line[i].strip()
+
+    
+
+                if display == True:
+                    print(line)
+                    print(" ")
+                    
+                if '' in line:
+                    line.remove('')
+
+                list_categories.append([count , line])
+                
+    return list_categories
+
+
+
+def technology(name_file, display ):
+    # Filepath too specific, need to change to relative path
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+    
+    list_categories = []
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+            
+            if display == True:
+                print("Technologies"  , count+1 , ":")
+
+        if line != "":
+
+            if line[0].isnumeric() and ">" in line and " 	" in line:
+                i = 0
+                try:
+                    line = line.split(" 	")[1]
+                except:
+                    print(line)
+                    break
+                
+
+                line = line.replace("\n" , " ")
+                line = line.replace("/" , " ")
+                line = line.replace("-" , " ")
+                line = line.strip()
+                line = re.sub(' +', ' ', line)
+                line = line.split(" 	")[0]
+                line = line.split(">")
+
+
+                if "(" in line[-1]:
+                    line[-1] = line[-1].split("(")[0] 
+
+
+                for i in range(len(line)):
+
+                    # remove multiple spaces
+                    line[i] = re.sub(' +', ' ', line[i])
+                    # remove trailing spaces
+                    line[i] = line[i].strip()
+
+    
+
+                if display == True:
+                    print(line)
+                    print(" ")
+                
+
+                list_categories.append([count , line])
+                
+    return list_categories
+
+          
+
+
+#################### Companies Functions #############################
+
+
+def extract_quantitative_data_technology(technologies, number_technology):
+    
+      
+    name_file = "iea"
+    dic_target = deployment_target(name_file , False)
+    dic_cost = cost_reduction_target(name_file , False)
+    dic_details = details(name_file , False)
+    cost_target_text = 'No information'
+    cost_text = 'No information'
+    
+    if number_technology in dic_details:
+        reference_text = dic_details[number_technology]
+            
+
+    if number_technology in dic_target:
+        cost_target_text = dic_target[number_technology]
+
+    if number_technology in dic_cost:
+        cost_text = dic_cost[number_technology]
+
+    return reference_text, cost_target_text, cost_text
+
+
+
+def big_ideas_encoding(number_technology):
+    
+    model = model_nlp()
+    
+    dic_big_ideas = load_data()
+    
+    dic_matches = {}
+    
+
+    name_file = 'iea'
+    dic_details = details(name_file , False)
+    k = 0 
+        
+        
+        
+    IEA_encoding = model.encode(dic_details[number_technology]  , convert_to_tensor=False , show_progress_bar = False).tolist()
+    for count_contest in dic_big_ideas:
+        if pd.isna(dic_big_ideas[count_contest]['encoded_description']) != True:
+            score = cosine_similarity2(IEA_encoding, clean_encoding(dic_big_ideas[count_contest]['encoded_description']))
+            if score > 0.45:
+                dic_matches[k] = dic_big_ideas[count_contest]
+                dic_matches[k]["score"] = score
+                k += 1
+                
+    
+                
+    if len(dic_matches) > 0:
+        return pd.DataFrame(dic_matches).T.sort_values("score" , ascending = False)[["Project_name" ,  "Description" ,"Project_date" , "University" , "Field" , "Team_members", "score" ]]  
+    else:
+        return "No related project found"  
+    
+    
+     
+
+
+
+
+
+
+################################### Extracted texts ###############################################################
+
+#@title Which patents are related to the technology?
+
+
+def finder():
+    name_file = 'iea'
+    res = technology("iea", False )
+    list_categories_tech = []
+    list_categories = key_words("iea" , False)
+    list_technologies = [ ( ", ".join(list_categories[i][1]) , i ) for i in range(len(list_categories))  ] 
+    dic_technologies = {}
+    for i in range(len(res)):
+        names = res[i][1]
+        if ", ".join(names) not in list_categories_tech:
+            list_categories_tech.append(", ".join(names))
+            dic_technologies[", ".join(names)] = []
+        dic_technologies[", ".join(names)].append( (", ".join(list_categories[i][1]) , i ))
+        
+
+    list_climate = [ ("Any related papers" , False ) , ("Climate related papers" , True)]
+
+    dic_categories = {}
+    for elem in list_technologies:
+        list_words = elem[0].split(",")[-3:]
+        for i in range(len(list_words)):
+            if "CCUS" in list_words[i]:
+                list_words[i] = list_words[i].replace("CCUS" , "carbon capture storage")
+        dic_categories[elem[1]]  = [  ", ".join([ " ".join(words.split()[:3]) for words in list_words ] )  ,  ", ".join([ " ".join(words.split()[:3]) for words in list_words[:-1] ]) , ", ".join([ " ".join(words.split()[:3]) for words in list_words[1:] ] ) ]  
+    
+    return dic_technologies, dic_categories, list_categories_tech, list_technologies

Climate_site/python_scripts/others.py

@@ -0,0 +1,7 @@
+import numpy as np
+
+def other(list):
+    return np.sum(list)
+
+def add(x,y):
+    return x+y

Climate_site/python_scripts/paper_functions.py

@@ -0,0 +1,1220 @@
+##packages code
+import streamlit as st
+from shapely.geometry import Point
+import pandas as pd
+from tqdm import tqdm
+
+import numpy as np
+import json, requests 
+import pandas as pd
+#from pandas.io.json import json_normalize
+
+import matplotlib.pyplot as plt
+import seaborn as sns
+from math import radians, cos, sin, asin, sqrt
+
+#import spacy
+#spacy_nlp = spacy.load("en_core_web_lg")
+
+#@st.cache_resource
+#def spacy_func():
+#    spacy_nlp = spacy.load("en_core_web_lg")
+#    return spacy_nlp
+
+from sentence_transformers import SentenceTransformer, util
+
+@st.cache_resource
+def model_nlp():
+    model = SentenceTransformer('all-MiniLM-L6-v2')
+    return model
+
+
+@st.cache_data  # 👈 Add the caching decorator
+def load_data():
+    url = "Synapse_project/Climate_site/python_scripts/institutions.tsv"
+    dic = pd.read_csv(url, delimiter = "\t" , index_col = 1).to_dict('index')
+    return dic
+
+dic_institutions = load_data()
+
+
+import unicodedata
+
+from metaphone import doublemetaphone
+from fuzzywuzzy import fuzz
+from difflib import SequenceMatcher
+import re
+
+import geopandas as gpd
+from geopandas import GeoDataFrame
+#################### General Functions #############################
+
+def URL(base_URL , entity_type , filters):
+    url = base_URL + entity_type + filters 
+    return url
+
+
+def get_data(url):
+    url = requests.get(url)
+    text = url.text
+    import json
+    data = json.loads(text)
+    return data
+
+## encoding the abstract
+
+
+def reconstruction_abstract(abstract_inverted_index):
+    # return the abstract is the abstract exists in the database, else, return None
+    
+    if abstract_inverted_index != None:
+        
+        list_values = list(abstract_inverted_index.values())
+        list_keys = list(abstract_inverted_index.keys())
+        #from the words in the abstract (keys of abstract_inverted_index) and their position in the text (values of abstract_inverted_index), reconstruct the abstract
+        
+        size_abstract = max([ max(elem) for elem in abstract_inverted_index.values() ] )
+        
+        abstract = [""]*(size_abstract +1)
+        
+        for i in range(len(list_values)):
+            for pos in list_values[i]:
+                abstract[pos] = list_keys[i]
+        
+        return " ".join(list(abstract))
+    
+    else:
+        return None
+            
+    
+## calculate efficiently the dot product between two vectors
+
+def norm(vector):
+    return np.sqrt(sum(x * x for x in vector))    
+
+def cosine_similarity2(vec_a, vec_b):
+        norm_a = norm(vec_a)
+        norm_b = norm(vec_b)
+        dot = sum(a * b for a, b in zip(vec_a, vec_b))
+        return dot / (norm_a * norm_b)
+
+## Extracted texts
+
+def print_extracted_text(name_file):
+
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+    count = 0
+    for index, line in enumerate(lines):
+        read_line = line.strip()
+        print(read_line)
+
+    file.close()
+    
+    
+    iea.txt
+
+def details(name_file , display):
+    
+    file = open("Synapse_project/Climate_site/python_scripts/iea.txt", "r")
+    lines = file.readlines()
+
+    mark = 0 
+    dic_details = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+
+
+        if mark == 1 and line != "" and line[0] == "*":
+            
+            if display == True:
+                print(count)
+                print(text)
+                print(" ")
+            dic_details[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line.split(" 	")[-1] == "Details" or line.split(" 	")[-1] == "Hide":
+            mark = 1
+            text = ""
+            
+    return dic_details
+
+
+
+
+
+
+
+def key_initiatives(name_file , display ):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+
+      
+    mark = 0 
+    dic_key_initiatives = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+
+            count += 1
+
+
+
+        if mark == 1 and line != "" and ( (line[0].isnumeric() and  ">" in line and " 	" in line) or line == "*Deployment targets:*" or line == "*Announced development targets:*"):
+            if display == True: 
+                print(count)
+                print(text)
+                print(" ")
+            
+            dic_key_initiatives[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line == "*Key initiatives:*":
+
+
+            mark = 1
+            text = ""
+            
+    return dic_key_initiatives
+
+
+
+
+def deployment_target(name_file , display):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+
+
+    mark = 0 
+    dic_target = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+
+
+
+        if mark == 1 and line != "" and  ((line[0].isnumeric() and ">" in line and " 	" in line)  or line == "*Announced cost reduction targets:*" or line == "*Announced development targets:*"):
+            
+            if display == True:
+                print(count)
+                print(text)
+                print(" ")
+                
+            dic_target[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line == "*Deployment targets:*" or line == "*Announced development targets:*":
+
+            mark = 1
+            text = ""
+            
+    return dic_target
+
+
+ 
+    
+def cost_reduction_target(name_file , display):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+    
+    mark = 0 
+    dic_cost = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+
+
+            count += 1
+
+
+
+        if mark == 1 and line != ""  and (line[0].isnumeric() and ">" in line and " 	" in line) :
+            
+            if display == True: 
+                print(count)
+                print(text)
+                print(" ")
+                
+            dic_cost[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line == "*Announced cost reduction targets:*":
+
+            mark = 1
+            text = ""
+            
+    return dic_cost
+
+
+
+def key_words(name_file, display ):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    
+    lines = file.readlines()
+    
+    list_categories = []
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+            
+            if display == True:
+                print("Technologies"  , count+1 , ":")
+
+        if line != "":
+
+            if line[0].isnumeric() and ">" in line and " 	" in line:
+                i = 0
+                try:
+                    line = line.split(" 	")[2]
+                except:
+                    print(line)
+                    break
+                
+                if "Details" not in lines[index] and "Moderate" not in lines[index]:
+                
+                    while "	" not in line:
+                        i += 1
+                        if "Details"==lines[index + i][:7] or "End-use"==lines[index + i][:7]:
+                            break
+                        else:
+                            line = line + "  " +  lines[index + i]
+
+                #if " Production" in line:
+                    #line = line.replace(" Production" , "")
+
+                line = line.replace("\n" , " ")
+                line = line.replace("/" , " ")
+                line = line.replace("-" , " ")
+                line = line.split(" 	")[0]
+
+                if "  " in line:
+                    line = line.replace("  ", " ")
+                line = line.split(">")
+
+
+                if "(" in line[-1]:
+                    line[-1] = line[-1].split("(")[0] 
+
+
+                for i in range(len(line)):
+
+                    # remove multiple spaces
+                    line[i] = re.sub(' +', ' ', line[i])
+                    # remove trailing spaces
+                    line[i] = line[i].strip()
+
+    
+
+                if display == True:
+                    print(line)
+                    print(" ")
+                    
+                if '' in line:
+                    line.remove('')
+
+                list_categories.append([count , line])
+                
+    return list_categories
+
+
+
+def technology(name_file, display ):
+    # Filepath too specific, need to change to relative path
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+    
+    list_categories = []
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+            
+            if display == True:
+                print("Technologies"  , count+1 , ":")
+
+        if line != "":
+
+            if line[0].isnumeric() and ">" in line and " 	" in line:
+                i = 0
+                try:
+                    line = line.split(" 	")[1]
+                except:
+                    print(line)
+                    break
+                
+
+                line = line.replace("\n" , " ")
+                line = line.replace("/" , " ")
+                line = line.replace("-" , " ")
+                line = line.strip()
+                line = re.sub(' +', ' ', line)
+                line = line.split(" 	")[0]
+                line = line.split(">")
+
+
+                if "(" in line[-1]:
+                    line[-1] = line[-1].split("(")[0] 
+
+
+                for i in range(len(line)):
+
+                    # remove multiple spaces
+                    line[i] = re.sub(' +', ' ', line[i])
+                    # remove trailing spaces
+                    line[i] = line[i].strip()
+
+    
+
+                if display == True:
+                    print(line)
+                    print(" ")
+                
+
+                list_categories.append([count , line])
+                
+    return list_categories
+
+
+#################### Paper Functions #############################
+
+def related_papers(number_technology  , carbon_related , display , key_words  ):
+    """Returns a dictionary of data related to the papers the user chooses
+    
+    This is done through generating URLs to connect with OpenAlex API and parsing through the json file
+    
+    Parameters
+    ----------
+    number_technology : int, the index of the technology with respect to the 23 IEA technologies
+    list_categories : str, an array of all the technologies related to the 23 IEA technologies
+    carbon_related: bool, a setting to select papers related to carbon capture
+    display:
+    key_words: str, an array of keywords selected based off technology/ user input
+
+    """   
+    dic = {}
+
+    max_count = 0
+    base_URL_OA = f'https://api.openalex.org/'
+    filter_works = f'works?'
+        
+
+    filter_openalex = f"search=" + key_words
+
+
+    if carbon_related == True:
+        filter_openalex += "&filter=concepts.id:https://openalex.org/C132651083|https://openalex.org/C115343472|https://openalex.org/C530467964&per_page=100&mailto=emma_scharfmann@berkeley.edu"
+    
+    else:
+        filter_openalex += "&per_page=100&mailto=emma_scharfmann@berkeley.edu"
+        
+        
+    filter_openalex = filter_openalex.replace(" " , "%20")
+    
+    url = URL(base_URL_OA , filter_works, filter_openalex) 
+    data = get_data(url)
+    count = data["meta"]["count"] 
+
+
+
+    if display == True:
+        print(  data["meta"]["count"] , [elem[i] for i in range(1,len(elem))]  )
+        print(url)
+    
+    for i in range(len(data["results"])):
+        dic[ data["results"][i]["id"]] = {}
+        dic[ data["results"][i]["id"]]["title"] = data["results"][i]["title"]
+        dic[ data["results"][i]["id"]]["abstract"] = reconstruction_abstract(data["results"][i]["abstract_inverted_index"])
+        dic[ data["results"][i]["id"]]["concepts"] = data["results"][i]["concepts"]
+        dic[ data["results"][i]["id"]]["date"] = data["results"][i]["publication_date"]
+        dic[ data["results"][i]["id"]]["authorships"] = data["results"][i]["authorships"]
+        dic[ data["results"][i]["id"]]["cited_by_count"] = data["results"][i]["cited_by_count"]
+
+        if len(data["results"][i]["authorships"]) > 0:
+            if data["results"][i]["authorships"][0]["institutions"] != []:
+                dic[ data["results"][i]["id"]]["countries"] = data["results"][i]["authorships"][0]["institutions"][0]["country_code"]
+                dic[ data["results"][i]["id"]]["institutions"] = data["results"][i]["authorships"][0]["institutions"][0]["display_name"]
+            else:
+                dic[ data["results"][i]["id"]]["countries"] = ""
+                dic[ data["results"][i]["id"]]["institutions"] = ""
+            dic[ data["results"][i]["id"]]["authors"] = data["results"][i]["authorships"][0]["author"]["display_name"]
+
+            for j in range(1 , len(data["results"][i]["authorships"])):
+                if data["results"][i]["authorships"][j]["institutions"] != []:
+                    dic[ data["results"][i]["id"]]["institutions"] += ", " + data["results"][i]["authorships"][j]["institutions"][0]["display_name"]
+                    dic[ data["results"][i]["id"]]["countries"] = data["results"][i]["authorships"][j]["institutions"][0]["country_code"]
+                dic[ data["results"][i]["id"]]["authors"] += ", " + data["results"][i]["authorships"][j]["author"]["display_name"]
+
+    if display == True: 
+        print(" ")
+        
+    return dic
+        
+
+## Ranking Papers
+
+def ranking_papers(number_technology,  dic_details   , display , size , carbon_related , search_words ):
+    """Returns a dictionary of data related to the papers the user chooses
+    
+    This is done through generating URLs to connect with OpenAlex API and parsing through the json file
+    
+    Parameters
+    ----------
+    number_technology : int, the index of the technology with respect to the 500+ IEA technologies
+    dic_details : str, dictionary where key is index of technology with respect to 500+ IEA technolgies and values are the details related to the technologies
+    
+    dic: bool, a setting to select papers related to carbon capture
+    list_categories: str, an array of keywords selected based off technology/ user input
+    display: bool, chooses to display information or not
+    size: str, an array of keywords selected based off technology/ user input
+
+    """   
+    model = model_nlp()
+    dic = related_papers(number_technology  , carbon_related , display , search_words  )
+    
+    
+    dic_scores_papers = {}
+    
+    
+
+    if display == True:
+        print("Technology " + str(number_technology))
+        print("Key words: " , list_categories[number_technology][1])
+            
+    if number_technology in dic_details:
+        reference_text = dic_details[number_technology]
+            
+        if display== True:
+            print("Technology details: " , reference_text)
+            print(" ")
+        encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist()
+        
+        
+        if len(dic) == 0:
+            return "No paper found, select other key words" 
+        
+        
+        for ids in list(dic.keys()):
+
+            dic_scores_papers[ids] = {}
+            
+            if dic[ids]["title"] != None:
+                encoded_title = model.encode(dic[ids]["title"], convert_to_tensor=False).tolist()
+                score_title = cosine_similarity2(encoded_title, encoded_text)
+            else:
+                score_title = None
+
+        
+
+            if dic[ids]["abstract"] != None:
+                encoded_abstract = model.encode(dic[ids]["abstract"], convert_to_tensor=False).tolist()
+                score_abstract = cosine_similarity2(encoded_abstract, encoded_text)
+            else:
+                score_abstract = None
+
+
+            #concepts = ""
+            #for elem in dic[ids]["concepts"]:
+            #    concepts += elem["display_name"] + " "
+            #encoded_concepts = model.encode(concepts, convert_to_tensor=False).tolist()
+            #score_concepts = cosine_similarity2(encoded_concepts, encoded_text)
+
+            dic_scores_papers[ids]["title comparison"] = score_title
+            dic_scores_papers[ids]["abstract comparison"] = score_abstract
+            #dic_scores_papers[ids]["concepts comparison"] = score_concepts
+            
+            dic_scores_papers[ids]["title"] = dic[ids]["title"]
+            dic_scores_papers[ids]["citations"] = dic[ids]["cited_by_count"]
+            
+
+            dic_scores_papers[ids]["date"] = dic[ids]["date"][:4]
+            if "institutions" in dic[ids]:
+                dic_scores_papers[ids]["institutions"] = dic[ids]["institutions"]
+            #    dic_scores_papers[ids]["countries"] = dic[ids]["countries"]
+                
+            else:
+                dic_scores_papers[ids]["institutions"] = None
+            #    dic_scores_papers[ids]["countries"] = None
+            
+            if "authors" in dic[ids] and dic[ids]["authors"] != None:
+                dic_scores_papers[ids]["number of co-authors"] = len(dic[ids]["authors"].split(","))
+                dic_scores_papers[ids]["authors"] = dic[ids]["authors"]
+            else:
+                dic_scores_papers[ids]["number of co-authors"] = None
+                dic_scores_papers[ids]["authors"] = None
+            
+
+        return dic , dic_scores_papers
+        
+
+
+
+
+def get_ranking_related_papers( technologies , number_technology , carbon_related , size , research_words):
+    name_file = "iea"
+    dic_details = details(name_file , False)
+    dic , dic_scores_papers = ranking_papers(number_technology,  dic_details   , False , size , carbon_related , research_words )
+    
+    if dic_scores_papers == {}:
+        return "No paper found"
+    
+    elif type(dic_scores_papers) == str:
+        return dic_scores_papers
+    
+    else:
+        return pd.DataFrame(dic_scores_papers).T.sort_values(by="abstract comparison" , ascending = False).head(size)
+     
+
+## Extract quantitative data
+
+def extract_sentences_with_numbers(text , text_name):
+    if text != None:
+        text = text.replace("CO 2" , "CO2")
+        text = text.replace("CO 3" , "CO3")
+        text = text.replace("CO(2)" , "CO2")
+        text = text.replace("CO(3)" , "CO3")
+        
+        print(text_name + ": " , text)
+        print(" ")
+
+
+
+        list_text = list(text)
+        for i in range(1,len(list_text)-1):
+            if list_text[i] == " " and list_text[i-1] == "." and list_text[i+1].isupper():
+                list_text[i] = "~"
+
+        text = "".join(list_text)
+        text = text.split("~")
+        for sentence in text:                
+
+            if any(char.isdigit() for char in sentence):
+                
+                
+                
+                if "CO2" in sentence:
+                    print("\x1b[31mCARBON RELATED:\x1b[0m", sentence)
+                    print(" ")
+
+
+                    
+                if "GJ" in sentence or "MJ" in sentence:
+                    print("\x1b[31mENERGY:\x1b[0m" , sentence)
+                    print(" ")
+
+
+                    
+                ##price
+                if "€"  in sentence or "$" in sentence or "EUR" in sentence or "dollars" in sentence.lower():
+                    print("\x1b[31mPRICE:\x1b[0m" , sentence)
+                    print(" ")
+
+
+                    
+                ##dates
+                digits = []
+                for word in sentence.replace("," , "").replace("%" , "").replace("." , " ").split():
+                    if word.isdigit() and 1850 < int(word) < 2200 :
+                        digits.append(word)
+                if digits != []:
+                    print("\x1b[31mDATE:\x1b[0m" , sentence)
+                    print(" ")
+
+
+                    
+                ##CO quantity
+                if "Mt" in sentence or "tC" in sentence or "t-C" in sentence:
+                    print("\x1b[31mCARBON QUANTITY:\x1b[0m" , sentence)
+                    print(" ")
+ 
+    print(" ")
+          
+    
+def extract_quantitative_data_technology(technologies, number_technology):
+    
+    model = model_nlp()
+    count = 0
+      
+    name_file = "iea"
+    dic_target = deployment_target(name_file , False)
+    dic_cost = cost_reduction_target(name_file , False)
+    dic_details = details(name_file , False)
+    cost_target_text = 'No information'
+    cost_text = 'No information'
+    
+    if number_technology in dic_details:
+        reference_text = dic_details[number_technology]
+            
+        #print("\033[96mFROM IEA website: ")
+        #print("\033[92mTechnology details: \x1b[0m" , reference_text)
+        #print(" ")
+        #encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist()
+
+    if number_technology in dic_target:
+        cost_target_text = dic_target[number_technology]
+        #print("\033[96mFROM IEA website: ")
+        #sentences = extract_sentences_with_numbers(cost_target_text , "\033[92mDeployment target and Announced development target\x1b[0m")
+
+    if number_technology in dic_cost:
+        cost_text = dic_cost[number_technology]
+        #print("\033[96mFROM IEA website: ")
+        #sentences = extract_sentences_with_numbers(cost_text , "\033[92mAnnounced cost reduction targets\x1b[0m")
+    
+    return reference_text, cost_target_text, cost_text
+
+
+def extract_quantitative_data_paper(work_id):
+
+    url = "https://api.openalex.org/works/" + str(work_id)
+    url_google = "https://explore.openalex.org/works/" + str(work_id)
+    
+    data = get_data(url)
+    date = data["publication_date"]
+    title = data["title"]
+    abstract = reconstruction_abstract(data["abstract_inverted_index"])
+    concepts = ", ".join( [elem["display_name"] for elem in data["concepts"]] )
+    authors = ", ".join( [elem["author"]["display_name"] for elem in data["authorships"]] )
+    institutions = ", ".join( set([elem["institutions"][0]["display_name"] for elem in data["authorships"] if len(elem["institutions"]) > 0]) )
+    return url_google , title , abstract , date , authors , institutions
+            
+
+    
+
+## Get papers related to cited projects
+
+def related_projects(technologies, number_technology , carbon_related , size):
+    
+    spacy_nlp = spacy_func()
+    model = model_nlp()
+
+    name_file = "iea"
+    dic_details = details(name_file , False)
+    dic_key_initiatives = key_initiatives(name_file , False )
+    
+    
+    dic2 = {}
+
+    reference_text = dic_details[number_technology]
+    print("\033[92mTechnology details: \x1b[0m" , reference_text)
+    print(" ")
+    encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist()
+     # reference_text key_initiative location_entities
+    if number_technology in dic_key_initiatives:
+        key_initiative = dic_key_initiatives[number_technology]
+        print("\033[92mTechnology key initiatives: \x1b[0m" ,  key_initiative)
+
+        location_entities = set()
+        doc = spacy_nlp(key_initiative.strip())
+        for j in doc.ents:
+            entry = str(j.lemma_).lower()
+            key_initiative = key_initiative.replace(str(j).lower(), "")
+            if j.label_ in ["ORG"]:
+                location_entities.add(j)
+        print(" ")
+        print("\033[92mExtracted projects and organizations names: \x1b[0m", location_entities)
+
+
+        base_URL_OA = f'https://api.openalex.org/'
+        filter_works = f'works?'
+        for element in location_entities:
+
+            filter_openalex = f"search=" 
+            filter_openalex += str(element) 
+            if carbon_related == True:
+                filter_openalex += "&filter=concepts.id:https://openalex.org/C2780021121&per_page=50&mailto=emma_scharfmann@berkeley.edu"
+            else:
+                filter_openalex += "&per_page=50&mailto=emma_scharfmann@berkeley.edu"
+
+            filter_openalex = filter_openalex.replace(" " , "%20")
+
+            url = URL(base_URL_OA , filter_works, filter_openalex) 
+
+            data = get_data(url)
+            count = len(data["results"])
+            if count < 40:
+
+                for k in range(count):
+                    
+                    if data["results"][k]["abstract_inverted_index"] != None:
+                        encoded_abstract = model.encode(reconstruction_abstract(data["results"][k]["abstract_inverted_index"]) ,  convert_to_tensor=False).tolist()
+                        abstract_similarity = cosine_similarity2(encoded_abstract, encoded_text)
+                    else:
+                        abstract_similarity = None
+                        
+                    if data["results"][k]["title"] != None:
+                        encoded_title = model.encode(data["results"][k]["title"] ,  convert_to_tensor=False).tolist()
+                        title_similarity = cosine_similarity2(encoded_title, encoded_text)
+                    else:
+                        title_similarity = None
+                        
+                    work_id = data["results"][k]["id"]
+                    dic2[work_id] = {}
+                    dic2[work_id]["project/institution name"] = str(element)
+                    dic2[work_id]["title comparison"] = title_similarity
+                    dic2[work_id]["abstract comparison"] = abstract_similarity
+                    dic2[work_id]["title"] = data["results"][k]["title"]
+                    #dic2[work_id]["abstract"] = reconstruction_abstract(data["results"][k]["abstract_inverted_index"])
+                    dic2[work_id]["date"] = data["results"][k]["publication_date"]
+
+            else:
+                list_categories = key_words( name_file, False)
+                key_word = list_categories[number_technology]
+                elem = key_word[1]
+
+                string = ""
+                for j in range(1,len(elem)):
+                    if elem[j] == "CCUS":
+                        elem[j] = "Carbon capture and storage"
+                    string += elem[j]  + " " 
+
+
+                string += str(element)
+                filter_openalex = f"search="  + string 
+
+                if carbon_related == True:
+                    filter_openalex += "&filter=concepts.id:https://openalex.org/C2780021121&per_page=50&mailto=emma_scharfmann@berkeley.edu"
+                else:
+                    filter_openalex += "&per_page=50&mailto=emma_scharfmann@berkeley.edu"
+                filter_openalex = filter_openalex.replace(" " , "%20")
+                url = URL(base_URL_OA , filter_works, filter_openalex) 
+                data = get_data(url)
+                count = len(data["results"])
+                for k in range(count):
+                    
+                    if data["results"][k]["abstract_inverted_index"] != None:
+                        encoded_abstract = model.encode(reconstruction_abstract(data["results"][k]["abstract_inverted_index"]) ,  convert_to_tensor=False).tolist()
+                        abstract_similarity = cosine_similarity2(encoded_abstract, encoded_text)
+                    else:
+                        abstract_similarity = None
+                        
+                    if data["results"][k]["title"] != None:
+                        encoded_title = model.encode(data["results"][k]["title"] ,  convert_to_tensor=False).tolist()
+                        title_similarity = cosine_similarity2(encoded_title, encoded_text)
+                    else:
+                        title_similarity = None
+                        
+                    work_id = data["results"][k]["id"]
+                    dic2[work_id] = {}
+                    dic2[work_id]["project/institution name"] = str(element)
+                    dic2[work_id]["title comparison"] = title_similarity
+                    dic2[work_id]["abstract comparison"] = abstract_similarity
+                    dic2[work_id]["title"] = data["results"][k]["title"]
+                    #dic2[work_id]["abstract"] = reconstruction_abstract(data["results"][k]["abstract_inverted_index"])
+                    dic2[work_id]["date"] = data["results"][k]["publication_date"]
+                    
+                    
+
+
+        if dic2 != {}:
+            return reference_text,key_initiative,location_entities,pd.DataFrame(dic2).T.sort_values(by="abstract comparison" , ascending = False).head(size)
+        else:
+            return reference_text,key_initiative,location_entities, "No paper found"
+
+
+## Mapping the authors
+
+#merge the nobiliary particles with the last name
+#ln_suff file can be modified if more or less nobiliary particles want to be suppressed
+
+ln_suff = ['oster',
+ 'nordre',
+ 'vaster',
+ 'aust',
+ 'vesle',
+ 'da',
+ 'van t',
+ 'af',
+ 'al',
+ 'setya',
+ 'zu',
+ 'la',
+ 'na',
+ 'mic',
+ 'ofver',
+ 'el',
+ 'vetle',
+ 'van het',
+ 'dos',
+ 'ui',
+ 'vest',
+ 'ab',
+ 'vste',
+ 'nord',
+ 'van der',
+ 'bin',
+ 'ibn',
+ 'war',
+ 'fitz',
+ 'alam',
+ 'di',
+ 'erch',
+ 'fetch',
+ 'nga',
+ 'ka',
+ 'soder',
+ 'lille',
+ 'upp',
+ 'ua',
+ 'te',
+ 'ni',
+ 'bint',
+ 'von und zu',
+ 'vast',
+ 'vestre',
+ 'over',
+ 'syd',
+ 'mac',
+ 'nin',
+ 'nic',
+ 'putri',
+ 'bet',
+ 'verch',
+ 'norr',
+ 'bath',
+ 'della',
+ 'van',
+ 'ben',
+ 'du',
+ 'stor',
+ 'das',
+ 'neder',
+ 'abu',
+ 'degli',
+ 'vre',
+ 'ait',
+ 'ny',
+ 'opp',
+ 'pour',
+ 'kil',
+ 'der',
+ 'oz',
+ 'von',
+ 'at',
+ 'nedre',
+ 'van den',
+ 'setia',
+ 'ap',
+ 'gil',
+ 'myljom',
+ 'van de',
+ 'stre',
+ 'dele',
+ 'mck',
+ 'de',
+ 'mellom',
+ 'mhic',
+ 'binti',
+ 'ath',
+ 'binte',
+ 'snder',
+ 'sre',
+ 'ned',
+ 'ter',
+ 'bar',
+ 'le',
+ 'mala',
+ 'ost',
+ 'syndre',
+ 'sr',
+ 'bat',
+ 'sndre',
+ 'austre',
+ 'putra',
+ 'putera',
+ 'av',
+ 'lu',
+ 'vetch',
+ 'ver',
+ 'puteri',
+ 'mc',
+ 'tre',
+ 'st']
+
+#suppress all the unwanted suffixes from a string. 
+#name_del file can be modified if more or less suffixes want to be suppressed 
+
+name_del = ['2nd', '3rd', 'Jr', 'Jr.', 'Junior', 'Sr', 'Sr.', 'Senior']
+
+
+def name_delete(string):
+    for elmt in name_del:
+        if f" {elmt}" in string:
+            return string.replace(f" {elmt}","")
+    return string
+
+def ln_suff_merge(string):
+    for suff in ln_suff:
+        if f" {suff} " in string or string.startswith(f"{suff} "):
+            return string.replace(f"{suff} ",suff.replace(" ",""))
+    return string
+
+#normalize a string dat that represents often a name. 
+
+def normalize(data):
+    normal = unicodedata.normalize('NFKD', data).encode('ASCII', 'ignore')
+    val = normal.decode("utf-8")
+    # delete unwanted elmt
+    val = name_delete(val)
+    # lower full name in upper
+    val = re.sub(r"[A-Z]{3,}", lambda x: x.group().lower(), val)
+    # add space in front of upper case
+    val = re.sub(r"(\w)([A-Z])", r"\1 \2", val)
+    # Lower case
+    val = val.lower()
+    # remove special characters
+    val = re.sub('[^A-Za-z0-9 ]+', ' ', val)
+    # remove multiple spaces
+    val = re.sub(' +', ' ', val)
+    # remove trailing spaces
+    val = val.strip()
+    # suffix merge
+    val = ln_suff_merge(val)
+
+    return val
+
+
+def main_authors(technologies,number_technology , carbon_related , size , research_words):
+    
+    name_file = "iea"
+    list_categories = key_words( name_file, False)
+    dic_details = details(name_file , False)
+    dic_papers , dic_papers_ranked = ranking_papers(number_technology,  dic_details   , False , size , carbon_related , research_words )
+    
+    dic_papers_co_authors = {}
+        
+
+
+    for paper in list(dic_papers_ranked.keys())[:size]:
+
+        for k in range(len(dic_papers[paper]["authorships"])):
+            coauthor_id = dic_papers[paper]["authorships"][k]["author"]["id"]
+            
+            
+            author_name = dic_papers[paper]["authorships"][k]["author"]["display_name"]
+            author_name_norm = normalize(dic_papers[paper]["authorships"][k]["author"]["display_name"]).split()
+
+            if len(author_name_norm ) > 0:
+                author_name_norm = author_name_norm[0] + " " + author_name_norm[-1]
+
+                if author_name_norm not in dic_papers_co_authors:
+                    dic_papers_co_authors[author_name_norm] = {}
+                    dic_papers_co_authors[author_name_norm]["Author's name(s)"] = author_name
+                    if coauthor_id != None :
+                        dic_papers_co_authors[author_name_norm]["Author's id(s)"] = coauthor_id[21:]
+                    else:
+                        dic_papers_co_authors[author_name_norm]["Author's id(s)"] = ""
+                    dic_papers_co_authors[author_name_norm]["Number of occurence within the " + str(size) + " most related papers"] = 1
+                    dic_papers_co_authors[author_name_norm]["Number of related citations"] = dic_papers[paper]["cited_by_count"]
+                else:
+                    dic_papers_co_authors[author_name_norm]["Number of occurence within the " + str(size) + " most related papers"] += 1
+                    if author_name not in dic_papers_co_authors[author_name_norm]["Author's name(s)"]:
+                        dic_papers_co_authors[author_name_norm]["Author's name(s)"] += ", " + author_name[0]
+                    if coauthor_id != None and coauthor_id[21:] not in dic_papers_co_authors[author_name_norm]["Author's id(s)"]:                                                              
+                        dic_papers_co_authors[author_name_norm]["Author's id(s)"] += ", " + coauthor_id[21:]
+                        dic_papers_co_authors[author_name_norm]["Number of related citations"] += dic_papers[paper]["cited_by_count"]
+
+
+
+    if dic_papers_co_authors != {}:
+        dic_papers_co_authors = {k: v for k, v in sorted(dic_papers_co_authors.items(), key=lambda item: item[1]["Number of occurence within the " + str(size) + " most related papers"] , reverse = True)}
+
+        for author_name_norm in list(dic_papers_co_authors.keys())[:size]:
+            list_ids = dic_papers_co_authors[author_name_norm]["Author's id(s)"].split(", ")
+            work_count = 0
+            cited_by_count = 0
+            institutions = ''
+            for elem in list_ids:
+                if len(elem) > 3:
+                    url = 'https://api.openalex.org/authors?filter=openalex_id:' + elem + "&page=1&per_page=200&mailto=emma_scharfmann@berkeley.edu"
+                    try:
+                      data = get_data(url)["results"][0]
+                    except:
+                      continue
+      
+                    work_count += data["works_count"]
+                    cited_by_count += data["cited_by_count"]
+                    if data["last_known_institution"] != None and len(data["last_known_institution"]) > 0 and data["last_known_institution"]["display_name"] != None:
+                        institutions += data["last_known_institution"]["display_name"]
+                    
+            dic_papers_co_authors[author_name_norm]["Last Known Institution"] = institutions
+            dic_papers_co_authors[author_name_norm]["Number of works"] = work_count
+            dic_papers_co_authors[author_name_norm]["Number of citations"] = cited_by_count
+
+            
+        return pd.DataFrame(dic_papers_co_authors, index = [ "Author's name(s)" ,  "Number of occurence within the " + str(size) + " most related papers" , "Last Known Institution" , "Number of works" , "Number of citations" ,  "Number of related citations", "Author's id(s)"]).T
+
+    else:
+        return ("Select another category")
+
+
+def map_authors(technologies, number_technology , carbon_related , size , research_words):
+
+    name_file = "iea"
+    dic_details = details(name_file , False)
+    dic_papers , dic_papers_ranked = ranking_papers(number_technology,  dic_details   , False , size , carbon_related , research_words )
+            
+    dic_papers_co_authors = {}
+    count = 0 
+
+    for paper in list(dic_papers_ranked.keys())[:size]:
+        
+
+        for k in range(len(dic_papers[paper]["authorships"])):
+            
+            if dic_papers[paper]["authorships"][k]["institutions"] != [] and "id" in dic_papers[paper]["authorships"][k]["institutions"][0] and dic_papers[paper]["authorships"][k]["institutions"][0]["id"] != None:
+                institution_id = dic_papers[paper]["authorships"][k]["institutions"][0]["id"][21:]
+                if institution_id in dic_institutions:
+                    data = dic_institutions[institution_id]
+
+                    dic_papers_co_authors[count] = {}
+                    dic_papers_co_authors[count]["longitude"] = data["longitude"]
+                    dic_papers_co_authors[count]["latitude"] = data["latitude"]
+                    dic_papers_co_authors[count]["abstract comparison"] = dic_papers_ranked[paper]["abstract comparison"]
+                    dic_papers_co_authors[count]["author"] = dic_papers[paper]["authorships"][k]["author"]["display_name"]
+                    dic_papers_co_authors[count]["institution"] = dic_papers[paper]["authorships"][k]["institutions"][0]["display_name"]
+                    dic_papers_co_authors[count]["date"] = dic_papers[paper]["date"]
+                    
+                    count += 1
+    
+                        
+    if dic_papers_co_authors == {}:
+        return "No papers"
+    
+
+    
+    map_df = pd.DataFrame(dic_papers_co_authors).T
+    map_df["longitude"]=map_df['longitude'].astype(float)
+    map_df['latitude']=map_df['latitude'].astype(float)
+    map_df = map_df[map_df["latitude"].notnull()]
+
+    return map_df
+       
+
+################################### Extracted texts ###############################################################
+
+#@title Which patents are related to the technology?
+def finder():
+    name_file = 'iea'
+    res = technology("iea", False )
+    list_categories_tech = []
+    list_categories = key_words("iea" , False)
+    list_technologies = [ ( ", ".join(list_categories[i][1]) , i ) for i in range(len(list_categories))  ] 
+    dic_technologies = {}
+    for i in range(len(res)):
+        names = res[i][1]
+        if ", ".join(names) not in list_categories_tech:
+            list_categories_tech.append(", ".join(names))
+            dic_technologies[", ".join(names)] = []
+        dic_technologies[", ".join(names)].append( (", ".join(list_categories[i][1]) , i ))
+        
+
+    list_climate = [ ("Any related papers" , False ) , ("Climate related papers" , True)]
+
+    dic_categories = {}
+    for elem in list_technologies:
+        list_words = elem[0].split(",")[-3:]
+        for i in range(len(list_words)):
+            if "CCUS" in list_words[i]:
+                list_words[i] = list_words[i].replace("CCUS" , "carbon capture storage")
+        dic_categories[elem[1]]  = [  ", ".join([ " ".join(words.split()[:3]) for words in list_words ] )  ,  ", ".join([ " ".join(words.split()[:3]) for words in list_words[:-1] ]) , ", ".join([ " ".join(words.split()[:3]) for words in list_words[1:] ] ) ]  
+    
+    return dic_technologies, dic_categories, list_categories_tech, list_technologies
+                                       
+# technologies_widget = widgets.Dropdown(options=list_categories_tech, 
+#                                             description="Choose the category:" , 
+#                                             style = {'description_width':'initial' } , 
+#                                             layout=Layout(width='500px'));
+
+# category_widget = widgets.RadioButtons( values = 1,
+#                                         description='Patents where the abstract contains:' , 
+#                                         style = {'description_width':'initial' } , 
+#                                         layout=Layout(width='1000px'));
+                                       
+# number_technology_widget = widgets.Dropdown(options=list_technologies, 
+#                                             description="Choose the technology:" , 
+#                                             style = {'description_width':'initial' } , 
+#                                             layout=Layout(width='500px'));
+
+# def update_category(*args):
+#     category_widget.options = dic_categories[number_technology_widget.value]
+    
+# def update_technology(*args):
+#     number_technology_widget.options = dic_technologies[technologies_widget.value]
+    
+                                       
+# number_technology_widget.observe(update_category)
+# technologies_widget.observe(update_technology)
+
+# print("Which patents are related to the technology?")
+# interact_manual(get_ranking_patents, 
+#                 technologies=technologies_widget,
+#                 number_technology= number_technology_widget ,    
+#                 carbon_related=widgets.Dropdown(options=list_climate, 
+#                                                 description="Select the type of patents:" , 
+#                                                 style = {'description_width':'initial' }   , 
+#                                                 layout=Layout(width='500px')  ) , 
+#                 size = widgets.IntSlider(min=3, 
+#                                          max=100, 
+#                                          value=10, 
+#                                          step=1, 
+#                                          description="Select the number of patents:" ,  
+#                                          style = {'description_width':'initial' } , 
+#                                          layout=Layout(width='500px')),
+#                category = category_widget);

Climate_site/python_scripts/paper_functions_own_details.py

@@ -0,0 +1,542 @@
+##packages code
+from shapely.geometry import Point
+import pandas as pd
+from tqdm import tqdm
+import streamlit as st
+import numpy as np
+import json, requests 
+import pandas as pd
+#from pandas.io.json import json_normalize
+
+import matplotlib.pyplot as plt
+import seaborn as sns
+from math import radians, cos, sin, asin, sqrt
+
+import spacy
+#!python -m spacy download en_core_web_lg
+
+
+from sentence_transformers import SentenceTransformer, util
+
+@st.cache_resource
+def model_nlp():
+    model = SentenceTransformer('all-MiniLM-L6-v2')
+    return model
+
+import unicodedata
+
+from metaphone import doublemetaphone
+from fuzzywuzzy import fuzz
+from difflib import SequenceMatcher
+import re
+
+import geopandas as gpd
+from geopandas import GeoDataFrame
+
+
+@st.cache_data  # 👈 Add the caching decorator
+def load_data():
+    url = "Synapse_project/Climate_site/python_scripts/institutions.tsv"
+    dic = pd.read_csv(url, delimiter = "\t" , index_col = 1).to_dict('index')
+    return dic
+
+dic_institutions = load_data()
+
+
+#################### General Functions #############################
+
+def URL(base_URL , entity_type , filters):
+    url = base_URL + entity_type + filters 
+    return url
+
+
+def get_data(url):
+    url = requests.get(url)
+    text = url.text
+    import json
+    data = json.loads(text)
+    return data
+
+## encoding the abstract
+
+
+def reconstruction_abstract(abstract_inverted_index):
+    # return the abstract is the abstract exists in the database, else, return None
+    
+    if abstract_inverted_index != None:
+        
+        list_values = list(abstract_inverted_index.values())
+        list_keys = list(abstract_inverted_index.keys())
+        #from the words in the abstract (keys of abstract_inverted_index) and their position in the text (values of abstract_inverted_index), reconstruct the abstract
+        
+        size_abstract = max([ max(elem) for elem in abstract_inverted_index.values() ] )
+        
+        abstract = [""]*(size_abstract +1)
+        
+        for i in range(len(list_values)):
+            for pos in list_values[i]:
+                abstract[pos] = list_keys[i]
+        
+        return " ".join(list(abstract))
+    
+    else:
+        return None
+            
+    
+## calculate efficiently the dot product between two vectors
+
+def norm(vector):
+    return np.sqrt(sum(x * x for x in vector))    
+
+def cosine_similarity2(vec_a, vec_b):
+        norm_a = norm(vec_a)
+        norm_b = norm(vec_b)
+        dot = sum(a * b for a, b in zip(vec_a, vec_b))
+        return dot / (norm_a * norm_b)
+
+
+
+
+#################### Paper Functions #############################
+
+def related_papers_own_research(  research_key_words , display  ):
+
+    dic = {}
+
+    max_count = 0
+    base_URL_OA = f'https://api.openalex.org/'
+    filter_works = f'works?'
+    
+        
+
+    filter_openalex = f"search=" + research_key_words + "&per_page=200&mailto=emma_scharfmann@berkeley.edu"
+        
+        
+    filter_openalex = filter_openalex.replace(" " , "%20")
+    
+    url = URL(base_URL_OA , filter_works, filter_openalex) 
+    data = get_data(url)
+    count = data["meta"]["count"] 
+
+
+
+    if display == True:
+        print(  data["meta"]["count"] , [elem[i] for i in range(1,len(elem))]  )
+        print(url)
+    
+    for i in range(len(data["results"])):
+        dic[ data["results"][i]["id"]] = {}
+        dic[ data["results"][i]["id"]]["title"] = data["results"][i]["title"]
+        dic[ data["results"][i]["id"]]["abstract"] = reconstruction_abstract(data["results"][i]["abstract_inverted_index"])
+        dic[ data["results"][i]["id"]]["concepts"] = data["results"][i]["concepts"]
+        dic[ data["results"][i]["id"]]["date"] = data["results"][i]["publication_date"]
+        dic[ data["results"][i]["id"]]["authorships"] = data["results"][i]["authorships"]
+        dic[ data["results"][i]["id"]]["cited_by_count"] = data["results"][i]["cited_by_count"]
+
+        if len(data["results"][i]["authorships"]) > 0:
+            if data["results"][i]["authorships"][0]["institutions"] != []:
+                dic[ data["results"][i]["id"]]["countries"] = data["results"][i]["authorships"][0]["institutions"][0]["country_code"]
+                dic[ data["results"][i]["id"]]["institutions"] = data["results"][i]["authorships"][0]["institutions"][0]["display_name"]
+            else:
+                dic[ data["results"][i]["id"]]["countries"] = ""
+                dic[ data["results"][i]["id"]]["institutions"] = ""
+            dic[ data["results"][i]["id"]]["authors"] = data["results"][i]["authorships"][0]["author"]["display_name"]
+
+            for j in range(1 , len(data["results"][i]["authorships"])):
+                if data["results"][i]["authorships"][j]["institutions"] != []:
+                    dic[ data["results"][i]["id"]]["institutions"] += ", " + data["results"][i]["authorships"][j]["institutions"][0]["display_name"]
+                    dic[ data["results"][i]["id"]]["countries"] = data["results"][i]["authorships"][j]["institutions"][0]["country_code"]
+                dic[ data["results"][i]["id"]]["authors"] += ", " + data["results"][i]["authorships"][j]["author"]["display_name"]
+
+    if display == True: 
+        print(" ")
+        
+    return dic
+        
+    
+        
+
+## Ranking Papers
+
+def ranking_own_research(research_key_words,  details  , display):
+    
+    model = model_nlp()
+    
+    dic_scores_papers = {}
+    dic = related_papers_own_research( research_key_words , False)
+    
+    reference_text = details
+        
+    if display== True:
+        print("Technology details: " , reference_text)
+        print(" ")
+    encoded_text = model.encode(reference_text, convert_to_tensor=False , show_progress_bar = False).tolist()
+    
+
+    
+    for ids in list(dic.keys()):
+
+        dic_scores_papers[ids] = {}
+        
+        if dic[ids]["title"] != None:
+            encoded_title = model.encode(dic[ids]["title"], convert_to_tensor=False , show_progress_bar = False).tolist()
+            score_title = cosine_similarity2(encoded_title, encoded_text)
+        else:
+            score_title = None
+
+    
+
+        if dic[ids]["abstract"] != None:
+            encoded_abstract = model.encode(dic[ids]["abstract"], convert_to_tensor=False , show_progress_bar = False).tolist()
+            score_abstract = cosine_similarity2(encoded_abstract, encoded_text)
+        else:
+            score_abstract = None
+
+
+        #concepts = ""
+        #for elem in dic[ids]["concepts"]:
+        #    concepts += elem["display_name"] + " "
+        #encoded_concepts = model.encode(concepts, convert_to_tensor=False).tolist()
+        #score_concepts = cosine_similarity2(encoded_concepts, encoded_text)
+
+        dic_scores_papers[ids]["title comparison"] = score_title
+        dic_scores_papers[ids]["abstract comparison"] = score_abstract
+        #dic_scores_papers[ids]["concepts comparision"] = score_concepts
+        
+        dic_scores_papers[ids]["title"] = dic[ids]["title"]
+        dic_scores_papers[ids]["citations"] = dic[ids]["cited_by_count"]
+        
+
+        dic_scores_papers[ids]["date"] = dic[ids]["date"][:4]
+        if "institutions" in dic[ids]:
+            dic_scores_papers[ids]["institutions"] = dic[ids]["institutions"]
+        #    dic_scores_papers[ids]["countries"] = dic[ids]["countries"]
+            
+        else:
+            dic_scores_papers[ids]["institutions"] = None
+        #    dic_scores_papers[ids]["countries"] = None
+        
+        if "authors" in dic[ids]:
+            dic_scores_papers[ids]["number of co-authors"] = len(dic[ids]["authors"].split(","))
+            dic_scores_papers[ids]["authors"] = dic[ids]["authors"]
+
+    return dic , dic_scores_papers
+    
+
+
+
+    
+    
+    
+def get_ranking_own_research(research_key_words,  details  , display , size):
+    
+    dic , dic_scores_papers = ranking_own_research(research_key_words,  details  , display)
+    
+    if dic_scores_papers == {}:
+        return "No paper found"
+    
+    elif type(dic_scores_papers) == str:
+        return dic_scores_papers
+    
+    else:
+        return pd.DataFrame(dic_scores_papers).T.sort_values(by="abstract comparison" , ascending = False).head(size)
+     
+      
+ 
+          
+    
+
+
+def extract_quantitative_data_paper(work_id):
+
+    url = "https://api.openalex.org/works/" + str(work_id)
+    url_google = "https://explore.openalex.org/works/" + str(work_id)
+    
+    data = get_data(url)
+    date = data["publication_date"]
+    title = data["title"]
+    abstract = reconstruction_abstract(data["abstract_inverted_index"])
+    concepts = ", ".join( [elem["display_name"] for elem in data["concepts"]] )
+    authors = ", ".join( [elem["author"]["display_name"] for elem in data["authorships"]] )
+    institutions = ", ".join( set([elem["institutions"][0]["display_name"] for elem in data["authorships"] if len(elem["institutions"]) > 0]) )
+    return url_google , title , abstract , date , authors , institutions
+            
+
+    
+
+
+## Mapping the authors
+
+#merge the nobiliary particles with the last name
+#ln_suff file can be modified if more or less nobiliary particles want to be suppressed
+
+ln_suff = ['oster',
+ 'nordre',
+ 'vaster',
+ 'aust',
+ 'vesle',
+ 'da',
+ 'van t',
+ 'af',
+ 'al',
+ 'setya',
+ 'zu',
+ 'la',
+ 'na',
+ 'mic',
+ 'ofver',
+ 'el',
+ 'vetle',
+ 'van het',
+ 'dos',
+ 'ui',
+ 'vest',
+ 'ab',
+ 'vste',
+ 'nord',
+ 'van der',
+ 'bin',
+ 'ibn',
+ 'war',
+ 'fitz',
+ 'alam',
+ 'di',
+ 'erch',
+ 'fetch',
+ 'nga',
+ 'ka',
+ 'soder',
+ 'lille',
+ 'upp',
+ 'ua',
+ 'te',
+ 'ni',
+ 'bint',
+ 'von und zu',
+ 'vast',
+ 'vestre',
+ 'over',
+ 'syd',
+ 'mac',
+ 'nin',
+ 'nic',
+ 'putri',
+ 'bet',
+ 'verch',
+ 'norr',
+ 'bath',
+ 'della',
+ 'van',
+ 'ben',
+ 'du',
+ 'stor',
+ 'das',
+ 'neder',
+ 'abu',
+ 'degli',
+ 'vre',
+ 'ait',
+ 'ny',
+ 'opp',
+ 'pour',
+ 'kil',
+ 'der',
+ 'oz',
+ 'von',
+ 'at',
+ 'nedre',
+ 'van den',
+ 'setia',
+ 'ap',
+ 'gil',
+ 'myljom',
+ 'van de',
+ 'stre',
+ 'dele',
+ 'mck',
+ 'de',
+ 'mellom',
+ 'mhic',
+ 'binti',
+ 'ath',
+ 'binte',
+ 'snder',
+ 'sre',
+ 'ned',
+ 'ter',
+ 'bar',
+ 'le',
+ 'mala',
+ 'ost',
+ 'syndre',
+ 'sr',
+ 'bat',
+ 'sndre',
+ 'austre',
+ 'putra',
+ 'putera',
+ 'av',
+ 'lu',
+ 'vetch',
+ 'ver',
+ 'puteri',
+ 'mc',
+ 'tre',
+ 'st']
+
+#suppress all the unwanted suffixes from a string. 
+#name_del file can be modified if more or less suffixes want to be suppressed 
+
+name_del = ['2nd', '3rd', 'Jr', 'Jr.', 'Junior', 'Sr', 'Sr.', 'Senior']
+
+
+def name_delete(string):
+    for elmt in name_del:
+        if f" {elmt}" in string:
+            return string.replace(f" {elmt}","")
+    return string
+
+def ln_suff_merge(string):
+    for suff in ln_suff:
+        if f" {suff} " in string or string.startswith(f"{suff} "):
+            return string.replace(f"{suff} ",suff.replace(" ",""))
+    return string
+
+#normalize a string dat that represents often a name. 
+
+def normalize(data):
+    normal = unicodedata.normalize('NFKD', data).encode('ASCII', 'ignore')
+    val = normal.decode("utf-8")
+    # delete unwanted elmt
+    val = name_delete(val)
+    # lower full name in upper
+    val = re.sub(r"[A-Z]{3,}", lambda x: x.group().lower(), val)
+    # add space in front of upper case
+    val = re.sub(r"(\w)([A-Z])", r"\1 \2", val)
+    # Lower case
+    val = val.lower()
+    # remove special characters
+    val = re.sub('[^A-Za-z0-9 ]+', ' ', val)
+    # remove multiple spaces
+    val = re.sub(' +', ' ', val)
+    # remove trailing spaces
+    val = val.strip()
+    # suffix merge
+    val = ln_suff_merge(val)
+
+    return val
+
+
+def main_authors( research_key_words  , details ,   size):
+        
+    dic_papers , dic_papers_ranked = ranking_own_research(  research_key_words  , details ,  False  )
+    
+    dic_papers_co_authors = {}
+        
+    for paper in list(dic_papers_ranked.keys())[:size]:
+
+        for k in range(len(dic_papers[paper]["authorships"])):
+            coauthor_id = dic_papers[paper]["authorships"][k]["author"]["id"]
+            
+            
+            author_name = dic_papers[paper]["authorships"][k]["author"]["display_name"]
+            author_name_norm = normalize(dic_papers[paper]["authorships"][k]["author"]["display_name"]).split()
+            if len (author_name_norm) > 0:
+                
+                author_name_norm = author_name_norm[0] + " " + author_name_norm[-1]
+
+                if author_name_norm not in dic_papers_co_authors:
+                    dic_papers_co_authors[author_name_norm] = {}
+                    dic_papers_co_authors[author_name_norm]["Author's name(s)"] = author_name
+                    if coauthor_id != None:
+                        dic_papers_co_authors[author_name_norm]["Author's id(s)"] = coauthor_id[21:]
+                    else:
+                        dic_papers_co_authors[author_name_norm]["Author's id(s)"] = ""
+                    dic_papers_co_authors[author_name_norm]["Number of occurence within the " + str(size) + " most related papers"] = 1
+                    dic_papers_co_authors[author_name_norm]["Number of related citations"] = dic_papers[paper]["cited_by_count"]
+                else:
+                    dic_papers_co_authors[author_name_norm]["Number of occurence within the " + str(size) + " most related papers"] += 1
+                    if author_name not in dic_papers_co_authors[author_name_norm]["Author's name(s)"]:
+                        dic_papers_co_authors[author_name_norm]["Author's name(s)"] += ", " + author_name
+                    if coauthor_id != None and coauthor_id[21:] not in dic_papers_co_authors[author_name_norm]["Author's id(s)"]:                                                              
+                        dic_papers_co_authors[author_name_norm]["Author's id(s)"] += ", " + coauthor_id[21:]
+                        dic_papers_co_authors[author_name_norm]["Number of related citations"] += dic_papers[paper]["cited_by_count"]
+
+
+
+    if dic_papers_co_authors != {}:
+        dic_papers_co_authors = {k: v for k, v in sorted(dic_papers_co_authors.items(), key=lambda item: item[1]["Number of occurence within the " + str(size) + " most related papers"] , reverse = True)}
+
+        for author_name_norm in list(dic_papers_co_authors.keys())[:size]:
+            list_ids = dic_papers_co_authors[author_name_norm]["Author's id(s)"].split(", ")
+            work_count = 0
+            cited_by_count = 0
+            institutions = ''
+            for elem in list_ids:
+                if len(elem) > 3:
+                    
+                    data = get_data("https://api.openalex.org/people/" + elem)
+                    work_count += data["works_count"]
+                    cited_by_count += data["cited_by_count"]
+                    if data["last_known_institution"] != None and data["last_known_institution"]["display_name"] != None:
+                        if institutions == '':
+                            institutions += data["last_known_institution"]["display_name"]
+                        else:
+                            institutions += ", " + data["last_known_institution"]["display_name"]
+                    
+            dic_papers_co_authors[author_name_norm]["Last Known Institution"] = institutions
+            dic_papers_co_authors[author_name_norm]["Number of works"] = work_count
+            dic_papers_co_authors[author_name_norm]["Number of citations"] = cited_by_count
+
+            
+        return pd.DataFrame(dic_papers_co_authors, index = [ "Author's name(s)" , "Author's id(s)" ,  "Number of occurence within the " + str(size) + " most related papers" , "Last Known Institution" , "Number of works" , "Number of citations" ,  "Number of related citations"]).T.style.hide(axis="index")
+
+    else:
+        return ("Select another category")
+
+
+def map_authors(research_key_words , details , size):
+
+    dic_papers , dic_papers_ranked = ranking_own_research(  research_key_words  , details ,  False  )
+
+    
+    dic_papers_co_authors = {}
+    count = 0
+    
+
+    for paper in list(dic_papers_ranked.keys())[:size]:
+        
+        for k in range(len(dic_papers[paper]["authorships"])):
+            
+            if dic_papers[paper]["authorships"][k]["institutions"] != [] and "id" in dic_papers[paper]["authorships"][k]["institutions"][0] and dic_papers[paper]["authorships"][k]["institutions"][0]["id"] != None:
+                institution_id = dic_papers[paper]["authorships"][k]["institutions"][0]["id"][21:]
+                if institution_id in dic_institutions:
+                    data = dic_institutions[institution_id]
+
+                    dic_papers_co_authors[count] = {}
+                    dic_papers_co_authors[count]["longitude"] = data["longitude"]
+                    dic_papers_co_authors[count]["latitude"] = data["latitude"]
+                    dic_papers_co_authors[count]["abstract comparison"] = dic_papers_ranked[paper]["abstract comparison"]
+                    dic_papers_co_authors[count]["author"] = dic_papers[paper]["authorships"][k]["author"]["display_name"]
+                    dic_papers_co_authors[count]["institution"] = dic_papers[paper]["authorships"][k]["institutions"][0]["display_name"]
+                    dic_papers_co_authors[count]["date"] = dic_papers[paper]["date"]
+                    
+                    count += 1
+                    
+    map_df = pd.DataFrame(dic_papers_co_authors).T
+                    
+    if dic_papers_co_authors == {}:
+        return map_df
+    
+
+    
+    
+    map_df["longitude"]=map_df['longitude'].astype(float)
+    map_df['latitude']=map_df['latitude'].astype(float)
+    map_df = map_df[map_df["latitude"].notnull()]
+
+    return map_df
+       
+
+
+

Climate_site/python_scripts/patent_functions.py

@@ -0,0 +1,892 @@
+##packages code
+import streamlit as st
+from shapely.geometry import Point
+import pandas as pd
+from tqdm import tqdm
+
+import numpy as np
+import json, requests 
+import pandas as pd
+#from pandas.io.json import json_normalize
+
+import matplotlib.pyplot as plt
+import seaborn as sns
+from math import radians, cos, sin, asin, sqrt
+
+
+from sentence_transformers import SentenceTransformer, util
+
+@st.cache_resource
+def model_nlp():
+    model = SentenceTransformer('all-MiniLM-L6-v2')
+    return model
+
+
+@st.cache_data  # 👈 Add the caching decorator
+def load_data():
+    url = "Synapse_project/Climate_site/python_scripts/institutions.tsv"
+    dic = pd.read_csv(url, delimiter = "\t" , index_col = 0).to_dict('index')
+    return dic
+
+dic_institutions = load_data()
+
+
+import unicodedata
+
+from metaphone import doublemetaphone
+from fuzzywuzzy import fuzz
+from difflib import SequenceMatcher
+import re
+
+import geopandas as gpd
+from geopandas import GeoDataFrame
+#################### General Functions #############################
+
+def URL(base_URL , entity_type , filters):
+    url = base_URL + entity_type + filters 
+    return url
+
+
+def get_data(url):
+    url = requests.get(url)
+    text = url.text
+    import json
+    data = json.loads(text)
+    return data
+
+## encoding the abstract
+
+
+def reconstruction_abstract(abstract_inverted_index):
+    # return the abstract is the abstract exists in the database, else, return None
+    
+    if abstract_inverted_index != None:
+        
+        list_values = list(abstract_inverted_index.values())
+        list_keys = list(abstract_inverted_index.keys())
+        #from the words in the abstract (keys of abstract_inverted_index) and their position in the text (values of abstract_inverted_index), reconstruct the abstract
+        
+        size_abstract = max([ max(elem) for elem in abstract_inverted_index.values() ] )
+        
+        abstract = [""]*(size_abstract +1)
+        
+        for i in range(len(list_values)):
+            for pos in list_values[i]:
+                abstract[pos] = list_keys[i]
+        
+        return " ".join(list(abstract))
+    
+    else:
+        return None
+            
+    
+## calculate efficiently the dot product between two vectors
+
+def norm(vector):
+    return np.sqrt(sum(x * x for x in vector))    
+
+def cosine_similarity2(vec_a, vec_b):
+        norm_a = norm(vec_a)
+        norm_b = norm(vec_b)
+        dot = sum(a * b for a, b in zip(vec_a, vec_b))
+        return dot / (norm_a * norm_b)
+
+## Extracted texts
+
+def print_extracted_text(name_file):
+
+    file = open('iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+    count = 0
+    for index, line in enumerate(lines):
+        read_line = line.strip()
+        print(read_line)
+
+    file.close()
+    
+    
+    iea.txt
+
+
+def details(name_file , display):
+    
+    file = open("Synapse_project/Climate_site/python_scripts/iea.txt", "r")
+    lines = file.readlines()
+
+    mark = 0 
+    dic_details = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+
+
+        if mark == 1 and line != "" and line[0] == "*":
+            
+            if display == True:
+                print(count)
+                print(text)
+                print(" ")
+            dic_details[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line.split(" 	")[-1] == "Details" or line.split(" 	")[-1] == "Hide":
+            mark = 1
+            text = ""
+            
+    return dic_details
+
+
+
+
+
+def key_initiatives(name_file , display ):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+
+      
+    mark = 0 
+    dic_key_initiatives = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+
+            count += 1
+
+
+
+        if mark == 1 and line != "" and ( (line[0].isnumeric() and  ">" in line and " 	" in line) or line == "*Deployment targets:*" or line == "*Announced development targets:*"):
+            if display == True: 
+                print(count)
+                print(text)
+                print(" ")
+            
+            dic_key_initiatives[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line == "*Key initiatives:*":
+
+
+            mark = 1
+            text = ""
+            
+    return dic_key_initiatives
+
+
+
+
+def deployment_target(name_file , display):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+
+
+    mark = 0 
+    dic_target = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+
+
+
+        if mark == 1 and line != "" and  ((line[0].isnumeric() and ">" in line and " 	" in line)  or line == "*Announced cost reduction targets:*" or line == "*Announced development targets:*"):
+            
+            if display == True:
+                print(count)
+                print(text)
+                print(" ")
+                
+            dic_target[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line == "*Deployment targets:*" or line == "*Announced development targets:*":
+
+            mark = 1
+            text = ""
+            
+    return dic_target
+
+
+ 
+    
+def cost_reduction_target(name_file , display):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+    
+    mark = 0 
+    dic_cost = {}
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+
+
+            count += 1
+
+
+
+        if mark == 1 and line != ""  and (line[0].isnumeric() and ">" in line and " 	" in line) :
+            
+            if display == True: 
+                print(count)
+                print(text)
+                print(" ")
+                
+            dic_cost[count] = text
+            mark = 0
+
+
+        if mark == 1:
+            text = text + line + " "
+
+        if line == "*Announced cost reduction targets:*":
+
+            mark = 1
+            text = ""
+            
+    return dic_cost
+
+
+
+def key_words(name_file, display ):
+    
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    
+    lines = file.readlines()
+    
+    list_categories = []
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+            
+            if display == True:
+                print("Technologies"  , count+1 , ":")
+
+        if line != "":
+
+            if line[0].isnumeric() and ">" in line and " 	" in line:
+                i = 0
+                try:
+                    line = line.split(" 	")[2]
+                except:
+                    print(line)
+                    break
+                
+                if "Details" not in lines[index] and "Moderate" not in lines[index]:
+                
+                    while "	" not in line:
+                        i += 1
+                        if "Details"==lines[index + i][:7] or "End-use"==lines[index + i][:7]:
+                            break
+                        else:
+                            line = line + "  " +  lines[index + i]
+
+                #if " Production" in line:
+                    #line = line.replace(" Production" , "")
+
+                line = line.replace("\n" , " ")
+                line = line.replace("/" , " ")
+                line = line.replace("-" , " ")
+                line = line.split(" 	")[0]
+
+                if "  " in line:
+                    line = line.replace("  ", " ")
+                line = line.split(">")
+
+
+                if "(" in line[-1]:
+                    line[-1] = line[-1].split("(")[0] 
+
+
+                for i in range(len(line)):
+
+                    # remove multiple spaces
+                    line[i] = re.sub(' +', ' ', line[i])
+                    # remove trailing spaces
+                    line[i] = line[i].strip()
+
+    
+
+                if display == True:
+                    print(line)
+                    print(" ")
+                    
+                if '' in line:
+                    line.remove('')
+
+                list_categories.append([count , line])
+                
+    return list_categories
+
+
+
+def technology(name_file, display ):
+    # Filepath too specific, need to change to relative path
+    file = open('Synapse_project/Climate_site/python_scripts/iea.txt', "r", encoding='utf8')
+    lines = file.readlines()
+    
+    list_categories = []
+    count = -1
+    for index, line in enumerate(lines):
+
+        line = line.strip()
+        
+        if line == "Close explanation":
+            break
+
+        if line != "" and  (line[0].isnumeric() and ">" in line and " 	" in line) :
+            count += 1
+            
+
+            if display == True:
+                print("Technologies"  , count+1 , ":")
+
+        if line != "":
+
+            if line[0].isnumeric() and ">" in line and " 	" in line:
+                i = 0
+                try:
+                    line = line.split(" 	")[1]
+                except:
+                    print(line)
+                    break
+                
+
+                line = line.replace("\n" , " ")
+                line = line.replace("/" , " ")
+                line = line.replace("-" , " ")
+                line = line.strip()
+                line = re.sub(' +', ' ', line)
+                line = line.split(" 	")[0]
+                line = line.split(">")
+
+
+                if "(" in line[-1]:
+                    line[-1] = line[-1].split("(")[0] 
+
+
+                for i in range(len(line)):
+
+                    # remove multiple spaces
+                    line[i] = re.sub(' +', ' ', line[i])
+                    # remove trailing spaces
+                    line[i] = line[i].strip()
+
+    
+
+                if display == True:
+                    print(line)
+                    print(" ")
+                
+
+                list_categories.append([count , line])
+                
+    return list_categories
+
+
+#################### Patent Functions #############################
+
+def related_patents(number_technology , research_words, carbon_related , display):
+    
+    name_file = "iea"
+    list_categories = key_words( name_file, False)
+
+    dic_patents = {}
+
+    max_count = 0
+    base_URL_PV = "https://api.patentsview.org/"
+    filter_works = "patents/query?"
+    filter_PV = "q={%22_and%22:[{%22_text_any%22:{%22patent_abstract%22:%22" 
+    filter_PV += research_words.replace("," , "")
+    
+    if carbon_related == True:
+        filter_PV += "%22}},{%22_eq%22:{%22cpc_group_id%22:%22Y02E%22}}]}&f=[%22patent_number%22,%22patent_title%22,%22assignee_country%22,%22patent_date%22,%22inventor_id%22,%22assignee_organization%22,%22inventor_longitude%22,%22inventor_latitude%22,%22inventor_last_name%22,%22inventor_first_name%22,%22cpc_subsection_title%22,%22assignee_city%22,%22patent_abstract%22,%22patent_kind%22,%22cpc_group_id%22,%22assignee_organization%22,%22citedby_patent_number%22]"
+    
+    else:
+        filter_PV += "%22}}]}&f=[%22patent_number%22,%22patent_title%22,%22assignee_country%22,%22patent_date%22,%22assignee_organization%22,%22inventor_longitude%22,%22inventor_latitude%22,%22inventor_last_name%22,%22inventor_id%22,%22inventor_first_name%22,%22cpc_subsection_title%22,%22assignee_city%22,%22patent_abstract%22,%22patent_kind%22,%22cpc_group_id%22,%22assignee_organization%22,%22citedby_patent_number%22]"
+
+        
+    filter_PV = filter_PV.replace(" " , "%20")
+
+        
+
+    url = URL(base_URL_PV , filter_works, filter_PV) 
+    data = get_data(url)
+
+
+    if display == True:
+        print(  data["total_patent_count"] , elem[-1] )
+        print(url)
+
+
+
+    
+    for i in range(data["count"]):
+        dic_patents[ "US-" + data["patents"][i]["patent_number"]] = {}
+        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["title"] = data["patents"][i]["patent_title"]
+        dic_patents["US-" + data["patents"][i]["patent_number"]]["abstract"] = data["patents"][i]["patent_abstract"]
+
+        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["assignee"] = str(data["patents"][i]["assignees"][0]["assignee_organization"])
+        dic_patents["US-" + data["patents"][i]["patent_number"]]["assignee_city"] = str(data["patents"][i]["assignees"][0]["assignee_city"])
+        dic_patents["US-" + data["patents"][i]["patent_number"]]["assignee_country"] = str(data["patents"][i]["assignees"][0]["assignee_country"])
+        for j in range(1, len(data["patents"][i]["assignees"])):
+            dic_patents[ "US-" + data["patents"][i]["patent_number"]]["assignee"] +=  ", " + str(data["patents"][i]["assignees"][j]["assignee_organization"])
+            dic_patents[ "US-" + data["patents"][i]["patent_number"]]["assignee_city"] +=  ", " + str(data["patents"][i]["assignees"][j]["assignee_city"])
+            dic_patents["US-" + data["patents"][i]["patent_number"]]["assignee_country"] +=  ", " + str(data["patents"][i]["assignees"][j]["assignee_country"])
+
+        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["list_inventors"] = data["patents"][i]["inventors"]
+
+        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["inventors"] = str(data["patents"][i]["inventors"][0]["inventor_first_name"]) + " " + str(data["patents"][i]["inventors"][0]["inventor_last_name"])
+        for j in range(1, len(data["patents"][i]["inventors"])):
+            dic_patents[ "US-" + data["patents"][i]["patent_number"]]["inventors"] +=  ", " + str(data["patents"][i]["inventors"][j]["inventor_first_name"]) + " " + str(data["patents"][i]["inventors"][j]["inventor_last_name"])
+
+
+        dic_patents["US-" + data["patents"][i]["patent_number"]]["date"] = data["patents"][i]["patent_date"]
+        dic_patents["US-" + data["patents"][i]["patent_number"]]["number_citations"] = len(data["patents"][i]["citedby_patents"])
+
+
+    if display == True:
+        print(" ")
+            
+            
+    return dic_patents
+
+
+
+## Ranking Patents
+
+def ranking_patents(number_technology , research_words, carbon_related , display):
+    
+    model = model_nlp()
+    
+    
+    name_file = "iea"
+    list_categories = key_words( name_file, False)
+    dic_details = details(name_file , False)
+    dic_patents = related_patents(number_technology , research_words, carbon_related , display)
+    
+
+    dic_scores = {}
+
+    if display == True:
+        print("Key words: " , list_categories[number_technology][1])
+        
+    if number_technology in dic_details:
+        reference_text = dic_details[number_technology]
+        if display == True:
+            print("Technology details: " , reference_text)
+            print(" ")
+        encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist()
+        
+        
+        if len(dic_patents ) == 0:
+            return "Select other key words"
+        
+        
+        else:
+            for ids in list(dic_patents.keys()):
+
+                dic_scores[ids] = {}
+
+                encoded_title = model.encode(dic_patents[ids]["title"], convert_to_tensor=False).tolist()
+                score_title = cosine_similarity2(encoded_title, encoded_text)
+
+
+                if dic_patents[ids]["abstract"] != None:
+                    encoded_abstract = model.encode(dic_patents[ids]["abstract"], convert_to_tensor=False).tolist()
+                    score_abstract = cosine_similarity2(encoded_abstract, encoded_text)
+                else:
+                    score_abstract = None
+
+                dic_scores[ids]["title comparision"] = score_title
+                dic_scores[ids]["abstract comparison"] = score_abstract
+                dic_scores[ids]["title"] = dic_patents[ids]["title"]
+                dic_scores[ids]["citations"] = dic_patents[ids]["number_citations"]
+                dic_scores[ids]["date"] = dic_patents[ids]["date"][:4]
+                dic_scores[ids]["assignee"] = dic_patents[ids]["assignee"]
+                #dic_scores[ids]["assignee_city"] = dic_patents[ids]["assignee_city"]
+                #dic_scores[ids]["assignee_country"] = dic_patents[ids]["assignee_country"]
+                dic_scores[ids]["inventors"] = dic_patents[ids]["inventors"]
+                dic_scores[ids]["number of co-inventors"] = len(dic_patents[ids]["inventors"].split(","))
+    return dic_patents , dic_scores
+
+
+def get_ranking_patents(technologies, number_technology ,category ,  carbon_related , size):
+    dic , dic_patents = ranking_patents(number_technology , category, carbon_related , False)
+    if dic_patents == {}:
+        return "No patent found, select other key words"
+
+    elif dic_patents == "Select other key words":
+        return dic_patents
+    
+    else:
+        return pd.DataFrame(dic_patents).T.sort_values(by="abstract comparison" , ascending = False)[:size]
+    
+    
+
+
+
+#normalize a string dat that represents often a name. 
+
+def normalize(data):
+    normal = unicodedata.normalize('NFKD', data).encode('ASCII', 'ignore')
+    val = normal.decode("utf-8")
+
+    # lower full name in upper
+    val = re.sub(r"[A-Z]{3,}", lambda x: x.group().lower(), val)
+    # add space in front of upper case
+    val = re.sub(r"(\w)([A-Z])", r"\1 \2", val)
+    # Lower case
+    val = val.lower()
+    # remove special characters
+    val = re.sub('[^A-Za-z0-9 ]+', ' ', val)
+    # remove multiple spaces
+    val = re.sub(' +', ' ', val)
+    # remove trailing spaces
+    val = val.strip()
+
+
+    return val
+
+
+def main_inventors(technologies, number_technology , carbon_related , category , size ):
+    display = False
+    dic_patents , dic_ranked = ranking_patents(number_technology , category, carbon_related , display)
+    dic_patents_co_inventors = {}
+    
+
+    for patent in list(dic_ranked.keys())[:size]:
+        for k in range(len(dic_patents[patent]["list_inventors"])):
+            
+            inventor_id = dic_patents[patent]["list_inventors"][k]["inventor_id"]
+            inventor_name = dic_patents[patent]["list_inventors"][k]["inventor_first_name"] + " " + dic_patents[patent]["list_inventors"][k]["inventor_last_name"]
+            inventor_name_norm = normalize(inventor_name).split()
+            inventor_name_norm = inventor_name_norm[0] + " " + inventor_name_norm[-1]
+
+            if inventor_name_norm not in dic_patents_co_inventors:
+                dic_patents_co_inventors[inventor_name_norm] = {}
+                dic_patents_co_inventors[inventor_name_norm]["Inventor's name"] = inventor_name 
+                dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"] =  inventor_id 
+                dic_patents_co_inventors[inventor_name_norm]["Number of occurence"] = 1
+                dic_patents_co_inventors[inventor_name_norm]["Number of related citations"] = dic_patents[patent]["number_citations"]
+
+            else:
+                if inventor_id not in dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"] :
+                    dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"] += ", " + inventor_id
+                if inventor_name not in dic_patents_co_inventors[inventor_name_norm]["Inventor's name"] :
+                    dic_patents_co_inventors[inventor_name_norm]["Inventor's name"] += ", " + inventor_name
+                dic_patents_co_inventors[inventor_name_norm]["Number of occurence"] += 1
+                dic_patents_co_inventors[inventor_name_norm]["Number of related citations"] += dic_patents[patent]["number_citations"]
+
+
+        dic_patents_co_inventors = {k: v for k, v in sorted(dic_patents_co_inventors.items(), key=lambda item: item[1]["Number of occurence"] , reverse = True)}
+    
+    if dic_patents_co_inventors == {}:
+    
+        
+        return "No patent, select other key words"
+    else:
+        
+        for inventor_name_norm in list(dic_patents_co_inventors.keys()):
+            list_inventors = dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"].split(", ")
+            work_count = 0
+            cited_by_count = 0
+            
+            for elem in list_inventors:
+                url = "https://api.patentsview.org/inventors/query?q={%22inventor_id%22:[%22" + elem + "%22]}&f=[%22inventor_total_num_patents%22,%22patent_num_cited_by_us_patents%22]"
+                data = get_data(url)["inventors"][0]
+                work_count += int(data["inventor_total_num_patents"])
+                for k in range(len(data["patents"])):
+                    cited_by_count += int(data["patents"][k]["patent_num_cited_by_us_patents"])
+                
+            dic_patents_co_inventors[inventor_name_norm]["Number of patents"] = work_count
+            dic_patents_co_inventors[inventor_name_norm]["Number of US patents citations"] = cited_by_count
+            
+        
+                
+                
+        return pd.DataFrame(dic_patents_co_inventors , index = ["Inventor's name", "PatentsView inventor's id", "Number of occurence"  , "Number of patents" ,"Number of US patents citations" ,  "Number of related citations"]).T
+
+
+    
+    
+def map_inventors(technologies, number_technology , carbon_related , category , size):
+    display = False
+    dic_patents , dic_ranked = ranking_patents(number_technology , category, carbon_related , display)
+    dic_patents_co_inventors = {}
+    count = 0
+    
+    for patent in list(dic_ranked.keys())[:size]:
+        for k in range(len(dic_patents[patent]["list_inventors"])):
+            
+            dic_patents_co_inventors[count] = {}
+            
+            dic_patents_co_inventors[count]["latitude"] = dic_patents[patent]["list_inventors"][k]["inventor_latitude"]
+            dic_patents_co_inventors[count]["longitude"] = dic_patents[patent]["list_inventors"][k]["inventor_longitude"]
+            dic_patents_co_inventors[count]["longitude"] = dic_patents[patent]["list_inventors"][k]["inventor_longitude"]
+            dic_patents_co_inventors[count]["inventor_name"]  = str(dic_patents[patent]["list_inventors"][k]["inventor_first_name"]) + " " + str(dic_patents[patent]["list_inventors"][k]["inventor_last_name"])
+            dic_patents_co_inventors[count]["patent_date"]  = dic_patents[patent]["date"]
+            count += 1
+                
+    if dic_patents_co_inventors == {}:
+        return "No patent, select other key words"
+    
+    
+    map_df = pd.DataFrame(dic_patents_co_inventors).T
+    map_df["longitude"]=map_df['longitude'].astype(float)
+    map_df['latitude']=map_df['latitude'].astype(float)
+    map_df = map_df[map_df["latitude"].notnull()]
+
+    return map_df
+
+    #geometry = [Point(xy) for xy in zip(map_df['Longitude'], map_df['Latitude'])]
+    #gdf = GeoDataFrame(map_df, geometry=geometry)   
+
+    #this is a simple map that goes with geopandas
+    #world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))
+    #gdf.plot(ax=world.plot( color='white', edgecolor='black' ), marker='o', color='red', markersize=15 , zorder = 1);
+    #plt.xlim([-180, 180])
+    #plt.ylim([-90, 90])
+
+
+    #plt.title("Main inventors: geographic location")
+    #plt.xlabel("Longitude")
+    #plt.ylabel("Latitude")
+    #plt.show()
+
+    
+## Extract quantitative data
+
+def extract_sentences_with_numbers(text , text_name):
+    
+    if text != None:
+        text = text.replace("CO 2" , "CO2")
+        text = text.replace("CO 3" , "CO3")
+        text = text.replace("CO(2)" , "CO2")
+        text = text.replace("CO(3)" , "CO3")
+        
+        print(text_name + ": " , text)
+        print(" ")
+
+
+
+        list_text = list(text)
+        for i in range(1,len(list_text)-1):
+            if list_text[i] == " " and list_text[i-1] == "." and list_text[i+1].isupper():
+                list_text[i] = "~"
+
+        text = "".join(list_text)
+        text = text.split("~")
+        for sentence in text:                
+
+            if any(char.isdigit() for char in sentence):
+                
+                
+                
+                if "CO2" in sentence:
+                    print("\x1b[31mCARBON RELATED:\x1b[0m", sentence)
+                    print(" ")
+
+
+                    
+                if "GJ" in sentence or "MJ" in sentence:
+                    print("\x1b[31mENERGY:\x1b[0m" , sentence)
+                    print(" ")
+
+
+                    
+                ##price
+                if "€"  in sentence or "$" in sentence or "EUR" in sentence or "dollars" in sentence.lower():
+                    print("\x1b[31mPRICE:\x1b[0m" , sentence)
+                    print(" ")
+
+
+                    
+                ##dates
+                digits = []
+                for word in sentence.replace("," , "").replace("%" , "").replace("." , " ").split():
+                    if word.isdigit() and 1850 < int(word) < 2200 :
+                        digits.append(word)
+                if digits != []:
+                    print("\x1b[31mDATE:\x1b[0m" , sentence)
+                    print(" ")
+
+
+                    
+                ##CO quantity
+                if "Mt" in sentence or "tC" in sentence or "t-C" in sentence:
+                    print("\x1b[31mCARBON QUANTITY:\x1b[0m" , sentence)
+                    print(" ")
+ 
+    print(" ")
+          
+    
+def extract_quantitative_data_technology(technologies, number_technology):
+    count = 0
+      
+    name_file = "iea"
+    dic_target = deployment_target(name_file , False)
+    dic_cost = cost_reduction_target(name_file , False)
+    dic_details = details(name_file , False)
+    cost_target_text = 'No information'
+    cost_text = 'No information'
+    
+    if number_technology in dic_details:
+        reference_text = dic_details[number_technology]
+            
+        #print("\033[96mFROM IEA website: ")
+        #print("\033[92mTechnology details: \x1b[0m" , reference_text)
+        #print(" ")
+        #encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist()
+
+    if number_technology in dic_target:
+        cost_target_text = dic_target[number_technology]
+        #print("\033[96mFROM IEA website: ")
+        #sentences = extract_sentences_with_numbers(cost_target_text , "\033[92mDeployment target and Announced development target\x1b[0m")
+
+    if number_technology in dic_cost:
+        cost_text = dic_cost[number_technology]
+        #print("\033[96mFROM IEA website: ")
+        #sentences = extract_sentences_with_numbers(cost_text , "\033[92mAnnounced cost reduction targets\x1b[0m")
+    
+    return reference_text, cost_target_text , cost_text
+
+
+        
+def extract_quantitative_data_patent(patent_id):
+    
+
+    patent_id = patent_id[3:]
+        
+    url = "https://api.patentsview.org/patents/query?q={%22patent_id%22:%22" + str(patent_id) + "%22}&f=[%22patent_number%22,%22patent_title%22,%22patent_abstract%22,%22patent_date%22,%22inventor_last_name%22,%22inventor_first_name%22,%22assignee_organization%22]"
+    url_google = "https://patents.google.com/patent/US" + str(patent_id)
+    
+    data = get_data(url)["patents"][0]
+    title = data["patent_title"]
+    abstract = data["patent_abstract"]
+    co_inventors = ", ".join([ data["inventors"][i]["inventor_first_name"] + " " + data["inventors"][i]["inventor_last_name"] for i in range(len(data["inventors"])) ]) 
+    assignees =  ", ".join([ str(data["assignees"][i]["assignee_organization"]) for i in range(len(data["assignees"])) ] )
+
+    return url_google , title , abstract , data["patent_date"] , co_inventors , assignees
+
+
+
+
+
+    
+        
+    
+
+################################### Extracted texts ###############################################################
+
+#@title Which patents are related to the technology?
+def finder():
+    name_file = 'iea'
+
+    res = technology("iea", False )
+    list_categories_tech = []
+    list_categories = key_words(name_file , False)
+    list_technologies = [ ( ", ".join(list_categories[i][1]) , i ) for i in range(len(list_categories))  ] 
+
+    dic_technologies = {}
+    for i in range(len(res)):
+        names = res[i][1]
+        if ", ".join(names) not in list_categories_tech:
+            list_categories_tech.append(", ".join(names))
+            dic_technologies[", ".join(names)] = []
+        dic_technologies[", ".join(names)].append( (", ".join(list_categories[i][1]) , i ))
+        
+
+    list_climate = [ ("Any related patents" , False ) , ("Climate related patents" , True)]
+
+    dic_categories = {}
+    for elem in list_technologies:
+        list_words = elem[0].split(",")[-3:]
+
+        dic_categories[elem[1]]  = [  " ".join(list_words[-1].split()[:3]) ,  " ".join(list_words[-2].split()[:3]) if len(list_words) > 1 else "" ,  " ".join(list_words[-3].split()[:3]) if len(list_words) > 2 else ""  ]  
+    
+    return dic_technologies, dic_categories, list_categories_tech, list_technologies
+                                       
+# technologies_widget = widgets.Dropdown(options=list_categories_tech, 
+#                                             description="Choose the category:" , 
+#                                             style = {'description_width':'initial' } , 
+#                                             layout=Layout(width='500px'));
+
+# category_widget = widgets.RadioButtons( values = 1,
+#                                         description='Patents where the abstract contains:' , 
+#                                         style = {'description_width':'initial' } , 
+#                                         layout=Layout(width='1000px'));
+                                       
+# number_technology_widget = widgets.Dropdown(options=list_technologies, 
+#                                             description="Choose the technology:" , 
+#                                             style = {'description_width':'initial' } , 
+#                                             layout=Layout(width='500px'));
+
+# def update_category(*args):
+#     category_widget.options = dic_categories[number_technology_widget.value]
+    
+# def update_technology(*args):
+#     number_technology_widget.options = dic_technologies[technologies_widget.value]
+    
+                                       
+# number_technology_widget.observe(update_category)
+# technologies_widget.observe(update_technology)
+
+# print("Which patents are related to the technology?")
+# interact_manual(get_ranking_patents, 
+#                 technologies=technologies_widget,
+#                 number_technology= number_technology_widget ,    
+#                 carbon_related=widgets.Dropdown(options=list_climate, 
+#                                                 description="Select the type of patents:" , 
+#                                                 style = {'description_width':'initial' }   , 
+#                                                 layout=Layout(width='500px')  ) , 
+#                 size = widgets.IntSlider(min=3, 
+#                                          max=100, 
+#                                          value=10, 
+#                                          step=1, 
+#                                          description="Select the number of patents:" ,  
+#                                          style = {'description_width':'initial' } , 
+#                                          layout=Layout(width='500px')),
+#                category = category_widget);

Climate_site/python_scripts/patent_functions_own_details.py

@@ -0,0 +1,475 @@
+##packages code
+from shapely.geometry import Point
+import pandas as pd
+from tqdm import tqdm
+import streamlit as st
+import numpy as np
+import json, requests 
+import pandas as pd
+#from pandas.io.json import json_normalize
+
+import matplotlib.pyplot as plt
+import seaborn as sns
+from math import radians, cos, sin, asin, sqrt
+
+import spacy
+#!python -m spacy download en_core_web_lg
+
+
+from sentence_transformers import SentenceTransformer, util
+
+@st.cache_resource
+def model_nlp():
+    model = SentenceTransformer('all-MiniLM-L6-v2')
+    return model
+
+import unicodedata
+
+from metaphone import doublemetaphone
+from fuzzywuzzy import fuzz
+from difflib import SequenceMatcher
+import re
+
+import geopandas as gpd
+from geopandas import GeoDataFrame
+
+
+@st.cache_data  # 👈 Add the caching decorator
+def load_data():
+    url = "Synapse_project/Climate_site/python_scripts/institutions.tsv"
+    dic = pd.read_csv(url, delimiter = "\t" , index_col = 0).to_dict('index')
+    return dic
+
+dic_institutions = load_data()
+
+
+#################### General Functions #############################
+
+def URL(base_URL , entity_type , filters):
+    url = base_URL + entity_type + filters 
+    return url
+
+
+def get_data(url):
+    url = requests.get(url)
+    text = url.text
+    import json
+    data = json.loads(text)
+    return data
+
+    
+## calculate efficiently the dot product between two vectors
+
+def norm(vector):
+    return np.sqrt(sum(x * x for x in vector))    
+
+def cosine_similarity2(vec_a, vec_b):
+        norm_a = norm(vec_a)
+        norm_b = norm(vec_b)
+        dot = sum(a * b for a, b in zip(vec_a, vec_b))
+        return dot / (norm_a * norm_b)
+
+
+
+
+#################### Patent Functions #############################
+
+
+def related_patents(research_words , display):
+    
+
+    dic_patents = {}
+
+    max_count = 0
+    base_URL_PV = "https://api.patentsview.org/"
+    filter_works = "patents/query?"
+    filter_PV = "q={%22_and%22:[{%22_text_all%22:{%22patent_abstract%22:%22" 
+    filter_PV += research_words
+
+    filter_PV += "%22}}]}&f=[%22patent_number%22,%22patent_title%22,%22assignee_country%22,%22patent_date%22,%22assignee_organization%22,%22inventor_longitude%22,%22inventor_latitude%22,%22inventor_last_name%22,%22inventor_id%22,%22inventor_first_name%22,%22cpc_subsection_title%22,%22assignee_city%22,%22patent_abstract%22,%22patent_kind%22,%22cpc_group_id%22,%22assignee_organization%22,%22citedby_patent_number%22]"
+
+    filter_PV = filter_PV.replace(" " , "%20")
+
+        
+
+    url = URL(base_URL_PV , filter_works, filter_PV) 
+    data = get_data(url)
+
+
+    if display == True:
+        print(  data["total_patent_count"] , elem[-1] )
+        print(url)
+
+
+
+    
+    for i in range(data["count"]):
+        dic_patents[ "US-" + data["patents"][i]["patent_number"]] = {}
+        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["title"] = data["patents"][i]["patent_title"]
+        dic_patents["US-" + data["patents"][i]["patent_number"]]["abstract"] = data["patents"][i]["patent_abstract"]
+
+        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["assignee"] = str(data["patents"][i]["assignees"][0]["assignee_organization"])
+        dic_patents["US-" + data["patents"][i]["patent_number"]]["assignee_city"] = str(data["patents"][i]["assignees"][0]["assignee_city"])
+        dic_patents["US-" + data["patents"][i]["patent_number"]]["assignee_country"] = str(data["patents"][i]["assignees"][0]["assignee_country"])
+        for j in range(1, len(data["patents"][i]["assignees"])):
+            dic_patents[ "US-" + data["patents"][i]["patent_number"]]["assignee"] +=  ", " + str(data["patents"][i]["assignees"][j]["assignee_organization"])
+            dic_patents[ "US-" + data["patents"][i]["patent_number"]]["assignee_city"] +=  ", " + str(data["patents"][i]["assignees"][j]["assignee_city"])
+            dic_patents["US-" + data["patents"][i]["patent_number"]]["assignee_country"] +=  ", " + str(data["patents"][i]["assignees"][j]["assignee_country"])
+
+        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["list_inventors"] = data["patents"][i]["inventors"]
+
+        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["inventors"] = str(data["patents"][i]["inventors"][0]["inventor_first_name"]) + " " + str(data["patents"][i]["inventors"][0]["inventor_last_name"])
+        for j in range(1, len(data["patents"][i]["inventors"])):
+            dic_patents[ "US-" + data["patents"][i]["patent_number"]]["inventors"] +=  ", " + str(data["patents"][i]["inventors"][j]["inventor_first_name"]) + " " + str(data["patents"][i]["inventors"][j]["inventor_last_name"])
+
+
+        dic_patents["US-" + data["patents"][i]["patent_number"]]["date"] = data["patents"][i]["patent_date"]
+        dic_patents["US-" + data["patents"][i]["patent_number"]]["number_citations"] = len(data["patents"][i]["citedby_patents"])
+
+
+    if display == True:
+        print(" ")
+            
+            
+    return dic_patents
+  
+  
+  
+  
+  
+def ranking_patents( research_words, details):
+    
+    model = model_nlp()
+    
+    dic_patents = related_patents( research_words , False)
+    
+
+    dic_scores = {}
+
+
+    
+    reference_text = details
+
+    encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist()
+        
+        
+    if len(dic_patents ) == 0:
+        return "Select other key words"
+        
+    else:
+        for ids in list(dic_patents.keys()):
+
+            dic_scores[ids] = {}
+
+            encoded_title = model.encode(dic_patents[ids]["title"], convert_to_tensor=False).tolist()
+            score_title = cosine_similarity2(encoded_title, encoded_text)
+
+
+            if dic_patents[ids]["abstract"] != None:
+                encoded_abstract = model.encode(dic_patents[ids]["abstract"], convert_to_tensor=False).tolist()
+                score_abstract = cosine_similarity2(encoded_abstract, encoded_text)
+            else:
+                score_abstract = None
+
+            dic_scores[ids]["title comparision"] = score_title
+            dic_scores[ids]["abstract comparison"] = score_abstract
+            dic_scores[ids]["title"] = dic_patents[ids]["title"]
+            dic_scores[ids]["citations"] = dic_patents[ids]["number_citations"]
+            dic_scores[ids]["date"] = dic_patents[ids]["date"][:4]
+            dic_scores[ids]["assignee"] = dic_patents[ids]["assignee"]
+            dic_scores[ids]["inventors"] = dic_patents[ids]["inventors"]
+            dic_scores[ids]["number of co-inventors"] = len(dic_patents[ids]["inventors"].split(","))
+            
+    return dic_patents , dic_scores
+
+
+
+
+def get_ranking_own_research(research_key_words,  details  , size):
+    
+    dic , dic_scores_papers = ranking_patents(research_key_words,  details )
+    
+    if dic_scores_papers == {}:
+        return "No paper found"
+    
+    elif type(dic_scores_papers) == str:
+        return dic_scores_papers
+    
+    else:
+        return pd.DataFrame(dic_scores_papers).T.sort_values(by="abstract comparison" , ascending = False).head(size)
+     
+ 
+
+def extract_quantitative_data_patent(patent_id):
+    
+
+    patent_id = patent_id[3:]
+        
+    url = "https://api.patentsview.org/patents/query?q={%22patent_id%22:%22" + str(patent_id) + "%22}&f=[%22patent_number%22,%22patent_title%22,%22patent_abstract%22,%22patent_date%22,%22inventor_last_name%22,%22inventor_first_name%22,%22assignee_organization%22]"
+    url_google = "https://patents.google.com/patent/US" + str(patent_id)
+    
+    data = get_data(url)["patents"][0]
+    title = data["patent_title"]
+    abstract = data["patent_abstract"]
+    co_inventors = ", ".join([ data["inventors"][i]["inventor_first_name"] + " " + data["inventors"][i]["inventor_last_name"] for i in range(len(data["inventors"])) ]) 
+    assignees =  ", ".join([ str(data["assignees"][i]["assignee_organization"]) for i in range(len(data["assignees"])) ] )
+
+    return url_google , title , abstract , data["patent_date"] , co_inventors , assignees
+
+
+
+
+    
+
+
+## Mapping the authors
+
+#merge the nobiliary particles with the last name
+#ln_suff file can be modified if more or less nobiliary particles want to be suppressed
+
+ln_suff = ['oster',
+ 'nordre',
+ 'vaster',
+ 'aust',
+ 'vesle',
+ 'da',
+ 'van t',
+ 'af',
+ 'al',
+ 'setya',
+ 'zu',
+ 'la',
+ 'na',
+ 'mic',
+ 'ofver',
+ 'el',
+ 'vetle',
+ 'van het',
+ 'dos',
+ 'ui',
+ 'vest',
+ 'ab',
+ 'vste',
+ 'nord',
+ 'van der',
+ 'bin',
+ 'ibn',
+ 'war',
+ 'fitz',
+ 'alam',
+ 'di',
+ 'erch',
+ 'fetch',
+ 'nga',
+ 'ka',
+ 'soder',
+ 'lille',
+ 'upp',
+ 'ua',
+ 'te',
+ 'ni',
+ 'bint',
+ 'von und zu',
+ 'vast',
+ 'vestre',
+ 'over',
+ 'syd',
+ 'mac',
+ 'nin',
+ 'nic',
+ 'putri',
+ 'bet',
+ 'verch',
+ 'norr',
+ 'bath',
+ 'della',
+ 'van',
+ 'ben',
+ 'du',
+ 'stor',
+ 'das',
+ 'neder',
+ 'abu',
+ 'degli',
+ 'vre',
+ 'ait',
+ 'ny',
+ 'opp',
+ 'pour',
+ 'kil',
+ 'der',
+ 'oz',
+ 'von',
+ 'at',
+ 'nedre',
+ 'van den',
+ 'setia',
+ 'ap',
+ 'gil',
+ 'myljom',
+ 'van de',
+ 'stre',
+ 'dele',
+ 'mck',
+ 'de',
+ 'mellom',
+ 'mhic',
+ 'binti',
+ 'ath',
+ 'binte',
+ 'snder',
+ 'sre',
+ 'ned',
+ 'ter',
+ 'bar',
+ 'le',
+ 'mala',
+ 'ost',
+ 'syndre',
+ 'sr',
+ 'bat',
+ 'sndre',
+ 'austre',
+ 'putra',
+ 'putera',
+ 'av',
+ 'lu',
+ 'vetch',
+ 'ver',
+ 'puteri',
+ 'mc',
+ 'tre',
+ 'st']
+
+#suppress all the unwanted suffixes from a string. 
+#name_del file can be modified if more or less suffixes want to be suppressed 
+
+name_del = ['2nd', '3rd', 'Jr', 'Jr.', 'Junior', 'Sr', 'Sr.', 'Senior']
+
+
+def name_delete(string):
+    for elmt in name_del:
+        if f" {elmt}" in string:
+            return string.replace(f" {elmt}","")
+    return string
+
+def ln_suff_merge(string):
+    for suff in ln_suff:
+        if f" {suff} " in string or string.startswith(f"{suff} "):
+            return string.replace(f"{suff} ",suff.replace(" ",""))
+    return string
+
+#normalize a string dat that represents often a name. 
+
+def normalize(data):
+    normal = unicodedata.normalize('NFKD', data).encode('ASCII', 'ignore')
+    val = normal.decode("utf-8")
+    # delete unwanted elmt
+    val = name_delete(val)
+    # lower full name in upper
+    val = re.sub(r"[A-Z]{3,}", lambda x: x.group().lower(), val)
+    # add space in front of upper case
+    val = re.sub(r"(\w)([A-Z])", r"\1 \2", val)
+    # Lower case
+    val = val.lower()
+    # remove special characters
+    val = re.sub('[^A-Za-z0-9 ]+', ' ', val)
+    # remove multiple spaces
+    val = re.sub(' +', ' ', val)
+    # remove trailing spaces
+    val = val.strip()
+    # suffix merge
+    val = ln_suff_merge(val)
+
+    return val
+
+
+def main_inventors( research_key_words , details , size):
+    
+    dic_patents , dic_ranked = ranking_patents(research_key_words , details)
+    dic_patents_co_inventors = {}
+    
+
+    for patent in list(dic_ranked.keys())[:size]:
+        for k in range(len(dic_patents[patent]["list_inventors"])):
+            
+            inventor_id = dic_patents[patent]["list_inventors"][k]["inventor_id"]
+            inventor_name = dic_patents[patent]["list_inventors"][k]["inventor_first_name"] + " " + dic_patents[patent]["list_inventors"][k]["inventor_last_name"]
+            inventor_name_norm = normalize(inventor_name).split()
+            inventor_name_norm = inventor_name_norm[0] + " " + inventor_name_norm[-1]
+
+            if inventor_name_norm not in dic_patents_co_inventors:
+                dic_patents_co_inventors[inventor_name_norm] = {}
+                dic_patents_co_inventors[inventor_name_norm]["Inventor's name"] = inventor_name 
+                dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"] =  inventor_id 
+                dic_patents_co_inventors[inventor_name_norm]["Number of occurence"] = 1
+                dic_patents_co_inventors[inventor_name_norm]["Number of related citations"] = dic_patents[patent]["number_citations"]
+
+            else:
+                if inventor_id not in dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"] :
+                    dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"] += ", " + inventor_id
+                if inventor_name not in dic_patents_co_inventors[inventor_name_norm]["Inventor's name"] :
+                    dic_patents_co_inventors[inventor_name_norm]["Inventor's name"] += ", " + inventor_name
+                dic_patents_co_inventors[inventor_name_norm]["Number of occurence"] += 1
+                dic_patents_co_inventors[inventor_name_norm]["Number of related citations"] += dic_patents[patent]["number_citations"]
+
+
+        dic_patents_co_inventors = {k: v for k, v in sorted(dic_patents_co_inventors.items(), key=lambda item: item[1]["Number of occurence"] , reverse = True)}
+    
+    if dic_patents_co_inventors == {}:
+    
+        
+        return "No patent, select other key words"
+    else:
+        
+        for inventor_name_norm in list(dic_patents_co_inventors.keys()):
+            list_inventors = dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"].split(", ")
+            work_count = 0
+            cited_by_count = 0
+            
+            for elem in list_inventors:
+                url = "https://api.patentsview.org/inventors/query?q={%22inventor_id%22:[%22" + elem + "%22]}&f=[%22inventor_total_num_patents%22,%22patent_num_cited_by_us_patents%22]"
+                data = get_data(url)["inventors"][0]
+                work_count += int(data["inventor_total_num_patents"])
+                for k in range(len(data["patents"])):
+                    cited_by_count += int(data["patents"][k]["patent_num_cited_by_us_patents"])
+                
+            dic_patents_co_inventors[inventor_name_norm]["Number of patents"] = work_count
+            dic_patents_co_inventors[inventor_name_norm]["Number of US patents citations"] = cited_by_count
+            
+        
+                
+                
+        return pd.DataFrame(dic_patents_co_inventors , index = ["Inventor's name", "PatentsView inventor's id", "Number of occurence"  , "Number of patents" ,"Number of US patents citations" ,  "Number of related citations"]).T.style.hide(axis="index")
+
+
+    
+def map_inventors(research_key_words , details , size):
+    display = False
+    dic_patents , dic_ranked = ranking_patents(research_key_words , details)
+    dic_patents_co_inventors = {}
+    count = 0
+    
+    for patent in list(dic_ranked.keys())[:size]:
+        for k in range(len(dic_patents[patent]["list_inventors"])):
+            
+            dic_patents_co_inventors[count] = {}
+            
+            dic_patents_co_inventors[count]["latitude"] = dic_patents[patent]["list_inventors"][k]["inventor_latitude"]
+            dic_patents_co_inventors[count]["longitude"] = dic_patents[patent]["list_inventors"][k]["inventor_longitude"]
+            dic_patents_co_inventors[count]["longitude"] = dic_patents[patent]["list_inventors"][k]["inventor_longitude"]
+            dic_patents_co_inventors[count]["inventor_name"]  = str(dic_patents[patent]["list_inventors"][k]["inventor_first_name"]) + " " + str(dic_patents[patent]["list_inventors"][k]["inventor_last_name"])
+            dic_patents_co_inventors[count]["patent_date"]  = dic_patents[patent]["date"]
+            count += 1
+                
+    if dic_patents_co_inventors == {}:
+        return "No patent, select other key words"
+    
+    
+    map_df = pd.DataFrame(dic_patents_co_inventors).T
+    map_df["longitude"]=map_df['longitude'].astype(float)
+    map_df['latitude']=map_df['latitude'].astype(float)
+    map_df = map_df[map_df["latitude"].notnull()]
+
+    return map_df
+

Climate_site/python_scripts/read_me.txt

@@ -0,0 +1,13 @@
+---
+title: SynapseProject
+emoji: 🌍
+colorFrom: yellow
+colorTo: blue
+sdk: streamlit
+sdk_version: 1.25.0
+app_file: app.py
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

Climate_site/python_scripts/related_scientist.py

@@ -0,0 +1,366 @@
+##packages code
+
+
+import pandas as pd
+import streamlit as st
+import numpy as np
+import json, requests 
+#from pandas.io.json import json_normalize
+from tqdm import tqdm
+
+
+@st.cache_data  # 👈 Add the caching decorator
+def load_data():
+    url = "Synapse_project/Climate_site/python_scripts/institutions.tsv"
+    dic = pd.read_csv(url, delimiter = "\t" , index_col = 1).to_dict('index')
+    return dic
+
+dic_institutions = load_data()
+
+
+dic_country_codes = {'Europe': ['AD',
+  'AL',
+  'AT',
+  'BE',
+  'BG',
+  'BY',
+  'CZ',
+  'DE',
+  'DK',
+  'EE',
+  'FI',
+  'FR',
+  'GR',
+  'HU',
+  'IE',
+  'IS',
+  'IT',
+  'LI',
+  'LT',
+  'LU',
+  'LV',
+  'MK',
+  'MT',
+  'NL',
+  'NO',
+  'PL',
+  'PT',
+  'RO',
+  'RU',
+  'SE',
+  'SI',
+  'SK',
+  'SM',
+  'UA',
+  'VA',
+  'BA',
+  'HR',
+  'MD',
+  'MC',
+  'ME',
+  'RS',
+  'ES',
+  'CH',
+  'GB'],
+ 'Asia': ['AF',
+  'AM',
+  'AZ',
+  'BD',
+  'BH',
+  'BN',
+  'BT',
+  'CN',
+  'CY',
+  'GE',
+  'ID',
+  'IL',
+  'IN',
+  'IQ',
+  'IR',
+  'JO',
+  'JP',
+  'KG',
+  'KP',
+  'KR',
+  'KW',
+  'LB',
+  'MM',
+  'MN',
+  'MV',
+  'MY',
+  'NP',
+  'OM',
+  'PH',
+  'PK',
+  'QA',
+  'SA',
+  'SG',
+  'SY',
+  'TH',
+  'TJ',
+  'TM',
+  'TR',
+  'UZ',
+  'VN',
+  'YE',
+  'KH',
+  'TL',
+  'KZ',
+  'LA',
+  'LK',
+  'AE'],
+ 'North America': ['AG',
+  'BB',
+  'BS',
+  'BZ',
+  'CA',
+  'CR',
+  'CU',
+  'DM',
+  'DO',
+  'GT',
+  'GT',
+  'HN',
+  'JM',
+  'MX',
+  'NI',
+  'PA',
+  'TT',
+  'US',
+  'SV',
+  'GD',
+  'KN',
+  'LC',
+  'VC'],
+ 'Africa': ['AO',
+  'BF',
+  'BI',
+  'BJ',
+  'BW',
+  'CD',
+  'CG',
+  'CI',
+  'CM',
+  'CV',
+  'DJ',
+  'EG',
+  'ER',
+  'ET',
+  'GA',
+  'GH',
+  'GM',
+  'GN',
+  'GW',
+  'KE',
+  'LR',
+  'LS',
+  'LY',
+  'MG',
+  'ML',
+  'MR',
+  'MU',
+  'MW',
+  'MZ',
+  'NA',
+  'NE',
+  'NG',
+  'RW',
+  'SC',
+  'SD',
+  'SL',
+  'SN',
+  'SO',
+  'ST',
+  'TG',
+  'TN',
+  'TZ',
+  'UG',
+  'ZM',
+  'ZW',
+  'DZ',
+  'CF',
+  'TD',
+  'KM',
+  'GQ',
+  'MA',
+  'ZA',
+  'SZ'],
+ 'South America': ['AR',
+  'BO',
+  'BR',
+  'CL',
+  'CO',
+  'EC',
+  'GY',
+  'PE',
+  'PY',
+  'SR',
+  'UY',
+  'VE'],
+ 'Oceania': ['AU',
+  'FJ',
+  'KI',
+  'MH',
+  'NR',
+  'NZ',
+  'PG',
+  'PW',
+  'SB',
+  'TO',
+  'TV',
+  'VU',
+  'FM',
+  'WS']}
+
+
+#################### General Functions #############################
+
+def URL(base_URL , entity_type , filters):
+    url = base_URL + entity_type + filters 
+    return url
+
+
+def get_data(url):
+    url = requests.get(url)
+    text = url.text
+    import json
+    data = json.loads(text)
+    return data
+
+## selecting the ids we want
+
+
+def author_id_from_name(name):
+    
+    dic_names = {}
+    
+    name = name.title()
+    
+    url = "https://api.openalex.org/authors?search=" + name + "&per_page=200&mailto=emma_scharfmann@berkeley.edu"
+    
+    data = get_data(url)["results"]
+
+    for k in range(len(data)):
+        author_id = data[k]["id"][21:]
+        dic_names[author_id] = {}
+        dic_names[author_id]["author_name"] = data[k]["display_name"]
+        
+        dic_names[author_id]["number_of_works"] = data[k]["works_count"]
+        dic_names[author_id]["number_of_citations"] = data[k]["cited_by_count"]
+        
+        dic_names[author_id]["field_of_study"] = ", ".join([ data[k]["x_concepts"][j]["display_name"] for j in range(min(5,len(data[k]["x_concepts"])))])
+
+        dic_names[author_id]["last_known_institution"] = None
+        dic_names[author_id]["country_code"] = None
+        
+        if data[k]["last_known_institution"] != None:
+            dic_names[author_id]["last_known_institution"] = data[k]["last_known_institution"]["display_name"]
+            dic_names[author_id]["country_code"] = data[k]["last_known_institution"]["country_code"]
+            
+        dic_names[author_id]["orcid"] = data[k]["orcid"]
+        
+            
+    return pd.DataFrame(dic_names).T , [ ( k , ", ".join([ elem["author_name"] , str(elem["last_known_institution"]) , str(elem["field_of_study"].split(", ")[0]) , "Number of works: " + str(elem["number_of_works"])  ] ) ) for k , elem in  dic_names.items()] 
+        
+        
+    
+ 
+        
+def from_author_id(main_author_ids, year, country_code , size):
+    
+    dic_main_workers = {} 
+
+    
+    for main_author_id in main_author_ids:
+    
+        url = "https://api.openalex.org/works?filter=author.id:" + main_author_id + ",publication_year:>" + str(year) + "&per_page=200&mailto=emma_scharfmann@berkeley.edu"
+
+        data = get_data(url)["results"]
+
+        for k in tqdm(range(len(data))):
+            work_id = data[k]["id"][21:]
+            for j in range(len(data[k]["authorships"])):
+                author_id = data[k]["authorships"][j]["author"]["id"][21:]
+
+                if author_id != main_author_id:
+                    if author_id not in dic_main_workers:
+                        dic_main_workers[author_id] = {}
+                        dic_main_workers[author_id]["author_name"] = data[k]["authorships"][j]["author"]["display_name"]
+                        dic_main_workers[author_id]["co_authors"] = 0
+                        dic_main_workers[author_id]["citations"] = 0
+                        dic_main_workers[author_id]["institution"] = None
+                        dic_main_workers[author_id]["country_code"] = None
+                        dic_main_workers[author_id]["id"] = None
+                        dic_main_workers[author_id]["longitude"] = None
+                        dic_main_workers[author_id]["latitude"] = None
+                        
+                    if dic_main_workers[author_id]["institution"] == None and data[k]["authorships"][j]["institutions"] != [] and "display_name" in data[k]["authorships"][j]["institutions"][0]:
+                        dic_main_workers[author_id]["institution"] = data[k]["authorships"][j]["institutions"][0]["display_name"]
+                    if dic_main_workers[author_id]["country_code"] == None and data[k]["authorships"][j]["institutions"] != [] and "country_code" in data[k]["authorships"][j]["institutions"][0]:
+                        dic_main_workers[author_id]["country_code"] = data[k]["authorships"][j]["institutions"][0]["country_code"]
+                    if dic_main_workers[author_id]["id"] == None and data[k]["authorships"][j]["institutions"] != [] and "id" in data[k]["authorships"][j]["institutions"][0] and data[k]["authorships"][j]["institutions"][0]["id"] != None:
+                        dic_main_workers[author_id]["id"] = data[k]["authorships"][j]["institutions"][0]["id"][21:]
+
+
+
+                    dic_main_workers[author_id]["co_authors"] += 1
+
+            url = "https://api.openalex.org/works?filter=referenced_works:" + work_id + ",publication_year:>" + str(year) + "&per_page=200&mailto=emma_scharfmann@berkeley.edu"
+            
+
+                
+            citations_data = get_data(url)["results"]
+            for i in range(len(citations_data)):
+                citing_work_id = citations_data[i]["id"][21:]
+                for j in range(len(citations_data[i]["authorships"])):
+                    citing_author_id = citations_data[i]["authorships"][j]["author"]["id"][21:]
+
+                    if citing_author_id != main_author_id:
+                        if citing_author_id not in dic_main_workers:
+                            dic_main_workers[citing_author_id] = {}
+                            dic_main_workers[citing_author_id]["author_name"] = citations_data[i]["authorships"][j]["author"]["display_name"]
+                            dic_main_workers[citing_author_id]["co_authors"] = 0
+                            dic_main_workers[citing_author_id]["citations"] = 0
+                            dic_main_workers[citing_author_id]["institution"] = None
+                            dic_main_workers[citing_author_id]["country_code"] = None
+                            dic_main_workers[citing_author_id]["id"] = None
+                            dic_main_workers[citing_author_id]["longitude"] = None
+                            dic_main_workers[citing_author_id]["latitude"] = None
+                            
+                        if dic_main_workers[citing_author_id]["institution"] == None and (citations_data[i]["authorships"][j]["institutions"] != [] and "display_name" in citations_data[i]["authorships"][j]["institutions"][0]):
+                                dic_main_workers[citing_author_id]["institution"] = citations_data[i]["authorships"][j]["institutions"][0]["display_name"]
+                        if dic_main_workers[citing_author_id]["country_code"] == None and (citations_data[i]["authorships"][j]["institutions"] != [] and "country_code" in citations_data[i]["authorships"][j]["institutions"][0] ):
+                            dic_main_workers[citing_author_id]["country_code"] = citations_data[i]["authorships"][j]["institutions"][0]["country_code"]
+                        if dic_main_workers[citing_author_id]["id"] == None and ( citations_data[i]["authorships"][j]["institutions"] != [] and "id" in citations_data[i]["authorships"][j]["institutions"][0] and citations_data[i]["authorships"][j]["institutions"][0]["id"] != None):
+                            dic_main_workers[citing_author_id]["id"] = citations_data[i]["authorships"][j]["institutions"][0]["id"][21:]
+                                
+                        
+
+                        dic_main_workers[citing_author_id]["citations"] += 1
+                        
+                        
+    for author_id in dic_main_workers:
+        if dic_main_workers[author_id]["id"] != None:
+            institution_id = dic_main_workers[author_id]["id"] 
+            if institution_id in dic_institutions:
+                geo_data = dic_institutions[institution_id]
+                dic_main_workers[author_id]["longitude"] = geo_data["longitude"]
+                dic_main_workers[author_id]["latitude"] = geo_data["latitude"]
+
+
+    res = pd.DataFrame(dic_main_workers).T.sort_values("citations" , ascending = False)[['author_name', 'co_authors', 'citations', 'institution', 'country_code']]
+    
+    res_geo = pd.DataFrame(dic_main_workers).T.sort_values("citations" , ascending = False)[['author_name', 'co_authors', 'citations', 'institution', 'country_code','longitude', 'latitude']]
+    res_geo = res_geo[res_geo["longitude"].notnull()]
+    if country_code != False:
+        
+        if country_code in [ "Europe" , "North America" , "Asia" , "South America", "Oceania" , "Africa" ]:
+            
+            return res[res["country_code"].isin(dic_country_codes[country_code])].head(size) , res_geo[res_geo["country_code"].isin(dic_country_codes[country_code])].head(size)
+            
+        else:
+            return res[res["country_code"] == country_code].head(size) , res_geo[res_geo["country_code"] == country_code].head(size)
+    else: 
+        return res.head(size), res_geo.head(size)
+
+