Refactor faculty scraping functionality and add new scrapers for various departments. Update main.py to comment out sum calculations and chart plotting. Introduce profile_to_vector_database notebook for faculty data processing. Remove simple.ipynb as it is no longer needed.

main.py

@@ -15,17 +15,19 @@ df = pd.DataFrame(data)
 
 df
 #%%
-# Calculate the sums
-sum_apples = df['Apples'].sum()
-sum_bananas = df['Bananas'].sum()
+# # Calculate the sums
+# sum_apples = df['Apples'].sum()
+# sum_bananas = df['Bananas'].sum()
 
-# Create a bar chart
-plt.bar(['Apples', 'Bananas'], [sum_apples, sum_bananas], color=['red', 'blue'])
+# # Create a bar chart
+# plt.bar(['Apples', 'Bananas'], [sum_apples, sum_bananas], color=['red', 'blue'])
 
-# Set a title
-plt.title('Comparison of total Apples and Bananas')
+# # Set a title
+# plt.title('Comparison of total Apples and Bananas')
 
-# Show the plot
-plt.show()
+# # Show the plot
+# plt.show()
 
 # See DataSpell help at https://www.jetbrains.com/help/dataspell/
+
+# %%

profile_to_vector_database.ipynb

@@ -0,0 +1,533 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "8a77807f92f26ee",
+   "metadata": {},
+   "source": [
+    "# Jupyter notebook sample"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "fc944c6d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%matplotlib inline\n",
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "7df9140457caed5e",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-05-05T16:47:28.619918Z",
+     "start_time": "2025-05-05T16:47:28.611208Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# imports\n",
+    "\n",
+    "import os\n",
+    "from tqdm import tqdm\n",
+    "from dotenv import load_dotenv\n",
+    "from huggingface_hub import login\n",
+    "from datasets import load_dataset\n",
+    "import chromadb\n",
+    "from sklearn.manifold import TSNE\n",
+    "from openai import OpenAI\n",
+    "from langchain_openai import OpenAIEmbeddings\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c4c650bb",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "ca866defaf4e79ba",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-05-05T16:44:41.846756Z",
+     "start_time": "2025-05-05T16:44:41.841991Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "load_dotenv(override=True)\n",
+    "api_key = os.getenv('OPENAI_API_KEY')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "aa576312",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<src.scraper.profile_scraper.FacultyProfileScraper at 0x1b95fcc50>"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import src.scraper.profile_scraper as profile_scraper\n",
+    "faculty = profile_scraper.FacultyProfileScraper(\"https://sph.uth.edu/faculty/?fac=O5VHhdEYnwuzkAQcMguuOA==\")\n",
+    "faculty"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "9378963b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "API key found and looks good so far!\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'Sure! Here is the processed text file content based on the provided website information for the faculty member, Yunxin Fu:\\n\\n---\\n\\n**Yunxin Fu**  \\nProfessor  \\nBiostatistics  \\n713/500-9813  \\nReuel Stallones Building  \\n1200 Pressler Street, Houston, TX 77030  \\n[View CV]\\n\\n**About**  \\nI was trained as a biostatistician specialized in computational biology, and have spent much of my career in developing population genetics theory and statistical methods for analyzing population samples of DNA sequences, including algorithms for simulating samples for the analysis of large scale data. Recently I have been involved in the analysis of polymorphism data from the 1000 Genomes Project and related large data sets. In addition to my continuous work on population genetics theory and evolution, my recent interests include within-individual polymorphism generated by both classical experiments and next generation sequencing for the purpose of understanding the mutational process during individual development, and population genetics of T. thermorphila.\\n\\n**Center Affiliation**  \\nHuman Genetics Center\\n\\n**Research Interests**  \\n- Big Data/Data Science  \\n- Biostatistics  \\n- Genetics and -omics  \\n- Infectious Disease  \\n\\n--- \\n\\nFeel free to let me know if you need any further modifications!'"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from src.summarize import summarize\n",
+    "summarize(faculty.text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "ce7fe0fa",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'image_url': 'https://web.sph.uth.edu/thumbs/addy.jpg',\n",
+       " 'name': 'Robert Addy',\n",
+       " 'profile': '<< Back\\nRobert Addy\\nFaculty Associate\\nHealth Promotion & Behavioral Sciences\\n713/500-9758\\nView CV\\nAbout\\nDr. Addy has over 25 years of research experience, including collecting, managing, and analyzing data in multiple contexts.  He has served as data manager for multiple projects involving single and multiple sites, as well as cross-sectional and longitudinal community-based studies in diverse settings (e.g. children, adolescents, adults, schools, clinics, and communities) and with diverse populations, including Hispanic, African-American, American Indian and Alaska Native populations.  He has extensive experience in compiling, cleaning, and processing data in preparation for analysis and distribution.  He is part of a teaching team that provides graduate level classes in program evaluation and data management at UTSPH.\\nCenter Affiliation\\nCenter for Health Promotion and Prevention Research\\nResearch Interests\\nAdolescent Health\\nBiostatistics\\nHealth Education/Behavioral Sciences\\nProgram Evaluation\\nSexual Health',\n",
+       " 'profile_url': 'https://sph.uth.edu/faculty/?fac=RxApW409EWwpnikiXFRn8g=='}"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# load json into python dictionary\n",
+    "import json \n",
+    "input_files = ['data/sph_faculty_list.jsonl', \n",
+    "              'data/gsbs_faculty_list.jsonl', \n",
+    "              'data/sbmi_faculty_list.jsonl']\n",
+    "faculty_list = []\n",
+    "for input_file in input_files:\n",
+    "    with open(input_file, 'r', encoding='utf-8') as f:\n",
+    "        for line in f:\n",
+    "            faculty_list.append(json.loads(line.strip()))\n",
+    "\n",
+    "len(faculty_list)\n",
+    "faculty_list[0]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "26d52ec1",
+   "metadata": {},
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "1b5eb8bb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# vectorize the faculty documents\n",
+    "# vectorizer = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')\n",
+    "vectorizer = OpenAIEmbeddings(model=\"text-embedding-ada-002\", \n",
+    "                              openai_api_key=os.getenv('OPENAI_API_KEY'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "6c24c978",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "DB = \"faculties_vectorstore\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "163fcada",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "client = chromadb.PersistentClient(path=DB)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "e5cffaf8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Check if the collection exists and delete it if it does\n",
+    "collection_name = \"faculties\"\n",
+    "\n",
+    "# For old versions of Chroma, use this line instead of the subsequent one\n",
+    "# existing_collection_names = [collection.name for collection in client.list_collections()]\n",
+    "existing_collection_names = client.list_collections()\n",
+    "\n",
+    "# if collection_name in [c.name for c in existing_collection_names]:\n",
+    "#     client.delete_collection(collection_name)\n",
+    "#     print(f\"Deleted existing collection: {collection_name}\")\n",
+    "    \n",
+    "# collection = client.create_collection(collection_name)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "8e835462",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# # Uncomment if you'd rather not wait for the full 400,000\n",
+    "# # NUMBER_OF_DOCUMENTS = 20000\n",
+    "# # vectors = vectorizer.encode(faculty_documents).astype(float).tolist()\n",
+    "# faculty_profile = []\n",
+    "# for faculty in faculty_list:\n",
+    "#     faculty_profile.append(faculty['profile'])\n",
+    "# vectors = vectorizer.embed_documents(faculty_profile)\n",
+    "# # metadata is everything except the about field\n",
+    "# metadatas = [{\"name\": faculty['name'], 'url': faculty['profile_url']} for faculty in faculty_list]\n",
+    "# ids = [f\"doc_{j}\" for j in range(len(faculty_profile))]\n",
+    "# collection.add(\n",
+    "#     ids=ids,\n",
+    "#     documents=faculty_profile,\n",
+    "#     embeddings=vectors,\n",
+    "#     metadatas=metadatas\n",
+    "# )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "8636a3a3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "collection = client.get_or_create_collection('faculties')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "7132cf50",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result = collection.get(include=['embeddings', 'documents', 'metadatas'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "ef088dc7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def find_similars(description):\n",
+    "    results = collection.query(query_embeddings=vectorizer.embed_query(description), n_results=10)\n",
+    "    documents = results['documents'][0][:]\n",
+    "    name = [m['name'] for m in results['metadatas'][0][:]]\n",
+    "    link = [m['url'] for m in results['metadatas'][0][:]]\n",
+    "    return documents, name, link"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "925c3625",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(['<< Back\\nElena Feofanova\\nAssistant Professor Non-Tenure Instruction\\nEpidemiology\\n713/500-9827\\nReuel Stallones Building\\n1200 Pressler Street, Houston, TX 77030\\nView CV\\nResearch Interests\\nEpidemiology',\n",
+       "  '<< Back\\nEric Jones\\nAssociate Professor\\nEpidemiology\\n915/975-8528\\nMCA Medical Center of the Americas Foundation\\n5130 Gateway East Blvd.  MCA 308\\nView CV\\nAbout\\nAs a social epidemiologist, I conduct research on social aspects of wellbeing. A major focus of my studies is on human behavior and wellbeing in extreme circumstances (like disasters, warfare. Migration, cancer), focusing on the role of relationships in stress, depression, anxiety, and functioning. My broad interest in collective action and social support, often using social network analysis, has also led me recently to study health care coordination, drug and violence prevention coalitions, informal and formal science education, and interorganizational dynamics. I continue to be involved in creating and maintaining online and desktop social network analysis software  Most of my research takes place in Mexico, Ecuador, and the United States.\\nMajor Research Programs\\n* Comparative Disaster Recovery - Variation in how individuals and families recover from disasters (Ecuador, Mexico, US, involving over 10 disasters).\\n* Hazards and Cultural Transformations - Variation in how leaders and their subjects have dealt with disasters in 98 societies around the world in the past 200 years, including sharing, land tenure, ideology, economics, politics, rules/restrictions, etc.\\n* Cultural Models of Nature – Variation in the way different cultural groups understand their relationships to the biophysical environment and changes in weather affecting their food production (Ecuador, with colleague sites in 12+ other countries).\\nCenter Affiliation\\nCenter for Health Promotion and Prevention Research\\nResearch Interests\\nBehavioral/Mental Health\\nBorder Health\\nCancer\\nEpidemiology\\nGlobal Health\\nHealth Equity\\nSocial Determinants of Health\\nPersonal Links\\nhttp://tmc.academia.edu/EricJones\\nhttps://www.researchgate.net/profile/Eric_Jones14',\n",
+       "  '<< Back\\nSusan Fisher-Hoch\\nProfessor Non-Tenure Research\\nEpidemiology\\n956/755-0635\\nRegional Academic Health Center\\nOne West University Blvd, SPH Building, S1.330, Brownsville, TX 78520\\nView CV\\nAbout\\nI am trained as a physician with a doctoral degree in epidemiology from London University with Membership of the Royal College of Pathology in virology.  After a long career at the Centers for Disease Control and Prevention working with viral hemorrhagic fevers, in poor communities in developing countries. I designed and directed the French Biosafety 4 Level principally for studies of Ebola and Lassa fever for which I received the Légion d’Honneur.  I joined the Brownsville campus of the School of Public Health to work in health disparity population health in 2001.  In 2004 I helped found the Cameron County Hispanic Cohort (n=5000) which I direct.  This is a randomly selected community based cohort of health disparity Mexican Americans.  Using these data I have collaborative programs in diabetes, cardiovascular disease and cancers understudied minorities with a wide range of collaborators from major institutions across the country, funded by several NIH and CPRIT grants, and having produced more than 60 publications.  I teach international epidemic investigation course on line based on real life outbreaks.\\nCenter Affiliation\\nHispanic Health Research Center\\nResearch Interests\\nCancer\\nCardiovascular and Chronic Diseases\\nClinical Trials\\nEpidemiology\\nInfectious Disease\\nPersonal Links\\nhttps://sph.uth.edu/research/centers/hispanic-health/about.htm',\n",
+       "  '<< Back\\nLu-Yu Hwang\\nProfessor Non-Tenure Instruction\\nEpidemiology\\n713/500-9382\\nReuel Stallones Building\\n1200 Pressler Street, Houston, TX 77030\\nView CV\\nCenter Affiliation\\nCenter for Infectious Diseases\\nResearch Interests\\nPediatrics; infectious disease; perinatal transmission; viral epidemiology; cancer epidemiology; hepatitis virus/liver cancer; HIV/AIDS, HTLV/leukemia; EBV/nasopharyngeal cancer; viral oncology.',\n",
+       "  'Fuqing Wu\\nRegular Member\\nAssistant Professor\\n713-500-9319\\n713-500-9319\\n[email\\xa0protected]\\nRAS - E703\\nThe University of Texas\\xa0 Health Science Center at Houston\\nDepartment of Environmental and Occupational Health Sciences\\nInfectious viral diseases remain one of the leading causes of mortality worldwide. Our group focuses on developing innovative environmental and genomic surveillance systems to understand population health and monitor the spread of infectious diseases. Using molecular virology, genomic sequencing, and mathematical modeling, our research works advance wastewater-based surveillance to track zoonotic pathogens, analyze the evolution of genomic variants, and forecast infection transmission dynamics. By integrating these tools, we aim to create early-warning systems for viral outbreaks and uncover key insights into pathogenic agents’ transmission and public health.\\nCurrent projects in my lab include:\\nWastewater-based Epidemiology: Developing methodologies for detecting pathogenic agents in community wastewater, capturing infection trends over time and space, and integrating surveillance data with mathematical models to predict outbreak dynamics and track viral evolution.\\nViral Metagenomics and Discovery: Employing cutting-edge metagenomic sequencing to explore human viral diversity, with the ultimate goal of creating a comprehensive global map of human viruses.\\nMicrobiome and Medicine: Investigating the interactions between viral infections, antiviral treatments, and the gut microbiome. Questions include how antibiotics affect gut microbiota and how everyday medications alter the microbiome.\\nStudents participating in tutorials/research in our lab will gain hands-on experience in molecular biology, genomic sequencing, bioinformatics, and mathematical modeling.\\nPubMed\\nMcGovern Medical School Faculty\\nEducation & Training\\nPh.D. - University of Arizona, 2017\\nResearch Opportunities\\nSeeking MS Students\\nSeeking PhD Students\\nPrograms\\nMicrobiology and Infectious Diseases\\nFaculty Development\\nMentoring Works\\nBack',\n",
+       "  '<< Back\\nHeather Essigmann\\nAssistant Professor Non-Tenure Instruction\\nEpidemiology\\n713/500-9280\\nReuel Stallones Building\\n1200 Pressler Street, Houston, TX 77030\\nView CV',\n",
+       "  'Paul Scheet\\nRegular Member\\nProfessor\\n713.745.2470\\n713.745.2470\\n[email\\xa0protected]\\nMDA CPB4.3301 (Unit 1340)\\nThe University of Texas\\xa0MD Anderson Cancer Center\\nDepartment of Epidemiology\\nMy main research involves the development of statistical methods for the analysis of population genetic data. Specifically, I work with models for linkage disequilibrium (LD), and develop applications for taking proper account of the dependence among nearby loci. These applications include haplotype inference, missing genotype imputation, disease mapping, genotype error detection and correction, the visualization of haplotype variation, and the integration of emerging data types such as next generation DNA sequencing. For these applications I maintain a widely-used software package called fastPHASE.\\nA rotation in my lab could provide a student with exposure to several state-of-the-art haplotype-based software packages (such as PHASE, fastPHASE, Beagle, and Mach) and, in the process, could help evaluate different models for LD for various applications and settings. These models rely on hidden Markov model (HMM) techniques and implicitly a student would gain some introduction to these as well as algorithms, such as Baum-Welch, Markov chain Monte Carlo and Expectation-Maximization. In addition, the student could perform hands-on data analyses and would gain experience in working with Perl and the statistical software R. A student with strong C programming skills could develop, test, and add a function for analysis to an existing software package.\\nThe Department of Epidemiology has space for students and postdocs and provides a rich academic atmosphere with a large number of faculty in the section of Computational Genetic Epidemiology. Ample computational support exists, including processors which support programs for genetic analysis, as well as standard programming and database management languages.\\nPubMed\\nMDACC Faculty\\nEducation & Training\\nPh.D. - University of Washington - 2006\\nPrograms\\nGenetics and Epigenetics\\nQuantitative Sciences\\nFaculty Development\\nMentoring Works\\nBack',\n",
+       "  '<< Back\\nCharles Darkoh\\nProfessor and R. Palmer Beasley, M.D. & Lu-Yu Hwang, M.D. Chair\\nEpidemiology\\n713/500-9368\\nReuel Stallones Building\\n1200 Pressler Street, Houston, TX 77030\\nView CV\\nAbout\\nMy laboratory focuses on the epidemiology, pathogenesis, and molecular basis of enteric infectious diseases with the goal to identify and understand mechanisms of action of novel bacterial virulence factors, pathways, and unique microbial products that can be harnessed for diagnostics and therapeutics. My long-term research interests include investigating the role of small molecules and other metabolites produced by the gut microbiota in the pathogenesis of enteric infectious diseases, mechanisms of host-pathogen interactions, antimicrobial resistance, host genetics and susceptibility to bacterial infections, and global health. I have active lines of research on Clostridium (Clostridioides) difficile infections, drug discovery, diagnostics, antimicrobial resistance, and global health.\\nCenter Affiliation\\nCenter for Infectious Diseases\\nResearch Interests\\nEpidemiology\\nGenetics and -omics\\nGlobal Health\\nInfectious Disease',\n",
+       "  '<< Back\\nJoseph McCormick\\nProfessor & James H. Steele, D.V.M. Professorship\\nEpidemiology\\n956/755-0605\\nRegional Academic Health Center\\nOne West University Blvd, SPH Building, S1.330, Brownsville, TX 78520\\nView CV\\nAbout\\nRaised on a farm in Indiana, I graduated from Florida Southern College in chemistry and mathematics and attended the Free University in Brussels in preparation for teaching sciences and mathematics in French in the Congo where I was also introduced to medicine. I obtained my MD at Duke in 1971 and an MS from Harvard (1970) under Dr. Weller (Nobel Laureate 1954).  I trained at CHOP under Dr. Koop (later US Surgeon General).  I joined the CDC EIS and worked in Brazil as a PAHO/CDC consultant for the meningitis outbreaks of 1974-6.  In 1977 I founded the CDC Lassa fever project in Sierra Leone. I investigated the first Ebola epidemic in 1976 and led the one in 1979.   In Sierra Leone I conducted definitive studies of the epidemiology and antiviral treatment of Lassa fever.  As Chief, Special Pathogens Branch, Viral Diseases, CDC in 1982, I ran the Biosafety level 4 lab for 9 years.  I organized and led the first AIDS investigation in Africa in 1983 and established the Project SIDA in Kinshasa, Congo.  In 1983 I isolated the virus that causes Hemorrhagic Fever with Renal Syndrome (Hantavirus) at CDC.\\nIn 1993, I became Chairman, CHS at the Aga Khan University Medical School where I established a CDC EIS like program, an MS in Epidemiology, and a family Medicine Residency.  I founded the UT Houston School of Public Health campus in Brownsville in 2001. The campus has published over300 peer reviewed articles characterizing the extraordinary level of health disparities in the region. I was instrumental in bringing in over $100 million in external funding.\\nI now research chronic and infectious diseases in the LRGV in a cohort of 5000 people with over 30 research collaborators. I am featured in several books for the lay reader (e.g.,The Coming Plague, The Hot Zone). My wife, Sue Fisher-Hoch and I wrote a popular account (Level 4, Virus Hunters of the CDC) of our professional adventures translated into 7 languages. I am an amateur pianist, and enjoy many outdoor activities.\\nCenter Affiliation\\nHispanic Health Research Center\\nResearch Interests\\nBehavioral/Mental Health\\nBorder Health\\nCancer\\nCardiovascular and Chronic Diseases\\nClinical Trials\\nEpidemiology\\nGlobal Health\\nInfectious Disease\\nPersonal Links\\nhttp://www.ncbi.nlm.nih.gov/myncbi/browse/collection/40492151/?sort=date&direction=ascending\\nhttps://en.wikipedia.org/wiki/Joseph_B._McCormick\\nhttps://scholar.google.com/citations?user=xgV24roAAAAJ\\nhttps://www.pbs.org/wgbh/pages/frontline/aids/interviews/mccormick.html',\n",
+       "  '<< Back\\nMicaela Sandoval\\nAssistant Professor Non-Tenure Research\\nEpidemiology\\n713/500-9000\\nReuel Stallones Building\\n1200 Pressler Street, Houston, TX 77030\\nView CV\\nCenter Affiliation\\nCenter for Infectious Diseases'],\n",
+       " ['Elena Feofanova',\n",
+       "  'Eric Jones',\n",
+       "  'Susan Fisher-Hoch',\n",
+       "  'Lu-Yu Hwang',\n",
+       "  'Fuqing  Wu',\n",
+       "  'Heather Essigmann',\n",
+       "  'Paul  Scheet',\n",
+       "  'Charles Darkoh',\n",
+       "  'Joseph McCormick',\n",
+       "  'Micaela Sandoval'],\n",
+       " ['https://sph.uth.edu/faculty/?fac=K2vQLVh2UlsEfc/vmmpr1Fs/4u7UYh/Fv1JRv6tu3Cw=',\n",
+       "  'https://sph.uth.edu/faculty/?fac=cR7MFX%2B3Wde9%2BEMVGY1Tr7oQx%2BsGhAOuruLsHO7UHu8=',\n",
+       "  'https://sph.uth.edu/faculty/?fac=L7LDTUNsMRSWWMc72/M6BszFbk4rvzn/sjkju%2BFRSyg=',\n",
+       "  'https://sph.uth.edu/faculty/?fac=rmT1omzsPnVmTSI/j5K7WQ==',\n",
+       "  'https://gsbs.uth.edu/directory/profile?id=3247cf0b-e4ab-4f46-be9a-17c4bfaad784',\n",
+       "  'https://sph.uth.edu/faculty/?fac=cl6ZUp8M430eVKolh7L3tysX2yOqWaMt/aVHwKhTm/g=',\n",
+       "  'https://gsbs.uth.edu/directory/profile?id=e2296b7d-017f-42c3-a45d-62339c0f2668',\n",
+       "  'https://sph.uth.edu/faculty/?fac=0Oe7NOsP%2Bwqxzm8fNMAyFA==',\n",
+       "  'https://sph.uth.edu/faculty/?fac=b0beWb7l7z1FViVZQPFKTm1SaQ3glg/R65TV2Px4lDs=',\n",
+       "  'https://sph.uth.edu/faculty/?fac=7easTM9ILRE6auEurA/AUCTpx%2BnpUB%2BOCR2rrOuQ6tg='])"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "find_similars(\"I am looking for a faculty member who is an expert in epidemiology.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "4b24eae0",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'To provide some context, here are some faculty members that might be relevant to your description.\\n\\nPotentially related faculty:\\nElena Feofanova\\n\\n        website: https://sph.uth.edu/faculty/?fac=K2vQLVh2UlsEfc/vmmpr1Fs/4u7UYh/Fv1JRv6tu3Cw=\\n\\n        << Back\\nElena Feofanova\\nAssistant Professor Non-Tenure Instruction\\nEpidemiology\\n713/500-9827\\nReuel Stallones Building\\n1200 Pressler Street, Houston, TX 77030\\nView CV\\nResearch Interests\\nEpidemiology\\n\\nPotentially related faculty:\\nEric Jones\\n\\n        website: https://sph.uth.edu/faculty/?fac=cR7MFX%2B3Wde9%2BEMVGY1Tr7oQx%2BsGhAOuruLsHO7UHu8=\\n\\n        << Back\\nEric Jones\\nAssociate Professor\\nEpidemiology\\n915/975-8528\\nMCA Medical Center of the Americas Foundation\\n5130 Gateway East Blvd.  MCA 308\\nView CV\\nAbout\\nAs a social epidemiologist, I conduct research on social aspects of wellbeing. A major focus of my studies is on human behavior and wellbeing in extreme circumstances (like disasters, warfare. Migration, cancer), focusing on the role of relationships in stress, depression, anxiety, and functioning. My broad interest in collective action and social support, often using social network analysis, has also led me recently to study health care coordination, drug and violence prevention coalitions, informal and formal science education, and interorganizational dynamics. I continue to be involved in creating and maintaining online and desktop social network analysis software  Most of my research takes place in Mexico, Ecuador, and the United States.\\nMajor Research Programs\\n* Comparative Disaster Recovery - Variation in how individuals and families recover from disasters (Ecuador, Mexico, US, involving over 10 disasters).\\n* Hazards and Cultural Transformations - Variation in how leaders and their subjects have dealt with disasters in 98 societies around the world in the past 200 years, including sharing, land tenure, ideology, economics, politics, rules/restrictions, etc.\\n* Cultural Models of Nature – Variation in the way different cultural groups understand their relationships to the biophysical environment and changes in weather affecting their food production (Ecuador, with colleague sites in 12+ other countries).\\nCenter Affiliation\\nCenter for Health Promotion and Prevention Research\\nResearch Interests\\nBehavioral/Mental Health\\nBorder Health\\nCancer\\nEpidemiology\\nGlobal Health\\nHealth Equity\\nSocial Determinants of Health\\nPersonal Links\\nhttp://tmc.academia.edu/EricJones\\nhttps://www.researchgate.net/profile/Eric_Jones14\\n\\nPotentially related faculty:\\nSusan Fisher-Hoch\\n\\n        website: https://sph.uth.edu/faculty/?fac=L7LDTUNsMRSWWMc72/M6BszFbk4rvzn/sjkju%2BFRSyg=\\n\\n        << Back\\nSusan Fisher-Hoch\\nProfessor Non-Tenure Research\\nEpidemiology\\n956/755-0635\\nRegional Academic Health Center\\nOne West University Blvd, SPH Building, S1.330, Brownsville, TX 78520\\nView CV\\nAbout\\nI am trained as a physician with a doctoral degree in epidemiology from London University with Membership of the Royal College of Pathology in virology.  After a long career at the Centers for Disease Control and Prevention working with viral hemorrhagic fevers, in poor communities in developing countries. I designed and directed the French Biosafety 4 Level principally for studies of Ebola and Lassa fever for which I received the Légion d’Honneur.  I joined the Brownsville campus of the School of Public Health to work in health disparity population health in 2001.  In 2004 I helped found the Cameron County Hispanic Cohort (n=5000) which I direct.  This is a randomly selected community based cohort of health disparity Mexican Americans.  Using these data I have collaborative programs in diabetes, cardiovascular disease and cancers understudied minorities with a wide range of collaborators from major institutions across the country, funded by several NIH and CPRIT grants, and having produced more than 60 publications.  I teach international epidemic investigation course on line based on real life outbreaks.\\nCenter Affiliation\\nHispanic Health Research Center\\nResearch Interests\\nCancer\\nCardiovascular and Chronic Diseases\\nClinical Trials\\nEpidemiology\\nInfectious Disease\\nPersonal Links\\nhttps://sph.uth.edu/research/centers/hispanic-health/about.htm\\n\\nPotentially related faculty:\\nLu-Yu Hwang\\n\\n        website: https://sph.uth.edu/faculty/?fac=rmT1omzsPnVmTSI/j5K7WQ==\\n\\n        << Back\\nLu-Yu Hwang\\nProfessor Non-Tenure Instruction\\nEpidemiology\\n713/500-9382\\nReuel Stallones Building\\n1200 Pressler Street, Houston, TX 77030\\nView CV\\nCenter Affiliation\\nCenter for Infectious Diseases\\nResearch Interests\\nPediatrics; infectious disease; perinatal transmission; viral epidemiology; cancer epidemiology; hepatitis virus/liver cancer; HIV/AIDS, HTLV/leukemia; EBV/nasopharyngeal cancer; viral oncology.\\n\\nPotentially related faculty:\\nFuqing  Wu\\n\\n        website: https://gsbs.uth.edu/directory/profile?id=3247cf0b-e4ab-4f46-be9a-17c4bfaad784\\n\\n        Fuqing Wu\\nRegular Member\\nAssistant Professor\\n713-500-9319\\n713-500-9319\\n[email\\xa0protected]\\nRAS - E703\\nThe University of Texas\\xa0 Health Science Center at Houston\\nDepartment of Environmental and Occupational Health Sciences\\nInfectious viral diseases remain one of the leading causes of mortality worldwide. Our group focuses on developing innovative environmental and genomic surveillance systems to understand population health and monitor the spread of infectious diseases. Using molecular virology, genomic sequencing, and mathematical modeling, our research works advance wastewater-based surveillance to track zoonotic pathogens, analyze the evolution of genomic variants, and forecast infection transmission dynamics. By integrating these tools, we aim to create early-warning systems for viral outbreaks and uncover key insights into pathogenic agents’ transmission and public health.\\nCurrent projects in my lab include:\\nWastewater-based Epidemiology: Developing methodologies for detecting pathogenic agents in community wastewater, capturing infection trends over time and space, and integrating surveillance data with mathematical models to predict outbreak dynamics and track viral evolution.\\nViral Metagenomics and Discovery: Employing cutting-edge metagenomic sequencing to explore human viral diversity, with the ultimate goal of creating a comprehensive global map of human viruses.\\nMicrobiome and Medicine: Investigating the interactions between viral infections, antiviral treatments, and the gut microbiome. Questions include how antibiotics affect gut microbiota and how everyday medications alter the microbiome.\\nStudents participating in tutorials/research in our lab will gain hands-on experience in molecular biology, genomic sequencing, bioinformatics, and mathematical modeling.\\nPubMed\\nMcGovern Medical School Faculty\\nEducation & Training\\nPh.D. - University of Arizona, 2017\\nResearch Opportunities\\nSeeking MS Students\\nSeeking PhD Students\\nPrograms\\nMicrobiology and Infectious Diseases\\nFaculty Development\\nMentoring Works\\nBack\\n\\nPotentially related faculty:\\nHeather Essigmann\\n\\n        website: https://sph.uth.edu/faculty/?fac=cl6ZUp8M430eVKolh7L3tysX2yOqWaMt/aVHwKhTm/g=\\n\\n        << Back\\nHeather Essigmann\\nAssistant Professor Non-Tenure Instruction\\nEpidemiology\\n713/500-9280\\nReuel Stallones Building\\n1200 Pressler Street, Houston, TX 77030\\nView CV\\n\\nPotentially related faculty:\\nPaul  Scheet\\n\\n        website: https://gsbs.uth.edu/directory/profile?id=e2296b7d-017f-42c3-a45d-62339c0f2668\\n\\n        Paul Scheet\\nRegular Member\\nProfessor\\n713.745.2470\\n713.745.2470\\n[email\\xa0protected]\\nMDA CPB4.3301 (Unit 1340)\\nThe University of Texas\\xa0MD Anderson Cancer Center\\nDepartment of Epidemiology\\nMy main research involves the development of statistical methods for the analysis of population genetic data. Specifically, I work with models for linkage disequilibrium (LD), and develop applications for taking proper account of the dependence among nearby loci. These applications include haplotype inference, missing genotype imputation, disease mapping, genotype error detection and correction, the visualization of haplotype variation, and the integration of emerging data types such as next generation DNA sequencing. For these applications I maintain a widely-used software package called fastPHASE.\\nA rotation in my lab could provide a student with exposure to several state-of-the-art haplotype-based software packages (such as PHASE, fastPHASE, Beagle, and Mach) and, in the process, could help evaluate different models for LD for various applications and settings. These models rely on hidden Markov model (HMM) techniques and implicitly a student would gain some introduction to these as well as algorithms, such as Baum-Welch, Markov chain Monte Carlo and Expectation-Maximization. In addition, the student could perform hands-on data analyses and would gain experience in working with Perl and the statistical software R. A student with strong C programming skills could develop, test, and add a function for analysis to an existing software package.\\nThe Department of Epidemiology has space for students and postdocs and provides a rich academic atmosphere with a large number of faculty in the section of Computational Genetic Epidemiology. Ample computational support exists, including processors which support programs for genetic analysis, as well as standard programming and database management languages.\\nPubMed\\nMDACC Faculty\\nEducation & Training\\nPh.D. - University of Washington - 2006\\nPrograms\\nGenetics and Epigenetics\\nQuantitative Sciences\\nFaculty Development\\nMentoring Works\\nBack\\n\\nPotentially related faculty:\\nCharles Darkoh\\n\\n        website: https://sph.uth.edu/faculty/?fac=0Oe7NOsP%2Bwqxzm8fNMAyFA==\\n\\n        << Back\\nCharles Darkoh\\nProfessor and R. Palmer Beasley, M.D. & Lu-Yu Hwang, M.D. Chair\\nEpidemiology\\n713/500-9368\\nReuel Stallones Building\\n1200 Pressler Street, Houston, TX 77030\\nView CV\\nAbout\\nMy laboratory focuses on the epidemiology, pathogenesis, and molecular basis of enteric infectious diseases with the goal to identify and understand mechanisms of action of novel bacterial virulence factors, pathways, and unique microbial products that can be harnessed for diagnostics and therapeutics. My long-term research interests include investigating the role of small molecules and other metabolites produced by the gut microbiota in the pathogenesis of enteric infectious diseases, mechanisms of host-pathogen interactions, antimicrobial resistance, host genetics and susceptibility to bacterial infections, and global health. I have active lines of research on Clostridium (Clostridioides) difficile infections, drug discovery, diagnostics, antimicrobial resistance, and global health.\\nCenter Affiliation\\nCenter for Infectious Diseases\\nResearch Interests\\nEpidemiology\\nGenetics and -omics\\nGlobal Health\\nInfectious Disease\\n\\nPotentially related faculty:\\nJoseph McCormick\\n\\n        website: https://sph.uth.edu/faculty/?fac=b0beWb7l7z1FViVZQPFKTm1SaQ3glg/R65TV2Px4lDs=\\n\\n        << Back\\nJoseph McCormick\\nProfessor & James H. Steele, D.V.M. Professorship\\nEpidemiology\\n956/755-0605\\nRegional Academic Health Center\\nOne West University Blvd, SPH Building, S1.330, Brownsville, TX 78520\\nView CV\\nAbout\\nRaised on a farm in Indiana, I graduated from Florida Southern College in chemistry and mathematics and attended the Free University in Brussels in preparation for teaching sciences and mathematics in French in the Congo where I was also introduced to medicine. I obtained my MD at Duke in 1971 and an MS from Harvard (1970) under Dr. Weller (Nobel Laureate 1954).  I trained at CHOP under Dr. Koop (later US Surgeon General).  I joined the CDC EIS and worked in Brazil as a PAHO/CDC consultant for the meningitis outbreaks of 1974-6.  In 1977 I founded the CDC Lassa fever project in Sierra Leone. I investigated the first Ebola epidemic in 1976 and led the one in 1979.   In Sierra Leone I conducted definitive studies of the epidemiology and antiviral treatment of Lassa fever.  As Chief, Special Pathogens Branch, Viral Diseases, CDC in 1982, I ran the Biosafety level 4 lab for 9 years.  I organized and led the first AIDS investigation in Africa in 1983 and established the Project SIDA in Kinshasa, Congo.  In 1983 I isolated the virus that causes Hemorrhagic Fever with Renal Syndrome (Hantavirus) at CDC.\\nIn 1993, I became Chairman, CHS at the Aga Khan University Medical School where I established a CDC EIS like program, an MS in Epidemiology, and a family Medicine Residency.  I founded the UT Houston School of Public Health campus in Brownsville in 2001. The campus has published over300 peer reviewed articles characterizing the extraordinary level of health disparities in the region. I was instrumental in bringing in over $100 million in external funding.\\nI now research chronic and infectious diseases in the LRGV in a cohort of 5000 people with over 30 research collaborators. I am featured in several books for the lay reader (e.g.,The Coming Plague, The Hot Zone). My wife, Sue Fisher-Hoch and I wrote a popular account (Level 4, Virus Hunters of the CDC) of our professional adventures translated into 7 languages. I am an amateur pianist, and enjoy many outdoor activities.\\nCenter Affiliation\\nHispanic Health Research Center\\nResearch Interests\\nBehavioral/Mental Health\\nBorder Health\\nCancer\\nCardiovascular and Chronic Diseases\\nClinical Trials\\nEpidemiology\\nGlobal Health\\nInfectious Disease\\nPersonal Links\\nhttp://www.ncbi.nlm.nih.gov/myncbi/browse/collection/40492151/?sort=date&direction=ascending\\nhttps://en.wikipedia.org/wiki/Joseph_B._McCormick\\nhttps://scholar.google.com/citations?user=xgV24roAAAAJ\\nhttps://www.pbs.org/wgbh/pages/frontline/aids/interviews/mccormick.html\\n\\nPotentially related faculty:\\nMicaela Sandoval\\n\\n        website: https://sph.uth.edu/faculty/?fac=7easTM9ILRE6auEurA/AUCTpx%2BnpUB%2BOCR2rrOuQ6tg=\\n\\n        << Back\\nMicaela Sandoval\\nAssistant Professor Non-Tenure Research\\nEpidemiology\\n713/500-9000\\nReuel Stallones Building\\n1200 Pressler Street, Houston, TX 77030\\nView CV\\nCenter Affiliation\\nCenter for Infectious Diseases\\n\\n'"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def make_context(similars):\n",
+    "    message = \"To provide some context, here are some faculty members that might be relevant to your description.\\n\\n\"\n",
+    "    documents, names, links = similars\n",
+    "    for similar, name, link in zip(documents, names, links):\n",
+    "        message += f'''Potentially related faculty:\\n{name}\\n\n",
+    "        website: {link}\\n\n",
+    "        {similar}\\n\\n'''\n",
+    "    return message\n",
+    "make_context(find_similars(\"I am looking for a faculty member who is an expert in epidemiology.\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "044b9390",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'role': 'user',\n",
+       " 'content': 'Here is my description: I am looking for a faculty member who is an expert in epidemiology.\\n\\nTo provide some context, here are some faculty members that might be relevant to your description.\\n\\nPotentially related faculty:\\nElena Feofanova\\n\\n        website: https://sph.uth.edu/faculty/?fac=K2vQLVh2UlsEfc/vmmpr1Fs/4u7UYh/Fv1JRv6tu3Cw=\\n\\n        << Back\\nElena Feofanova\\nAssistant Professor Non-Tenure Instruction\\nEpidemiology\\n713/500-9827\\nReuel Stallones Building\\n1200 Pressler Street, Houston, TX 77030\\nView CV\\nResearch Interests\\nEpidemiology\\n\\nPotentially related faculty:\\nEric Jones\\n\\n        website: https://sph.uth.edu/faculty/?fac=cR7MFX%2B3Wde9%2BEMVGY1Tr7oQx%2BsGhAOuruLsHO7UHu8=\\n\\n        << Back\\nEric Jones\\nAssociate Professor\\nEpidemiology\\n915/975-8528\\nMCA Medical Center of the Americas Foundation\\n5130 Gateway East Blvd.  MCA 308\\nView CV\\nAbout\\nAs a social epidemiologist, I conduct research on social aspects of wellbeing. A major focus of my studies is on human behavior and wellbeing in extreme circumstances (like disasters, warfare. Migration, cancer), focusing on the role of relationships in stress, depression, anxiety, and functioning. My broad interest in collective action and social support, often using social network analysis, has also led me recently to study health care coordination, drug and violence prevention coalitions, informal and formal science education, and interorganizational dynamics. I continue to be involved in creating and maintaining online and desktop social network analysis software  Most of my research takes place in Mexico, Ecuador, and the United States.\\nMajor Research Programs\\n* Comparative Disaster Recovery - Variation in how individuals and families recover from disasters (Ecuador, Mexico, US, involving over 10 disasters).\\n* Hazards and Cultural Transformations - Variation in how leaders and their subjects have dealt with disasters in 98 societies around the world in the past 200 years, including sharing, land tenure, ideology, economics, politics, rules/restrictions, etc.\\n* Cultural Models of Nature – Variation in the way different cultural groups understand their relationships to the biophysical environment and changes in weather affecting their food production (Ecuador, with colleague sites in 12+ other countries).\\nCenter Affiliation\\nCenter for Health Promotion and Prevention Research\\nResearch Interests\\nBehavioral/Mental Health\\nBorder Health\\nCancer\\nEpidemiology\\nGlobal Health\\nHealth Equity\\nSocial Determinants of Health\\nPersonal Links\\nhttp://tmc.academia.edu/EricJones\\nhttps://www.researchgate.net/profile/Eric_Jones14\\n\\nPotentially related faculty:\\nSusan Fisher-Hoch\\n\\n        website: https://sph.uth.edu/faculty/?fac=L7LDTUNsMRSWWMc72/M6BszFbk4rvzn/sjkju%2BFRSyg=\\n\\n        << Back\\nSusan Fisher-Hoch\\nProfessor Non-Tenure Research\\nEpidemiology\\n956/755-0635\\nRegional Academic Health Center\\nOne West University Blvd, SPH Building, S1.330, Brownsville, TX 78520\\nView CV\\nAbout\\nI am trained as a physician with a doctoral degree in epidemiology from London University with Membership of the Royal College of Pathology in virology.  After a long career at the Centers for Disease Control and Prevention working with viral hemorrhagic fevers, in poor communities in developing countries. I designed and directed the French Biosafety 4 Level principally for studies of Ebola and Lassa fever for which I received the Légion d’Honneur.  I joined the Brownsville campus of the School of Public Health to work in health disparity population health in 2001.  In 2004 I helped found the Cameron County Hispanic Cohort (n=5000) which I direct.  This is a randomly selected community based cohort of health disparity Mexican Americans.  Using these data I have collaborative programs in diabetes, cardiovascular disease and cancers understudied minorities with a wide range of collaborators from major institutions across the country, funded by several NIH and CPRIT grants, and having produced more than 60 publications.  I teach international epidemic investigation course on line based on real life outbreaks.\\nCenter Affiliation\\nHispanic Health Research Center\\nResearch Interests\\nCancer\\nCardiovascular and Chronic Diseases\\nClinical Trials\\nEpidemiology\\nInfectious Disease\\nPersonal Links\\nhttps://sph.uth.edu/research/centers/hispanic-health/about.htm\\n\\nPotentially related faculty:\\nLu-Yu Hwang\\n\\n        website: https://sph.uth.edu/faculty/?fac=rmT1omzsPnVmTSI/j5K7WQ==\\n\\n        << Back\\nLu-Yu Hwang\\nProfessor Non-Tenure Instruction\\nEpidemiology\\n713/500-9382\\nReuel Stallones Building\\n1200 Pressler Street, Houston, TX 77030\\nView CV\\nCenter Affiliation\\nCenter for Infectious Diseases\\nResearch Interests\\nPediatrics; infectious disease; perinatal transmission; viral epidemiology; cancer epidemiology; hepatitis virus/liver cancer; HIV/AIDS, HTLV/leukemia; EBV/nasopharyngeal cancer; viral oncology.\\n\\nPotentially related faculty:\\nFuqing  Wu\\n\\n        website: https://gsbs.uth.edu/directory/profile?id=3247cf0b-e4ab-4f46-be9a-17c4bfaad784\\n\\n        Fuqing Wu\\nRegular Member\\nAssistant Professor\\n713-500-9319\\n713-500-9319\\n[email\\xa0protected]\\nRAS - E703\\nThe University of Texas\\xa0 Health Science Center at Houston\\nDepartment of Environmental and Occupational Health Sciences\\nInfectious viral diseases remain one of the leading causes of mortality worldwide. Our group focuses on developing innovative environmental and genomic surveillance systems to understand population health and monitor the spread of infectious diseases. Using molecular virology, genomic sequencing, and mathematical modeling, our research works advance wastewater-based surveillance to track zoonotic pathogens, analyze the evolution of genomic variants, and forecast infection transmission dynamics. By integrating these tools, we aim to create early-warning systems for viral outbreaks and uncover key insights into pathogenic agents’ transmission and public health.\\nCurrent projects in my lab include:\\nWastewater-based Epidemiology: Developing methodologies for detecting pathogenic agents in community wastewater, capturing infection trends over time and space, and integrating surveillance data with mathematical models to predict outbreak dynamics and track viral evolution.\\nViral Metagenomics and Discovery: Employing cutting-edge metagenomic sequencing to explore human viral diversity, with the ultimate goal of creating a comprehensive global map of human viruses.\\nMicrobiome and Medicine: Investigating the interactions between viral infections, antiviral treatments, and the gut microbiome. Questions include how antibiotics affect gut microbiota and how everyday medications alter the microbiome.\\nStudents participating in tutorials/research in our lab will gain hands-on experience in molecular biology, genomic sequencing, bioinformatics, and mathematical modeling.\\nPubMed\\nMcGovern Medical School Faculty\\nEducation & Training\\nPh.D. - University of Arizona, 2017\\nResearch Opportunities\\nSeeking MS Students\\nSeeking PhD Students\\nPrograms\\nMicrobiology and Infectious Diseases\\nFaculty Development\\nMentoring Works\\nBack\\n\\nPotentially related faculty:\\nHeather Essigmann\\n\\n        website: https://sph.uth.edu/faculty/?fac=cl6ZUp8M430eVKolh7L3tysX2yOqWaMt/aVHwKhTm/g=\\n\\n        << Back\\nHeather Essigmann\\nAssistant Professor Non-Tenure Instruction\\nEpidemiology\\n713/500-9280\\nReuel Stallones Building\\n1200 Pressler Street, Houston, TX 77030\\nView CV\\n\\nPotentially related faculty:\\nPaul  Scheet\\n\\n        website: https://gsbs.uth.edu/directory/profile?id=e2296b7d-017f-42c3-a45d-62339c0f2668\\n\\n        Paul Scheet\\nRegular Member\\nProfessor\\n713.745.2470\\n713.745.2470\\n[email\\xa0protected]\\nMDA CPB4.3301 (Unit 1340)\\nThe University of Texas\\xa0MD Anderson Cancer Center\\nDepartment of Epidemiology\\nMy main research involves the development of statistical methods for the analysis of population genetic data. Specifically, I work with models for linkage disequilibrium (LD), and develop applications for taking proper account of the dependence among nearby loci. These applications include haplotype inference, missing genotype imputation, disease mapping, genotype error detection and correction, the visualization of haplotype variation, and the integration of emerging data types such as next generation DNA sequencing. For these applications I maintain a widely-used software package called fastPHASE.\\nA rotation in my lab could provide a student with exposure to several state-of-the-art haplotype-based software packages (such as PHASE, fastPHASE, Beagle, and Mach) and, in the process, could help evaluate different models for LD for various applications and settings. These models rely on hidden Markov model (HMM) techniques and implicitly a student would gain some introduction to these as well as algorithms, such as Baum-Welch, Markov chain Monte Carlo and Expectation-Maximization. In addition, the student could perform hands-on data analyses and would gain experience in working with Perl and the statistical software R. A student with strong C programming skills could develop, test, and add a function for analysis to an existing software package.\\nThe Department of Epidemiology has space for students and postdocs and provides a rich academic atmosphere with a large number of faculty in the section of Computational Genetic Epidemiology. Ample computational support exists, including processors which support programs for genetic analysis, as well as standard programming and database management languages.\\nPubMed\\nMDACC Faculty\\nEducation & Training\\nPh.D. - University of Washington - 2006\\nPrograms\\nGenetics and Epigenetics\\nQuantitative Sciences\\nFaculty Development\\nMentoring Works\\nBack\\n\\nPotentially related faculty:\\nCharles Darkoh\\n\\n        website: https://sph.uth.edu/faculty/?fac=0Oe7NOsP%2Bwqxzm8fNMAyFA==\\n\\n        << Back\\nCharles Darkoh\\nProfessor and R. Palmer Beasley, M.D. & Lu-Yu Hwang, M.D. Chair\\nEpidemiology\\n713/500-9368\\nReuel Stallones Building\\n1200 Pressler Street, Houston, TX 77030\\nView CV\\nAbout\\nMy laboratory focuses on the epidemiology, pathogenesis, and molecular basis of enteric infectious diseases with the goal to identify and understand mechanisms of action of novel bacterial virulence factors, pathways, and unique microbial products that can be harnessed for diagnostics and therapeutics. My long-term research interests include investigating the role of small molecules and other metabolites produced by the gut microbiota in the pathogenesis of enteric infectious diseases, mechanisms of host-pathogen interactions, antimicrobial resistance, host genetics and susceptibility to bacterial infections, and global health. I have active lines of research on Clostridium (Clostridioides) difficile infections, drug discovery, diagnostics, antimicrobial resistance, and global health.\\nCenter Affiliation\\nCenter for Infectious Diseases\\nResearch Interests\\nEpidemiology\\nGenetics and -omics\\nGlobal Health\\nInfectious Disease\\n\\nPotentially related faculty:\\nJoseph McCormick\\n\\n        website: https://sph.uth.edu/faculty/?fac=b0beWb7l7z1FViVZQPFKTm1SaQ3glg/R65TV2Px4lDs=\\n\\n        << Back\\nJoseph McCormick\\nProfessor & James H. Steele, D.V.M. Professorship\\nEpidemiology\\n956/755-0605\\nRegional Academic Health Center\\nOne West University Blvd, SPH Building, S1.330, Brownsville, TX 78520\\nView CV\\nAbout\\nRaised on a farm in Indiana, I graduated from Florida Southern College in chemistry and mathematics and attended the Free University in Brussels in preparation for teaching sciences and mathematics in French in the Congo where I was also introduced to medicine. I obtained my MD at Duke in 1971 and an MS from Harvard (1970) under Dr. Weller (Nobel Laureate 1954).  I trained at CHOP under Dr. Koop (later US Surgeon General).  I joined the CDC EIS and worked in Brazil as a PAHO/CDC consultant for the meningitis outbreaks of 1974-6.  In 1977 I founded the CDC Lassa fever project in Sierra Leone. I investigated the first Ebola epidemic in 1976 and led the one in 1979.   In Sierra Leone I conducted definitive studies of the epidemiology and antiviral treatment of Lassa fever.  As Chief, Special Pathogens Branch, Viral Diseases, CDC in 1982, I ran the Biosafety level 4 lab for 9 years.  I organized and led the first AIDS investigation in Africa in 1983 and established the Project SIDA in Kinshasa, Congo.  In 1983 I isolated the virus that causes Hemorrhagic Fever with Renal Syndrome (Hantavirus) at CDC.\\nIn 1993, I became Chairman, CHS at the Aga Khan University Medical School where I established a CDC EIS like program, an MS in Epidemiology, and a family Medicine Residency.  I founded the UT Houston School of Public Health campus in Brownsville in 2001. The campus has published over300 peer reviewed articles characterizing the extraordinary level of health disparities in the region. I was instrumental in bringing in over $100 million in external funding.\\nI now research chronic and infectious diseases in the LRGV in a cohort of 5000 people with over 30 research collaborators. I am featured in several books for the lay reader (e.g.,The Coming Plague, The Hot Zone). My wife, Sue Fisher-Hoch and I wrote a popular account (Level 4, Virus Hunters of the CDC) of our professional adventures translated into 7 languages. I am an amateur pianist, and enjoy many outdoor activities.\\nCenter Affiliation\\nHispanic Health Research Center\\nResearch Interests\\nBehavioral/Mental Health\\nBorder Health\\nCancer\\nCardiovascular and Chronic Diseases\\nClinical Trials\\nEpidemiology\\nGlobal Health\\nInfectious Disease\\nPersonal Links\\nhttp://www.ncbi.nlm.nih.gov/myncbi/browse/collection/40492151/?sort=date&direction=ascending\\nhttps://en.wikipedia.org/wiki/Joseph_B._McCormick\\nhttps://scholar.google.com/citations?user=xgV24roAAAAJ\\nhttps://www.pbs.org/wgbh/pages/frontline/aids/interviews/mccormick.html\\n\\nPotentially related faculty:\\nMicaela Sandoval\\n\\n        website: https://sph.uth.edu/faculty/?fac=7easTM9ILRE6auEurA/AUCTpx%2BnpUB%2BOCR2rrOuQ6tg=\\n\\n        << Back\\nMicaela Sandoval\\nAssistant Professor Non-Tenure Research\\nEpidemiology\\n713/500-9000\\nReuel Stallones Building\\n1200 Pressler Street, Houston, TX 77030\\nView CV\\nCenter Affiliation\\nCenter for Infectious Diseases\\n\\n'}"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def messages_for(description, similars):\n",
+    "    user_prompt = f\"Here is my description: {description}\\n\\n\"\n",
+    "    user_prompt += make_context(similars)\n",
+    "    return {\"role\": \"user\", \"content\": user_prompt}\n",
+    "messages_for(\"I am looking for a faculty member who is an expert in epidemiology.\", find_similars(\"I am looking for a faculty member who is an expert in epidemiology.\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "0c9e6350",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# environment\n",
+    "\n",
+    "load_dotenv(override=True)\n",
+    "os.environ['OPENAI_API_KEY'] = os.getenv('OPENAI_API_KEY', 'your-key-if-not-using-env')\n",
+    "os.environ['HF_TOKEN'] = os.getenv('HF_TOKEN', 'your-key-if-not-using-env')\n",
+    "\n",
+    "openai = OpenAI()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "cd319467",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<generator object gpt_4o_mini_rag at 0x1bdd28dc0>\n"
+     ]
+    }
+   ],
+   "source": [
+    "# print in markdown format\n",
+    "from IPython.display import Markdown, display\n",
+    "def gpt_4o_mini_rag(description, history):\n",
+    "\n",
+    "    system_message = \"You are a academic advisor. You estimate the relevance of faculty members to a given description. Suggest relevant faculty members. Don't forget to include a link to the faculty member's profile. You should give explanation for your choice in markdown format.\"\n",
+    "    system_message = {\"role\": \"system\", \"content\": system_message}\n",
+    "    similars = find_similars(description=description)\n",
+    "    messages = messages_for(description, similars)\n",
+    "    messages = [system_message] + history + [messages]\n",
+    "    \n",
+    "    stream = openai.chat.completions.create(\n",
+    "        model=\"gpt-4o-mini\", \n",
+    "        messages=messages,\n",
+    "        seed=42,\n",
+    "        stream=True\n",
+    "    )\n",
+    "    \n",
+    "    response = \"\"\n",
+    "    for chunk in stream:\n",
+    "        response += chunk.choices[0].delta.content or ''\n",
+    "        yield response\n",
+    "\n",
+    "\n",
+    "\n",
+    "def display_markdown_response(description):\n",
+    "    response = gpt_4o_mini_rag(description, [])\n",
+    "    print(response)\n",
+    "    # display(Markdown(response))\n",
+    "\n",
+    "display_markdown_response(\"I am keen on clinical trial. Can you give me faculty members who are experts in develop statistical methods for clinical trial?\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "05423e60",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/guchunhui/Documents/sandbox/data_generator/.venv/lib/python3.11/site-packages/gradio/blocks.py:1153: UserWarning: Cannot load gradio/monochrome. Caught Exception: 401 Client Error: Unauthorized for url: https://huggingface.co/api/spaces/gradio/monochrome (Request ID: Root=1-68329c92-678e41f1740e159154584e10;6a619f00-c33e-4633-931d-eab62cf234bc)\n",
+      "\n",
+      "Invalid credentials in Authorization header\n",
+      "  warnings.warn(f\"Cannot load {theme}. Caught Exception: {str(e)}\")\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "* Running on local URL:  http://127.0.0.1:7864\n",
+      "* To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7864/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": []
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import gradio as gr\n",
+    "MODEL = \"gpt-4o-mini\"\n",
+    "\n",
+    "\n",
+    "\n",
+    "gr.ChatInterface(fn=gpt_4o_mini_rag, \n",
+    "                  title=\"Faculty Advisor Chat\",\n",
+    "                    description=\"Ask about faculty members and their expertise\",\n",
+    "                    theme='earneleh/paris',\n",
+    "                    examples=[\n",
+    "                        \"I am looking for a faculty member who is an expert in epidemiology\",\n",
+    "                        \"Can you recommend someone who works on clinical trials?\",\n",
+    "                        \"Who specializes in machine learning?\"\n",
+    "    ],\n",
+    "                 type=\"messages\").launch()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

src/{GSBS_faculty_scraper.py → scraper/GSBS_faculty_scraper.py}

@@ -6,7 +6,7 @@ from typing_extensions import override
 import time
 from urllib.parse import urljoin
 from summarize import summarize
-from faculty_scraper import FacultyScraper
+from src.scraper.faculty_scraper import FacultyScraper
 
 class GSBSFacultyScraper(FacultyScraper):
     """A class to scrape faculty information from GSBS websites."""
@@ -208,7 +208,6 @@ class GSBSFacultyScraper(FacultyScraper):
         
         print(f"Completed scraping {page_num} page(s), total faculty: {len(faculty_list)}")
         self.faculty_list = faculty_list
-        return faculty_list
 
 
 def main():

src/{faculty_scraper.py → scraper/faculty_scraper.py}

@@ -1,8 +1,10 @@
+import os
+import time
 import requests
 from bs4 import BeautifulSoup
 from typing import List, Dict, Optional
 from urllib.parse import urljoin
-import profile_scraper
+import src.scraper.profile_scraper as profile_scraper
 from summarize import summarize
 import json
 
@@ -28,6 +30,10 @@ class FacultyScraper:
         if not self.faculty_list:
             print("No faculty data to save. Please scrape data first.")
             return
+        
+                # remove the file if it exists
+        if os.path.exists(output_file):
+            os.remove(output_file)
             
         with open(output_file, 'w', encoding='utf-8') as f:
             for faculty_member in self.faculty_list:
@@ -80,7 +86,6 @@ class FacultyScraper:
             print(f"Processed {i+1}/{len(faculty_divs)} faculty members")
         
         self.faculty_list = faculty_list
-        return faculty_list
             
     def _get_faculty_profile(self, profile_url: str) -> str:
         """
@@ -104,11 +109,6 @@ class FacultyScraper:
         '''
         pass
 
-    def scrape_faculty_list(self) -> List[Dict]:
-        '''
-        Scrape the faculty list from the faculty divs
-        '''
-        pass
 
     def _extract_image_url(self, faculty_div) -> Optional[str]:
         '''

src/{sbmi_faculty_scraper.py → scraper/sbmi_faculty_scraper.py}

@@ -6,7 +6,7 @@ from typing_extensions import override
 import time
 from urllib.parse import urljoin
 from summarize import summarize
-from faculty_scraper import FacultyScraper
+from src.scraper.faculty_scraper import FacultyScraper
 
 class SBMIFacultyScraper(FacultyScraper):
     """A class to scrape faculty information from SBMI websites."""
@@ -31,30 +31,8 @@ class SBMIFacultyScraper(FacultyScraper):
         Returns:
             List: List of faculty divs
         """
-        # Try specific SBMI patterns first, then fallback to common patterns
-        faculty_divs = (
-            # SBMI-specific selectors
-            self.soup.find_all('div', class_='faculty-listing-item') or
-            self.soup.find_all('div', class_='faculty-member-card') or
-            self.soup.find_all('article', class_='faculty-profile') or
-            
-            # UTHealth common patterns
-            self.soup.find_all('div', class_='people-card') or
-            self.soup.find_all('div', class_='person-profile') or
-            
-            # Generic patterns
-            self.soup.find_all('div', class_='faculty-card') or
-            self.soup.find_all('div', class_='faculty-member') or
-            self.soup.find_all('div', class_='person-card') or
-            self.soup.find_all('div', class_='cell') or
-            self.soup.find_all('article', class_='faculty') or
-            self.soup.find_all('div', class_='directory-entry') or
-            
-            # Look for any div containing faculty-related links
-            self.soup.find_all('div', lambda value: value and any(
-                keyword in str(value).lower() for keyword in ['faculty', 'people', 'staff', 'profile']
-            ))
-        )
+        # Look for divs with onclick attributes that contain faculty URLs
+        faculty_divs = self.soup.find_all('div', attrs={'onclick': lambda x: x and 'window.location' in x and 'faculty-and-staff' in x})
         
         # Filter out empty or invalid divs
         return [div for div in faculty_divs if div and div.get_text().strip()]
@@ -70,30 +48,14 @@ class SBMIFacultyScraper(FacultyScraper):
         Returns:
             Optional[str]: Faculty name if found, None otherwise
         """
-        # Try different common selectors for names
-        name_selectors = [
-            'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
-            '.faculty-name', '.name', '.person-name', '.full-name',
-            '.title', '.entry-title', '.profile-name',
-            'strong', 'b', '.font-weight-bold', '.fw-bold'
-        ]
-        
-        for selector in name_selectors:
-            name_element = faculty_div.select_one(selector)
-            if name_element:
-                name = name_element.get_text().strip()
-                # Clean up common title patterns and suffixes
-                name = self._clean_name(name)
-                if name and len(name) > 2 and ' ' in name:  # Ensure it's a full name
-                    return name
-        
-        # If no name found in structured elements, look for links with names
-        links = faculty_div.find_all('a')
-        for link in links:
-            link_text = link.get_text().strip()
-            cleaned_name = self._clean_name(link_text)
-            if cleaned_name and len(cleaned_name) > 2 and ' ' in cleaned_name:
-                return cleaned_name
+        # Look for the name in the fac-nam span
+        name_element = faculty_div.select_one('.fac-nam strong')
+        if name_element:
+            name = name_element.get_text().strip()
+            # Clean up common title patterns and suffixes
+            name = self._clean_name(name)
+            if name and len(name) > 2 and ' ' in name:  # Ensure it's a full name
+                return name
                 
         return None
     
@@ -120,118 +82,89 @@ class SBMIFacultyScraper(FacultyScraper):
         
         return name.strip()
     
-    @override
-    def _extract_profile_url(self, faculty_div) -> Optional[str]:
+    def _extract_areas_of_expertise(self, faculty_div) -> List[str]:
         """
-        Extract the profile URL from the faculty div.
+        Extract areas of expertise from the faculty div.
         
         Args:
             faculty_div: BeautifulSoup element containing faculty info
             
         Returns:
-            Optional[str]: Profile URL if found, None otherwise
+            List[str]: List of areas of expertise
         """
-        # Look for various types of links that might lead to faculty profiles
-        link_selectors = [
-            'a[href*="faculty/"]',
-            'a[href*="/profile/"]',
-            'a[href*="/people/"]',
-            'a[href*="/staff/"]',
-            'a[href*="sbmi.uth.edu"]',
-            '.profile-link a',
-            '.faculty-link a',
-            'a'  # fallback to any link
-        ]
+        areas = []
+        # Look for text after "Areas of Expertise" and before "click for full bio"
+        text_content = faculty_div.get_text()
+        if "Areas of Expertise" in text_content:
+            expertise_section = text_content.split("Areas of Expertise")[1].split("click for full bio")[0]
+            # Split by "»" and clean up each area
+            for line in expertise_section.split("»"):
+                area = line.strip()
+                if area and not area.startswith("click for full bio"):
+                    areas.append(area.strip())
+        return areas
+    
+    @override
+    def _extract_position(self, faculty_div) -> Optional[str]:
+        """
+        Extract faculty position/title.
         
-        for selector in link_selectors:
-            links = faculty_div.select(selector)
-            for link in links:
-                href = link.get('href')
-                if href and not href.startswith(('mailto:', 'tel:', '#', 'javascript:')):
-                    # Prefer links that look like faculty profiles
-                    if any(keyword in href.lower() for keyword in ['faculty', 'profile', 'people']):
-                        return self.get_absolute_url(self.base_url, href)
+        Args:
+            faculty_div: BeautifulSoup element containing faculty info
             
-            # If we found any valid link, use the first one as fallback
-            if links and links[0].get('href'):
-                href = links[0]['href']
-                if not href.startswith(('mailto:', 'tel:', '#', 'javascript:')):
-                    return self.get_absolute_url(self.base_url, href)
-        
+        Returns:
+            Optional[str]: Position if found, None otherwise
+        """
+        # Look for the position in the em tag after the name
+        position_element = faculty_div.select_one('.fac-nam + em')
+        if position_element:
+            return position_element.get_text().strip()
+                
         return None
     
     @override
-    def _extract_image_url(self, faculty_div) -> Optional[str]:
+    def _extract_profile_url(self, faculty_div) -> Optional[str]:
         """
-        Extract faculty image URL.
+        Extract the profile URL from the faculty div.
         
         Args:
             faculty_div: BeautifulSoup element containing faculty info
             
         Returns:
-            Optional[str]: Image URL if found, None otherwise
+            Optional[str]: Profile URL if found, None otherwise
         """
-        # Try different common patterns for faculty images
-        img_tag = faculty_div.find('img')
-        if img_tag:
-            src = img_tag.get('src') or img_tag.get('data-src')
-            if src and not any(placeholder in src.lower() for placeholder in 
-                             ['placeholder', 'default', 'no-image', 'blank', 'missing']):
-                return self.get_absolute_url(self.base_url, src)
-        
-        # Check for background images in style attributes
-        elements_with_style = faculty_div.find_all(attrs={'style': True})
-        for element in elements_with_style:
-            style = element.get('style', '')
-            if 'background-image' in style and 'url(' in style:
-                start = style.find('url(') + 4
-                end = style.find(')', start)
-                if start != -1 and end != -1:
-                    img_url = style[start:end].strip('\'"')
-                    if img_url and not img_url.startswith('data:'):
-                        return self.get_absolute_url(self.base_url, img_url)
+        # Extract URL from onclick attribute
+        onclick = faculty_div.get('onclick', '')
+        if 'window.location' in onclick:
+            # Extract URL between quotes
+            start = onclick.find("'") + 1
+            end = onclick.find("'", start)
+            if start != -1 and end != -1:
+                url = onclick[start:end]
+                if url and not url.startswith(('mailto:', 'tel:', '#', 'javascript:')):
+                    return url
         
         return None
     
-    def _extract_position(self, faculty_div) -> Optional[str]:
+    @override
+    def _extract_image_url(self, faculty_div) -> Optional[str]:
         """
-        Extract faculty position/title.
+        Extract faculty image URL.
         
         Args:
             faculty_div: BeautifulSoup element containing faculty info
             
         Returns:
-            Optional[str]: Position if found, None otherwise
+            Optional[str]: Image URL if found, None otherwise
         """
-        # Try different selectors for position/title
-        position_selectors = [
-            '.position', '.title', '.role', '.job-title',
-            '.faculty-title', '.academic-title', '.professional-title',
-            '.department', '.affiliation', '.faculty-position',
-            'p.title', 'span.title', 'div.title',
-            '.faculty-info p', '.profile-info p'
-        ]
-        
-        for selector in position_selectors:
-            elements = faculty_div.select(selector)
-            for element in elements:
-                text = element.get_text().strip()
-                # Filter out obvious non-position text
-                if (text and len(text) > 3 and 
-                    not any(skip in text.lower() for skip in 
-                           ['email', 'phone', 'office', 'room', '@', 'tel:', 'mailto:'])):
-                    return text
-        
-        # Look for text patterns that might be positions
-        text_content = faculty_div.get_text()
-        lines = [line.strip() for line in text_content.split('\n') if line.strip()]
+        # Look for img tag within the photo div
+        img_tag = faculty_div.select_one('.photo img')
+        if img_tag:
+            src = img_tag.get('src')
+            if src and not any(placeholder in src.lower() for placeholder in 
+                             ['placeholder', 'default', 'no-image', 'blank', 'missing']):
+                return self.get_absolute_url(self.base_url, src)
         
-        for line in lines[1:4]:  # Skip first line (likely name), check next few
-            if (len(line) > 5 and 
-                any(keyword in line.lower() for keyword in 
-                   ['professor', 'director', 'chair', 'faculty', 'researcher', 'scientist'])):
-                return line
-                
         return None
     
     @override
@@ -284,6 +217,11 @@ class SBMIFacultyScraper(FacultyScraper):
             if position:
                 faculty_info['position'] = position
 
+            # Extract areas of expertise
+            areas = self._extract_areas_of_expertise(faculty_div)
+            if areas:
+                faculty_info['areas_of_expertise'] = areas
+
             # Extract profile URL and get profile summary
             profile_url = self._extract_profile_url(faculty_div)
             if profile_url:
@@ -312,11 +250,11 @@ def main():
     base_url = 'https://sbmi.uth.edu/faculty-and-staff/'
     
     # Create scraper instance and run
-    scraper = SBMIFacultyScraper(base_url, debug=True)
+    scraper = SBMIFacultyScraper(base_url, debug=False)
     faculty_list = scraper.scrape_faculty_list()
-    
+    file_name = 'data/sbmi_faculty_list.jsonl'
     # Save the faculty list as jsonl file
-    scraper.save_to_jsonl('data/sbmi_faculty_list.jsonl')
+    scraper.save_to_jsonl(file_name)
 
 
 if __name__ == "__main__":

src/{sph_faculty_scraper.py → scraper/sph_faculty_scraper.py}

@@ -6,7 +6,7 @@ from typing_extensions import override
 import time
 from urllib.parse import urljoin
 from summarize import summarize
-from faculty_scraper import FacultyScraper
+from src.scraper.faculty_scraper import FacultyScraper
 
 class SPHFacultyScraper(FacultyScraper):
     """A class to scrape faculty information from SPH websites."""
@@ -98,12 +98,9 @@ def main():
     base_url = 'https://sph.uth.edu/faculty/'
     
     # Create scraper instance and run
-    scraper = SPHFacultyScraper(base_url, debug=True)
-    faculty_list = scraper.scrape_faculty_list()
-    # dump the faculty list as jsonl file
-    with open('data/faculty_list.jsonl', 'w') as f:
-        for faculty in faculty_list:
-            f.write(json.dumps(faculty) + '\n')
+    scraper = SPHFacultyScraper(base_url, debug=False)
+    scraper.scrape_faculty_list()
+    scraper.save_to_jsonl('data/sph_faculty_list.jsonl')
 
 
 if __name__ == "__main__":

src/summarize.py

@@ -6,7 +6,7 @@ from dotenv import load_dotenv
 from bs4 import BeautifulSoup
 from IPython.display import Markdown, display
 from openai import OpenAI
-import profile_scraper as profile_scraper
+import src.scraper.profile_scraper as profile_scraper
 # Load environment variables in a file called .env
 
 load_dotenv(override=True)
@@ -28,18 +28,15 @@ else:
 
 # Define our system prompt - you can experiment with this later, changing the last sentence to 'Respond in markdown in Spanish."
 
-system_prompt = "You are[ an assistant that analyzes the contents of a website \
-and provides a short summary, ignoring text that might be navigation related. \
-Respond in markdown."
+system_prompt = '''You are an assistant helping to remove unrelated html elements and put everything in a single text file'''
 
 # A function that writes a User Prompt that asks for summaries of websites:
 
 def user_prompt_for(website):
-    user_prompt = f"You are looking at a website titled {website.title}"
+    user_prompt = f"You are looking at a faculty member's website"
     user_prompt += "\nThe contents of this website is as follows; \
-please provide a short summary of this website in markdown. \
-If it includes news or announcements, then summarize these too. And tell me whether the device has to be bought in US.\n\n"
-    user_prompt += website.text
+please provide a processed text file\n"
+    user_prompt += website
     return user_prompt
 
 # See how this function creates exactly the format above
@@ -50,12 +47,20 @@ def messages_for(website):
         {"role": "user", "content": user_prompt_for(website)}
     ]
 
+def remove_prefix_and_suffix(text):
+    # only keep text between ``` and ```
+    text = text.split("```")[1]
+    return text
+
 # And now: call the OpenAI API. You will get very familiar with this!
 
-def summarize(url):
-    website = profile_scraper.FacultyProfileScraper(url)
+def summarize(website):
     response = openai.chat.completions.create(
         model = "gpt-4o-mini",
         messages = messages_for(website)
     )
-    return response.choices[0].message.content
+
+
+    return response.choices[0].message.content
+
+