{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "537645d1", "metadata": {}, "outputs": [], "source": [ "!pip install -q requests torch bitsandbytes transformers sentencepiece accelerate gradio httpx==0.27.2" ] }, { "cell_type": "code", "execution_count": null, "id": "49b928f5", "metadata": {}, "outputs": [], "source": [ "from huggingface_hub import login\n", "from google.colab import userdata\n", "import torch\n", "import gradio as gr\n", "import re\n", "from transformers import (\n", " AutoTokenizer,\n", " AutoModelForCausalLM,\n", " AutoModelForSpeechSeq2Seq,\n", " AutoProcessor,\n", " pipeline,\n", " BitsAndBytesConfig\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "4e332f0e", "metadata": {}, "outputs": [], "source": [ "hf_token = userdata.get('HF_TOKEN')\n", "login(hf_token, add_to_git_credential=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "f752fe77", "metadata": {}, "outputs": [], "source": [ "class CustomTextStreamer:\n", " \"\"\"Custom text streamer to handle generated text from the model\"\"\"\n", " def __init__(self, tokenizer, skip_prompt=True, skip_special_tokens=True):\n", " self.tokenizer = tokenizer\n", " self.skip_prompt = skip_prompt\n", " self.skip_special_tokens = skip_special_tokens\n", " self.buffer = \"\"\n", " self.started = False\n", "\n", " def put(self, value):\n", " # Handle both tensor and list inputs\n", " if isinstance(value, torch.Tensor):\n", " value = value.tolist()\n", "\n", " # Make sure value is a list of lists (batched tokens)\n", " if not isinstance(value[0], list):\n", " value = [value]\n", "\n", " for token_ids in value:\n", " text = self.tokenizer.decode(token_ids, skip_special_tokens=self.skip_special_tokens)\n", " if not self.started and '# ' in text:\n", " self.started = True\n", " text = text[text.index('# '):]\n", " self.buffer += text\n", "\n", " def end(self):\n", " return self.buffer" ] }, { "cell_type": "code", "execution_count": null, "id": "fcb3d8c5", "metadata": {}, "outputs": [], "source": [ "def transcribe_audio(audio_path):\n", " \"\"\"Transcribe audio using Whisper model\"\"\"\n", " torch.cuda.empty_cache()\n", " AUDIO_MODEL = \"openai/whisper-large-v3\"\n", " device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", "\n", " # Load model and processor\n", " speech_model = AutoModelForSpeechSeq2Seq.from_pretrained(\n", " AUDIO_MODEL,\n", " torch_dtype=torch.float16 if device == \"cuda\" else torch.float32,\n", " low_cpu_mem_usage=True,\n", " use_safetensors=True,\n", " device_map=\"auto\"\n", " )\n", "\n", " processor = AutoProcessor.from_pretrained(AUDIO_MODEL)\n", "\n", " # Create ASR pipeline\n", " pipe = pipeline(\n", " \"automatic-speech-recognition\",\n", " model=speech_model,\n", " tokenizer=processor.tokenizer,\n", " feature_extractor=processor.feature_extractor,\n", " torch_dtype=torch.float16 if device == \"cuda\" else torch.float32,\n", " model_kwargs={\"use_flash_attention_2\": device == \"cuda\"}\n", " )\n", "\n", " # Handle different input formats\n", " actual_path = audio_path['path'] if isinstance(audio_path, dict) else audio_path\n", "\n", " # Transcribe the audio\n", " result = pipe(actual_path, return_timestamps=True)\n", " return result['text']\n" ] }, { "cell_type": "code", "execution_count": null, "id": "bd1e45ac", "metadata": {}, "outputs": [], "source": [ "def clean_notes(raw_notes):\n", " \"\"\"Clean up generated notes to follow the correct format\"\"\"\n", " # Keep everything starting from \"# Topic\"\n", " if '# Topic' in raw_notes:\n", " cleaned = raw_notes[raw_notes.index('# Topic'):]\n", " else:\n", " cleaned = raw_notes\n", "\n", " # Remove any special tokens and formatting\n", " cleaned = re.sub(r'<\\|.*?\\|>', '', cleaned)\n", " cleaned = re.sub(r'(system|user|assistant):', '', cleaned, flags=re.IGNORECASE)\n", " cleaned = re.sub(r'Today Date:.*?\\n', '', cleaned)\n", "\n", " # Normalize markdown headers\n", " cleaned = cleaned.strip()\n", " cleaned = re.sub(r'^#\\s+', '# ', cleaned, flags=re.MULTILINE)\n", " cleaned = re.sub(r'^##\\s+', '## ', cleaned, flags=re.MULTILINE)\n", "\n", " return cleaned" ] }, { "cell_type": "code", "execution_count": null, "id": "d7b84da7", "metadata": {}, "outputs": [], "source": [ "`def generate_notes(transcript):\n", " \"\"\"Generate lecture notes from transcript using LLM\"\"\"\n", " LLAMA = \"meta-llama/Meta-Llama-3.1-8B-Instruct\"\n", " system_prompt = (\n", " \"Generate ONLY the lecture notes in markdown format with EXACTLY these sections:\\n\"\n", " \"# Topic\\n\"\n", " \"## Instructor\\n\"\n", " \"## Summary\\n\"\n", " \"## Key Concepts (bullet points)\\n\"\n", " \"## Key Points (bullet points)\\n\"\n", " \"## Takeaways (bullet points)\\n\\n\"\n", " \"Do NOT include:\\n\"\n", " \"- Any introductory text\\n\"\n", " \"- The transcript\\n\"\n", " \"- Any explanation about the format\\n\"\n", " \"- Any text outside the specified sections\\n\"\n", " \"The first line of your response MUST be '# Topic: [actual topic name]'\"\n", " )\n", "\n", " user_prompt = f\"Please generate comprehensive notes from this lecture transcript:\\n\\n{transcript}\"\n", "\n", " messages = [\n", " {\"role\": \"system\", \"content\": system_prompt},\n", " {\"role\": \"user\", \"content\": user_prompt}\n", " ]\n", "\n", " # Configure model loading with quantization\n", " quant_config = BitsAndBytesConfig(\n", " load_in_4bit=True,\n", " bnb_4bit_use_double_quant=True,\n", " bnb_4bit_compute_dtype=torch.bfloat16,\n", " bnb_4bit_quant_type=\"nf4\"\n", " )\n", "\n", " # Load tokenizer and model\n", " tokenizer = AutoTokenizer.from_pretrained(LLAMA)\n", " if tokenizer.pad_token is None:\n", " tokenizer.pad_token = tokenizer.eos_token\n", "\n", " model = AutoModelForCausalLM.from_pretrained(\n", " LLAMA,\n", " device_map='auto',\n", " quantization_config=quant_config,\n", " torch_dtype=torch.float16,\n", " )\n", "\n", " # Prepare input\n", " inputs = tokenizer.apply_chat_template(messages, return_tensors=\"pt\")\n", " device = 'cuda' if torch.cuda.is_available() else 'cpu'\n", " input_ids = inputs.to(device)\n", "\n", " # Generate text\n", " with torch.no_grad():\n", " try:\n", " context_manager = torch.backends.cuda.sdp_kernel(enable_flash=True) if torch.cuda.is_available() else contextlib.nullcontext()\n", " with context_manager:\n", " output_ids = model.generate(\n", " input_ids=input_ids,\n", " max_new_tokens=2000,\n", " do_sample=True,\n", " temperature=0.7,\n", " top_p=0.9,\n", " eos_token_id=tokenizer.eos_token_id,\n", " pad_token_id=tokenizer.pad_token_id,\n", " )\n", "\n", " # Process the output directly\n", " generated_text = tokenizer.decode(output_ids[0][input_ids.shape[1]:], skip_special_tokens=True)\n", "\n", " # Find and extract the notes\n", " if '# Topic' in generated_text:\n", " generated_text = generated_text[generated_text.index('# Topic'):]\n", " except Exception as e:\n", " print(f\"Generation error with flash attention: {e}\")\n", " # Fallback without flash attention\n", " output_ids = model.generate(\n", " input_ids=input_ids,\n", " max_new_tokens=2000,\n", " do_sample=True,\n", " temperature=0.7,\n", " top_p=0.9,\n", " eos_token_id=tokenizer.eos_token_id,\n", " pad_token_id=tokenizer.pad_token_id,\n", " )\n", "\n", " # Process the output directly\n", " generated_text = tokenizer.decode(output_ids[0][input_ids.shape[1]:], skip_special_tokens=True)\n", "\n", " # Find and extract the notes\n", " if '# Topic' in generated_text:\n", " generated_text = generated_text[generated_text.index('# Topic'):]\n", "\n", " # Clean up notes and ensure topic has a name\n", " notes = clean_notes(generated_text)\n", "\n", " # Check if the topic has no name and try to extract it from the transcript\n", " if notes.startswith(\"# Topic\\n\"):\n", " # Try to identify a topic from the transcript\n", " topic_candidates = [\"RAG\", \"Retrieval Augmented Generation\", \"LLM\", \"NLP\", \"Large Language Models\"]\n", " for candidate in topic_candidates:\n", " if candidate.lower() in transcript.lower():\n", " notes = notes.replace(\"# Topic\\n\", f\"# Topic: {candidate}\\n\")\n", " break\n", "\n", " # If no candidate found, use a generic topic\n", " if notes.startswith(\"# Topic\\n\"):\n", " notes = notes.replace(\"# Topic\\n\", \"# Topic: AI Lecture\\n\")\n", "\n", " return notes" ] }, { "cell_type": "code", "execution_count": null, "id": "fcd9cc3a", "metadata": {}, "outputs": [], "source": [ "def process_audio(audio_path, progress=gr.Progress()):\n", " \"\"\"Main function to process audio and generate notes\"\"\"\n", " try:\n", " progress(0.2, desc=\"Transcribing audio...\")\n", " transcript = transcribe_audio(audio_path)\n", " if not transcript:\n", " return \"Error: Could not transcribe audio\"\n", "\n", " progress(0.6, desc=\"Generating notes...\")\n", " notes = generate_notes(transcript)\n", " progress(1.0, desc=\"Done!\")\n", " return notes\n", " except Exception as e:\n", " import traceback\n", " return f\"Error: {str(e)}\\n\\n{traceback.format_exc()}\"" ] }, { "cell_type": "code", "execution_count": null, "id": "1f2e170d", "metadata": {}, "outputs": [], "source": [ "def create_ui():\n", " with gr.Blocks(theme=gr.themes.Soft()) as demo:\n", " gr.Markdown(\"\"\"\n", " # 📝 AI Lecture Notes Generator\n", " Upload a lecture audio file to generate clean, structured notes.\n", " \"\"\")\n", "\n", " with gr.Row():\n", " with gr.Column():\n", " audio_input = gr.Audio(\n", " type=\"filepath\",\n", " label=\"Upload Lecture Audio\",\n", " elem_classes=[\"audio-input\"]\n", " )\n", " btn = gr.Button(\"Generate Notes\", variant=\"primary\")\n", "\n", " with gr.Column():\n", " # Use Markdown component instead of Textbox for proper rendering\n", " notes_output = gr.Markdown(\n", " label=\"Generated Notes\",\n", " elem_classes=[\"notes-output\"]\n", " )\n", "\n", " btn.click(\n", " fn=process_audio,\n", " inputs=audio_input,\n", " outputs=notes_output,\n", " api_name=\"generate_notes\"\n", " )\n", "\n", " demo.css = \"\"\"\n", " .audio-input { min-height: 100px; }\n", " .notes-output {\n", " font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;\n", " background: #f000000;\n", " border-radius: 8px;\n", " padding: 15px;\n", " margin-top: 10px;\n", " }\n", " .notes-output h1 {\n", " font-size: 1.8em;\n", " margin-top: 0.5em;\n", " margin-bottom: 0.5em;\n", " }\n", " .notes-output h2 {\n", " font-size: 1.4em;\n", " margin-top: 1em;\n", " margin-bottom: 0.5em;\n", " }\n", " .notes-output ul {\n", " margin-left: 1.5em;\n", " }\n", " \"\"\"\n", " return demo" ] }, { "cell_type": "code", "execution_count": null, "id": "c2bb8aef", "metadata": {}, "outputs": [], "source": [ "if __name__ == \"__main__\":\n", " demo = create_ui()\n", " demo.launch(debug=True)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.7" } }, "nbformat": 4, "nbformat_minor": 5 }