Prathamesh Sarjerao Vaidya commited on
Commit
294cbe8
·
1 Parent(s): 5942c3e

made an update

Browse files
.github/workflows/check.yml CHANGED
@@ -1,4 +1,4 @@
1
- name: Check file size and sync to Google Drive
2
  on:
3
  pull_request:
4
  branches: [main]
@@ -8,105 +8,10 @@ jobs:
8
  check-file-size:
9
  runs-on: ubuntu-latest
10
  steps:
11
- - uses: actions/checkout@v3
12
  with:
13
  lfs: true
14
  - name: Check large files
15
  uses: ActionsDesk/lfs-warning@v2.0
16
  with:
17
  filesizelimit: 1073741824 # 1GB
18
-
19
- convert-and-sync:
20
- runs-on: ubuntu-latest
21
- needs: check-file-size
22
- if: github.event_name == 'pull_request'
23
- outputs:
24
- skip_pdf: ${{ steps.check_md_changes.outputs.skip_pdf }}
25
- steps:
26
- - uses: actions/checkout@v3
27
- with:
28
- lfs: true
29
- fetch-depth: 0 # Need full history for git diff
30
-
31
- - name: Pull LFS files
32
- run: |
33
- git lfs install
34
- git lfs pull
35
-
36
- # NEW STEP: Check if MD files were modified
37
- - name: Check for MD file changes
38
- id: check_md_changes
39
- run: |
40
- echo "Checking for markdown file changes..."
41
-
42
- # Get the commit message
43
- COMMIT_MSG="${{ github.event.head_commit.message }}"
44
- if [ -z "$COMMIT_MSG" ]; then
45
- COMMIT_MSG=$(git log -1 --pretty=%B)
46
- fi
47
- echo "Commit message: $COMMIT_MSG"
48
-
49
- # Check if commit message indicates MD changes
50
- MD_IN_COMMIT=$(echo "$COMMIT_MSG" | grep -i "\.md\|markdown\|documentation\|docs\|readme" || true)
51
-
52
- # Check if any MD files were actually modified in the diff
53
- if [ "${{ github.event_name }}" = "pull_request" ]; then
54
- MD_FILES_CHANGED=$(git diff --name-only origin/main...HEAD | grep "\.md$" || true)
55
- else
56
- MD_FILES_CHANGED=$(git diff --name-only HEAD~1 HEAD | grep "\.md$" || true)
57
- fi
58
-
59
- echo "MD files in commit message: $MD_IN_COMMIT"
60
- echo "MD files changed: $MD_FILES_CHANGED"
61
-
62
- # Skip PDF conversion if no MD files changed AND no MD-related keywords in commit
63
- if [ -z "$MD_FILES_CHANGED" ] && [ -z "$MD_IN_COMMIT" ]; then
64
- echo "skip_pdf=true" >> $GITHUB_OUTPUT
65
- echo "Skipping PDF conversion - no MD files modified"
66
- else
67
- echo "skip_pdf=false" >> $GITHUB_OUTPUT
68
- echo "MD files detected - will convert to PDF"
69
- echo "Changed MD files: $MD_FILES_CHANGED"
70
- fi
71
-
72
- - name: Setup Python
73
- uses: actions/setup-python@v4
74
- with:
75
- python-version: '3.11'
76
-
77
- # CONDITIONAL STEP: Only run if MD files changed
78
- - name: Setup system dependencies
79
- if: steps.check_md_changes.outputs.skip_pdf == 'false'
80
- run: |
81
- echo "Setting up system dependencies for PDF conversion..."
82
- chmod +x .github/workflows/scripts/setup_system.sh && .github/workflows/scripts/setup_system.sh
83
-
84
- # CONDITIONAL STEP: Only run if MD files changed
85
- - name: Convert MD to PDF
86
- if: steps.check_md_changes.outputs.skip_pdf == 'false'
87
- run: |
88
- echo "Converting MD files to PDF..."
89
- chmod +x .github/workflows/scripts/convert_md_to_pdf.sh && .github/workflows/scripts/convert_md_to_pdf.sh
90
-
91
- # CONDITIONAL STEP: Only run if MD files changed
92
- - name: Upload PDF artifacts
93
- if: steps.check_md_changes.outputs.skip_pdf == 'false'
94
- uses: actions/upload-artifact@v4
95
- with:
96
- name: converted-pdfs
97
- path: "**/*.pdf"
98
- retention-days: 30
99
-
100
- - name: Install Google API dependencies
101
- run: pip install google-auth google-auth-oauthlib google-auth-httplib2 google-api-python-client
102
-
103
- - name: Upload to Google Drive
104
- env:
105
- GOOGLE_OAUTH_TOKEN: ${{ secrets.GOOGLE_OAUTH_TOKEN }}
106
- run: |
107
- if [ "${{ steps.check_md_changes.outputs.skip_pdf }}" = "true" ]; then
108
- echo "Skipped PDF conversion - uploading existing files only"
109
- else
110
- echo "Uploading files including new PDFs to Google Drive"
111
- fi
112
- python .github/workflows/scripts/upload_to_drive.py
 
1
+ name: Check file size
2
  on:
3
  pull_request:
4
  branches: [main]
 
8
  check-file-size:
9
  runs-on: ubuntu-latest
10
  steps:
11
+ - uses: actions/checkout@v4
12
  with:
13
  lfs: true
14
  - name: Check large files
15
  uses: ActionsDesk/lfs-warning@v2.0
16
  with:
17
  filesizelimit: 1073741824 # 1GB
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
.github/workflows/main.yml CHANGED
@@ -1,14 +1,14 @@
1
- name: Sync to Hugging Face hub and Google Drive
2
  on:
3
  push:
4
  branches: [main]
5
  workflow_dispatch:
6
 
7
  jobs:
8
- sync-to-hub-and-drive:
9
  runs-on: ubuntu-latest
10
  steps:
11
- - uses: actions/checkout@v3
12
  with:
13
  fetch-depth: 0
14
  lfs: true
@@ -18,84 +18,10 @@ jobs:
18
  git lfs install
19
  git lfs pull
20
 
21
- # NEW STEP: Check for MD file changes
22
- - name: Check for MD file changes
23
- id: check_md_changes
24
- run: |
25
- echo "Checking for markdown file changes..."
26
-
27
- # Get the commit message
28
- COMMIT_MSG=$(git log -1 --pretty=%B)
29
- echo "Commit message: $COMMIT_MSG"
30
-
31
- # Check if commit message indicates MD changes
32
- MD_IN_COMMIT=$(echo "$COMMIT_MSG" | grep -i "\.md\|markdown\|documentation\|docs\|readme" || true)
33
-
34
- # Check if any MD files were actually modified in the last commit
35
- MD_FILES_CHANGED=$(git diff --name-only HEAD~1 HEAD | grep "\.md$" || true)
36
-
37
- echo "MD files in commit message: $MD_IN_COMMIT"
38
- echo "MD files changed: $MD_FILES_CHANGED"
39
-
40
- # Skip PDF conversion if no MD files changed AND no MD-related keywords in commit
41
- if [ -z "$MD_FILES_CHANGED" ] && [ -z "$MD_IN_COMMIT" ]; then
42
- echo "skip_pdf=true" >> $GITHUB_OUTPUT
43
- echo "Skipping PDF conversion - no MD files modified"
44
- else
45
- echo "skip_pdf=false" >> $GITHUB_OUTPUT
46
- echo "MD files detected - will convert to PDF"
47
- echo "Changed MD files: $MD_FILES_CHANGED"
48
- fi
49
-
50
- - name: Setup Python
51
- uses: actions/setup-python@v4
52
- with:
53
- python-version: '3.11'
54
-
55
- # CONDITIONAL STEP: Only run if MD files changed
56
- - name: Setup system dependencies
57
- if: steps.check_md_changes.outputs.skip_pdf == 'false'
58
- run: |
59
- echo "Setting up system dependencies for PDF conversion..."
60
- chmod +x .github/workflows/scripts/setup_system.sh && .github/workflows/scripts/setup_system.sh
61
-
62
- # CONDITIONAL STEP: Only run if MD files changed
63
- - name: Convert MD to PDF
64
- if: steps.check_md_changes.outputs.skip_pdf == 'false'
65
- run: |
66
- echo "Converting MD files to PDF..."
67
- chmod +x .github/workflows/scripts/convert_md_to_pdf.sh && .github/workflows/scripts/convert_md_to_pdf.sh
68
-
69
- # CONDITIONAL STEP: Only run if MD files changed
70
- - name: Upload PDF artifacts
71
- if: steps.check_md_changes.outputs.skip_pdf == 'false'
72
- uses: actions/upload-artifact@v4
73
- with:
74
- name: converted-pdfs
75
- path: "**/*.pdf"
76
- retention-days: 30
77
-
78
- - name: Install Google API dependencies
79
- run: pip install google-auth google-auth-oauthlib google-auth-httplib2 google-api-python-client
80
-
81
- - name: Upload to Google Drive
82
- env:
83
- # Primary authentication method (OAuth)
84
- GOOGLE_OAUTH_TOKEN: ${{ secrets.GOOGLE_OAUTH_TOKEN }}
85
- # Fallback authentication method (Service Account)
86
- GOOGLE_SERVICE_ACCOUNT_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
87
- run: |
88
- if [ "${{ steps.check_md_changes.outputs.skip_pdf }}" = "true" ]; then
89
- echo "Skipped PDF conversion - uploading existing files only"
90
- else
91
- echo "Uploading files including new PDFs to Google Drive"
92
- fi
93
- python .github/workflows/scripts/upload_to_drive.py
94
-
95
- - name: Push to Hugging Face hub
96
  env:
97
  HF_TOKEN: ${{ secrets.HF_TOKEN }}
98
  run: |
99
  git config user.name "github-actions"
100
  git config user.email "github-actions@github.com"
101
- git push --force https://prathameshv07:$HF_TOKEN@huggingface.co/spaces/prathameshv07/Multilingual-Audio-Intelligence-System main
 
1
+ name: Sync to Hugging Face Space
2
  on:
3
  push:
4
  branches: [main]
5
  workflow_dispatch:
6
 
7
  jobs:
8
+ sync-to-hub:
9
  runs-on: ubuntu-latest
10
  steps:
11
+ - uses: actions/checkout@v4
12
  with:
13
  fetch-depth: 0
14
  lfs: true
 
18
  git lfs install
19
  git lfs pull
20
 
21
+ - name: Push to Hugging Face Space
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  env:
23
  HF_TOKEN: ${{ secrets.HF_TOKEN }}
24
  run: |
25
  git config user.name "github-actions"
26
  git config user.email "github-actions@github.com"
27
+ git push --force https://prathameshv07:$HF_TOKEN@huggingface.co/spaces/prathameshv07/Multilingual-Audio-Intelligence-System main
.github/workflows/puppeteer-config.json DELETED
@@ -1,27 +0,0 @@
1
- {
2
- "args": [
3
- "--no-sandbox",
4
- "--disable-setuid-sandbox",
5
- "--disable-dev-shm-usage",
6
- "--disable-gpu",
7
- "--disable-web-security",
8
- "--disable-features=VizDisplayCompositor",
9
- "--run-all-compositor-stages-before-draw",
10
- "--disable-background-timer-throttling",
11
- "--disable-backgrounding-occluded-windows",
12
- "--disable-renderer-backgrounding",
13
- "--disable-field-trial-config",
14
- "--disable-ipc-flooding-protection",
15
- "--no-first-run",
16
- "--no-default-browser-check",
17
- "--disable-default-apps",
18
- "--disable-extensions",
19
- "--disable-plugins",
20
- "--disable-sync",
21
- "--disable-translate",
22
- "--hide-scrollbars",
23
- "--mute-audio",
24
- "--no-zygote",
25
- "--single-process"
26
- ]
27
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
.github/workflows/scripts/convert_md_to_pdf.sh DELETED
@@ -1,114 +0,0 @@
1
- #!/bin/bash
2
- set -e
3
-
4
- SCRIPTS_DIR="$GITHUB_WORKSPACE/.github/workflows/scripts"
5
-
6
- echo "Converting MD files to PDF..."
7
- find . -name "*.md" -not -path "./.git/*" | while read file; do
8
- dir="$(dirname "$file")"
9
- filename="$(basename "$file" .md)"
10
- pdf_path="$dir/$filename.pdf"
11
-
12
- echo "Processing $file..."
13
- echo "Directory: $dir"
14
- echo "Filename (without extension): $filename"
15
- echo "Target PDF path: $pdf_path"
16
-
17
- if [ ! -f "$file" ]; then
18
- echo "ERROR: File $file does not exist"
19
- continue
20
- fi
21
-
22
- if [ ! -r "$file" ]; then
23
- echo "ERROR: File $file is not readable"
24
- continue
25
- fi
26
-
27
- echo "File size: $(wc -c < "$file") bytes"
28
-
29
- # Preprocess the markdown file
30
- cd "$dir"
31
- processed_file=$(python3 "$SCRIPTS_DIR/preprocess_markdown.py" "$(basename "$file")" 2>&1) || {
32
- echo "Preprocessing failed, using original file"
33
- processed_file="$(basename "$file")"
34
- }
35
-
36
- if [ ! -f "$processed_file" ]; then
37
- echo "Processed file $processed_file does not exist, using original"
38
- processed_file="$(basename "$file")"
39
- fi
40
-
41
- echo "Using file for conversion: $processed_file"
42
-
43
- # Method 1: Try XeLaTeX with enhanced settings
44
- pandoc "$processed_file" \
45
- -o "$pdf_path" \
46
- --pdf-engine=xelatex \
47
- --include-in-header="$SCRIPTS_DIR/latex-header.tex" \
48
- --variable mainfont="DejaVu Sans" \
49
- --variable sansfont="DejaVu Sans" \
50
- --variable monofont="DejaVu Sans Mono" \
51
- --variable colorlinks=true \
52
- --variable linkcolor=blue \
53
- --variable urlcolor=blue \
54
- --variable toccolor=gray \
55
- --resource-path="$dir:$SCRIPTS_DIR" \
56
- --standalone \
57
- --toc \
58
- --number-sections \
59
- --highlight-style=pygments \
60
- --wrap=auto \
61
- --dpi=300 \
62
- --verbose 2>&1 || {
63
-
64
- echo "XeLaTeX failed, trying HTML->PDF conversion..."
65
-
66
- # Method 2: HTML to PDF conversion
67
- pandoc "$processed_file" \
68
- -t html5 \
69
- --standalone \
70
- --embed-resources \
71
- --css="$SCRIPTS_DIR/styles.css" \
72
- --toc \
73
- --number-sections \
74
- --highlight-style=pygments \
75
- -o "$dir/$filename.html" 2>&1
76
-
77
- if [ -f "$dir/$filename.html" ]; then
78
- weasyprint "$dir/$filename.html" "$pdf_path" --presentational-hints 2>&1 || {
79
- wkhtmltopdf \
80
- --page-size A4 \
81
- --margin-top 0.5in \
82
- --margin-right 0.5in \
83
- --margin-bottom 0.5in \
84
- --margin-left 0.5in \
85
- --encoding UTF-8 \
86
- --no-outline \
87
- --enable-local-file-access \
88
- "$dir/$filename.html" "$pdf_path" 2>&1 || {
89
- echo "All conversion methods failed for $file"
90
- continue
91
- }
92
- }
93
- rm -f "$dir/$filename.html"
94
- else
95
- echo "Failed to create HTML file for $file"
96
- continue
97
- fi
98
- }
99
-
100
- # Clean up
101
- if [ "$processed_file" != "$(basename "$file")" ]; then
102
- rm -f "$processed_file"
103
- fi
104
- rm -f mermaid_*.png mermaid_*.svg mermaid_*.mmd
105
-
106
- if [ -f "$pdf_path" ]; then
107
- echo "✅ Successfully converted $file to $pdf_path"
108
- echo "PDF file size: $(wc -c < "$pdf_path") bytes"
109
- else
110
- echo "❌ Failed to convert $file"
111
- fi
112
-
113
- cd "$GITHUB_WORKSPACE"
114
- done
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
.github/workflows/scripts/latex-header.tex DELETED
@@ -1,34 +0,0 @@
1
- \usepackage{graphicx}
2
- \usepackage{float}
3
- \usepackage{adjustbox}
4
- \usepackage{caption}
5
- \usepackage{subcaption}
6
- \usepackage[a4paper,top=2.25cm,bottom=2.25cm,left=1.9cm,right=1.9cm]{geometry}
7
- \usepackage{fancyhdr}
8
- \usepackage{xcolor}
9
- \usepackage{hyperref}
10
- \usepackage{fontspec}
11
- \usepackage{unicode-math}
12
-
13
- % Set fonts with emoji support
14
- \setmainfont{DejaVu Sans}
15
- \setsansfont{DejaVu Sans}
16
- \setmonofont{DejaVu Sans Mono}
17
-
18
- % Better image positioning and scaling
19
- \floatplacement{figure}{H}
20
- \let\oldincludegraphics\includegraphics
21
- \renewcommand{\includegraphics}[2][]{%
22
- \adjustbox{max width=\textwidth,center}{\oldincludegraphics[#1]{#2}}%
23
- }
24
-
25
- % Try to set a font with emoji support as fallback
26
- \newfontfamily\emojifont{Noto Color Emoji}[Renderer=Harfbuzz]
27
-
28
- % Hyperlink colors
29
- \hypersetup{
30
- colorlinks=true,
31
- linkcolor=blue,
32
- urlcolor=blue,
33
- citecolor=blue
34
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
.github/workflows/scripts/preprocess_markdown.py DELETED
@@ -1,213 +0,0 @@
1
- #!/usr/bin/env python3
2
- import re
3
- import os
4
- import sys
5
- import subprocess
6
- from pathlib import Path
7
-
8
- def process_mermaid_diagrams(content, file_dir):
9
- """Convert mermaid diagrams to images"""
10
- mermaid_pattern = r'```mermaid\n(.*?)\n```'
11
-
12
- def replace_mermaid(match):
13
- mermaid_code = match.group(1)
14
- # Create a unique filename for this diagram
15
- diagram_hash = str(abs(hash(mermaid_code)))
16
- mermaid_file = f"{file_dir}/mermaid_{diagram_hash}.mmd"
17
- svg_file = f"{file_dir}/mermaid_{diagram_hash}.svg"
18
- png_file = f"{file_dir}/mermaid_{diagram_hash}.png"
19
-
20
- # Write mermaid code to file
21
- try:
22
- with open(mermaid_file, 'w', encoding='utf-8') as f:
23
- f.write(mermaid_code)
24
- except Exception as e:
25
- print(f"Error writing mermaid file: {e}")
26
- return f'\n```\n{mermaid_code}\n```\n'
27
-
28
- try:
29
- # Method 1: Try with config file (newer versions)
30
- config_file = os.path.join(file_dir, '..', '..', '.github', 'workflows', 'puppeteer-config.json')
31
- if os.path.exists(config_file):
32
- result = subprocess.run([
33
- 'mmdc', '-i', mermaid_file, '-o', svg_file,
34
- '--theme', 'default', '--backgroundColor', 'white',
35
- '--configFile', config_file,
36
- '--puppeteerConfig', config_file
37
- ], check=True, capture_output=True, text=True, timeout=60)
38
- else:
39
- # Method 2: Try without puppeteer config (fallback)
40
- result = subprocess.run([
41
- 'mmdc', '-i', mermaid_file, '-o', svg_file,
42
- '--theme', 'default', '--backgroundColor', 'white',
43
- '--puppeteerConfig', '{"args": ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", "--disable-gpu", "--single-process"]}'
44
- ], check=True, capture_output=True, text=True, timeout=60)
45
-
46
- # Convert SVG to PNG for better PDF compatibility
47
- subprocess.run([
48
- 'rsvg-convert', '-f', 'png', '-o', png_file,
49
- '--width', '1200', '--height', '800', svg_file
50
- ], check=True, capture_output=True, text=True)
51
-
52
- # Clean up intermediate files
53
- try:
54
- os.remove(mermaid_file)
55
- if os.path.exists(svg_file):
56
- os.remove(svg_file)
57
- except:
58
- pass
59
-
60
- # Return markdown image syntax
61
- return (
62
- f'\n<div class="mermaid-container">\n\n'
63
- f'![Architecture Diagram]({os.path.basename(png_file)})\n\n'
64
- f'</div>\n'
65
- )
66
-
67
- except subprocess.CalledProcessError as e:
68
- print(f"Error converting mermaid diagram: {e}")
69
- print(f"Command output: {e.stderr if e.stderr else 'No stderr'}")
70
-
71
- # Fallback: Try basic mmdc command without any config
72
- try:
73
- print("Trying basic mmdc command...")
74
- subprocess.run([
75
- 'mmdc', '-i', mermaid_file, '-o', svg_file,
76
- '--puppeteerConfig', '{"args": ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", "--disable-gpu", "--single-process"]}'
77
- ], check=True, capture_output=True, text=True, timeout=60)
78
-
79
- # Convert to PNG
80
- subprocess.run([
81
- 'rsvg-convert', '-f', 'png', '-o', png_file,
82
- '--width', '1200', '--height', '800', svg_file
83
- ], check=True, capture_output=True, text=True)
84
-
85
- # Clean up
86
- try:
87
- os.remove(mermaid_file)
88
- if os.path.exists(svg_file):
89
- os.remove(svg_file)
90
- except:
91
- pass
92
-
93
- return (
94
- f'\n<div class="mermaid-container">\n\n'
95
- f'![Architecture Diagram]({os.path.basename(png_file)})\n\n'
96
- f'</div>\n'
97
- )
98
-
99
- except subprocess.CalledProcessError as e2:
100
- print(f"Basic mmdc also failed: {e2}")
101
- try:
102
- os.remove(mermaid_file)
103
- except:
104
- pass
105
-
106
- # Return original mermaid code if all rendering fails
107
- print("All Mermaid rendering methods failed, keeping original code")
108
- return f'\n```mermaid\n{mermaid_code}\n```\n'
109
-
110
- except Exception as e:
111
- print(f"Unexpected error with mermaid: {e}")
112
- try:
113
- os.remove(mermaid_file)
114
- except:
115
- pass
116
- return f'\n```mermaid\n{mermaid_code}\n```\n'
117
-
118
- return re.sub(mermaid_pattern, replace_mermaid, content, flags=re.DOTALL)
119
-
120
-
121
- def clean_emojis_and_fix_images(content, file_dir):
122
- """Remove/replace emojis and fix image paths"""
123
- emoji_replacements = {
124
- '🎵': '[Audio]',
125
- '🎬': '[Video]',
126
- '📝': '[Document]',
127
- '📊': '[Analytics]',
128
- '🧠': '[AI]',
129
- '🎥': '[Media]',
130
- '📄': '[File]',
131
- '✅': '[Success]',
132
- '❌': '[Failed]'
133
- }
134
-
135
- for emoji, replacement in emoji_replacements.items():
136
- content = content.replace(emoji, replacement)
137
-
138
- # Pattern to match markdown images
139
- img_pattern = r'!\[([^\]]*)\]\(([^)]+)\)'
140
-
141
- def replace_image(match):
142
- alt_text = match.group(1)
143
- img_path = match.group(2)
144
-
145
- if not img_path.startswith(('http://', 'https://', '/')):
146
- abs_img_path = os.path.join(file_dir, img_path)
147
- if os.path.exists(abs_img_path):
148
- img_path = os.path.relpath(abs_img_path, file_dir)
149
-
150
- return (
151
- f'<img src="{img_path}" alt="{alt_text}" '
152
- f'style="max-width: 100%; height: auto; display: block; margin: 1em auto;" />'
153
- )
154
-
155
- content = re.sub(img_pattern, replace_image, content)
156
-
157
- # Fix existing HTML img tags
158
- content = re.sub(
159
- r'<img\s+([^>]*?)\s*/?>',
160
- lambda m: (
161
- f'<img {m.group(1)} '
162
- f'style="max-width: 100%; height: auto; display: block; margin: 1em auto;" />'
163
- ),
164
- content
165
- )
166
-
167
- return content
168
-
169
- def main():
170
- if len(sys.argv) != 2:
171
- print("Usage: python preprocess_markdown.py <markdown_file>")
172
- sys.exit(1)
173
-
174
- md_file = sys.argv[1]
175
-
176
- if not os.path.exists(md_file):
177
- print(f"Error: File {md_file} does not exist")
178
- sys.exit(1)
179
-
180
- try:
181
- file_dir = os.path.dirname(os.path.abspath(md_file))
182
-
183
- with open(md_file, 'r', encoding='utf-8') as f:
184
- content = f.read()
185
-
186
- print(f"Processing file: {md_file}")
187
- print(f"File directory: {file_dir}")
188
- print(f"Content length: {len(content)} characters")
189
-
190
- # Process mermaid diagrams
191
- content = process_mermaid_diagrams(content, file_dir)
192
- print(f"Mermaid processing complete. Content length: {len(content)}")
193
-
194
- # Clean emojis and fix image paths
195
- content = clean_emojis_and_fix_images(content, file_dir)
196
- print(f"Image path fixing complete. Content length: {len(content)}")
197
-
198
- # Write processed content
199
- processed_file = md_file.replace('.md', '_processed.md')
200
- with open(processed_file, 'w', encoding='utf-8') as f:
201
- f.write(content)
202
-
203
- print(f"Processed file saved as: {processed_file}")
204
- print(processed_file)
205
-
206
- except Exception as e:
207
- print(f"Error processing {md_file}: {e}")
208
- import traceback
209
- traceback.print_exc()
210
- sys.exit(1)
211
-
212
- if __name__ == "__main__":
213
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
.github/workflows/scripts/setup_system.sh DELETED
@@ -1,38 +0,0 @@
1
- #!/bin/bash
2
- set -e
3
-
4
- echo "Installing system dependencies..."
5
- sudo apt-get update
6
- sudo apt-get install -y \
7
- texlive-full \
8
- texlive-xetex \
9
- texlive-luatex \
10
- pandoc \
11
- librsvg2-bin \
12
- python3-pip \
13
- nodejs \
14
- npm \
15
- imagemagick \
16
- ghostscript \
17
- wkhtmltopdf
18
-
19
- echo "Installing Node.js dependencies for Mermaid..."
20
- npm install -g @mermaid-js/mermaid-cli@latest
21
- npm install -g puppeteer
22
- sudo apt-get install -y google-chrome-stable
23
-
24
- echo "Installing Python dependencies..."
25
- pip install --upgrade pip
26
- pip install \
27
- weasyprint \
28
- markdown \
29
- pymdown-extensions \
30
- pillow \
31
- cairosvg \
32
- pdfkit \
33
- google-auth \
34
- google-auth-oauthlib \
35
- google-auth-httplib2 \
36
- google-api-python-client
37
-
38
- echo "System setup complete!"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
.github/workflows/scripts/styles.css DELETED
@@ -1,109 +0,0 @@
1
- body {
2
- font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
3
- max-width: 210mm;
4
- margin: 0 auto;
5
- padding: 20mm;
6
- line-height: 1.6;
7
- color: #333;
8
- background: white;
9
- }
10
-
11
- img {
12
- max-width: 100%;
13
- height: auto;
14
- display: block;
15
- margin: 1em auto;
16
- border-radius: 4px;
17
- box-shadow: 0 2px 8px rgba(0,0,0,0.1);
18
- }
19
-
20
- pre {
21
- background: #f8f9fa;
22
- padding: 1em;
23
- border-radius: 6px;
24
- border-left: 4px solid #007acc;
25
- overflow-x: auto;
26
- font-family: 'Consolas', 'Monaco', 'Courier New', monospace;
27
- font-size: 0.9em;
28
- }
29
-
30
- code {
31
- background: #f1f3f4;
32
- padding: 0.2em 0.4em;
33
- border-radius: 3px;
34
- font-family: 'Consolas', 'Monaco', 'Courier New', monospace;
35
- font-size: 0.9em;
36
- }
37
-
38
- h1, h2, h3, h4, h5, h6 {
39
- color: #2c3e50;
40
- margin-top: 2em;
41
- margin-bottom: 1em;
42
- page-break-after: avoid;
43
- }
44
-
45
- h1 {
46
- border-bottom: 3px solid #3498db;
47
- padding-bottom: 0.5em;
48
- }
49
-
50
- h2 {
51
- border-bottom: 2px solid #95a5a6;
52
- padding-bottom: 0.3em;
53
- }
54
-
55
- table {
56
- border-collapse: collapse;
57
- width: 100%;
58
- margin: 1em 0;
59
- }
60
-
61
- th, td {
62
- border: 1px solid #ddd;
63
- padding: 0.75em;
64
- text-align: left;
65
- }
66
-
67
- th {
68
- background-color: #f8f9fa;
69
- font-weight: bold;
70
- }
71
-
72
- blockquote {
73
- border-left: 4px solid #3498db;
74
- margin: 1em 0;
75
- padding: 0.5em 1em;
76
- background: #f8f9fa;
77
- border-radius: 0 4px 4px 0;
78
- }
79
-
80
- .mermaid-container {
81
- text-align: center;
82
- margin: 2em 0;
83
- page-break-inside: avoid;
84
- }
85
-
86
- .mermaid-container img {
87
- max-width: 100%;
88
- height: auto;
89
- }
90
-
91
- @media print {
92
- body {
93
- margin: 0;
94
- padding: 15mm;
95
- }
96
-
97
- img {
98
- max-height: 80vh;
99
- page-break-inside: avoid;
100
- }
101
-
102
- h1, h2, h3, h4, h5, h6 {
103
- page-break-after: avoid;
104
- }
105
-
106
- pre, blockquote {
107
- page-break-inside: avoid;
108
- }
109
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
.github/workflows/scripts/upload_to_drive.py DELETED
@@ -1,141 +0,0 @@
1
- #!/usr/bin/env python3
2
- import os
3
- import json
4
- import mimetypes
5
- import fnmatch
6
- import sys
7
- from google.oauth2.credentials import Credentials
8
- from google.oauth2 import service_account
9
- from google.auth.transport.requests import Request
10
- from googleapiclient.discovery import build
11
- from googleapiclient.http import MediaFileUpload
12
-
13
- FOLDER_ID = '1-8HJcWxsUUQIj9OMXQeoeULS06RA9Hg9'
14
-
15
- def get_drive_service():
16
- """Get Google Drive service with OAuth/Service Account fallback"""
17
- # Try OAuth first
18
- oauth_token = os.environ.get('GOOGLE_OAUTH_TOKEN')
19
- if oauth_token:
20
- try:
21
- token_info = json.loads(oauth_token)
22
- credentials = Credentials.from_authorized_user_info(token_info)
23
- if credentials.expired and credentials.refresh_token:
24
- credentials.refresh(Request())
25
- return build('drive', 'v3', credentials=credentials)
26
- except Exception as e:
27
- print(f"OAuth failed: {e}")
28
-
29
- # Fallback to Service Account
30
- sa_key = os.environ.get('GOOGLE_SERVICE_ACCOUNT_KEY')
31
- if sa_key:
32
- try:
33
- credentials_info = json.loads(sa_key)
34
- credentials = service_account.Credentials.from_service_account_info(
35
- credentials_info, scopes=['https://www.googleapis.com/auth/drive.file']
36
- )
37
- return build('drive', 'v3', credentials=credentials)
38
- except Exception as e:
39
- print(f"Service Account failed: {e}")
40
-
41
- print("Both authentication methods failed!")
42
- sys.exit(1)
43
-
44
- def upload_file(file_path, parent_folder_id, service):
45
- """Upload or update a file"""
46
- file_name = os.path.basename(file_path)
47
-
48
- # Check if file exists
49
- query = f"name='{file_name}' and '{parent_folder_id}' in parents and trashed=false"
50
- results = service.files().list(q=query).execute()
51
- items = results.get('files', [])
52
-
53
- media = MediaFileUpload(file_path, mimetype=mimetypes.guess_type(file_path)[0] or 'application/octet-stream')
54
-
55
- if items:
56
- # Update existing
57
- service.files().update(fileId=items[0]['id'], media_body=media).execute()
58
- print(f'Updated: {file_name}')
59
- else:
60
- # Create new
61
- file_metadata = {'name': file_name, 'parents': [parent_folder_id]}
62
- service.files().create(body=file_metadata, media_body=media).execute()
63
- print(f'Uploaded: {file_name}')
64
-
65
- def create_folder(folder_name, parent_id, service):
66
- """Create folder if it doesn't exist"""
67
- query = f"name='{folder_name}' and '{parent_id}' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false"
68
- results = service.files().list(q=query).execute()
69
- items = results.get('files', [])
70
-
71
- if items:
72
- return items[0]['id']
73
-
74
- folder_metadata = {
75
- 'name': folder_name,
76
- 'parents': [parent_id],
77
- 'mimeType': 'application/vnd.google-apps.folder'
78
- }
79
- folder = service.files().create(body=folder_metadata).execute()
80
- return folder.get('id')
81
-
82
- def upload_directory(local_path, parent_id, service):
83
- """Upload directory with exclusions"""
84
- exclude_dirs = {'.git', '.github', 'node_modules', '__pycache__'}
85
- exclude_patterns = ['*.md'] # Skip markdown files
86
-
87
- uploaded = skipped = errors = 0
88
-
89
- for root, dirs, files in os.walk(local_path):
90
- # Filter directories
91
- dirs[:] = [d for d in dirs if d not in exclude_dirs]
92
-
93
- # Create folder structure
94
- rel_path = os.path.relpath(root, local_path)
95
- current_folder_id = parent_id
96
-
97
- if rel_path != '.':
98
- for part in rel_path.split(os.sep):
99
- current_folder_id = create_folder(part, current_folder_id, service)
100
-
101
- # Upload files
102
- for file in files:
103
- # Check exclusions
104
- if any(fnmatch.fnmatch(file, pattern) for pattern in exclude_patterns):
105
- print(f'Skipping: {file}')
106
- skipped += 1
107
- continue
108
-
109
- try:
110
- upload_file(os.path.join(root, file), current_folder_id, service)
111
- uploaded += 1
112
- except Exception as e:
113
- print(f'Error uploading {file}: {e}')
114
- errors += 1
115
-
116
- return uploaded, skipped, errors
117
-
118
- def main():
119
- print("Starting Google Drive upload...")
120
-
121
- # Get service and test access
122
- service = get_drive_service()
123
-
124
- try:
125
- service.files().list(q=f"'{FOLDER_ID}' in parents", pageSize=1).execute()
126
- print("Folder access confirmed")
127
- except Exception as e:
128
- print(f"Cannot access folder {FOLDER_ID}: {e}")
129
- sys.exit(1)
130
-
131
- # Upload files
132
- uploaded, skipped, errors = upload_directory('.', FOLDER_ID, service)
133
-
134
- print(f"\nSummary: {uploaded} uploaded, {skipped} skipped, {errors} errors")
135
-
136
- if errors > 0:
137
- sys.exit(1)
138
- print("Upload completed successfully!")
139
-
140
- if __name__ == "__main__":
141
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
demo_audio/Car_Trouble.mp3 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf02f5b91eac9f997bd5b34b0efc978871273b16feb988d4d5dfcf3d45a4f8ae
3
+ size 738449
demo_audio/Film_Podcast.mp3 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf129ae36a824a575a90da05f703502a424b43c0cc86416c6c2dd1cbf2c1062b
3
+ size 3687747
demo_audio/Tamil_Wikipedia_Interview.ogg ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30b578d696c204c178cb3ea6754b63fb47a7fc56e2e9b7d33fd499359a88fefb
3
+ size 32676479
demo_audio/Yuri_Kizaki.mp3 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:acd4b3289bf3e59b576f43a43dba3490951e2b8ec90f8d63489e98c937a89f4a
3
+ size 519003
static/imgs/banner.png ADDED

Git LFS Details

  • SHA256: 82d55557be2da7a05d864bf4403ec7cba10d5ef1326feb0eba57d4c2d9be02d7
  • Pointer size: 130 Bytes
  • Size of remote file: 89 kB
static/imgs/demo_mode_banner.png ADDED

Git LFS Details

  • SHA256: eba7c5900f15485fbfadbfd9fbc027a259e31bbfdecfc661e2c752e1186f3709
  • Pointer size: 130 Bytes
  • Size of remote file: 81.4 kB
static/imgs/demo_res_summary.png ADDED

Git LFS Details

  • SHA256: ad4e9f0f178b691af590614dadc9e8f302b274c35ebccbf0dbf99b9d9069c071
  • Pointer size: 130 Bytes
  • Size of remote file: 31.1 kB
static/imgs/demo_res_transcript_translate.png ADDED

Git LFS Details

  • SHA256: 4c42bfb013fda394dbd554cf4516f119ff78bfec14c758b06413c557d3fbf49e
  • Pointer size: 130 Bytes
  • Size of remote file: 81.9 kB
static/imgs/demo_res_visual.png ADDED

Git LFS Details

  • SHA256: 79b2abcd242cd7a797882c751eacf08eaa7b1abec8bf99a266dbafccf1cd2eb9
  • Pointer size: 130 Bytes
  • Size of remote file: 52.2 kB
static/imgs/full_mode_banner.png ADDED

Git LFS Details

  • SHA256: d4644edad345ae6b5089da6c31c199f4d7db80b82839ecad9738c570a0b4c549
  • Pointer size: 130 Bytes
  • Size of remote file: 53 kB