Prathamesh Sarjerao Vaidya commited on
Commit ·
a63aab0
1
Parent(s): 94b1ab5
updated the github action to for support emoji conversion to pdf
Browse files- .github/workflows/check.yml +84 -17
- .github/workflows/main.yml +83 -21
.github/workflows/check.yml
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
name: Check file size and sync to Google Drive
|
| 2 |
-
on
|
| 3 |
pull_request:
|
| 4 |
branches: [main]
|
| 5 |
workflow_dispatch:
|
|
@@ -31,22 +31,78 @@ jobs:
|
|
| 31 |
git lfs install
|
| 32 |
git lfs pull
|
| 33 |
|
| 34 |
-
# Install pandoc
|
| 35 |
-
- name: Install pandoc
|
| 36 |
run: |
|
| 37 |
sudo apt-get update
|
| 38 |
-
sudo apt-get install -y
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
|
| 40 |
# Convert MD files to PDF
|
| 41 |
- name: Convert MD to PDF
|
| 42 |
run: |
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
echo "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
done
|
| 51 |
|
| 52 |
# Set up Python for Google Drive upload
|
|
@@ -112,14 +168,29 @@ jobs:
|
|
| 112 |
).execute()
|
| 113 |
print(f'Uploaded PR Preview: {file_name}')
|
| 114 |
|
| 115 |
-
def upload_directory(local_path, parent_folder_id, drive_service, exclude_dirs=None):
|
| 116 |
if exclude_dirs is None:
|
| 117 |
exclude_dirs = ['.git', '.github', 'node_modules', '__pycache__']
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
|
| 119 |
for root, dirs, files in os.walk(local_path):
|
| 120 |
dirs[:] = [d for d in dirs if d not in exclude_dirs]
|
| 121 |
|
| 122 |
for file in files:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
file_path = os.path.join(root, file)
|
| 124 |
try:
|
| 125 |
upload_file(file_path, parent_folder_id, drive_service)
|
|
@@ -129,11 +200,7 @@ jobs:
|
|
| 129 |
print("Starting PR preview upload to Google Drive...")
|
| 130 |
upload_directory('.', FOLDER_ID, service)
|
| 131 |
|
| 132 |
-
|
| 133 |
-
print("Uploading converted PDF files...")
|
| 134 |
-
upload_directory('pdf_output', FOLDER_ID, service)
|
| 135 |
-
|
| 136 |
-
print("PR preview upload completed!")
|
| 137 |
EOF
|
| 138 |
|
| 139 |
python upload_to_drive.py
|
|
|
|
| 1 |
name: Check file size and sync to Google Drive
|
| 2 |
+
on:
|
| 3 |
pull_request:
|
| 4 |
branches: [main]
|
| 5 |
workflow_dispatch:
|
|
|
|
| 31 |
git lfs install
|
| 32 |
git lfs pull
|
| 33 |
|
| 34 |
+
# Install pandoc and fonts for proper Unicode/emoji support
|
| 35 |
+
- name: Install pandoc and dependencies
|
| 36 |
run: |
|
| 37 |
sudo apt-get update
|
| 38 |
+
sudo apt-get install -y \
|
| 39 |
+
pandoc \
|
| 40 |
+
texlive-xetex \
|
| 41 |
+
texlive-fonts-recommended \
|
| 42 |
+
texlive-fonts-extra \
|
| 43 |
+
fonts-dejavu \
|
| 44 |
+
fonts-liberation \
|
| 45 |
+
python3-pip
|
| 46 |
+
|
| 47 |
+
# Install weasyprint as fallback
|
| 48 |
+
pip install weasyprint
|
| 49 |
|
| 50 |
# Convert MD files to PDF
|
| 51 |
- name: Convert MD to PDF
|
| 52 |
run: |
|
| 53 |
+
find . -name "*.md" -not -path "./.git/*" | while read file; do
|
| 54 |
+
# Get the directory and filename
|
| 55 |
+
dir="$(dirname "$file")"
|
| 56 |
+
filename="$(basename "$file" .md)"
|
| 57 |
+
pdf_path="$dir/$filename.pdf"
|
| 58 |
+
|
| 59 |
+
echo "Converting $file to $pdf_path"
|
| 60 |
+
|
| 61 |
+
# Convert MD to PDF with proper Unicode support and image handling
|
| 62 |
+
pandoc "$file" \
|
| 63 |
+
-o "$pdf_path" \
|
| 64 |
+
--pdf-engine=xelatex \
|
| 65 |
+
--variable mainfont="DejaVu Sans" \
|
| 66 |
+
--variable sansfont="DejaVu Sans" \
|
| 67 |
+
--variable monofont="DejaVu Sans Mono" \
|
| 68 |
+
--include-in-header=<(echo '\usepackage{graphicx}') \
|
| 69 |
+
--include-in-header=<(echo '\usepackage{float}') \
|
| 70 |
+
--include-in-header=<(echo '\floatplacement{figure}{H}') \
|
| 71 |
+
--resource-path="$dir" \
|
| 72 |
+
--standalone \
|
| 73 |
+
--toc \
|
| 74 |
+
--number-sections \
|
| 75 |
+
--highlight-style=github \
|
| 76 |
+
-V geometry:margin=1in \
|
| 77 |
+
-V colorlinks=true \
|
| 78 |
+
-V linkcolor=blue \
|
| 79 |
+
-V urlcolor=blue \
|
| 80 |
+
-V toccolor=gray \
|
| 81 |
+
--wrap=auto \
|
| 82 |
+
--dpi=300 \
|
| 83 |
+
--verbose || {
|
| 84 |
+
echo "Error converting $file with xelatex, trying with weasyprint..."
|
| 85 |
+
# Fallback to HTML->PDF conversion for complex documents
|
| 86 |
+
pandoc "$file" \
|
| 87 |
+
-t html5 \
|
| 88 |
+
--standalone \
|
| 89 |
+
--self-contained \
|
| 90 |
+
--css=<(echo 'body{font-family:Arial,sans-serif;max-width:800px;margin:0 auto;padding:20px;line-height:1.6}img{max-width:100%;height:auto}pre{background:#f5f5f5;padding:10px;border-radius:5px;overflow-x:auto}') \
|
| 91 |
+
-o "$dir/$filename.html"
|
| 92 |
+
|
| 93 |
+
# Use weasyprint for HTML to PDF conversion
|
| 94 |
+
weasyprint "$dir/$filename.html" "$pdf_path" || {
|
| 95 |
+
echo "Both conversions failed for $file, skipping..."
|
| 96 |
+
continue
|
| 97 |
+
}
|
| 98 |
+
rm -f "$dir/$filename.html"
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
if [ -f "$pdf_path" ]; then
|
| 102 |
+
echo "Successfully converted $file to $pdf_path"
|
| 103 |
+
else
|
| 104 |
+
echo "Failed to convert $file"
|
| 105 |
+
fi
|
| 106 |
done
|
| 107 |
|
| 108 |
# Set up Python for Google Drive upload
|
|
|
|
| 168 |
).execute()
|
| 169 |
print(f'Uploaded PR Preview: {file_name}')
|
| 170 |
|
| 171 |
+
def upload_directory(local_path, parent_folder_id, drive_service, exclude_dirs=None, exclude_files=None):
|
| 172 |
if exclude_dirs is None:
|
| 173 |
exclude_dirs = ['.git', '.github', 'node_modules', '__pycache__']
|
| 174 |
+
if exclude_files is None:
|
| 175 |
+
exclude_files = ['*.md'] # Skip markdown files
|
| 176 |
+
|
| 177 |
+
import fnmatch
|
| 178 |
|
| 179 |
for root, dirs, files in os.walk(local_path):
|
| 180 |
dirs[:] = [d for d in dirs if d not in exclude_dirs]
|
| 181 |
|
| 182 |
for file in files:
|
| 183 |
+
# Skip excluded file patterns (like *.md)
|
| 184 |
+
should_skip = False
|
| 185 |
+
for pattern in exclude_files:
|
| 186 |
+
if fnmatch.fnmatch(file, pattern):
|
| 187 |
+
should_skip = True
|
| 188 |
+
break
|
| 189 |
+
|
| 190 |
+
if should_skip:
|
| 191 |
+
print(f'Skipping {file} (excluded file type)')
|
| 192 |
+
continue
|
| 193 |
+
|
| 194 |
file_path = os.path.join(root, file)
|
| 195 |
try:
|
| 196 |
upload_file(file_path, parent_folder_id, drive_service)
|
|
|
|
| 200 |
print("Starting PR preview upload to Google Drive...")
|
| 201 |
upload_directory('.', FOLDER_ID, service)
|
| 202 |
|
| 203 |
+
print("PR preview upload completed - MD files were skipped, PDFs were uploaded!")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 204 |
EOF
|
| 205 |
|
| 206 |
python upload_to_drive.py
|
.github/workflows/main.yml
CHANGED
|
@@ -19,27 +19,79 @@ jobs:
|
|
| 19 |
git lfs install
|
| 20 |
git lfs pull
|
| 21 |
|
| 22 |
-
# Install pandoc
|
| 23 |
-
- name: Install pandoc
|
| 24 |
run: |
|
| 25 |
sudo apt-get update
|
| 26 |
-
sudo apt-get install -y
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
# Convert MD files to PDF
|
| 29 |
- name: Convert MD to PDF
|
| 30 |
run: |
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
pdf_path="
|
| 36 |
|
| 37 |
-
|
| 38 |
-
mkdir -p "$(dirname "$pdf_path")"
|
| 39 |
|
| 40 |
-
# Convert MD to PDF
|
| 41 |
-
pandoc "$file"
|
| 42 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
done
|
| 44 |
|
| 45 |
# Set up Python for Google Drive upload
|
|
@@ -110,31 +162,41 @@ jobs:
|
|
| 110 |
).execute()
|
| 111 |
print(f'Uploaded: {file_name} (ID: {file.get("id")})')
|
| 112 |
|
| 113 |
-
def upload_directory(local_path, parent_folder_id, drive_service, exclude_dirs=None):
|
| 114 |
if exclude_dirs is None:
|
| 115 |
exclude_dirs = ['.git', '.github', 'node_modules', '__pycache__']
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
|
| 117 |
for root, dirs, files in os.walk(local_path):
|
| 118 |
# Remove excluded directories
|
| 119 |
dirs[:] = [d for d in dirs if d not in exclude_dirs]
|
| 120 |
|
| 121 |
for file in files:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
file_path = os.path.join(root, file)
|
| 123 |
try:
|
| 124 |
upload_file(file_path, parent_folder_id, drive_service)
|
| 125 |
except Exception as e:
|
| 126 |
print(f'Error uploading {file_path}: {e}')
|
| 127 |
|
| 128 |
-
# Upload all files to Google Drive
|
| 129 |
print("Starting upload to Google Drive...")
|
| 130 |
upload_directory('.', FOLDER_ID, service)
|
| 131 |
|
| 132 |
-
|
| 133 |
-
if os.path.exists('pdf_output'):
|
| 134 |
-
print("Uploading converted PDF files...")
|
| 135 |
-
upload_directory('pdf_output', FOLDER_ID, service)
|
| 136 |
-
|
| 137 |
-
print("Upload completed!")
|
| 138 |
EOF
|
| 139 |
|
| 140 |
# Upload to Google Drive
|
|
|
|
| 19 |
git lfs install
|
| 20 |
git lfs pull
|
| 21 |
|
| 22 |
+
# Install pandoc and fonts for proper Unicode/emoji support
|
| 23 |
+
- name: Install pandoc and dependencies
|
| 24 |
run: |
|
| 25 |
sudo apt-get update
|
| 26 |
+
sudo apt-get install -y \
|
| 27 |
+
pandoc \
|
| 28 |
+
texlive-xetex \
|
| 29 |
+
texlive-fonts-recommended \
|
| 30 |
+
texlive-fonts-extra \
|
| 31 |
+
fonts-dejavu \
|
| 32 |
+
fonts-liberation \
|
| 33 |
+
python3-pip
|
| 34 |
+
|
| 35 |
+
# Install weasyprint as fallback
|
| 36 |
+
pip install weasyprint
|
| 37 |
|
| 38 |
# Convert MD files to PDF
|
| 39 |
- name: Convert MD to PDF
|
| 40 |
run: |
|
| 41 |
+
find . -name "*.md" -not -path "./.git/*" | while read file; do
|
| 42 |
+
# Get the directory and filename
|
| 43 |
+
dir="$(dirname "$file")"
|
| 44 |
+
filename="$(basename "$file" .md)"
|
| 45 |
+
pdf_path="$dir/$filename.pdf"
|
| 46 |
|
| 47 |
+
echo "Converting $file to $pdf_path"
|
|
|
|
| 48 |
|
| 49 |
+
# Convert MD to PDF with proper Unicode support and image handling
|
| 50 |
+
pandoc "$file" \
|
| 51 |
+
-o "$pdf_path" \
|
| 52 |
+
--pdf-engine=xelatex \
|
| 53 |
+
--variable mainfont="DejaVu Sans" \
|
| 54 |
+
--variable sansfont="DejaVu Sans" \
|
| 55 |
+
--variable monofont="DejaVu Sans Mono" \
|
| 56 |
+
--include-in-header=<(echo '\usepackage{graphicx}') \
|
| 57 |
+
--include-in-header=<(echo '\usepackage{float}') \
|
| 58 |
+
--include-in-header=<(echo '\floatplacement{figure}{H}') \
|
| 59 |
+
--resource-path="$dir" \
|
| 60 |
+
--standalone \
|
| 61 |
+
--toc \
|
| 62 |
+
--number-sections \
|
| 63 |
+
--highlight-style=github \
|
| 64 |
+
-V geometry:margin=1in \
|
| 65 |
+
-V colorlinks=true \
|
| 66 |
+
-V linkcolor=blue \
|
| 67 |
+
-V urlcolor=blue \
|
| 68 |
+
-V toccolor=gray \
|
| 69 |
+
--wrap=auto \
|
| 70 |
+
--dpi=300 \
|
| 71 |
+
--verbose || {
|
| 72 |
+
echo "Error converting $file with xelatex, trying with weasyprint..."
|
| 73 |
+
# Fallback to HTML->PDF conversion for complex documents
|
| 74 |
+
pandoc "$file" \
|
| 75 |
+
-t html5 \
|
| 76 |
+
--standalone \
|
| 77 |
+
--self-contained \
|
| 78 |
+
--css=<(echo 'body{font-family:Arial,sans-serif;max-width:800px;margin:0 auto;padding:20px;line-height:1.6}img{max-width:100%;height:auto}pre{background:#f5f5f5;padding:10px;border-radius:5px;overflow-x:auto}') \
|
| 79 |
+
-o "$dir/$filename.html"
|
| 80 |
+
|
| 81 |
+
# Install weasyprint for HTML to PDF conversion
|
| 82 |
+
pip install weasyprint
|
| 83 |
+
weasyprint "$dir/$filename.html" "$pdf_path" || {
|
| 84 |
+
echo "Both conversions failed for $file, skipping..."
|
| 85 |
+
continue
|
| 86 |
+
}
|
| 87 |
+
rm -f "$dir/$filename.html"
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
if [ -f "$pdf_path" ]; then
|
| 91 |
+
echo "Successfully converted $file to $pdf_path"
|
| 92 |
+
else
|
| 93 |
+
echo "Failed to convert $file"
|
| 94 |
+
fi
|
| 95 |
done
|
| 96 |
|
| 97 |
# Set up Python for Google Drive upload
|
|
|
|
| 162 |
).execute()
|
| 163 |
print(f'Uploaded: {file_name} (ID: {file.get("id")})')
|
| 164 |
|
| 165 |
+
def upload_directory(local_path, parent_folder_id, drive_service, exclude_dirs=None, exclude_files=None):
|
| 166 |
if exclude_dirs is None:
|
| 167 |
exclude_dirs = ['.git', '.github', 'node_modules', '__pycache__']
|
| 168 |
+
if exclude_files is None:
|
| 169 |
+
exclude_files = ['*.md'] # Skip markdown files
|
| 170 |
+
|
| 171 |
+
import fnmatch
|
| 172 |
|
| 173 |
for root, dirs, files in os.walk(local_path):
|
| 174 |
# Remove excluded directories
|
| 175 |
dirs[:] = [d for d in dirs if d not in exclude_dirs]
|
| 176 |
|
| 177 |
for file in files:
|
| 178 |
+
# Skip excluded file patterns (like *.md)
|
| 179 |
+
should_skip = False
|
| 180 |
+
for pattern in exclude_files:
|
| 181 |
+
if fnmatch.fnmatch(file, pattern):
|
| 182 |
+
should_skip = True
|
| 183 |
+
break
|
| 184 |
+
|
| 185 |
+
if should_skip:
|
| 186 |
+
print(f'Skipping {file} (excluded file type)')
|
| 187 |
+
continue
|
| 188 |
+
|
| 189 |
file_path = os.path.join(root, file)
|
| 190 |
try:
|
| 191 |
upload_file(file_path, parent_folder_id, drive_service)
|
| 192 |
except Exception as e:
|
| 193 |
print(f'Error uploading {file_path}: {e}')
|
| 194 |
|
| 195 |
+
# Upload all files to Google Drive (excluding MD files)
|
| 196 |
print("Starting upload to Google Drive...")
|
| 197 |
upload_directory('.', FOLDER_ID, service)
|
| 198 |
|
| 199 |
+
print("Upload completed - MD files were skipped, PDFs were uploaded!")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 200 |
EOF
|
| 201 |
|
| 202 |
# Upload to Google Drive
|