Prathamesh Sarjerao Vaidya commited on
Commit
a63aab0
·
1 Parent(s): 94b1ab5

updated the github action to for support emoji conversion to pdf

Browse files
.github/workflows/check.yml CHANGED
@@ -1,5 +1,5 @@
1
  name: Check file size and sync to Google Drive
2
- on
3
  pull_request:
4
  branches: [main]
5
  workflow_dispatch:
@@ -31,22 +31,78 @@ jobs:
31
  git lfs install
32
  git lfs pull
33
 
34
- # Install pandoc for MD to PDF conversion
35
- - name: Install pandoc
36
  run: |
37
  sudo apt-get update
38
- sudo apt-get install -y pandoc texlive-latex-base texlive-fonts-recommended texlive-latex-extra
 
 
 
 
 
 
 
 
 
 
39
 
40
  # Convert MD files to PDF
41
  - name: Convert MD to PDF
42
  run: |
43
- mkdir -p pdf_output
44
- find . -name "*.md" -not -path "./.git/*" -not -path "./pdf_output/*" | while read file; do
45
- relative_path="${file#./}"
46
- pdf_path="pdf_output/${relative_path%.md}.pdf"
47
- mkdir -p "$(dirname "$pdf_path")"
48
- pandoc "$file" -o "$pdf_path" --pdf-engine=pdflatex
49
- echo "Converted $file to $pdf_path"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  done
51
 
52
  # Set up Python for Google Drive upload
@@ -112,14 +168,29 @@ jobs:
112
  ).execute()
113
  print(f'Uploaded PR Preview: {file_name}')
114
 
115
- def upload_directory(local_path, parent_folder_id, drive_service, exclude_dirs=None):
116
  if exclude_dirs is None:
117
  exclude_dirs = ['.git', '.github', 'node_modules', '__pycache__']
 
 
 
 
118
 
119
  for root, dirs, files in os.walk(local_path):
120
  dirs[:] = [d for d in dirs if d not in exclude_dirs]
121
 
122
  for file in files:
 
 
 
 
 
 
 
 
 
 
 
123
  file_path = os.path.join(root, file)
124
  try:
125
  upload_file(file_path, parent_folder_id, drive_service)
@@ -129,11 +200,7 @@ jobs:
129
  print("Starting PR preview upload to Google Drive...")
130
  upload_directory('.', FOLDER_ID, service)
131
 
132
- if os.path.exists('pdf_output'):
133
- print("Uploading converted PDF files...")
134
- upload_directory('pdf_output', FOLDER_ID, service)
135
-
136
- print("PR preview upload completed!")
137
  EOF
138
 
139
  python upload_to_drive.py
 
1
  name: Check file size and sync to Google Drive
2
+ on:
3
  pull_request:
4
  branches: [main]
5
  workflow_dispatch:
 
31
  git lfs install
32
  git lfs pull
33
 
34
+ # Install pandoc and fonts for proper Unicode/emoji support
35
+ - name: Install pandoc and dependencies
36
  run: |
37
  sudo apt-get update
38
+ sudo apt-get install -y \
39
+ pandoc \
40
+ texlive-xetex \
41
+ texlive-fonts-recommended \
42
+ texlive-fonts-extra \
43
+ fonts-dejavu \
44
+ fonts-liberation \
45
+ python3-pip
46
+
47
+ # Install weasyprint as fallback
48
+ pip install weasyprint
49
 
50
  # Convert MD files to PDF
51
  - name: Convert MD to PDF
52
  run: |
53
+ find . -name "*.md" -not -path "./.git/*" | while read file; do
54
+ # Get the directory and filename
55
+ dir="$(dirname "$file")"
56
+ filename="$(basename "$file" .md)"
57
+ pdf_path="$dir/$filename.pdf"
58
+
59
+ echo "Converting $file to $pdf_path"
60
+
61
+ # Convert MD to PDF with proper Unicode support and image handling
62
+ pandoc "$file" \
63
+ -o "$pdf_path" \
64
+ --pdf-engine=xelatex \
65
+ --variable mainfont="DejaVu Sans" \
66
+ --variable sansfont="DejaVu Sans" \
67
+ --variable monofont="DejaVu Sans Mono" \
68
+ --include-in-header=<(echo '\usepackage{graphicx}') \
69
+ --include-in-header=<(echo '\usepackage{float}') \
70
+ --include-in-header=<(echo '\floatplacement{figure}{H}') \
71
+ --resource-path="$dir" \
72
+ --standalone \
73
+ --toc \
74
+ --number-sections \
75
+ --highlight-style=github \
76
+ -V geometry:margin=1in \
77
+ -V colorlinks=true \
78
+ -V linkcolor=blue \
79
+ -V urlcolor=blue \
80
+ -V toccolor=gray \
81
+ --wrap=auto \
82
+ --dpi=300 \
83
+ --verbose || {
84
+ echo "Error converting $file with xelatex, trying with weasyprint..."
85
+ # Fallback to HTML->PDF conversion for complex documents
86
+ pandoc "$file" \
87
+ -t html5 \
88
+ --standalone \
89
+ --self-contained \
90
+ --css=<(echo 'body{font-family:Arial,sans-serif;max-width:800px;margin:0 auto;padding:20px;line-height:1.6}img{max-width:100%;height:auto}pre{background:#f5f5f5;padding:10px;border-radius:5px;overflow-x:auto}') \
91
+ -o "$dir/$filename.html"
92
+
93
+ # Use weasyprint for HTML to PDF conversion
94
+ weasyprint "$dir/$filename.html" "$pdf_path" || {
95
+ echo "Both conversions failed for $file, skipping..."
96
+ continue
97
+ }
98
+ rm -f "$dir/$filename.html"
99
+ }
100
+
101
+ if [ -f "$pdf_path" ]; then
102
+ echo "Successfully converted $file to $pdf_path"
103
+ else
104
+ echo "Failed to convert $file"
105
+ fi
106
  done
107
 
108
  # Set up Python for Google Drive upload
 
168
  ).execute()
169
  print(f'Uploaded PR Preview: {file_name}')
170
 
171
+ def upload_directory(local_path, parent_folder_id, drive_service, exclude_dirs=None, exclude_files=None):
172
  if exclude_dirs is None:
173
  exclude_dirs = ['.git', '.github', 'node_modules', '__pycache__']
174
+ if exclude_files is None:
175
+ exclude_files = ['*.md'] # Skip markdown files
176
+
177
+ import fnmatch
178
 
179
  for root, dirs, files in os.walk(local_path):
180
  dirs[:] = [d for d in dirs if d not in exclude_dirs]
181
 
182
  for file in files:
183
+ # Skip excluded file patterns (like *.md)
184
+ should_skip = False
185
+ for pattern in exclude_files:
186
+ if fnmatch.fnmatch(file, pattern):
187
+ should_skip = True
188
+ break
189
+
190
+ if should_skip:
191
+ print(f'Skipping {file} (excluded file type)')
192
+ continue
193
+
194
  file_path = os.path.join(root, file)
195
  try:
196
  upload_file(file_path, parent_folder_id, drive_service)
 
200
  print("Starting PR preview upload to Google Drive...")
201
  upload_directory('.', FOLDER_ID, service)
202
 
203
+ print("PR preview upload completed - MD files were skipped, PDFs were uploaded!")
 
 
 
 
204
  EOF
205
 
206
  python upload_to_drive.py
.github/workflows/main.yml CHANGED
@@ -19,27 +19,79 @@ jobs:
19
  git lfs install
20
  git lfs pull
21
 
22
- # Install pandoc for MD to PDF conversion
23
- - name: Install pandoc
24
  run: |
25
  sudo apt-get update
26
- sudo apt-get install -y pandoc texlive-latex-base texlive-fonts-recommended texlive-latex-extra
 
 
 
 
 
 
 
 
 
 
27
 
28
  # Convert MD files to PDF
29
  - name: Convert MD to PDF
30
  run: |
31
- mkdir -p pdf_output
32
- find . -name "*.md" -not -path "./.git/*" -not -path "./pdf_output/*" | while read file; do
33
- # Get the relative path and change extension to .pdf
34
- relative_path="${file#./}"
35
- pdf_path="pdf_output/${relative_path%.md}.pdf"
36
 
37
- # Create directory structure in pdf_output
38
- mkdir -p "$(dirname "$pdf_path")"
39
 
40
- # Convert MD to PDF
41
- pandoc "$file" -o "$pdf_path" --pdf-engine=pdflatex
42
- echo "Converted $file to $pdf_path"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
  done
44
 
45
  # Set up Python for Google Drive upload
@@ -110,31 +162,41 @@ jobs:
110
  ).execute()
111
  print(f'Uploaded: {file_name} (ID: {file.get("id")})')
112
 
113
- def upload_directory(local_path, parent_folder_id, drive_service, exclude_dirs=None):
114
  if exclude_dirs is None:
115
  exclude_dirs = ['.git', '.github', 'node_modules', '__pycache__']
 
 
 
 
116
 
117
  for root, dirs, files in os.walk(local_path):
118
  # Remove excluded directories
119
  dirs[:] = [d for d in dirs if d not in exclude_dirs]
120
 
121
  for file in files:
 
 
 
 
 
 
 
 
 
 
 
122
  file_path = os.path.join(root, file)
123
  try:
124
  upload_file(file_path, parent_folder_id, drive_service)
125
  except Exception as e:
126
  print(f'Error uploading {file_path}: {e}')
127
 
128
- # Upload all files to Google Drive
129
  print("Starting upload to Google Drive...")
130
  upload_directory('.', FOLDER_ID, service)
131
 
132
- # Upload PDF files if they exist
133
- if os.path.exists('pdf_output'):
134
- print("Uploading converted PDF files...")
135
- upload_directory('pdf_output', FOLDER_ID, service)
136
-
137
- print("Upload completed!")
138
  EOF
139
 
140
  # Upload to Google Drive
 
19
  git lfs install
20
  git lfs pull
21
 
22
+ # Install pandoc and fonts for proper Unicode/emoji support
23
+ - name: Install pandoc and dependencies
24
  run: |
25
  sudo apt-get update
26
+ sudo apt-get install -y \
27
+ pandoc \
28
+ texlive-xetex \
29
+ texlive-fonts-recommended \
30
+ texlive-fonts-extra \
31
+ fonts-dejavu \
32
+ fonts-liberation \
33
+ python3-pip
34
+
35
+ # Install weasyprint as fallback
36
+ pip install weasyprint
37
 
38
  # Convert MD files to PDF
39
  - name: Convert MD to PDF
40
  run: |
41
+ find . -name "*.md" -not -path "./.git/*" | while read file; do
42
+ # Get the directory and filename
43
+ dir="$(dirname "$file")"
44
+ filename="$(basename "$file" .md)"
45
+ pdf_path="$dir/$filename.pdf"
46
 
47
+ echo "Converting $file to $pdf_path"
 
48
 
49
+ # Convert MD to PDF with proper Unicode support and image handling
50
+ pandoc "$file" \
51
+ -o "$pdf_path" \
52
+ --pdf-engine=xelatex \
53
+ --variable mainfont="DejaVu Sans" \
54
+ --variable sansfont="DejaVu Sans" \
55
+ --variable monofont="DejaVu Sans Mono" \
56
+ --include-in-header=<(echo '\usepackage{graphicx}') \
57
+ --include-in-header=<(echo '\usepackage{float}') \
58
+ --include-in-header=<(echo '\floatplacement{figure}{H}') \
59
+ --resource-path="$dir" \
60
+ --standalone \
61
+ --toc \
62
+ --number-sections \
63
+ --highlight-style=github \
64
+ -V geometry:margin=1in \
65
+ -V colorlinks=true \
66
+ -V linkcolor=blue \
67
+ -V urlcolor=blue \
68
+ -V toccolor=gray \
69
+ --wrap=auto \
70
+ --dpi=300 \
71
+ --verbose || {
72
+ echo "Error converting $file with xelatex, trying with weasyprint..."
73
+ # Fallback to HTML->PDF conversion for complex documents
74
+ pandoc "$file" \
75
+ -t html5 \
76
+ --standalone \
77
+ --self-contained \
78
+ --css=<(echo 'body{font-family:Arial,sans-serif;max-width:800px;margin:0 auto;padding:20px;line-height:1.6}img{max-width:100%;height:auto}pre{background:#f5f5f5;padding:10px;border-radius:5px;overflow-x:auto}') \
79
+ -o "$dir/$filename.html"
80
+
81
+ # Install weasyprint for HTML to PDF conversion
82
+ pip install weasyprint
83
+ weasyprint "$dir/$filename.html" "$pdf_path" || {
84
+ echo "Both conversions failed for $file, skipping..."
85
+ continue
86
+ }
87
+ rm -f "$dir/$filename.html"
88
+ }
89
+
90
+ if [ -f "$pdf_path" ]; then
91
+ echo "Successfully converted $file to $pdf_path"
92
+ else
93
+ echo "Failed to convert $file"
94
+ fi
95
  done
96
 
97
  # Set up Python for Google Drive upload
 
162
  ).execute()
163
  print(f'Uploaded: {file_name} (ID: {file.get("id")})')
164
 
165
+ def upload_directory(local_path, parent_folder_id, drive_service, exclude_dirs=None, exclude_files=None):
166
  if exclude_dirs is None:
167
  exclude_dirs = ['.git', '.github', 'node_modules', '__pycache__']
168
+ if exclude_files is None:
169
+ exclude_files = ['*.md'] # Skip markdown files
170
+
171
+ import fnmatch
172
 
173
  for root, dirs, files in os.walk(local_path):
174
  # Remove excluded directories
175
  dirs[:] = [d for d in dirs if d not in exclude_dirs]
176
 
177
  for file in files:
178
+ # Skip excluded file patterns (like *.md)
179
+ should_skip = False
180
+ for pattern in exclude_files:
181
+ if fnmatch.fnmatch(file, pattern):
182
+ should_skip = True
183
+ break
184
+
185
+ if should_skip:
186
+ print(f'Skipping {file} (excluded file type)')
187
+ continue
188
+
189
  file_path = os.path.join(root, file)
190
  try:
191
  upload_file(file_path, parent_folder_id, drive_service)
192
  except Exception as e:
193
  print(f'Error uploading {file_path}: {e}')
194
 
195
+ # Upload all files to Google Drive (excluding MD files)
196
  print("Starting upload to Google Drive...")
197
  upload_directory('.', FOLDER_ID, service)
198
 
199
+ print("Upload completed - MD files were skipped, PDFs were uploaded!")
 
 
 
 
 
200
  EOF
201
 
202
  # Upload to Google Drive