re-type commited on
Commit
086f2ec
·
verified ·
1 Parent(s): d243d0b

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +883 -335
app.py CHANGED
@@ -1,3 +1,4 @@
 
1
  import gradio as gr
2
  import torch
3
  import pickle
@@ -7,70 +8,73 @@ import os
7
  import re
8
  import logging
9
  import numpy as np
10
- from tensorflow.keras.preprocessing.sequence import pad_sequences
11
  from tensorflow.keras.models import load_model
12
  import ml_simplified_tree
13
  import tempfile
14
  import shutil
 
15
  from pathlib import Path
16
- from huggingface_hub import hf_hub_download
17
 
18
  # --- Global Variables ---
19
- MAFFT_PATH = "mafft/mafftdir/bin/mafft"
20
- IQTREE_PATH = "iqtree/bin/iqtree2"
21
 
22
  # --- Logging ---
23
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
24
 
25
  # --- Paths ---
 
 
 
26
  model_repo = "GGproject10/best_boundary_aware_model"
27
- csv_path = "f_cleaned.csv"
28
- classifier_model_dir = "model"
29
 
30
- # Get HF token from environment
31
  hf_token = os.getenv("HF_TOKEN")
32
 
33
  # --- Load Models ---
34
  boundary_model = None
35
  keras_model = None
36
  kmer_to_index = None
37
- classifier_model = None
38
- classifier_kmer_to_index = None
39
- classifier_maxlen = None
40
- LABELS = ["Random", "F", "P", "N", "M", "HN", "L"]
41
 
 
42
  try:
43
- boundary_path = hf_hub_download(repo_id=model_repo, filename="best_boundary_aware_model.pth", token=hf_token)
 
 
 
 
44
  if os.path.exists(boundary_path):
45
  boundary_model = GenePredictor(boundary_path)
46
- logging.info("Boundary model loaded successfully.")
 
 
47
  except Exception as e:
48
- logging.error(f"Failed to load boundary model: {e}")
49
 
 
50
  try:
51
- keras_path = hf_hub_download(repo_id=model_repo, filename="best_model.keras", token=hf_token)
52
- kmer_path = hf_hub_download(repo_id=model_repo, filename="kmer_to_index.pkl", token=hf_token)
 
 
 
 
 
 
 
 
 
53
  if os.path.exists(keras_path) and os.path.exists(kmer_path):
54
  keras_model = load_model(keras_path)
55
  with open(kmer_path, "rb") as f:
56
  kmer_to_index = pickle.load(f)
57
- logging.info("Keras model and k-mer index loaded.")
58
- except Exception as e:
59
- logging.error(f"Failed to load Keras model: {e}")
60
-
61
- try:
62
- classifier_path = os.path.join(classifier_model_dir, "best_model.keras")
63
- classifier_kmer_path = os.path.join(classifier_model_dir, "kmer_to_index.pkl")
64
- classifier_maxlen_path = os.path.join(classifier_model_dir, "maxlen.txt")
65
- if os.path.exists(classifier_path) and os.path.exists(classifier_kmer_path) and os.path.exists(classifier_maxlen_path):
66
- classifier_model = load_model(classifier_path)
67
- with open(classifier_kmer_path, "rb") as f:
68
- classifier_kmer_to_index = pickle.load(f)
69
- with open(classifier_maxlen_path, "r") as f:
70
- classifier_maxlen = int(f.read().strip())
71
- logging.info("Classifier model loaded.")
72
  except Exception as e:
73
- logging.error(f"Failed to load classifier model: {e}")
74
 
75
  # --- Initialize Tree Analyzer ---
76
  analyzer = None
@@ -78,14 +82,15 @@ try:
78
  analyzer = ml_simplified_tree.PhylogeneticTreeAnalyzer()
79
  if os.path.exists(csv_path):
80
  if analyzer.load_data(csv_path):
81
- logging.info("Tree analyzer initialized.")
 
82
  try:
83
  if not analyzer.train_ai_model():
84
- logging.warning("AI model training failed.")
85
  except Exception as e:
86
  logging.warning(f"AI model training failed: {e}")
87
  else:
88
- logging.error("Failed to load CSV data.")
89
  analyzer = None
90
  else:
91
  logging.error(f"CSV file not found: {csv_path}")
@@ -94,278 +99,474 @@ except Exception as e:
94
  logging.error(f"Failed to initialize tree analyzer: {e}")
95
  analyzer = None
96
 
97
- # --- Predictor Functions (from predictor.py) ---
98
- class GenePredictor:
99
- """Minimal implementation to load and use the boundary-aware model"""
100
- def __init__(self, model_path):
101
- try:
102
- self.model = torch.load(model_path, map_location=torch.device('cpu'))
103
- logging.info(f"Loaded boundary model from {model_path}")
104
- except Exception as e:
105
- logging.error(f"Failed to load boundary model: {e}")
106
- self.model = None
107
-
108
- def predict(self, sequence):
109
- if self.model is None:
110
- return None, None, 0.0
111
- # Placeholder: Implement actual prediction logic
112
- return [0] * len(sequence), [0.5] * len(sequence), 0.5
113
-
114
- def extract_gene_regions(self, predictions, sequence):
115
- if predictions is None:
116
- return []
117
- return [{"sequence": sequence, "start": 0, "end": len(sequence), "confidence": 0.5}]
118
-
119
- def preprocess_sequence_for_ndv_f_gene(sequence):
120
- """Enhanced preprocessing specifically for NDV F gene sequences"""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
121
  try:
122
- sequence = sequence.upper().strip()
123
- sequence = re.sub(r'[^ATCGN]', '', sequence)
124
- if len(sequence) < 1000:
125
- logging.warning(f"Sequence length ({len(sequence)}) shorter than typical NDV F gene (1662-1800 nt)")
126
- if not sequence.startswith('ATG'):
127
- logging.warning("Sequence doesn't start with ATG start codon")
128
- atg_pos = sequence.find('ATG')
129
- if atg_pos != -1:
130
- sequence = sequence[atg_pos:]
131
- logging.info(f"Found ATG at position {atg_pos}, using sequence from there")
132
- if len(sequence) % 3 != 0:
133
- sequence = sequence[:len(sequence) - (len(sequence) % 3)]
134
- logging.info(f"Trimmed sequence to maintain reading frame: {len(sequence)} nt")
135
- return sequence
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
136
  except Exception as e:
137
- logging.error(f"Sequence preprocessing failed: {e}")
138
- return sequence
139
 
140
- def enhanced_keras_prediction(sequence, keras_model, kmer_to_index, kmer_size=6):
141
- """Enhanced Keras prediction with better handling for NDV F gene"""
142
  try:
143
- if not keras_model or not kmer_to_index:
144
- return "Keras model not available"
145
- processed_seq = preprocess_sequence_for_ndv_f_gene(sequence)
146
- if len(processed_seq) < kmer_size:
147
- return f"Sequence too short for k-mer prediction (minimum {kmer_size} nucleotides required)"
148
- kmers = [processed_seq[i:i+kmer_size] for i in range(len(processed_seq)-kmer_size+1)]
149
- indices = []
150
- unknown_kmers = 0
151
- for kmer in kmers:
152
- if kmer in kmer_to_index:
153
- indices.append(kmer_to_index[kmer])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
154
  else:
155
- indices.append(0)
156
- unknown_kmers += 1
157
- logging.info(f"Generated {len(kmers)} k-mers, {unknown_kmers} unknown k-mers")
158
- input_arr = np.array([indices])
159
- prediction = keras_model.predict(input_arr, verbose=0)[0]
160
- max_prob = np.max(prediction)
161
- mean_prob = np.mean(prediction)
162
- confidence_score = max_prob
163
- consistency_score = 1.0 - np.std(prediction)
164
- result = {
165
- 'raw_prediction': prediction.tolist(),
166
- 'max_probability': float(max_prob),
167
- 'mean_probability': float(mean_prob),
168
- 'confidence_score': float(confidence_score),
169
- 'consistency_score': float(consistency_score),
170
- 'sequence_length': len(processed_seq),
171
- 'kmers_generated': len(kmers),
172
- 'unknown_kmers': unknown_kmers,
173
- 'kmer_coverage': 1.0 - (unknown_kmers / len(kmers)) if kmers else 0.0
174
- }
175
- return result
176
  except Exception as e:
177
- logging.error(f"Enhanced Keras prediction failed: {e}")
178
- return f"Enhanced Keras prediction failed: {str(e)}"
179
 
180
- def enhanced_classify_sequence(sequence, classifier_model, classifier_kmer_to_index, classifier_maxlen, labels):
181
- """Enhanced classification with NDV F gene specific improvements"""
182
  try:
183
- if not classifier_model or not classifier_kmer_to_index or classifier_maxlen is None:
184
- return {
185
- "status": "error",
186
- "message": "Classification model not available",
187
- "confidence": None,
188
- "predicted_label": None,
189
- "details": {}
190
- }
191
- processed_seq = preprocess_sequence_for_ndv_f_gene(sequence)
192
- if len(processed_seq) < 1000:
193
- return {
194
- "status": "warning",
195
- "message": f"Sequence shorter than typical NDV F gene ({len(processed_seq)} < 1000 nt)",
196
- "confidence": None,
197
- "predicted_label": None,
198
- "details": {"sequence_length": len(processed_seq)}
199
- }
200
- kmer_size = 6
201
- tokens = [processed_seq[i:i+kmer_size] for i in range(len(processed_seq)-kmer_size+1)]
202
- encoded = []
203
- unknown_count = 0
204
- for kmer in tokens:
205
- if kmer in classifier_kmer_to_index:
206
- encoded.append(classifier_kmer_to_index[kmer])
207
- else:
208
- encoded.append(0)
209
- unknown_count += 1
210
- padded = pad_sequences([encoded], maxlen=classifier_maxlen, padding='post')
211
- pred = classifier_model.predict(padded, verbose=0)
212
- predicted_class = int(np.argmax(pred))
213
- confidence = float(np.max(pred))
214
- predicted_label = labels[predicted_class] if predicted_class < len(labels) else "Unknown"
215
- kmer_coverage = 1.0 - (unknown_count / len(tokens)) if tokens else 0.0
216
- prediction_entropy = -np.sum(pred[0] * np.log(pred[0] + 1e-10))
217
- details = {
218
- "sequence_length": len(processed_seq),
219
- "kmers_generated": len(tokens),
220
- "unknown_kmers": unknown_count,
221
- "kmer_coverage": kmer_coverage,
222
- "prediction_entropy": float(prediction_entropy),
223
- "all_probabilities": {labels[i]: float(pred[0][i]) for i in range(len(labels)) if i < len(pred[0])},
224
- "starts_with_atg": processed_seq.startswith('ATG'),
225
- "length_in_frame": len(processed_seq) % 3 == 0
226
- }
227
- if predicted_label == "F":
228
- f_gene_score = confidence
229
- if kmer_coverage > 0.8: f_gene_score *= 1.1
230
- if processed_seq.startswith('ATG'): f_gene_score *= 1.05
231
- if len(processed_seq) % 3 == 0: f_gene_score *= 1.05
232
- if 1500 <= len(processed_seq) <= 2000: f_gene_score *= 1.1
233
- details["enhanced_f_score"] = min(f_gene_score, 1.0)
234
- if f_gene_score > 0.7:
235
- return {"status": "success", "message": "NDV F gene detected with high confidence", "confidence": confidence, "predicted_label": predicted_label, "details": details}
236
- elif f_gene_score > 0.5:
237
- return {"status": "success", "message": "NDV F gene detected with moderate confidence", "confidence": confidence, "predicted_label": predicted_label, "details": details}
238
- else:
239
- return {"status": "warning", "message": "Possible F gene but low confidence - check sequence quality", "confidence": confidence, "predicted_label": predicted_label, "details": details}
240
- elif predicted_label == "Random":
241
- if kmer_coverage < 0.5:
242
- return {"status": "error", "message": f"Poor sequence quality detected (coverage: {kmer_coverage:.1%}). Check for sequencing errors.", "confidence": confidence, "predicted_label": predicted_label, "details": details}
243
- else:
244
- return {"status": "error", "message": "Sequence does not appear to be NDV F gene. Verify input sequence.", "confidence": confidence, "predicted_label": predicted_label, "details": details}
245
  else:
246
- return {"status": "error", "message": f"Detected as {predicted_label} gene, not F gene. Please provide NDV F gene sequence.", "confidence": confidence, "predicted_label": predicted_label, "details": details}
 
 
 
 
 
 
 
 
 
247
  except Exception as e:
248
- logging.error(f"Enhanced classification failed: {e}")
249
- return {"status": "error", "message": f"Classification failed: {str(e)}", "confidence": None, "predicted_label": None, "details": {"error": str(e)}}
250
-
251
- def validate_ndv_f_gene_sequence(sequence):
252
- """Additional validation specific to NDV F gene characteristics"""
253
- issues = []
254
- suggestions = []
255
- if len(sequence) < 1500:
256
- issues.append(f"Sequence length ({len(sequence)}) shorter than typical NDV F gene (1662-1800 nt)")
257
- suggestions.append("Verify complete F gene sequence was provided")
258
- elif len(sequence) > 2000:
259
- issues.append(f"Sequence length ({len(sequence)}) longer than typical NDV F gene")
260
- suggestions.append("Check if sequence contains additional regions beyond F gene")
261
- if not sequence.startswith('ATG'):
262
- issues.append("Sequence doesn't start with ATG start codon")
263
- suggestions.append("Ensure sequence starts from the translation start site")
264
- if len(sequence) % 3 != 0:
265
- issues.append("Sequence length not divisible by 3 (reading frame issue)")
266
- suggestions.append("Check for insertions/deletions or trim to proper reading frame")
267
- if len(sequence) >= 3:
268
- last_codon = sequence[-3:]
269
- stop_codons = ['TAA', 'TAG', 'TGA']
270
- if last_codon not in stop_codons:
271
- issues.append(f"Sequence doesn't end with stop codon (ends with {last_codon})")
272
- suggestions.append("Verify complete F gene sequence including stop codon")
273
- gc_content = (sequence.count('G') + sequence.count('C')) / len(sequence) * 100
274
- if gc_content < 30 or gc_content > 70:
275
- issues.append(f"Unusual GC content: {gc_content:.1f}% (typical range: 35-65%)")
276
- suggestions.append("Verify sequence quality and correct nucleotide composition")
277
- n_content = sequence.count('N') / len(sequence) * 100
278
- if n_content > 5:
279
- issues.append(f"High ambiguous nucleotide content: {n_content:.1f}% N's")
280
- suggestions.append("Consider resequencing regions with ambiguous nucleotides")
281
- return issues, suggestions
282
-
283
- def enhanced_run_pipeline(dna_input, keras_model, kmer_to_index, classifier_model,
284
- classifier_kmer_to_index, classifier_maxlen, labels,
285
- similarity_score=95.0, build_ml_tree=False):
286
- """Enhanced pipeline with improved F gene prediction"""
287
  try:
288
- dna_input = dna_input.upper().strip()
289
- if not dna_input:
290
- return "Empty input", "", "", "", "", "", "", "", "", None, None, None, "No input provided"
291
- if not re.match('^[ACTGN]+$', dna_input):
292
- dna_input = ''.join(c if c in 'ACTGN' else 'N' for c in dna_input)
293
- logging.info("DNA sequence sanitized")
294
- validation_issues, validation_suggestions = validate_ndv_f_gene_sequence(dna_input)
295
- keras_result = enhanced_keras_prediction(dna_input, keras_model, kmer_to_index)
296
- if isinstance(keras_result, dict):
297
- keras_output = f"Prediction confidence: {keras_result['confidence_score']:.3f}\n"
298
- keras_output += f"K-mer coverage: {keras_result['kmer_coverage']:.1%}\n"
299
- keras_output += f"Sequence length: {keras_result['sequence_length']} nt"
300
- if keras_result['kmer_coverage'] < 0.8:
301
- keras_output += "\n⚠️ Low k-mer coverage - may affect accuracy"
302
  else:
303
- keras_output = str(keras_result)
304
- classifier_result = enhanced_classify_sequence(
305
- dna_input, classifier_model, classifier_kmer_to_index, classifier_maxlen, labels
306
- )
307
- classifier_status = classifier_result["status"]
308
- classifier_message = classifier_result["message"]
309
- classifier_label = classifier_result["predicted_label"]
310
- classifier_confidence = classifier_result["confidence"]
311
- if validation_issues:
312
- classifier_message += f"\n\n⚠️ Sequence validation issues:\n" + "\n".join(f"• {issue}" for issue in validation_issues[:3])
313
- if validation_suggestions:
314
- classifier_message += f"\n\n💡 Suggestions:\n" + "\n".join(f"• {sug}" for sug in validation_suggestions[:3])
315
- if classifier_result.get("details"):
316
- details = classifier_result["details"]
317
- if "all_probabilities" in details:
318
- probs = details["all_probabilities"]
319
- classifier_message += f"\n\nPrediction probabilities:"
320
- for label, prob in sorted(probs.items(), key=lambda x: x[1], reverse=True)[:3]:
321
- classifier_message += f"\n• {label}: {prob:.1%}"
322
- boundary_output = f"Enhanced preprocessing applied. Length: {len(dna_input)} bp"
323
- if validation_issues:
324
- boundary_output += f"\n{len(validation_issues)} validation issues detected"
325
- return (
326
- boundary_output,
327
- keras_output,
328
- classifier_status,
329
- classifier_message,
330
- classifier_label or "Unknown",
331
- f"{classifier_confidence:.3f}" if classifier_confidence else "N/A",
332
- "ML tree not requested" if not build_ml_tree else "ML tree processing...",
333
- "Enhanced analysis completed",
334
- "<p>Enhanced F gene analysis completed</p>",
335
- None, None, None,
336
- f"Enhanced pipeline completed. Processed {len(dna_input)} bp sequence."
337
- )
338
  except Exception as e:
339
- error_msg = f"Enhanced pipeline failed: {str(e)}"
340
- logging.error(error_msg)
341
- return (
342
- error_msg, "", "error", error_msg, "Error", "0.000",
343
- "", "", f"<p>Error: {error_msg}</p>",
344
- None, None, None, error_msg
345
- )
346
 
347
- # --- Tool Detection and Helper Functions ---
348
- def check_tool_availability():
349
- mafft_available = shutil.which(MAFFT_PATH) is not None
350
- iqtree_available = shutil.which(IQTREE_PATH) is not None
351
- return mafft_available, iqtree_available, MAFFT_PATH, IQTREE_PATH
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
352
 
353
- def install_dependencies_guide():
354
- return """
355
- 🔧 INSTALLATION GUIDE:
356
- - MAFFT: `conda install -c bioconda mafft`
357
- - IQ-TREE: `conda install -c bioconda iqtree`
358
- """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
359
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
360
  def read_fasta_file(file_obj):
361
  try:
362
  if file_obj is None:
363
  return ""
 
 
364
  if hasattr(file_obj, 'name'):
365
  with open(file_obj.name, "r") as f:
366
  content = f.read()
367
  else:
368
  content = file_obj.read().decode("utf-8") if hasattr(file_obj, "read") else str(file_obj)
 
369
  lines = content.strip().split("\n")
370
  seq_lines = [line.strip() for line in lines if not line.startswith(">")]
371
  return ''.join(seq_lines)
@@ -373,105 +574,452 @@ def read_fasta_file(file_obj):
373
  logging.error(f"Failed to read FASTA file: {e}")
374
  return ""
375
 
 
376
  def run_pipeline_from_file(fasta_file_obj, similarity_score, build_ml_tree):
377
  try:
378
  dna_input = read_fasta_file(fasta_file_obj)
379
  if not dna_input:
380
- return "Failed to read FASTA file", "", "", "", "", "", "", "", "", None, None, None, "No input sequence"
381
- return enhanced_run_pipeline(
382
- dna_input, keras_model, kmer_to_index, classifier_model,
383
- classifier_kmer_to_index, classifier_maxlen, LABELS,
384
- similarity_score, build_ml_tree
385
- )
386
  except Exception as e:
387
  error_msg = f"Pipeline error: {str(e)}"
388
  logging.error(error_msg)
389
- return error_msg, "", "", "", "", "", "", "", "", None, None, None, error_msg
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
390
 
391
  # --- Gradio Interface ---
392
  def create_interface():
 
 
 
393
  custom_css = """
394
- .gradio-container { max-width: 1200px !important; }
395
- .tab-nav button { font-size: 16px !important; }
396
- .output-html { height: 600px !important; overflow: auto; }
 
 
 
 
 
 
 
397
  """
 
398
  with gr.Blocks(css=custom_css, title="F Gene Analysis Pipeline") as iface:
399
  gr.Markdown("""
400
  # 🧬 F Gene Analysis Pipeline
401
- This tool analyzes F genes with enhanced prediction and validation.
402
- **Instructions:** Enter sequence or upload FASTA, set similarity threshold, and run.
 
 
 
 
 
 
 
 
 
403
  """)
 
404
  with gr.Tab("🔬 Analysis Pipeline"):
405
  with gr.Row():
406
  with gr.Column(scale=2):
407
- dna_input = gr.Textbox(label="DNA Sequence", placeholder="Enter DNA sequence...", lines=5)
408
- fasta_file = gr.File(label="Or Upload FASTA File", file_types=[".fasta", ".fa", ".txt"])
 
 
 
 
 
 
 
 
 
 
 
 
409
  with gr.Row():
410
- similarity_score = gr.Slider(minimum=1, maximum=99, value=95.0, step=1.0, label="Similarity Threshold (%)")
411
- build_ml_tree = gr.Checkbox(label="Build ML Tree", value=False)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
412
  with gr.Row():
413
- run_btn = gr.Button("🚀 Run Analysis", variant="primary")
414
  clear_btn = gr.Button("����️ Clear", variant="secondary")
 
415
  with gr.Column(scale=1):
416
- status_display = gr.Textbox(label="Status", value="Ready", interactive=False, lines=3)
417
- gr.Markdown("### Available Models\n" + "\n".join([
418
- "✅ Boundary Detection Model" if boundary_model else "❌ Boundary Detection Model",
419
- "✅ Keras Model" if keras_model else "❌ Keras Model",
420
- "✅ Classifier Model" if classifier_model else "❌ Classifier Model",
421
- "✅ Tree Analysis" if analyzer else "❌ Tree Analysis"
422
- ]))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
423
  with gr.Tab("📊 Results"):
424
  with gr.Row():
425
  with gr.Column():
426
- boundary_output = gr.Textbox(label="🎯 F Gene Extraction", lines=5, interactive=False)
427
- keras_output = gr.Textbox(label="🔍 Gene Validation", lines=5, interactive=False)
428
- classifier_status = gr.Textbox(label="🧬 Classification Status", lines=1, interactive=False)
429
- classifier_message = gr.Textbox(label="📝 Classification Message", lines=6, interactive=False)
430
- classifier_label = gr.Textbox(label="🏷️ Predicted Label", lines=1, interactive=False)
431
- classifier_confidence = gr.Textbox(label="📊 Confidence Score", lines=1, interactive=False)
 
 
 
 
 
 
 
432
  with gr.Column():
433
- ml_tree_output = gr.Textbox(label="🌳 Maximum Likelihood Tree", lines=5, interactive=False)
434
- simplified_ml_output = gr.Textbox(label="📈 Simplified Phylogenetic Analysis", lines=3, interactive=False)
435
- tree_html = gr.HTML(label="🌲 Phylogenetic Tree", value="<p>No tree generated.</p>")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
436
  with gr.Row():
437
- aligned_file = gr.File(label="Aligned Sequences", interactive=False)
438
- phy_file = gr.File(label="Tree File", interactive=False)
439
- html_file = gr.File(label="Tree HTML", interactive=False)
 
 
 
 
 
 
 
 
 
 
 
 
440
  with gr.Tab("ℹ️ Help & Info"):
441
  gr.Markdown("""
442
  ## About This Tool
443
- Analyzes F genes with enhanced preprocessing, validation, and phylogenetic analysis.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
444
  ### Dependencies
445
- Install MAFFT and IQ-TREE via `conda install -c bioconda mafft iqtree`.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
446
  ### Troubleshooting
447
- - Ensure model files are downloaded.
448
- - Check sequence length (>1000 bp for F gene).
 
 
 
 
 
 
 
 
 
 
 
 
 
449
  """)
 
 
 
 
 
 
 
 
450
  def run_analysis_combined(dna_seq, file_obj, sim_score, build_tree):
 
451
  if file_obj is not None:
452
  return run_pipeline_from_file(file_obj, sim_score, build_tree)
453
  else:
454
- return enhanced_run_pipeline(
455
- dna_seq, keras_model, kmer_to_index, classifier_model,
456
- classifier_kmer_to_index, classifier_maxlen, LABELS,
457
- sim_score, build_tree
458
- )
459
  def clear_inputs():
460
- return "", None, 95.0, False, "Ready"
 
 
461
  run_btn.click(
462
  fn=run_analysis_combined,
463
  inputs=[dna_input, fasta_file, similarity_score, build_ml_tree],
464
  outputs=[
465
- boundary_output, keras_output, classifier_status, classifier_message,
466
- classifier_label, classifier_confidence, ml_tree_output, simplified_ml_output,
467
- tree_html, aligned_file, phy_file, html_file, status_display
468
  ]
469
  )
470
- clear_btn.click(fn=clear_inputs, outputs=[dna_input, fasta_file, similarity_score, build_ml_tree, status_display])
471
- example_btn = gr.Button("Load Example", variant="secondary")
472
- example_btn.click(fn=lambda: ("ATG" + "A" * 1650, "Example loaded"), outputs=[dna_input, status_display])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
473
  return iface
474
 
 
475
  if __name__ == "__main__":
 
476
  interface = create_interface()
477
- interface.launch(server_name="0.0.0.0", server_port=7860, share=False, debug=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # app.py
2
  import gradio as gr
3
  import torch
4
  import pickle
 
8
  import re
9
  import logging
10
  import numpy as np
11
+ from predictor import GenePredictor
12
  from tensorflow.keras.models import load_model
13
  import ml_simplified_tree
14
  import tempfile
15
  import shutil
16
+ import sys
17
  from pathlib import Path
 
18
 
19
  # --- Global Variables ---
20
+ MAFFT_PATH = "mafft/mafftdir/bin/mafft" # Update this path as needed
21
+ IQTREE_PATH = "iqtree/bin/iqtree2" # Update this path as needed
22
 
23
  # --- Logging ---
24
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
25
 
26
  # --- Paths ---
27
+ from huggingface_hub import hf_hub_download
28
+
29
+ # Model repository and file paths
30
  model_repo = "GGproject10/best_boundary_aware_model"
31
+ csv_path = "f cleaned.csv"
 
32
 
33
+ # Get HF token from environment (if available)
34
  hf_token = os.getenv("HF_TOKEN")
35
 
36
  # --- Load Models ---
37
  boundary_model = None
38
  keras_model = None
39
  kmer_to_index = None
 
 
 
 
40
 
41
+ # Try to load boundary model from Hugging Face Hub
42
  try:
43
+ boundary_path = hf_hub_download(
44
+ repo_id=model_repo,
45
+ filename="best_boundary_aware_model.pth",
46
+ token=hf_token
47
+ )
48
  if os.path.exists(boundary_path):
49
  boundary_model = GenePredictor(boundary_path)
50
+ logging.info("Boundary model loaded successfully from Hugging Face Hub.")
51
+ else:
52
+ logging.warning(f"Boundary model file not found after download")
53
  except Exception as e:
54
+ logging.error(f"Failed to load boundary model from HF Hub: {e}")
55
 
56
+ # Try to load Keras model from Hugging Face Hub
57
  try:
58
+ keras_path = hf_hub_download(
59
+ repo_id=model_repo,
60
+ filename="best_model.keras",
61
+ token=hf_token
62
+ )
63
+ kmer_path = hf_hub_download(
64
+ repo_id=model_repo,
65
+ filename="kmer_to_index.pkl",
66
+ token=hf_token
67
+ )
68
+
69
  if os.path.exists(keras_path) and os.path.exists(kmer_path):
70
  keras_model = load_model(keras_path)
71
  with open(kmer_path, "rb") as f:
72
  kmer_to_index = pickle.load(f)
73
+ logging.info("Keras model and k-mer index loaded successfully from Hugging Face Hub.")
74
+ else:
75
+ logging.warning(f"Keras model or kmer files not found after download")
 
 
 
 
 
 
 
 
 
 
 
 
76
  except Exception as e:
77
+ logging.error(f"Failed to load Keras model from HF Hub: {e}")
78
 
79
  # --- Initialize Tree Analyzer ---
80
  analyzer = None
 
82
  analyzer = ml_simplified_tree.PhylogeneticTreeAnalyzer()
83
  if os.path.exists(csv_path):
84
  if analyzer.load_data(csv_path):
85
+ logging.info("Tree analyzer initialized successfully")
86
+ # Try to train AI model (optional)
87
  try:
88
  if not analyzer.train_ai_model():
89
+ logging.warning("AI model training failed; proceeding with basic analysis.")
90
  except Exception as e:
91
  logging.warning(f"AI model training failed: {e}")
92
  else:
93
+ logging.error("Failed to load CSV data for tree analyzer")
94
  analyzer = None
95
  else:
96
  logging.error(f"CSV file not found: {csv_path}")
 
99
  logging.error(f"Failed to initialize tree analyzer: {e}")
100
  analyzer = None
101
 
102
+ # --- Enhanced Tool Detection ---
103
+ def check_tool_availability():
104
+ """Enhanced check for MAFFT and IQ-TREE availability with multiple fallback options"""
105
+
106
+ # Check MAFFT
107
+ mafft_available = False
108
+ mafft_cmd = None
109
+
110
+ # Try multiple MAFFT locations
111
+ mafft_candidates = [
112
+ MAFFT_PATH,
113
+ 'mafft',
114
+ '/usr/bin/mafft',
115
+ '/usr/local/bin/mafft',
116
+ 'mafft.bat', # Windows
117
+ ]
118
+
119
+ for candidate in mafft_candidates:
120
+ if candidate and (os.path.exists(candidate) or shutil.which(candidate) is not None):
121
+ mafft_available = True
122
+ mafft_cmd = candidate
123
+ logging.info(f"Found MAFFT at: {candidate}")
124
+ break
125
+
126
+ # Check IQ-TREE
127
+ iqtree_available = False
128
+ iqtree_cmd = None
129
+
130
+ # Try multiple IQ-TREE locations and names
131
+ iqtree_candidates = [
132
+ IQTREE_PATH,
133
+ 'iqtree2',
134
+ 'iqtree',
135
+ '/usr/bin/iqtree2',
136
+ '/usr/local/bin/iqtree2',
137
+ '/usr/bin/iqtree',
138
+ '/usr/local/bin/iqtree',
139
+ 'iqtree2.exe', # Windows
140
+ 'iqtree.exe', # Windows
141
+ ]
142
+
143
+ for candidate in iqtree_candidates:
144
+ if candidate and (os.path.exists(candidate) or shutil.which(candidate) is not None):
145
+ iqtree_available = True
146
+ iqtree_cmd = candidate
147
+ logging.info(f"Found IQ-TREE at: {candidate}")
148
+ break
149
+
150
+ return mafft_available, iqtree_available, mafft_cmd, iqtree_cmd
151
+
152
+ def install_dependencies_guide():
153
+ """Provide installation guidance for missing dependencies"""
154
+ guide = """
155
+ 🔧 INSTALLATION GUIDE FOR MISSING DEPENDENCIES:
156
+
157
+ For MAFFT:
158
+ - Ubuntu/Debian: sudo apt-get install mafft
159
+ - CentOS/RHEL: sudo yum install mafft
160
+ - macOS: brew install mafft
161
+ - Windows: Download from https://mafft.cbrc.jp/alignment/software/
162
+
163
+ For IQ-TREE:
164
+ - Ubuntu/Debian: sudo apt-get install iqtree
165
+ - CentOS/RHEL: sudo yum install iqtree
166
+ - macOS: brew install iqtree
167
+ - Windows: Download from http://www.iqtree.org/
168
+
169
+ Alternative: Use conda/mamba:
170
+ - conda install -c bioconda mafft iqtree
171
+
172
+ Docker option:
173
+ - docker run -it --rm -v $(pwd):/data quay.io/biocontainers/mafft:7.490--h779adbc_0
174
+ - docker run -it --rm -v $(pwd):/data quay.io/biocontainers/iqtree:2.1.4_beta--hdcc8f71_0
175
+ """
176
+ return guide
177
+
178
+ def run_mafft_alignment(input_fasta, output_fasta, mafft_cmd):
179
+ """Run MAFFT alignment with enhanced error handling"""
180
  try:
181
+ # MAFFT command with more robust options
182
+ cmd = [
183
+ mafft_cmd,
184
+ '--auto', # Automatic strategy selection
185
+ '--quiet', # Reduce output verbosity
186
+ input_fasta
187
+ ]
188
+
189
+ logging.info(f"Running MAFFT: {' '.join(cmd)}")
190
+
191
+ # Run MAFFT with enhanced error handling
192
+ result = subprocess.run(
193
+ cmd,
194
+ capture_output=True,
195
+ text=True,
196
+ timeout=600, # Increased timeout to 10 minutes
197
+ cwd=os.getcwd() # Ensure working directory is set
198
+ )
199
+
200
+ if result.returncode == 0:
201
+ # Write aligned sequences to output file
202
+ with open(output_fasta, 'w') as f:
203
+ f.write(result.stdout)
204
+ logging.info(f"MAFFT alignment completed: {output_fasta}")
205
+
206
+ # Verify output file
207
+ if os.path.exists(output_fasta) and os.path.getsize(output_fasta) > 0:
208
+ return True, output_fasta
209
+ else:
210
+ return False, "MAFFT completed but output file is empty"
211
+ else:
212
+ error_msg = result.stderr.strip() if result.stderr else "Unknown MAFFT error"
213
+ logging.error(f"MAFFT failed: {error_msg}")
214
+ return False, f"MAFFT error: {error_msg}"
215
+
216
+ except subprocess.TimeoutExpired:
217
+ logging.error("MAFFT timeout")
218
+ return False, "MAFFT timeout (>10 minutes). Try with fewer sequences."
219
+ except FileNotFoundError:
220
+ return False, f"MAFFT executable not found: {mafft_cmd}"
221
  except Exception as e:
222
+ logging.error(f"MAFFT execution failed: {e}")
223
+ return False, f"MAFFT execution failed: {str(e)}"
224
 
225
+ def run_iqtree_analysis(aligned_fasta, output_prefix, iqtree_cmd):
226
+ """Run IQ-TREE with enhanced options and error handling"""
227
  try:
228
+ # Enhanced IQ-TREE command
229
+ cmd = [
230
+ iqtree_cmd,
231
+ '-s', aligned_fasta,
232
+ '-m', 'MFP', # ModelFinder Plus for automatic model selection
233
+ '-bb', '1000', # Bootstrap replicates
234
+ '-alrt', '1000', # SH-aLRT test
235
+ '-nt', 'AUTO', # Auto detect threads
236
+ '--prefix', output_prefix,
237
+ '-redo', # Overwrite existing files
238
+ '--quiet' # Reduce verbosity
239
+ ]
240
+
241
+ logging.info(f"Running IQ-TREE: {' '.join(cmd)}")
242
+
243
+ # Run IQ-TREE with enhanced error handling
244
+ result = subprocess.run(
245
+ cmd,
246
+ capture_output=True,
247
+ text=True,
248
+ timeout=1200, # 20 minute timeout for larger datasets
249
+ cwd=os.getcwd()
250
+ )
251
+
252
+ if result.returncode == 0:
253
+ tree_file = f"{output_prefix}.treefile"
254
+ if os.path.exists(tree_file) and os.path.getsize(tree_file) > 0:
255
+ logging.info(f"IQ-TREE analysis completed: {tree_file}")
256
+ return True, tree_file
257
  else:
258
+ logging.error("IQ-TREE completed but tree file not found or empty")
259
+ return False, "Tree file not generated or empty"
260
+ else:
261
+ error_msg = result.stderr.strip() if result.stderr else "Unknown IQ-TREE error"
262
+ logging.error(f"IQ-TREE failed: {error_msg}")
263
+ return False, f"IQ-TREE error: {error_msg}"
264
+
265
+ except subprocess.TimeoutExpired:
266
+ logging.error("IQ-TREE timeout")
267
+ return False, "IQ-TREE timeout (>20 minutes). Try with fewer sequences or simpler model."
268
+ except FileNotFoundError:
269
+ return False, f"IQ-TREE executable not found: {iqtree_cmd}"
 
 
 
 
 
 
 
 
 
270
  except Exception as e:
271
+ logging.error(f"IQ-TREE execution failed: {e}")
272
+ return False, f"IQ-TREE execution failed: {str(e)}"
273
 
274
+ def create_simple_neighbor_joining_tree(sequences_dict):
275
+ """Create a simple distance-based tree when ML tools are not available"""
276
  try:
277
+ # This is a simplified implementation
278
+ # In a real scenario, you'd want to use a proper NJ implementation
279
+ import random
280
+
281
+ seq_names = list(sequences_dict.keys())
282
+ n_seqs = len(seq_names)
283
+
284
+ if n_seqs < 2:
285
+ return None, "Need at least 2 sequences for tree construction"
286
+
287
+ # Create a simple Newick tree structure
288
+ if n_seqs == 2:
289
+ tree_str = f"({seq_names[0]}:0.1,{seq_names[1]}:0.1);"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
290
  else:
291
+ # Simple clustering approach
292
+ tree_str = "(" + ",".join([f"{name}:0.1" for name in seq_names[:5]]) + ");"
293
+
294
+ # Save to temporary file
295
+ tree_file = "simple_tree.nwk"
296
+ with open(tree_file, 'w') as f:
297
+ f.write(tree_str)
298
+
299
+ return tree_file, "Simple distance-based tree created"
300
+
301
  except Exception as e:
302
+ return None, f"Simple tree creation failed: {str(e)}"
303
+
304
+ def create_multi_fasta_with_query(query_sequence, query_id="Query_F_Gene"):
305
+ """Create a multi-FASTA file with query sequence and reference sequences"""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
306
  try:
307
+ # Create temporary FASTA file
308
+ temp_fasta = tempfile.NamedTemporaryFile(mode='w', suffix='.fasta', delete=False)
309
+
310
+ # Add query sequence
311
+ temp_fasta.write(f">{query_id}\n{query_sequence}\n")
312
+
313
+ # Add reference sequences from existing aligned FASTA if available
314
+ ref_fasta_path = "f_gene_sequences_aligned.fasta"
315
+ if os.path.exists(ref_fasta_path):
316
+ with open(ref_fasta_path, 'r') as ref_file:
317
+ temp_fasta.write(ref_file.read())
318
+ logging.info(f"Added reference sequences from {ref_fasta_path}")
 
 
319
  else:
320
+ # If no reference file, try to create from CSV data
321
+ if analyzer and hasattr(analyzer, 'data'):
322
+ count = 0
323
+ for idx, row in analyzer.data.iterrows():
324
+ if 'sequence' in row and len(str(row['sequence'])) > 50:
325
+ seq_id = row.get('id', f"Ref_{count}")
326
+ sequence = str(row['sequence']).upper()
327
+ temp_fasta.write(f">{seq_id}\n{sequence}\n")
328
+ count += 1
329
+ if count >= 20: # Limit to prevent too large datasets
330
+ break
331
+ logging.info(f"Added {count} reference sequences from CSV")
332
+
333
+ temp_fasta.close()
334
+ return temp_fasta.name
335
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
336
  except Exception as e:
337
+ logging.error(f"Failed to create multi-FASTA: {e}")
338
+ return None
 
 
 
 
 
339
 
340
+ def build_maximum_likelihood_tree(f_gene_sequence):
341
+ """Build maximum likelihood phylogenetic tree with comprehensive fallback options"""
342
+ try:
343
+ # Check tool availability with enhanced detection
344
+ mafft_available, iqtree_available, mafft_cmd, iqtree_cmd = check_tool_availability()
345
+
346
+ # Prepare status message
347
+ status_msg = "🔍 Checking dependencies...\n"
348
+
349
+ if not mafft_available:
350
+ status_msg += "❌ MAFFT not found\n"
351
+ else:
352
+ status_msg += f"✅ MAFFT found: {mafft_cmd}\n"
353
+
354
+ if not iqtree_available:
355
+ status_msg += "❌ IQ-TREE not found\n"
356
+ else:
357
+ status_msg += f"✅ IQ-TREE found: {iqtree_cmd}\n"
358
+
359
+ # If neither tool is available, provide installation guide
360
+ if not mafft_available and not iqtree_available:
361
+ guide = install_dependencies_guide()
362
+ return False, f"{status_msg}\n{guide}", None, None
363
+
364
+ # If only one tool is missing, provide specific guidance
365
+ if not mafft_available:
366
+ return False, f"{status_msg}\n❌ MAFFT is required for sequence alignment. Please install MAFFT first.", None, None
367
+
368
+ if not iqtree_available:
369
+ status_msg += "\n⚠️ IQ-TREE not available. Attempting simple tree construction...\n"
370
+
371
+ # Try to create a simple tree as fallback
372
+ multi_fasta = create_multi_fasta_with_query(f_gene_sequence)
373
+ if multi_fasta:
374
+ # Read sequences
375
+ sequences = {}
376
+ current_seq = ""
377
+ current_name = ""
378
+
379
+ with open(multi_fasta, 'r') as f:
380
+ for line in f:
381
+ line = line.strip()
382
+ if line.startswith('>'):
383
+ if current_name and current_seq:
384
+ sequences[current_name] = current_seq
385
+ current_name = line[1:]
386
+ current_seq = ""
387
+ else:
388
+ current_seq += line
389
+ if current_name and current_seq:
390
+ sequences[current_name] = current_seq
391
+
392
+ simple_tree, simple_msg = create_simple_neighbor_joining_tree(sequences)
393
+ os.unlink(multi_fasta)
394
+
395
+ if simple_tree:
396
+ return True, f"{status_msg}✅ {simple_msg}", None, simple_tree
397
+ else:
398
+ return False, f"{status_msg}❌ {simple_msg}", None, None
399
+ else:
400
+ return False, f"{status_msg}❌ Failed to create input sequences", None, None
401
+
402
+ # Both tools available - proceed with full ML analysis
403
+ # Create output directory
404
+ output_dir = "ml_tree_output"
405
+ os.makedirs(output_dir, exist_ok=True)
406
+
407
+ # Step 1: Create multi-FASTA file with query and reference sequences
408
+ logging.info("Creating multi-FASTA file...")
409
+ multi_fasta = create_multi_fasta_with_query(f_gene_sequence)
410
+ if not multi_fasta:
411
+ return False, f"{status_msg}❌ Failed to create input FASTA", None, None
412
+
413
+ # Step 2: Run MAFFT alignment
414
+ logging.info("Running MAFFT alignment...")
415
+ aligned_fasta = os.path.join(output_dir, "aligned_sequences.fasta")
416
+ mafft_success, mafft_result = run_mafft_alignment(multi_fasta, aligned_fasta, mafft_cmd)
417
+
418
+ # Clean up temporary file
419
+ os.unlink(multi_fasta)
420
+
421
+ if not mafft_success:
422
+ return False, f"{status_msg}❌ MAFFT failed: {mafft_result}", None, None
423
+
424
+ # Step 3: Run IQ-TREE analysis
425
+ logging.info("Running IQ-TREE analysis...")
426
+ tree_prefix = os.path.join(output_dir, "ml_tree")
427
+ iqtree_success, iqtree_result = run_iqtree_analysis(aligned_fasta, tree_prefix, iqtree_cmd)
428
+
429
+ if not iqtree_success:
430
+ return False, f"{status_msg}❌ IQ-TREE failed: {iqtree_result}", aligned_fasta, None
431
+
432
+ # Step 4: Prepare output files
433
+ tree_file = iqtree_result
434
+ log_file = f"{tree_prefix}.log"
435
+
436
+ # Copy to standard names for compatibility
437
+ standard_aligned = "f_gene_sequences_aligned.fasta"
438
+ standard_tree = "f_gene_sequences.phy.treefile"
439
+
440
+ if os.path.exists(aligned_fasta):
441
+ shutil.copy2(aligned_fasta, standard_aligned)
442
+ if os.path.exists(tree_file):
443
+ shutil.copy2(tree_file, standard_tree)
444
+
445
+ success_msg = f"{status_msg}✅ Maximum likelihood tree built successfully!\n"
446
+ success_msg += f"- Alignment: {os.path.basename(aligned_fasta)}\n"
447
+ success_msg += f"- Tree: {os.path.basename(tree_file)}\n"
448
+
449
+ if os.path.exists(log_file):
450
+ try:
451
+ with open(log_file, 'r') as f:
452
+ log_content = f.read()
453
+ # Extract model information
454
+ if "Best-fit model:" in log_content:
455
+ model_lines = [line for line in log_content.split('\n') if "Best-fit model:" in line]
456
+ if model_lines:
457
+ success_msg += f"- {model_lines[0].strip()}\n"
458
+ except Exception as e:
459
+ logging.warning(f"Could not read log file: {e}")
460
+
461
+ logging.info("Maximum likelihood tree construction completed")
462
+ return True, success_msg, aligned_fasta, tree_file
463
+
464
+ except Exception as e:
465
+ logging.error(f"ML tree construction failed: {e}")
466
+ return False, f"ML tree construction failed: {str(e)}", None, None
467
 
468
+ # --- Tree Analysis Function (Based on old Gradio API) ---
469
+ def analyze_sequence_for_tree(sequence: str, matching_percentage: float) -> str:
470
+ """
471
+ Analyze sequence and create phylogenetic tree using the working Gradio API pattern
472
+ """
473
+ try:
474
+ if not analyzer:
475
+ return "Error: Tree analyzer not initialized. Please check if the CSV data file is available."
476
+
477
+ if not sequence:
478
+ return "Error: Please provide a sequence."
479
+
480
+ if not (1 <= matching_percentage <= 99):
481
+ return "Error: Matching percentage must be between 1 and 99."
482
+
483
+ # Find query sequence
484
+ if not analyzer.find_query_sequence(sequence):
485
+ return "Error: Invalid query sequence or sequence not found in dataset."
486
+
487
+ # Set matching percentage
488
+ analyzer.matching_percentage = matching_percentage
489
+
490
+ # Find similar sequences
491
+ matched_ids, actual_percentage = analyzer.find_similar_sequences(matching_percentage)
492
+
493
+ if not matched_ids:
494
+ return f"No similar sequences found at {matching_percentage}% similarity. Try lowering the threshold."
495
+
496
+ logging.info(f"Found {len(matched_ids)} similar sequences at {actual_percentage:.1f}% similarity")
497
+
498
+ # Build tree structure
499
+ tree_structure = analyzer.build_tree_structure(matched_ids)
500
+ if not tree_structure:
501
+ return "Error: Failed to build tree structure."
502
+
503
+ # Create interactive tree
504
+ fig = analyzer.create_interactive_tree(matched_ids, actual_percentage)
505
+ if not fig:
506
+ return "Error: Failed to create tree visualization."
507
+
508
+ # Generate HTML content
509
+ html_content = fig.to_html(full_html=True, include_plotlyjs='cdn')
510
+
511
+ # Save to output folder
512
+ output_dir = "output"
513
+ os.makedirs(output_dir, exist_ok=True)
514
+
515
+ # Create a safe filename
516
+ safe_seq_name = re.sub(r'[^a-zA-Z0-9]', '_', sequence[:20])
517
+ html_filename = os.path.join(output_dir, f"tree_{safe_seq_name}_{matching_percentage}.html")
518
+
519
+ with open(html_filename, "w", encoding='utf-8') as f:
520
+ f.write(html_content)
521
+
522
+ logging.info(f"Tree HTML saved to {html_filename}")
523
+
524
+ return html_content
525
+
526
+ except Exception as e:
527
+ error_msg = f"Tree analysis error: {str(e)}"
528
+ logging.error(error_msg)
529
+ import traceback
530
+ logging.error(f"Full traceback: {traceback.format_exc()}")
531
+ return error_msg
532
 
533
+ # --- Keras Prediction ---
534
+ def predict_with_keras(sequence):
535
+ try:
536
+ if not keras_model or not kmer_to_index:
537
+ return f"Keras model not available. Input sequence: {sequence[:100]}..."
538
+
539
+ if len(sequence) < 6:
540
+ return "Sequence too short for k-mer prediction (minimum 6 nucleotides required)."
541
+
542
+ # Generate k-mers
543
+ kmers = [sequence[i:i+6] for i in range(len(sequence)-5)]
544
+ indices = [kmer_to_index.get(kmer, 0) for kmer in kmers]
545
+
546
+ # Prepare input
547
+ input_arr = np.array([indices])
548
+ prediction = keras_model.predict(input_arr, verbose=0)[0]
549
+
550
+ # Format prediction as probabilities/scores (not a sequence)
551
+ result = ''.join([str(round(p, 3)) for p in prediction])
552
+ return result
553
+ except Exception as e:
554
+ logging.error(f"Keras prediction failed: {e}")
555
+ return f"Keras prediction failed: {str(e)}"
556
+
557
+ # --- FASTA Reader ---
558
  def read_fasta_file(file_obj):
559
  try:
560
  if file_obj is None:
561
  return ""
562
+
563
+ # Handle file object
564
  if hasattr(file_obj, 'name'):
565
  with open(file_obj.name, "r") as f:
566
  content = f.read()
567
  else:
568
  content = file_obj.read().decode("utf-8") if hasattr(file_obj, "read") else str(file_obj)
569
+
570
  lines = content.strip().split("\n")
571
  seq_lines = [line.strip() for line in lines if not line.startswith(">")]
572
  return ''.join(seq_lines)
 
574
  logging.error(f"Failed to read FASTA file: {e}")
575
  return ""
576
 
577
+ # --- Full Pipeline ---
578
  def run_pipeline_from_file(fasta_file_obj, similarity_score, build_ml_tree):
579
  try:
580
  dna_input = read_fasta_file(fasta_file_obj)
581
  if not dna_input:
582
+ return "Failed to read FASTA file", "", "", "", "", None, None, None, "No input sequence"
583
+ return run_pipeline(dna_input, similarity_score, build_ml_tree)
 
 
 
 
584
  except Exception as e:
585
  error_msg = f"Pipeline error: {str(e)}"
586
  logging.error(error_msg)
587
+ return error_msg, "", "", "", "", None, None, None, error_msg
588
+
589
+ def run_pipeline(dna_input, similarity_score=95.0, build_ml_tree=False):
590
+ try:
591
+ # Clean input
592
+ dna_input = dna_input.upper().strip()
593
+ if not dna_input:
594
+ return "Empty input", "", "", "", "", None, None, None, "No input provided"
595
+
596
+ # Sanitize DNA sequence
597
+ if not re.match('^[ACTGN]+$', dna_input):
598
+ dna_input = ''.join(c if c in 'ACTGN' else 'N' for c in dna_input)
599
+ logging.info("DNA sequence sanitized")
600
+
601
+ # Step 1: Boundary Prediction - Extract F gene sequence
602
+ processed_sequence = dna_input # This will be the sequence used for downstream analysis
603
+ boundary_output = ""
604
+
605
+ if boundary_model:
606
+ try:
607
+ predictions, probs, confidence = boundary_model.predict(dna_input)
608
+ regions = boundary_model.extract_gene_regions(predictions, dna_input)
609
+ if regions:
610
+ processed_sequence = regions[0]["sequence"] # Use the extracted gene region
611
+ boundary_output = processed_sequence # Output the actual F gene sequence
612
+ logging.info(f"F gene extracted: {len(processed_sequence)} bp (confidence: {confidence:.3f})")
613
+ else:
614
+ boundary_output = f"No F gene regions found in input sequence"
615
+ processed_sequence = dna_input
616
+ logging.warning("No gene regions found, using full sequence")
617
+ logging.info("Boundary model prediction completed")
618
+ except Exception as e:
619
+ logging.error(f"Boundary model failed: {e}")
620
+ boundary_output = f"Boundary model error: {str(e)}"
621
+ processed_sequence = dna_input # Fall back to original sequence
622
+ else:
623
+ boundary_output = f"Boundary model not available. Using original input: {len(dna_input)} bp"
624
+ processed_sequence = dna_input
625
+
626
+ # Step 2: Keras Prediction (F gene validation)
627
+ keras_output = ""
628
+ if processed_sequence and len(processed_sequence) >= 6:
629
+ keras_prediction = predict_with_keras(processed_sequence)
630
+ # Interpret keras prediction as F gene validation
631
+ if keras_prediction and not keras_prediction.startswith(("Keras", "Sequence too short")):
632
+ # You might want to add logic here to interpret the prediction scores
633
+ # For now, just show the prediction
634
+ keras_output = f"F gene validation scores: {keras_prediction[:100]}..."
635
+ else:
636
+ keras_output = keras_prediction
637
+ else:
638
+ keras_output = "Skipped: sequence too short for F gene validation"
639
+
640
+ # Step 3: Maximum Likelihood Tree (MAFFT + IQ-TREE)
641
+ aligned_file = None
642
+ phy_file = None
643
+ ml_tree_output = ""
644
+
645
+ if build_ml_tree and processed_sequence and len(processed_sequence) >= 50:
646
+ try:
647
+ logging.info("Starting maximum likelihood tree construction...")
648
+ ml_success, ml_message, ml_aligned, ml_tree = build_maximum_likelihood_tree(processed_sequence)
649
+
650
+ if ml_success:
651
+ ml_tree_output = ml_message
652
+ aligned_file = ml_aligned
653
+ phy_file = ml_tree
654
+ else:
655
+ ml_tree_output = ml_message # This now includes detailed error information
656
+
657
+ except Exception as e:
658
+ ml_tree_output = f"❌ ML Tree construction failed: {str(e)}"
659
+ logging.error(f"ML Tree failed: {e}")
660
+ elif build_ml_tree:
661
+ ml_tree_output = "❌ F gene sequence too short for ML tree construction (minimum 50 bp)"
662
+ else:
663
+ ml_tree_output = "ML tree construction skipped (not requested)"
664
+
665
+ # Step 4: ML Simplified Tree (using the existing approach)
666
+ html_file = None
667
+ tree_html_content = "No tree generated"
668
+ simplified_ml_output = ""
669
+
670
+ if analyzer and processed_sequence and len(processed_sequence) >= 10:
671
+ try:
672
+ logging.info(f"Starting simplified ML tree analysis with F gene sequence length: {len(processed_sequence)}")
673
+
674
+ # Use the existing tree analysis function with user-specified similarity
675
+ tree_result = analyze_sequence_for_tree(processed_sequence, matching_percentage=similarity_score)
676
+
677
+ if tree_result and not tree_result.startswith("Error:"):
678
+ # Success - we have HTML content
679
+ tree_html_content = tree_result
680
+ simplified_ml_output = "✅ Simplified phylogenetic tree generated successfully!"
681
+
682
+ # Check if HTML file was created
683
+ output_dir = "output"
684
+ if os.path.exists(output_dir):
685
+ html_files = [f for f in os.listdir(output_dir) if f.endswith('.html')]
686
+ if html_files:
687
+ html_file = os.path.join(output_dir, html_files[-1]) # Get the latest
688
+ simplified_ml_output += f"\n- Tree file: {html_files[-1]}"
689
+
690
+ # Count sequences analyzed
691
+ if analyzer.find_query_sequence(processed_sequence):
692
+ matched_ids, perc = analyzer.find_similar_sequences(similarity_score)
693
+ simplified_ml_output += f"\n- {len(matched_ids)} sequences analyzed"
694
+ simplified_ml_output += f"\n- Similarity threshold: {perc:.1f}%"
695
+ else:
696
+
697
+ simplified_ml_output = f"❌ Simplified ML tree failed: {tree_result}"
698
+ tree_html_content = f"<p>Error: {tree_result}</p>"
699
+
700
+ except Exception as e:
701
+ logging.error(f"Simplified ML tree analysis failed: {e}")
702
+ simplified_ml_output = f"❌ Simplified ML tree analysis failed: {str(e)}"
703
+ tree_html_content = f"<p>Error: {str(e)}</p>"
704
+ else:
705
+ if not analyzer:
706
+ simplified_ml_output = "❌ Tree analyzer not available"
707
+ else:
708
+ simplified_ml_output = "❌ F gene sequence too short for tree analysis (minimum 10 bp)"
709
+
710
+ # Return all results
711
+ return (
712
+ boundary_output, # F gene extraction result
713
+ keras_output, # F gene validation result
714
+ ml_tree_output, # ML tree construction status
715
+ simplified_ml_output, # Simplified tree analysis status
716
+ tree_html_content, # HTML content for tree display
717
+ aligned_file, # Path to aligned FASTA file
718
+ phy_file, # Path to phylogenetic tree file
719
+ html_file, # Path to HTML tree file
720
+ f"Pipeline completed. F gene length: {len(processed_sequence)} bp" # Summary
721
+ )
722
+
723
+ except Exception as e:
724
+ error_msg = f"Pipeline execution failed: {str(e)}"
725
+ logging.error(error_msg)
726
+ import traceback
727
+ logging.error(f"Full traceback: {traceback.format_exc()}")
728
+ return (
729
+ error_msg, "", "", "", f"<p>Error: {error_msg}</p>",
730
+ None, None, None, error_msg
731
+ )
732
 
733
  # --- Gradio Interface ---
734
  def create_interface():
735
+ """Create the Gradio interface with enhanced layout and features"""
736
+
737
+ # Custom CSS for better styling
738
  custom_css = """
739
+ .gradio-container {
740
+ max-width: 1200px !important;
741
+ }
742
+ .tab-nav button {
743
+ font-size: 16px !important;
744
+ }
745
+ .output-html {
746
+ height: 600px !important;
747
+ overflow: auto;
748
+ }
749
  """
750
+
751
  with gr.Blocks(css=custom_css, title="F Gene Analysis Pipeline") as iface:
752
  gr.Markdown("""
753
  # 🧬 F Gene Analysis Pipeline
754
+
755
+ This tool provides comprehensive analysis of F genes including:
756
+ - **Gene Boundary Detection**: Extract F gene sequences from larger genomic sequences
757
+ - **Gene Validation**: Validate extracted sequences using machine learning
758
+ - **Phylogenetic Analysis**: Build maximum likelihood trees and simplified phylogenetic trees
759
+
760
+ **Instructions:**
761
+ 1. Enter your sequence directly or upload a FASTA file
762
+ 2. Adjust similarity threshold for phylogenetic analysis (1-99%)
763
+ 3. Choose whether to build maximum likelihood trees (requires MAFFT & IQ-TREE)
764
+ 4. Click "Run Analysis" to start the pipeline
765
  """)
766
+
767
  with gr.Tab("🔬 Analysis Pipeline"):
768
  with gr.Row():
769
  with gr.Column(scale=2):
770
+ # Input section
771
+ gr.Markdown("### Input Sequence")
772
+ dna_input = gr.Textbox(
773
+ label="DNA Sequence",
774
+ placeholder="Enter your DNA sequence here (ATCG format)...",
775
+ lines=5,
776
+ max_lines=10
777
+ )
778
+
779
+ fasta_file = gr.File(
780
+ label="Or Upload FASTA File",
781
+ file_types=[".fasta", ".fa", ".fas", ".txt"]
782
+ )
783
+
784
  with gr.Row():
785
+ similarity_score = gr.Slider(
786
+ minimum=1,
787
+ maximum=99,
788
+ value=95.0,
789
+ step=1.0,
790
+ label="Similarity Threshold (%)",
791
+ info="Minimum similarity for phylogenetic analysis"
792
+ )
793
+
794
+ build_ml_tree = gr.Checkbox(
795
+ label="Build ML Tree",
796
+ value=False,
797
+ info="Build maximum likelihood tree (requires MAFFT & IQ-TREE)"
798
+ )
799
+
800
+ # Action buttons
801
  with gr.Row():
802
+ run_btn = gr.Button("🚀 Run Analysis", variant="primary", size="lg")
803
  clear_btn = gr.Button("����️ Clear", variant="secondary")
804
+
805
  with gr.Column(scale=1):
806
+ # Status and info
807
+ gr.Markdown("### Analysis Status")
808
+ status_display = gr.Textbox(
809
+ label="Status",
810
+ value="Ready to analyze",
811
+ interactive=False,
812
+ lines=3
813
+ )
814
+
815
+ # Model status
816
+ gr.Markdown("### Available Models")
817
+ model_status = []
818
+ if boundary_model:
819
+ model_status.append("✅ Boundary Detection Model")
820
+ else:
821
+ model_status.append("❌ Boundary Detection Model")
822
+
823
+ if keras_model:
824
+ model_status.append("✅ Gene Validation Model")
825
+ else:
826
+ model_status.append("❌ Gene Validation Model")
827
+
828
+ if analyzer:
829
+ model_status.append("✅ Tree Analysis Module")
830
+ else:
831
+ model_status.append("❌ Tree Analysis Module")
832
+
833
+ gr.Markdown("\n".join(model_status))
834
+
835
  with gr.Tab("📊 Results"):
836
  with gr.Row():
837
  with gr.Column():
838
+ # Text outputs
839
+ boundary_output = gr.Textbox(
840
+ label="🎯 F Gene Extraction",
841
+ lines=5,
842
+ interactive=False
843
+ )
844
+
845
+ keras_output = gr.Textbox(
846
+ label="🔍 Gene Validation",
847
+ lines=3,
848
+ interactive=False
849
+ )
850
+
851
  with gr.Column():
852
+ ml_tree_output = gr.Textbox(
853
+ label="🌳 Maximum Likelihood Tree",
854
+ lines=5,
855
+ interactive=False
856
+ )
857
+
858
+ simplified_ml_output = gr.Textbox(
859
+ label="📈 Simplified Phylogenetic Analysis",
860
+ lines=3,
861
+ interactive=False
862
+ )
863
+
864
+ # Tree visualization
865
+ gr.Markdown("### 🌲 Phylogenetic Tree Visualization")
866
+ tree_html = gr.HTML(
867
+ label="Interactive Tree",
868
+ value="<p>No tree generated yet. Run analysis to see results.</p>"
869
+ )
870
+
871
+ # File downloads
872
+ gr.Markdown("### 📁 Download Results")
873
  with gr.Row():
874
+ aligned_file = gr.File(
875
+ label="Aligned Sequences (FASTA)",
876
+ interactive=False
877
+ )
878
+
879
+ phy_file = gr.File(
880
+ label="Phylogenetic Tree File",
881
+ interactive=False
882
+ )
883
+
884
+ html_file = gr.File(
885
+ label="Interactive Tree (HTML)",
886
+ interactive=False
887
+ )
888
+
889
  with gr.Tab("ℹ️ Help & Info"):
890
  gr.Markdown("""
891
  ## About This Tool
892
+
893
+ ### F Gene Analysis Pipeline
894
+ This comprehensive pipeline analyzes F genes through multiple computational approaches:
895
+
896
+ #### 🎯 Gene Boundary Detection
897
+ - Uses deep learning to identify and extract F gene sequences from larger genomic sequences
898
+ - Provides confidence scores for detected boundaries
899
+ - Automatically trims sequences to focus on the F gene region
900
+
901
+ #### 🔍 Gene Validation
902
+ - Employs k-mer based machine learning models to validate extracted sequences
903
+ - Provides probability scores indicating likelihood of being a genuine F gene
904
+ - Uses 6-mer frequency patterns for classification
905
+
906
+ #### 🌳 Phylogenetic Analysis
907
+
908
+ **Maximum Likelihood Trees:**
909
+ - Requires MAFFT (sequence alignment) and IQ-TREE (phylogenetic reconstruction)
910
+ - Performs model selection and bootstrap analysis
911
+ - Generates publication-quality phylogenetic trees
912
+ - Provides detailed evolutionary analysis
913
+
914
+ **Simplified Trees:**
915
+ - Uses built-in algorithms for quick phylogenetic analysis
916
+ - Interactive visualization with similarity-based clustering
917
+ - Faster alternative when external tools are not available
918
+
919
+ ### Input Requirements
920
+ - **DNA Sequences**: ATCG format, minimum 50 bp for meaningful analysis
921
+ - **FASTA Files**: Standard FASTA format with single or multiple sequences
922
+ - **Similarity Threshold**: 1-99% for controlling phylogenetic analysis sensitivity
923
+
924
  ### Dependencies
925
+
926
+ **Required for ML Trees:**
927
+ ```bash
928
+ # Ubuntu/Debian
929
+ sudo apt-get install mafft iqtree
930
+
931
+ # macOS
932
+ brew install mafft iqtree
933
+
934
+ # Conda
935
+ conda install -c bioconda mafft iqtree
936
+ ```
937
+
938
+ ### Output Files
939
+ - **Aligned FASTA**: Multiple sequence alignment in FASTA format
940
+ - **Tree File**: Newick format phylogenetic tree
941
+ - **HTML Tree**: Interactive visualization for web browsers
942
+
943
  ### Troubleshooting
944
+
945
+ **Common Issues:**
946
+ - *"No similar sequences found"*: Lower the similarity threshold
947
+ - *"Sequence too short"*: Provide sequences longer than 50 bp
948
+ - *"MAFFT/IQ-TREE not found"*: Install required dependencies
949
+ - *"Model not available"*: Check model files are properly downloaded
950
+
951
+ **Performance Tips:**
952
+ - Use sequences between 100-2000 bp for optimal performance
953
+ - Limit to <50 sequences for faster tree construction
954
+ - Lower similarity thresholds find more distant relatives
955
+ - Higher thresholds focus on closely related sequences
956
+
957
+ ### Citation
958
+ If you use this tool in your research, please cite the appropriate methods and tools used.
959
  """)
960
+
961
+ # Event handlers
962
+ def run_analysis_text(dna_seq, sim_score, build_tree):
963
+ return run_pipeline(dna_seq, sim_score, build_tree)
964
+
965
+ def run_analysis_file(file_obj, sim_score, build_tree):
966
+ return run_pipeline_from_file(file_obj, sim_score, build_tree)
967
+
968
  def run_analysis_combined(dna_seq, file_obj, sim_score, build_tree):
969
+ # Priority: file upload over text input
970
  if file_obj is not None:
971
  return run_pipeline_from_file(file_obj, sim_score, build_tree)
972
  else:
973
+ return run_pipeline(dna_seq, sim_score, build_tree)
974
+
 
 
 
975
  def clear_inputs():
976
+ return "", None, 95.0, False, "Ready to analyze"
977
+
978
+ # Connect events
979
  run_btn.click(
980
  fn=run_analysis_combined,
981
  inputs=[dna_input, fasta_file, similarity_score, build_ml_tree],
982
  outputs=[
983
+ boundary_output, keras_output, ml_tree_output,
984
+ simplified_ml_output, tree_html, aligned_file,
985
+ phy_file, html_file, status_display
986
  ]
987
  )
988
+
989
+ clear_btn.click(
990
+ fn=clear_inputs,
991
+ outputs=[dna_input, fasta_file, similarity_score, build_ml_tree, status_display]
992
+ )
993
+
994
+ # Example data loading
995
+ gr.Markdown("### 🧪 Example Data")
996
+ example_btn = gr.Button("Load Example F Gene Sequence", variant="secondary")
997
+
998
+ def load_example():
999
+ example_seq = "ATGAAACTGTCAACACTCACTGAGTACATTAGCCAAGTTCTCAAGACTGAGTGTTTACCTTTGTGAATACACTGAGTCCTTGTCAACGTTCGGCTGCAGTCACACTGATGGTCTTGTCTTCAGGAGCAACTGCAGTCTGTGCTGTGTACTATAGTGCTAAGAGTGATAATGCACTGTTCAGTACCTTTGACAGTGTGTCTCTGTCACCTGGTGCTATGCAGAGCTGCGATGAGATCTACATTGGTCTGATCGATAAGACTGAGTCCAAGGGTGTTGCTGTGTGTACTGTAGAGTGTGATAGTGTTGCCTGCACTGTGTCTATGGCTGATCTTGAGGCTCTGCTTATGTCAACACTGAGTGTGAAATGTTCATTTGCTACTTCAAGACTGATGTGAAGACTGTGTATTGTACTCAGTCATGCAGAGTGAAGTCCTTGAGCCACTTGCTTTGTACAATGTGGGTGATGAGATGTTGTGCTGCAGTGTCAAGGGGCCACAGTCTTGCCTTGATAGTGCGATTGCTGTGATGATGTGCACTTCAATGAGTGGTCGAGATGCTGCTGTGTGTAAGGATGCTGCTGTGTGTAAGAAGGATGCTGCTGTGTGTAAGA"
1000
+ return example_seq, "Example F gene sequence loaded"
1001
+
1002
+ example_btn.click(
1003
+ fn=load_example,
1004
+ outputs=[dna_input, status_display]
1005
+ )
1006
+
1007
  return iface
1008
 
1009
+ # --- Main Execution ---
1010
  if __name__ == "__main__":
1011
+ # Initialize and launch interface
1012
  interface = create_interface()
1013
+
1014
+ # Launch with enhanced configuration
1015
+ interface.launch(
1016
+ server_name="0.0.0.0", # Allow external connections
1017
+ server_port=7860, # Default Gradio port
1018
+ share=False, # Set to True for public sharing
1019
+ debug=True, # Enable debug mode
1020
+ show_error=True, # Show detailed errors
1021
+ max_threads=4, # Limit concurrent threads
1022
+ auth=None, # Add authentication if needed: ("username", "password")
1023
+ ssl_verify=False, # For development environments
1024
+ quiet=False # Show startup messages
1025
+ )