re-type commited on
Commit
b15c785
·
verified ·
1 Parent(s): e736ddf

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +267 -7
app.py CHANGED
@@ -7,7 +7,7 @@ import os
7
  import re
8
  import logging
9
  import numpy as np
10
- from predictor import GenePredictor, enhanced_run_pipeline
11
  from tensorflow.keras.models import load_model
12
  import ml_simplified_tree
13
  import tempfile
@@ -24,7 +24,7 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(
24
 
25
  # --- Paths ---
26
  model_repo = "GGproject10/best_boundary_aware_model"
27
- csv_path = "f cleaned.csv"
28
  classifier_model_dir = "model"
29
 
30
  # Get HF token from environment
@@ -94,8 +94,268 @@ except Exception as e:
94
  logging.error(f"Failed to initialize tree analyzer: {e}")
95
  analyzer = None
96
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
  # --- Tool Detection and Helper Functions ---
98
- # (Keep existing functions: check_tool_availability, install_dependencies_guide, etc., as they are unchanged)
 
 
 
 
 
 
 
 
 
 
99
 
100
  def read_fasta_file(file_obj):
101
  try:
@@ -119,8 +379,8 @@ def run_pipeline_from_file(fasta_file_obj, similarity_score, build_ml_tree):
119
  if not dna_input:
120
  return "Failed to read FASTA file", "", "", "", "", "", "", "", "", None, None, None, "No input sequence"
121
  return enhanced_run_pipeline(
122
- dna_input, keras_model, kmer_to_index, classifier_model,
123
- classifier_kmer_to_index, classifier_maxlen, LABELS,
124
  similarity_score, build_ml_tree
125
  )
126
  except Exception as e:
@@ -192,8 +452,8 @@ def create_interface():
192
  return run_pipeline_from_file(file_obj, sim_score, build_tree)
193
  else:
194
  return enhanced_run_pipeline(
195
- dna_seq, keras_model, kmer_to_index, classifier_model,
196
- classifier_kmer_to_index, classifier_maxlen, LABELS,
197
  sim_score, build_tree
198
  )
199
  def clear_inputs():
 
7
  import re
8
  import logging
9
  import numpy as np
10
+ from tensorflow.keras.preprocessing.sequence import pad_sequences
11
  from tensorflow.keras.models import load_model
12
  import ml_simplified_tree
13
  import tempfile
 
24
 
25
  # --- Paths ---
26
  model_repo = "GGproject10/best_boundary_aware_model"
27
+ csv_path = "f_cleaned.csv"
28
  classifier_model_dir = "model"
29
 
30
  # Get HF token from environment
 
94
  logging.error(f"Failed to initialize tree analyzer: {e}")
95
  analyzer = None
96
 
97
+ # --- Predictor Functions (from predictor.py) ---
98
+ class GenePredictor:
99
+ """Minimal implementation to load and use the boundary-aware model"""
100
+ def __init__(self, model_path):
101
+ try:
102
+ self.model = torch.load(model_path, map_location=torch.device('cpu'))
103
+ logging.info(f"Loaded boundary model from {model_path}")
104
+ except Exception as e:
105
+ logging.error(f"Failed to load boundary model: {e}")
106
+ self.model = None
107
+
108
+ def predict(self, sequence):
109
+ if self.model is None:
110
+ return None, None, 0.0
111
+ # Placeholder: Implement actual prediction logic
112
+ return [0] * len(sequence), [0.5] * len(sequence), 0.5
113
+
114
+ def extract_gene_regions(self, predictions, sequence):
115
+ if predictions is None:
116
+ return []
117
+ return [{"sequence": sequence, "start": 0, "end": len(sequence), "confidence": 0.5}]
118
+
119
+ def preprocess_sequence_for_ndv_f_gene(sequence):
120
+ """Enhanced preprocessing specifically for NDV F gene sequences"""
121
+ try:
122
+ sequence = sequence.upper().strip()
123
+ sequence = re.sub(r'[^ATCGN]', '', sequence)
124
+ if len(sequence) < 1000:
125
+ logging.warning(f"Sequence length ({len(sequence)}) shorter than typical NDV F gene (1662-1800 nt)")
126
+ if not sequence.startswith('ATG'):
127
+ logging.warning("Sequence doesn't start with ATG start codon")
128
+ atg_pos = sequence.find('ATG')
129
+ if atg_pos != -1:
130
+ sequence = sequence[atg_pos:]
131
+ logging.info(f"Found ATG at position {atg_pos}, using sequence from there")
132
+ if len(sequence) % 3 != 0:
133
+ sequence = sequence[:len(sequence) - (len(sequence) % 3)]
134
+ logging.info(f"Trimmed sequence to maintain reading frame: {len(sequence)} nt")
135
+ return sequence
136
+ except Exception as e:
137
+ logging.error(f"Sequence preprocessing failed: {e}")
138
+ return sequence
139
+
140
+ def enhanced_keras_prediction(sequence, keras_model, kmer_to_index, kmer_size=6):
141
+ """Enhanced Keras prediction with better handling for NDV F gene"""
142
+ try:
143
+ if not keras_model or not kmer_to_index:
144
+ return "Keras model not available"
145
+ processed_seq = preprocess_sequence_for_ndv_f_gene(sequence)
146
+ if len(processed_seq) < kmer_size:
147
+ return f"Sequence too short for k-mer prediction (minimum {kmer_size} nucleotides required)"
148
+ kmers = [processed_seq[i:i+kmer_size] for i in range(len(processed_seq)-kmer_size+1)]
149
+ indices = []
150
+ unknown_kmers = 0
151
+ for kmer in kmers:
152
+ if kmer in kmer_to_index:
153
+ indices.append(kmer_to_index[kmer])
154
+ else:
155
+ indices.append(0)
156
+ unknown_kmers += 1
157
+ logging.info(f"Generated {len(kmers)} k-mers, {unknown_kmers} unknown k-mers")
158
+ input_arr = np.array([indices])
159
+ prediction = keras_model.predict(input_arr, verbose=0)[0]
160
+ max_prob = np.max(prediction)
161
+ mean_prob = np.mean(prediction)
162
+ confidence_score = max_prob
163
+ consistency_score = 1.0 - np.std(prediction)
164
+ result = {
165
+ 'raw_prediction': prediction.tolist(),
166
+ 'max_probability': float(max_prob),
167
+ 'mean_probability': float(mean_prob),
168
+ 'confidence_score': float(confidence_score),
169
+ 'consistency_score': float(consistency_score),
170
+ 'sequence_length': len(processed_seq),
171
+ 'kmers_generated': len(kmers),
172
+ 'unknown_kmers': unknown_kmers,
173
+ 'kmer_coverage': 1.0 - (unknown_kmers / len(kmers)) if kmers else 0.0
174
+ }
175
+ return result
176
+ except Exception as e:
177
+ logging.error(f"Enhanced Keras prediction failed: {e}")
178
+ return f"Enhanced Keras prediction failed: {str(e)}"
179
+
180
+ def enhanced_classify_sequence(sequence, classifier_model, classifier_kmer_to_index, classifier_maxlen, labels):
181
+ """Enhanced classification with NDV F gene specific improvements"""
182
+ try:
183
+ if not classifier_model or not classifier_kmer_to_index or classifier_maxlen is None:
184
+ return {
185
+ "status": "error",
186
+ "message": "Classification model not available",
187
+ "confidence": None,
188
+ "predicted_label": None,
189
+ "details": {}
190
+ }
191
+ processed_seq = preprocess_sequence_for_ndv_f_gene(sequence)
192
+ if len(processed_seq) < 1000:
193
+ return {
194
+ "status": "warning",
195
+ "message": f"Sequence shorter than typical NDV F gene ({len(processed_seq)} < 1000 nt)",
196
+ "confidence": None,
197
+ "predicted_label": None,
198
+ "details": {"sequence_length": len(processed_seq)}
199
+ }
200
+ kmer_size = 6
201
+ tokens = [processed_seq[i:i+kmer_size] for i in range(len(processed_seq)-kmer_size+1)]
202
+ encoded = []
203
+ unknown_count = 0
204
+ for kmer in tokens:
205
+ if kmer in classifier_kmer_to_index:
206
+ encoded.append(classifier_kmer_to_index[kmer])
207
+ else:
208
+ encoded.append(0)
209
+ unknown_count += 1
210
+ padded = pad_sequences([encoded], maxlen=classifier_maxlen, padding='post')
211
+ pred = classifier_model.predict(padded, verbose=0)
212
+ predicted_class = int(np.argmax(pred))
213
+ confidence = float(np.max(pred))
214
+ predicted_label = labels[predicted_class] if predicted_class < len(labels) else "Unknown"
215
+ kmer_coverage = 1.0 - (unknown_count / len(tokens)) if tokens else 0.0
216
+ prediction_entropy = -np.sum(pred[0] * np.log(pred[0] + 1e-10))
217
+ details = {
218
+ "sequence_length": len(processed_seq),
219
+ "kmers_generated": len(tokens),
220
+ "unknown_kmers": unknown_count,
221
+ "kmer_coverage": kmer_coverage,
222
+ "prediction_entropy": float(prediction_entropy),
223
+ "all_probabilities": {labels[i]: float(pred[0][i]) for i in range(len(labels)) if i < len(pred[0])},
224
+ "starts_with_atg": processed_seq.startswith('ATG'),
225
+ "length_in_frame": len(processed_seq) % 3 == 0
226
+ }
227
+ if predicted_label == "F":
228
+ f_gene_score = confidence
229
+ if kmer_coverage > 0.8: f_gene_score *= 1.1
230
+ if processed_seq.startswith('ATG'): f_gene_score *= 1.05
231
+ if len(processed_seq) % 3 == 0: f_gene_score *= 1.05
232
+ if 1500 <= len(processed_seq) <= 2000: f_gene_score *= 1.1
233
+ details["enhanced_f_score"] = min(f_gene_score, 1.0)
234
+ if f_gene_score > 0.7:
235
+ return {"status": "success", "message": "NDV F gene detected with high confidence", "confidence": confidence, "predicted_label": predicted_label, "details": details}
236
+ elif f_gene_score > 0.5:
237
+ return {"status": "success", "message": "NDV F gene detected with moderate confidence", "confidence": confidence, "predicted_label": predicted_label, "details": details}
238
+ else:
239
+ return {"status": "warning", "message": "Possible F gene but low confidence - check sequence quality", "confidence": confidence, "predicted_label": predicted_label, "details": details}
240
+ elif predicted_label == "Random":
241
+ if kmer_coverage < 0.5:
242
+ return {"status": "error", "message": f"Poor sequence quality detected (coverage: {kmer_coverage:.1%}). Check for sequencing errors.", "confidence": confidence, "predicted_label": predicted_label, "details": details}
243
+ else:
244
+ return {"status": "error", "message": "Sequence does not appear to be NDV F gene. Verify input sequence.", "confidence": confidence, "predicted_label": predicted_label, "details": details}
245
+ else:
246
+ return {"status": "error", "message": f"Detected as {predicted_label} gene, not F gene. Please provide NDV F gene sequence.", "confidence": confidence, "predicted_label": predicted_label, "details": details}
247
+ except Exception as e:
248
+ logging.error(f"Enhanced classification failed: {e}")
249
+ return {"status": "error", "message": f"Classification failed: {str(e)}", "confidence": None, "predicted_label": None, "details": {"error": str(e)}}
250
+
251
+ def validate_ndv_f_gene_sequence(sequence):
252
+ """Additional validation specific to NDV F gene characteristics"""
253
+ issues = []
254
+ suggestions = []
255
+ if len(sequence) < 1500:
256
+ issues.append(f"Sequence length ({len(sequence)}) shorter than typical NDV F gene (1662-1800 nt)")
257
+ suggestions.append("Verify complete F gene sequence was provided")
258
+ elif len(sequence) > 2000:
259
+ issues.append(f"Sequence length ({len(sequence)}) longer than typical NDV F gene")
260
+ suggestions.append("Check if sequence contains additional regions beyond F gene")
261
+ if not sequence.startswith('ATG'):
262
+ issues.append("Sequence doesn't start with ATG start codon")
263
+ suggestions.append("Ensure sequence starts from the translation start site")
264
+ if len(sequence) % 3 != 0:
265
+ issues.append("Sequence length not divisible by 3 (reading frame issue)")
266
+ suggestions.append("Check for insertions/deletions or trim to proper reading frame")
267
+ if len(sequence) >= 3:
268
+ last_codon = sequence[-3:]
269
+ stop_codons = ['TAA', 'TAG', 'TGA']
270
+ if last_codon not in stop_codons:
271
+ issues.append(f"Sequence doesn't end with stop codon (ends with {last_codon})")
272
+ suggestions.append("Verify complete F gene sequence including stop codon")
273
+ gc_content = (sequence.count('G') + sequence.count('C')) / len(sequence) * 100
274
+ if gc_content < 30 or gc_content > 70:
275
+ issues.append(f"Unusual GC content: {gc_content:.1f}% (typical range: 35-65%)")
276
+ suggestions.append("Verify sequence quality and correct nucleotide composition")
277
+ n_content = sequence.count('N') / len(sequence) * 100
278
+ if n_content > 5:
279
+ issues.append(f"High ambiguous nucleotide content: {n_content:.1f}% N's")
280
+ suggestions.append("Consider resequencing regions with ambiguous nucleotides")
281
+ return issues, suggestions
282
+
283
+ def enhanced_run_pipeline(dna_input, keras_model, kmer_to_index, classifier_model,
284
+ classifier_kmer_to_index, classifier_maxlen, labels,
285
+ similarity_score=95.0, build_ml_tree=False):
286
+ """Enhanced pipeline with improved F gene prediction"""
287
+ try:
288
+ dna_input = dna_input.upper().strip()
289
+ if not dna_input:
290
+ return "Empty input", "", "", "", "", "", "", "", "", None, None, None, "No input provided"
291
+ if not re.match('^[ACTGN]+$', dna_input):
292
+ dna_input = ''.join(c if c in 'ACTGN' else 'N' for c in dna_input)
293
+ logging.info("DNA sequence sanitized")
294
+ validation_issues, validation_suggestions = validate_ndv_f_gene_sequence(dna_input)
295
+ keras_result = enhanced_keras_prediction(dna_input, keras_model, kmer_to_index)
296
+ if isinstance(keras_result, dict):
297
+ keras_output = f"Prediction confidence: {keras_result['confidence_score']:.3f}\n"
298
+ keras_output += f"K-mer coverage: {keras_result['kmer_coverage']:.1%}\n"
299
+ keras_output += f"Sequence length: {keras_result['sequence_length']} nt"
300
+ if keras_result['kmer_coverage'] < 0.8:
301
+ keras_output += "\n⚠️ Low k-mer coverage - may affect accuracy"
302
+ else:
303
+ keras_output = str(keras_result)
304
+ classifier_result = enhanced_classify_sequence(
305
+ dna_input, classifier_model, classifier_kmer_to_index, classifier_maxlen, labels
306
+ )
307
+ classifier_status = classifier_result["status"]
308
+ classifier_message = classifier_result["message"]
309
+ classifier_label = classifier_result["predicted_label"]
310
+ classifier_confidence = classifier_result["confidence"]
311
+ if validation_issues:
312
+ classifier_message += f"\n\n⚠️ Sequence validation issues:\n" + "\n".join(f"• {issue}" for issue in validation_issues[:3])
313
+ if validation_suggestions:
314
+ classifier_message += f"\n\n💡 Suggestions:\n" + "\n".join(f"• {sug}" for sug in validation_suggestions[:3])
315
+ if classifier_result.get("details"):
316
+ details = classifier_result["details"]
317
+ if "all_probabilities" in details:
318
+ probs = details["all_probabilities"]
319
+ classifier_message += f"\n\nPrediction probabilities:"
320
+ for label, prob in sorted(probs.items(), key=lambda x: x[1], reverse=True)[:3]:
321
+ classifier_message += f"\n• {label}: {prob:.1%}"
322
+ boundary_output = f"Enhanced preprocessing applied. Length: {len(dna_input)} bp"
323
+ if validation_issues:
324
+ boundary_output += f"\n{len(validation_issues)} validation issues detected"
325
+ return (
326
+ boundary_output,
327
+ keras_output,
328
+ classifier_status,
329
+ classifier_message,
330
+ classifier_label or "Unknown",
331
+ f"{classifier_confidence:.3f}" if classifier_confidence else "N/A",
332
+ "ML tree not requested" if not build_ml_tree else "ML tree processing...",
333
+ "Enhanced analysis completed",
334
+ "<p>Enhanced F gene analysis completed</p>",
335
+ None, None, None,
336
+ f"Enhanced pipeline completed. Processed {len(dna_input)} bp sequence."
337
+ )
338
+ except Exception as e:
339
+ error_msg = f"Enhanced pipeline failed: {str(e)}"
340
+ logging.error(error_msg)
341
+ return (
342
+ error_msg, "", "error", error_msg, "Error", "0.000",
343
+ "", "", f"<p>Error: {error_msg}</p>",
344
+ None, None, None, error_msg
345
+ )
346
+
347
  # --- Tool Detection and Helper Functions ---
348
+ def check_tool_availability():
349
+ mafft_available = shutil.which(MAFFT_PATH) is not None
350
+ iqtree_available = shutil.which(IQTREE_PATH) is not None
351
+ return mafft_available, iqtree_available, MAFFT_PATH, IQTREE_PATH
352
+
353
+ def install_dependencies_guide():
354
+ return """
355
+ 🔧 INSTALLATION GUIDE:
356
+ - MAFFT: `conda install -c bioconda mafft`
357
+ - IQ-TREE: `conda install -c bioconda iqtree`
358
+ """
359
 
360
  def read_fasta_file(file_obj):
361
  try:
 
379
  if not dna_input:
380
  return "Failed to read FASTA file", "", "", "", "", "", "", "", "", None, None, None, "No input sequence"
381
  return enhanced_run_pipeline(
382
+ dna_input, keras_model, kmer_to_index, classifier_model,
383
+ classifier_kmer_to_index, classifier_maxlen, LABELS,
384
  similarity_score, build_ml_tree
385
  )
386
  except Exception as e:
 
452
  return run_pipeline_from_file(file_obj, sim_score, build_tree)
453
  else:
454
  return enhanced_run_pipeline(
455
+ dna_seq, keras_model, kmer_to_index, classifier_model,
456
+ classifier_kmer_to_index, classifier_maxlen, LABELS,
457
  sim_score, build_tree
458
  )
459
  def clear_inputs():