re-type commited on
Commit
e736ddf
·
verified ·
1 Parent(s): 14ca026

Update predictor.py

Browse files
Files changed (1) hide show
  1. predictor.py +577 -249
predictor.py CHANGED
@@ -1,259 +1,587 @@
 
 
 
 
 
 
 
 
1
  import numpy as np
2
- import re
3
  import logging
4
- from tensorflow.keras.preprocessing.sequence import pad_sequences
5
- import torch
6
 
7
  # Configure logging
8
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
9
 
10
- class GenePredictor:
11
- """Minimal implementation to load and use the boundary-aware model"""
12
- def __init__(self, model_path):
13
- try:
14
- self.model = torch.load(model_path, map_location=torch.device('cpu'))
15
- logging.info(f"Loaded boundary model from {model_path}")
16
- except Exception as e:
17
- logging.error(f"Failed to load boundary model: {e}")
18
- self.model = None
19
-
20
- def predict(self, sequence):
21
- if self.model is None:
22
- return None, None, 0.0
23
- # Placeholder: Implement actual prediction logic based on your model
24
- # This is a stub; replace with your model's prediction method
25
- return [0] * len(sequence), [0.5] * len(sequence), 0.5
26
-
27
- def extract_gene_regions(self, predictions, sequence):
28
- if predictions is None:
29
- return []
30
- # Placeholder: Extract regions based on predictions
31
- return [{"sequence": sequence, "start": 0, "end": len(sequence), "confidence": 0.5}]
32
-
33
- def preprocess_sequence_for_ndv_f_gene(sequence):
34
- """Enhanced preprocessing specifically for NDV F gene sequences"""
35
- try:
36
- sequence = sequence.upper().strip()
37
- sequence = re.sub(r'[^ATCGN]', '', sequence)
38
- if len(sequence) < 1000:
39
- logging.warning(f"Sequence length ({len(sequence)}) shorter than typical NDV F gene (1662-1800 nt)")
40
- if not sequence.startswith('ATG'):
41
- logging.warning("Sequence doesn't start with ATG start codon")
42
- atg_pos = sequence.find('ATG')
43
- if atg_pos != -1:
44
- sequence = sequence[atg_pos:]
45
- logging.info(f"Found ATG at position {atg_pos}, using sequence from there")
46
- if len(sequence) % 3 != 0:
47
- sequence = sequence[:len(sequence) - (len(sequence) % 3)]
48
- logging.info(f"Trimmed sequence to maintain reading frame: {len(sequence)} nt")
49
- return sequence
50
- except Exception as e:
51
- logging.error(f"Sequence preprocessing failed: {e}")
52
- return sequence
53
-
54
- def enhanced_keras_prediction(sequence, keras_model, kmer_to_index, kmer_size=6):
55
- """Enhanced Keras prediction with better handling for NDV F gene"""
56
- try:
57
- if not keras_model or not kmer_to_index:
58
- return "Keras model not available"
59
- processed_seq = preprocess_sequence_for_ndv_f_gene(sequence)
60
- if len(processed_seq) < kmer_size:
61
- return f"Sequence too short for k-mer prediction (minimum {kmer_size} nucleotides required)"
62
- kmers = [processed_seq[i:i+kmer_size] for i in range(len(processed_seq)-kmer_size+1)]
63
- indices = []
64
- unknown_kmers = 0
65
- for kmer in kmers:
66
- if kmer in kmer_to_index:
67
- indices.append(kmer_to_index[kmer])
68
- else:
69
- indices.append(0)
70
- unknown_kmers += 1
71
- logging.info(f"Generated {len(kmers)} k-mers, {unknown_kmers} unknown k-mers")
72
- input_arr = np.array([indices])
73
- prediction = keras_model.predict(input_arr, verbose=0)[0]
74
- max_prob = np.max(prediction)
75
- mean_prob = np.mean(prediction)
76
- confidence_score = max_prob
77
- consistency_score = 1.0 - np.std(prediction)
78
- result = {
79
- 'raw_prediction': prediction.tolist(),
80
- 'max_probability': float(max_prob),
81
- 'mean_probability': float(mean_prob),
82
- 'confidence_score': float(confidence_score),
83
- 'consistency_score': float(consistency_score),
84
- 'sequence_length': len(processed_seq),
85
- 'kmers_generated': len(kmers),
86
- 'unknown_kmers': unknown_kmers,
87
- 'kmer_coverage': 1.0 - (unknown_kmers / len(kmers)) if kmers else 0.0
 
 
 
 
 
 
 
 
 
88
  }
89
- return result
90
- except Exception as e:
91
- logging.error(f"Enhanced Keras prediction failed: {e}")
92
- return f"Enhanced Keras prediction failed: {str(e)}"
93
-
94
- def enhanced_classify_sequence(sequence, classifier_model, classifier_kmer_to_index, classifier_maxlen, labels):
95
- """Enhanced classification with NDV F gene specific improvements"""
96
- try:
97
- if not classifier_model or not classifier_kmer_to_index or classifier_maxlen is None:
98
- return {
99
- "status": "error",
100
- "message": "Classification model not available",
101
- "confidence": None,
102
- "predicted_label": None,
103
- "details": {}
104
- }
105
- processed_seq = preprocess_sequence_for_ndv_f_gene(sequence)
106
- if len(processed_seq) < 1000:
107
- return {
108
- "status": "warning",
109
- "message": f"Sequence shorter than typical NDV F gene ({len(processed_seq)} < 1000 nt)",
110
- "confidence": None,
111
- "predicted_label": None,
112
- "details": {"sequence_length": len(processed_seq)}
113
- }
114
- kmer_size = 6
115
- tokens = [processed_seq[i:i+kmer_size] for i in range(len(processed_seq)-kmer_size+1)]
116
- encoded = []
117
- unknown_count = 0
118
- for kmer in tokens:
119
- if kmer in classifier_kmer_to_index:
120
- encoded.append(classifier_kmer_to_index[kmer])
121
- else:
122
- encoded.append(0)
123
- unknown_count += 1
124
- padded = pad_sequences([encoded], maxlen=classifier_maxlen, padding='post')
125
- pred = classifier_model.predict(padded, verbose=0)
126
- predicted_class = int(np.argmax(pred))
127
- confidence = float(np.max(pred))
128
- predicted_label = labels[predicted_class] if predicted_class < len(labels) else "Unknown"
129
- kmer_coverage = 1.0 - (unknown_count / len(tokens)) if tokens else 0.0
130
- prediction_entropy = -np.sum(pred[0] * np.log(pred[0] + 1e-10))
131
- details = {
132
- "sequence_length": len(processed_seq),
133
- "kmers_generated": len(tokens),
134
- "unknown_kmers": unknown_count,
135
- "kmer_coverage": kmer_coverage,
136
- "prediction_entropy": float(prediction_entropy),
137
- "all_probabilities": {labels[i]: float(pred[0][i]) for i in range(len(labels)) if i < len(pred[0])},
138
- "starts_with_atg": processed_seq.startswith('ATG'),
139
- "length_in_frame": len(processed_seq) % 3 == 0
140
  }
141
- if predicted_label == "F":
142
- f_gene_score = confidence
143
- if kmer_coverage > 0.8: f_gene_score *= 1.1
144
- if processed_seq.startswith('ATG'): f_gene_score *= 1.05
145
- if len(processed_seq) % 3 == 0: f_gene_score *= 1.05
146
- if 1500 <= len(processed_seq) <= 2000: f_gene_score *= 1.1
147
- details["enhanced_f_score"] = min(f_gene_score, 1.0)
148
- if f_gene_score > 0.7:
149
- return {"status": "success", "message": "NDV F gene detected with high confidence", "confidence": confidence, "predicted_label": predicted_label, "details": details}
150
- elif f_gene_score > 0.5:
151
- return {"status": "success", "message": "NDV F gene detected with moderate confidence", "confidence": confidence, "predicted_label": predicted_label, "details": details}
152
- else:
153
- return {"status": "warning", "message": "Possible F gene but low confidence - check sequence quality", "confidence": confidence, "predicted_label": predicted_label, "details": details}
154
- elif predicted_label == "Random":
155
- if kmer_coverage < 0.5:
156
- return {"status": "error", "message": f"Poor sequence quality detected (coverage: {kmer_coverage:.1%}). Check for sequencing errors.", "confidence": confidence, "predicted_label": predicted_label, "details": details}
157
- else:
158
- return {"status": "error", "message": "Sequence does not appear to be NDV F gene. Verify input sequence.", "confidence": confidence, "predicted_label": predicted_label, "details": details}
159
- else:
160
- return {"status": "error", "message": f"Detected as {predicted_label} gene, not F gene. Please provide NDV F gene sequence.", "confidence": confidence, "predicted_label": predicted_label, "details": details}
161
- except Exception as e:
162
- logging.error(f"Enhanced classification failed: {e}")
163
- return {"status": "error", "message": f"Classification failed: {str(e)}", "confidence": None, "predicted_label": None, "details": {"error": str(e)}}
164
-
165
- def validate_ndv_f_gene_sequence(sequence):
166
- """Additional validation specific to NDV F gene characteristics"""
167
- issues = []
168
- suggestions = []
169
- if len(sequence) < 1500:
170
- issues.append(f"Sequence length ({len(sequence)}) shorter than typical NDV F gene (1662-1800 nt)")
171
- suggestions.append("Verify complete F gene sequence was provided")
172
- elif len(sequence) > 2000:
173
- issues.append(f"Sequence length ({len(sequence)}) longer than typical NDV F gene")
174
- suggestions.append("Check if sequence contains additional regions beyond F gene")
175
- if not sequence.startswith('ATG'):
176
- issues.append("Sequence doesn't start with ATG start codon")
177
- suggestions.append("Ensure sequence starts from the translation start site")
178
- if len(sequence) % 3 != 0:
179
- issues.append("Sequence length not divisible by 3 (reading frame issue)")
180
- suggestions.append("Check for insertions/deletions or trim to proper reading frame")
181
- if len(sequence) >= 3:
182
- last_codon = sequence[-3:]
183
- stop_codons = ['TAA', 'TAG', 'TGA']
184
- if last_codon not in stop_codons:
185
- issues.append(f"Sequence doesn't end with stop codon (ends with {last_codon})")
186
- suggestions.append("Verify complete F gene sequence including stop codon")
187
- gc_content = (sequence.count('G') + sequence.count('C')) / len(sequence) * 100
188
- if gc_content < 30 or gc_content > 70:
189
- issues.append(f"Unusual GC content: {gc_content:.1f}% (typical range: 35-65%)")
190
- suggestions.append("Verify sequence quality and correct nucleotide composition")
191
- n_content = sequence.count('N') / len(sequence) * 100
192
- if n_content > 5:
193
- issues.append(f"High ambiguous nucleotide content: {n_content:.1f}% N's")
194
- suggestions.append("Consider resequencing regions with ambiguous nucleotides")
195
- return issues, suggestions
196
-
197
- def enhanced_run_pipeline(dna_input, keras_model, kmer_to_index, classifier_model,
198
- classifier_kmer_to_index, classifier_maxlen, labels,
199
- similarity_score=95.0, build_ml_tree=False):
200
- """Enhanced pipeline with improved F gene prediction"""
201
- try:
202
- dna_input = dna_input.upper().strip()
203
- if not dna_input:
204
- return "Empty input", "", "", "", "", "", "", "", "", None, None, None, "No input provided"
205
- if not re.match('^[ACTGN]+$', dna_input):
206
- dna_input = ''.join(c if c in 'ACTGN' else 'N' for c in dna_input)
207
- logging.info("DNA sequence sanitized")
208
- validation_issues, validation_suggestions = validate_ndv_f_gene_sequence(dna_input)
209
- keras_result = enhanced_keras_prediction(dna_input, keras_model, kmer_to_index)
210
- if isinstance(keras_result, dict):
211
- keras_output = f"Prediction confidence: {keras_result['confidence_score']:.3f}\n"
212
- keras_output += f"K-mer coverage: {keras_result['kmer_coverage']:.1%}\n"
213
- keras_output += f"Sequence length: {keras_result['sequence_length']} nt"
214
- if keras_result['kmer_coverage'] < 0.8:
215
- keras_output += "\n⚠️ Low k-mer coverage - may affect accuracy"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
216
  else:
217
- keras_output = str(keras_result)
218
- classifier_result = enhanced_classify_sequence(
219
- dna_input, classifier_model, classifier_kmer_to_index, classifier_maxlen, labels
220
- )
221
- classifier_status = classifier_result["status"]
222
- classifier_message = classifier_result["message"]
223
- classifier_label = classifier_result["predicted_label"]
224
- classifier_confidence = classifier_result["confidence"]
225
- if validation_issues:
226
- classifier_message += f"\n\n⚠️ Sequence validation issues:\n" + "\n".join(f"• {issue}" for issue in validation_issues[:3])
227
- if validation_suggestions:
228
- classifier_message += f"\n\n💡 Suggestions:\n" + "\n".join(f"• {sug}" for sug in validation_suggestions[:3])
229
- if classifier_result.get("details"):
230
- details = classifier_result["details"]
231
- if "all_probabilities" in details:
232
- probs = details["all_probabilities"]
233
- classifier_message += f"\n\nPrediction probabilities:"
234
- for label, prob in sorted(probs.items(), key=lambda x: x[1], reverse=True)[:3]:
235
- classifier_message += f"\n• {label}: {prob:.1%}"
236
- boundary_output = f"Enhanced preprocessing applied. Length: {len(dna_input)} bp"
237
- if validation_issues:
238
- boundary_output += f"\n{len(validation_issues)} validation issues detected"
239
- return (
240
- boundary_output,
241
- keras_output,
242
- classifier_status,
243
- classifier_message,
244
- classifier_label or "Unknown",
245
- f"{classifier_confidence:.3f}" if classifier_confidence else "N/A",
246
- "ML tree not requested" if not build_ml_tree else "ML tree processing...",
247
- "Enhanced analysis completed",
248
- "<p>Enhanced F gene analysis completed</p>",
249
- None, None, None,
250
- f"Enhanced pipeline completed. Processed {len(dna_input)} bp sequence."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
251
  )
252
- except Exception as e:
253
- error_msg = f"Enhanced pipeline failed: {str(e)}"
254
- logging.error(error_msg)
255
- return (
256
- error_msg, "", "error", error_msg, "Error", "0.000",
257
- "", "", f"<p>Error: {error_msg}</p>",
258
- None, None, None, error_msg
259
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Enhanced Gene Predictor optimized for Newcastle Disease Virus F Gene
4
+ """
5
+
6
+ import torch
7
+ import torch.nn as nn
8
+ import torch.nn.functional as F
9
  import numpy as np
10
+ from typing import List, Tuple, Dict, Optional
11
  import logging
12
+ import re
 
13
 
14
  # Configure logging
15
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
16
 
17
+ # ============================= MODEL COMPONENTS =============================
18
+
19
+ class ViralGenePredictor(nn.Module):
20
+ """Enhanced model specifically optimized for viral gene prediction."""
21
+
22
+ def __init__(self, input_dim: int = 16, hidden_dim: int = 512,
23
+ num_layers: int = 4, dropout: float = 0.2):
24
+ super().__init__()
25
+
26
+ # Multi-scale convolutional layers for capturing viral patterns
27
+ self.conv_layers = nn.ModuleList([
28
+ nn.Conv1d(input_dim, hidden_dim//8, kernel_size=k, padding=k//2)
29
+ for k in [3, 7, 15, 31, 63] # Added larger kernel for long-range dependencies
30
+ ])
31
+
32
+ # Deeper LSTM for better sequence modeling
33
+ self.lstm = nn.LSTM(hidden_dim//8 * 5, hidden_dim//2, num_layers,
34
+ batch_first=True, bidirectional=True, dropout=dropout)
35
+
36
+ # Enhanced normalization and attention
37
+ self.norm1 = nn.LayerNorm(hidden_dim)
38
+ self.norm2 = nn.LayerNorm(hidden_dim)
39
+ self.dropout = nn.Dropout(dropout)
40
+
41
+ # Multi-head attention with more heads for complex patterns
42
+ self.boundary_attention = nn.MultiheadAttention(hidden_dim, num_heads=16, batch_first=True)
43
+ self.self_attention = nn.MultiheadAttention(hidden_dim, num_heads=8, batch_first=True)
44
+
45
+ # Enhanced classifiers with residual connections
46
+ self.gene_classifier = nn.Sequential(
47
+ nn.Linear(hidden_dim, hidden_dim),
48
+ nn.ReLU(),
49
+ nn.Dropout(dropout),
50
+ nn.Linear(hidden_dim, hidden_dim//2),
51
+ nn.ReLU(),
52
+ nn.Dropout(dropout),
53
+ nn.Linear(hidden_dim//2, 2)
54
+ )
55
+
56
+ self.start_classifier = nn.Sequential(
57
+ nn.Linear(hidden_dim, hidden_dim//2),
58
+ nn.ReLU(),
59
+ nn.Dropout(dropout),
60
+ nn.Linear(hidden_dim//2, 2)
61
+ )
62
+
63
+ self.end_classifier = nn.Sequential(
64
+ nn.Linear(hidden_dim, hidden_dim//2),
65
+ nn.ReLU(),
66
+ nn.Dropout(dropout),
67
+ nn.Linear(hidden_dim//2, 2)
68
+ )
69
+
70
+ def forward(self, x: torch.Tensor, lengths: Optional[torch.Tensor] = None) -> Dict[str, torch.Tensor]:
71
+ batch_size, seq_len, _ = x.shape
72
+ x_conv = x.transpose(1, 2)
73
+
74
+ # Multi-scale convolution
75
+ conv_features = [F.relu(conv(x_conv)) for conv in self.conv_layers]
76
+ features = torch.cat(conv_features, dim=1).transpose(1, 2)
77
+
78
+ # LSTM processing
79
+ if lengths is not None:
80
+ packed = nn.utils.rnn.pack_padded_sequence(
81
+ features, lengths.cpu(), batch_first=True, enforce_sorted=False
82
+ )
83
+ lstm_out, _ = self.lstm(packed)
84
+ lstm_out, _ = nn.utils.rnn.pad_packed_sequence(lstm_out, batch_first=True)
85
+ else:
86
+ lstm_out, _ = self.lstm(features)
87
+
88
+ # Layer normalization and attention
89
+ lstm_out = self.norm1(lstm_out)
90
+
91
+ # Self-attention for internal dependencies
92
+ self_attended, _ = self.self_attention(lstm_out, lstm_out, lstm_out)
93
+ self_attended = self.dropout(self_attended)
94
+
95
+ # Boundary attention
96
+ boundary_attended, _ = self.boundary_attention(self_attended, self_attended, self_attended)
97
+ boundary_attended = self.norm2(boundary_attended + self_attended) # Residual connection
98
+ final_features = self.dropout(boundary_attended)
99
+
100
+ return {
101
+ 'gene': self.gene_classifier(final_features),
102
+ 'start': self.start_classifier(final_features),
103
+ 'end': self.end_classifier(final_features)
104
  }
105
+
106
+ # ============================= ENHANCED DATA PREPROCESSING =============================
107
+
108
+ class ViralDNAProcessor:
109
+ """Enhanced DNA sequence processor optimized for viral genes."""
110
+
111
+ def __init__(self):
112
+ self.base_to_idx = {'A': 0, 'C': 1, 'G': 2, 'T': 3, 'N': 4}
113
+ self.idx_to_base = {v: k for k, v in self.base_to_idx.items()}
114
+
115
+ # Expanded codon recognition for viral genes
116
+ self.start_codons = {'ATG', 'GTG', 'TTG', 'ATT', 'ATA', 'CTG'} # Added viral-specific
117
+ self.stop_codons = {'TAA', 'TAG', 'TGA'}
118
+
119
+ # Newcastle virus specific patterns (F gene characteristics)
120
+ self.viral_signals = {
121
+ 'kozak_consensus': r'[AG]CC[AG]CCATGG', # Kozak sequence
122
+ 'shine_dalgarno': r'AGGAGG',
123
+ 'f_gene_motifs': ['ATGGGC', 'CTGCTG', 'TTCTTC'] # Common F gene motifs
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
124
  }
125
+
126
+ def encode_sequence(self, sequence: str) -> torch.Tensor:
127
+ sequence = sequence.upper()
128
+ encoded = [self.base_to_idx.get(base, self.base_to_idx['N']) for base in sequence]
129
+ return torch.tensor(encoded, dtype=torch.long)
130
+
131
+ def create_enhanced_features(self, sequence: str) -> torch.Tensor:
132
+ sequence = sequence.upper()
133
+ seq_len = len(sequence)
134
+ encoded = self.encode_sequence(sequence)
135
+
136
+ # One-hot encoding (5 features)
137
+ one_hot = torch.zeros(seq_len, 5)
138
+ one_hot.scatter_(1, encoded.unsqueeze(1), 1)
139
+ features = [one_hot]
140
+
141
+ # Enhanced start codon indicators (6 features)
142
+ start_indicators = torch.zeros(seq_len, 6)
143
+ for i in range(seq_len - 2):
144
+ codon = sequence[i:i+3]
145
+ if codon == 'ATG':
146
+ start_indicators[i:i+3, 0] = 1.0
147
+ elif codon == 'GTG':
148
+ start_indicators[i:i+3, 1] = 0.95
149
+ elif codon == 'TTG':
150
+ start_indicators[i:i+3, 2] = 0.9
151
+ elif codon == 'ATT':
152
+ start_indicators[i:i+3, 3] = 0.8
153
+ elif codon == 'ATA':
154
+ start_indicators[i:i+3, 4] = 0.75
155
+ elif codon == 'CTG':
156
+ start_indicators[i:i+3, 5] = 0.7
157
+ features.append(start_indicators)
158
+
159
+ # Stop codon indicators (3 features)
160
+ stop_indicators = torch.zeros(seq_len, 3)
161
+ for i in range(seq_len - 2):
162
+ codon = sequence[i:i+3]
163
+ if codon == 'TAA':
164
+ stop_indicators[i:i+3, 0] = 1.0
165
+ elif codon == 'TAG':
166
+ stop_indicators[i:i+3, 1] = 1.0
167
+ elif codon == 'TGA':
168
+ stop_indicators[i:i+3, 2] = 1.0
169
+ features.append(stop_indicators)
170
+
171
+ # Enhanced GC content with multiple window sizes (1 feature)
172
+ gc_content = torch.zeros(seq_len, 1)
173
+ for window_size in [21, 51, 101]: # Multiple windows
174
+ for i in range(seq_len):
175
+ start = max(0, i - window_size//2)
176
+ end = min(seq_len, i + window_size//2)
177
+ window = sequence[start:end]
178
+ gc_count = window.count('G') + window.count('C')
179
+ gc_ratio = gc_count / len(window) if len(window) > 0 else 0
180
+ gc_content[i, 0] = max(gc_content[i, 0], gc_ratio)
181
+ features.append(gc_content)
182
+
183
+ # Position encoding (2 features)
184
+ pos_encoding = torch.zeros(seq_len, 2)
185
+ positions = torch.arange(seq_len, dtype=torch.float)
186
+ pos_encoding[:, 0] = torch.sin(positions / 10000)
187
+ pos_encoding[:, 1] = torch.cos(positions / 10000)
188
+ features.append(pos_encoding)
189
+
190
+ return torch.cat(features, dim=1) # Total: 5 + 6 + 3 + 1 + 1 = 16
191
+
192
+ # ============================= VIRAL-SPECIFIC POST-PROCESSING =============================
193
+
194
+ class ViralPostProcessor:
195
+ """Post-processor specifically optimized for viral genes like Newcastle F gene."""
196
+
197
+ def __init__(self, min_gene_length: int = 300, max_gene_length: int = 6000):
198
+ # Adjusted for viral F genes (typically 1600-1700 bp)
199
+ self.min_gene_length = min_gene_length
200
+ self.max_gene_length = max_gene_length
201
+ self.start_codons = {'ATG', 'GTG', 'TTG', 'ATT', 'ATA', 'CTG'}
202
+ self.stop_codons = {'TAA', 'TAG', 'TGA'}
203
+
204
+ # F gene specific characteristics
205
+ self.optimal_f_length = 1662 # Typical F gene length
206
+ self.f_gene_patterns = ['ATGGGC', 'CTGCTG', 'TTCTTC']
207
+
208
+ def process_predictions(self, gene_probs: np.ndarray, start_probs: np.ndarray,
209
+ end_probs: np.ndarray, sequence: str = None) -> np.ndarray:
210
+ """Enhanced processing with viral-specific optimizations."""
211
+
212
+ # More sensitive thresholds for viral genes
213
+ gene_pred = (gene_probs[:, 1] > 0.4).astype(int) # Lowered threshold
214
+ start_pred = (start_probs[:, 1] > 0.3).astype(int) # More sensitive
215
+ end_pred = (end_probs[:, 1] > 0.4).astype(int)
216
+
217
+ if sequence is not None:
218
+ processed = self._refine_with_viral_patterns(
219
+ gene_pred, start_pred, end_pred, sequence
220
+ )
221
  else:
222
+ processed = self._refine_with_boundaries(gene_pred, start_pred, end_pred)
223
+
224
+ processed = self._apply_viral_constraints(processed, sequence)
225
+ return processed
226
+
227
+ def _refine_with_viral_patterns(self, gene_pred: np.ndarray,
228
+ start_pred: np.ndarray, end_pred: np.ndarray,
229
+ sequence: str) -> np.ndarray:
230
+ refined = gene_pred.copy()
231
+ sequence = sequence.upper()
232
+
233
+ # Find all potential start and stop positions
234
+ start_positions = self._find_start_codons(sequence)
235
+ stop_positions = self._find_stop_codons(sequence)
236
+
237
+ changes = np.diff(np.concatenate(([0], gene_pred, [0])))
238
+ gene_starts = np.where(changes == 1)[0]
239
+ gene_ends = np.where(changes == -1)[0]
240
+
241
+ refined = np.zeros_like(gene_pred)
242
+
243
+ for g_start, g_end in zip(gene_starts, gene_ends):
244
+ best_start, best_end = self._optimize_gene_boundaries(
245
+ g_start, g_end, start_positions, stop_positions,
246
+ start_pred, end_pred, sequence
247
+ )
248
+
249
+ gene_length = best_end - best_start
250
+ if (self.min_gene_length <= gene_length <= self.max_gene_length and
251
+ best_start < best_end):
252
+ refined[best_start:best_end] = 1
253
+
254
+ return refined
255
+
256
+ def _find_start_codons(self, sequence: str) -> List[Tuple[int, str, float]]:
257
+ """Find all start codons with their positions and weights."""
258
+ positions = []
259
+ for i in range(len(sequence) - 2):
260
+ codon = sequence[i:i+3]
261
+ if codon in self.start_codons:
262
+ # Weight based on codon strength
263
+ weight = {
264
+ 'ATG': 1.0, 'GTG': 0.95, 'TTG': 0.9,
265
+ 'ATT': 0.8, 'ATA': 0.75, 'CTG': 0.7
266
+ }.get(codon, 0.5)
267
+ positions.append((i, codon, weight))
268
+ return positions
269
+
270
+ def _find_stop_codons(self, sequence: str) -> List[int]:
271
+ """Find all stop codon positions."""
272
+ positions = []
273
+ for i in range(len(sequence) - 2):
274
+ if sequence[i:i+3] in self.stop_codons:
275
+ positions.append(i + 3) # End position
276
+ return positions
277
+
278
+ def _optimize_gene_boundaries(self, g_start, g_end, start_positions, stop_positions,
279
+ start_pred, end_pred, sequence):
280
+ """Optimize gene boundaries using multiple criteria."""
281
+
282
+ # Find best start
283
+ best_start = g_start
284
+ start_window = 150 # Larger window for viral genes
285
+
286
+ candidates = [
287
+ (pos, codon, weight) for pos, codon, weight in start_positions
288
+ if abs(pos - g_start) <= start_window
289
+ ]
290
+
291
+ if candidates:
292
+ start_scores = []
293
+ for pos, codon, weight in candidates:
294
+ if pos < len(start_pred):
295
+ # Multi-criteria scoring
296
+ boundary_score = start_pred[pos] if pos < len(start_pred) else 0
297
+ distance_penalty = abs(pos - g_start) / start_window * 0.3
298
+
299
+ # F gene pattern bonus
300
+ pattern_bonus = 0
301
+ for pattern in self.f_gene_patterns:
302
+ if sequence[pos:pos+len(pattern)] == pattern:
303
+ pattern_bonus = 0.2
304
+ break
305
+
306
+ score = weight * 0.4 + boundary_score * 0.4 - distance_penalty + pattern_bonus
307
+ start_scores.append((score, pos))
308
+
309
+ if start_scores:
310
+ best_start = max(start_scores, key=lambda x: x[0])[1]
311
+
312
+ # Find best end
313
+ best_end = g_end
314
+ end_window = 200
315
+
316
+ valid_ends = [
317
+ pos for pos in stop_positions
318
+ if best_start < pos <= g_end + end_window
319
+ ]
320
+
321
+ if valid_ends:
322
+ end_scores = []
323
+ for pos in valid_ends:
324
+ gene_length = pos - best_start
325
+ if self.min_gene_length <= gene_length <= self.max_gene_length:
326
+ # Scoring criteria
327
+ boundary_score = end_pred[pos] if pos < len(end_pred) else 0
328
+ frame_bonus = 0.3 if gene_length % 3 == 0 else 0
329
+
330
+ # Length preference (closer to typical F gene length)
331
+ length_score = 1.0 - abs(gene_length - self.optimal_f_length) / self.optimal_f_length
332
+ length_score = max(0, length_score) * 0.2
333
+
334
+ score = boundary_score * 0.5 + frame_bonus + length_score
335
+ end_scores.append((score, pos))
336
+
337
+ if end_scores:
338
+ best_end = max(end_scores, key=lambda x: x[0])[1]
339
+
340
+ return best_start, best_end
341
+
342
+ def _refine_with_boundaries(self, gene_pred: np.ndarray, start_pred: np.ndarray,
343
+ end_pred: np.ndarray) -> np.ndarray:
344
+ """Fallback boundary refinement when sequence is not available."""
345
+ refined = gene_pred.copy()
346
+ changes = np.diff(np.concatenate(([0], gene_pred, [0])))
347
+ gene_starts = np.where(changes == 1)[0]
348
+ gene_ends = np.where(changes == -1)[0]
349
+
350
+ for g_start, g_end in zip(gene_starts, gene_ends):
351
+ # Refine start
352
+ start_window = slice(max(0, g_start-50), min(len(start_pred), g_start+50))
353
+ start_candidates = np.where(start_pred[start_window])[0]
354
+ if len(start_candidates) > 0:
355
+ relative_positions = start_candidates + max(0, g_start-50)
356
+ distances = np.abs(relative_positions - g_start)
357
+ best_start_idx = np.argmin(distances)
358
+ new_start = relative_positions[best_start_idx]
359
+ refined[g_start:new_start] = 0 if new_start > g_start else refined[g_start:new_start]
360
+ refined[new_start:g_end] = 1
361
+ g_start = new_start
362
+
363
+ # Refine end
364
+ end_window = slice(max(0, g_end-75), min(len(end_pred), g_end+75))
365
+ end_candidates = np.where(end_pred[end_window])[0]
366
+ if len(end_candidates) > 0:
367
+ relative_positions = end_candidates + max(0, g_end-75)
368
+ valid_ends = [pos for pos in relative_positions
369
+ if self.min_gene_length <= pos - g_start <= self.max_gene_length]
370
+ if valid_ends:
371
+ distances = np.abs(np.array(valid_ends) - g_end)
372
+ new_end = valid_ends[np.argmin(distances)]
373
+ refined[g_start:new_end] = 1
374
+ refined[new_end:g_end] = 0 if new_end < g_end else refined[new_end:g_end]
375
+
376
+ return refined
377
+
378
+ def _apply_viral_constraints(self, predictions: np.ndarray, sequence: str = None) -> np.ndarray:
379
+ """Apply viral-specific constraints."""
380
+ processed = predictions.copy()
381
+ changes = np.diff(np.concatenate(([0], predictions, [0])))
382
+ starts = np.where(changes == 1)[0]
383
+ ends = np.where(changes == -1)[0]
384
+
385
+ for start, end in zip(starts, ends):
386
+ gene_length = end - start
387
+
388
+ # Length constraints
389
+ if gene_length < self.min_gene_length or gene_length > self.max_gene_length:
390
+ processed[start:end] = 0
391
+ continue
392
+
393
+ # Frame constraint for viral genes
394
+ if sequence is not None and gene_length % 3 != 0:
395
+ # Try to adjust to maintain frame
396
+ new_length = ((gene_length // 3) + 1) * 3 # Round up for viral genes
397
+ if new_length <= self.max_gene_length:
398
+ new_end = start + new_length
399
+ if new_end <= len(processed):
400
+ processed[start:new_end] = 1
401
+ processed[new_end:end] = 0
402
+ else:
403
+ new_length = (gene_length // 3) * 3
404
+ if new_length >= self.min_gene_length:
405
+ new_end = start + new_length
406
+ processed[new_end:end] = 0
407
+ else:
408
+ processed[start:end] = 0
409
+
410
+ return processed
411
+
412
+ # ============================= ENHANCED PREDICTION CLASS =============================
413
+
414
+ class EnhancedGenePredictor:
415
+ """Enhanced gene predictor optimized for Newcastle virus F gene."""
416
+
417
+ def __init__(self, model_path: str = 'model/enhanced_viral_model.pth',
418
+ device: str = 'cuda' if torch.cuda.is_available() else 'cpu'):
419
+ self.device = device
420
+ self.model = ViralGenePredictor(input_dim=16).to(device)
421
+
422
+ try:
423
+ self.model.load_state_dict(torch.load(model_path, map_location=device))
424
+ logging.info(f"Loaded enhanced model from {model_path}")
425
+ except Exception as e:
426
+ logging.warning(f"Could not load model from {model_path}: {e}")
427
+ logging.info("Using randomly initialized model - please train before use")
428
+
429
+ self.model.eval()
430
+ self.processor = ViralDNAProcessor()
431
+ self.post_processor = ViralPostProcessor()
432
+
433
+ def predict(self, sequence: str, use_ensemble: bool = True) -> Tuple[np.ndarray, Dict[str, np.ndarray], float]:
434
+ """Enhanced prediction with optional ensemble approach."""
435
+ sequence = sequence.upper()
436
+ if not re.match('^[ACTGN]+$', sequence):
437
+ logging.warning("Sequence contains invalid characters. Using 'N' for unknowns.")
438
+ sequence = ''.join(c if c in 'ACTGN' else 'N' for c in sequence)
439
+
440
+ features = self.processor.create_enhanced_features(sequence).unsqueeze(0).to(self.device)
441
+
442
+ with torch.no_grad():
443
+ if use_ensemble:
444
+ # Ensemble prediction with multiple dropout samples
445
+ predictions = []
446
+ self.model.train() # Enable dropout
447
+ for _ in range(5): # Multiple samples
448
+ outputs = self.model(features)
449
+ gene_probs = F.softmax(outputs['gene'], dim=-1).cpu().numpy()[0]
450
+ start_probs = F.softmax(outputs['start'], dim=-1).cpu().numpy()[0]
451
+ end_probs = F.softmax(outputs['end'], dim=-1).cpu().numpy()[0]
452
+ predictions.append((gene_probs, start_probs, end_probs))
453
+
454
+ self.model.eval()
455
+
456
+ # Average predictions
457
+ gene_probs = np.mean([p[0] for p in predictions], axis=0)
458
+ start_probs = np.mean([p[1] for p in predictions], axis=0)
459
+ end_probs = np.mean([p[2] for p in predictions], axis=0)
460
+ else:
461
+ outputs = self.model(features)
462
+ gene_probs = F.softmax(outputs['gene'], dim=-1).cpu().numpy()[0]
463
+ start_probs = F.softmax(outputs['start'], dim=-1).cpu().numpy()[0]
464
+ end_probs = F.softmax(outputs['end'], dim=-1).cpu().numpy()[0]
465
+
466
+ predictions = self.post_processor.process_predictions(
467
+ gene_probs, start_probs, end_probs, sequence
468
  )
469
+
470
+ confidence = np.mean(gene_probs[:, 1][predictions == 1]) if np.any(predictions == 1) else 0.0
471
+
472
+ return predictions, {'gene': gene_probs, 'start': start_probs, 'end': end_probs}, confidence
473
+
474
+ def extract_gene_regions(self, predictions: np.ndarray, sequence: str) -> List[Dict]:
475
+ """Extract gene regions with enhanced viral-specific information."""
476
+ regions = []
477
+ changes = np.diff(np.concatenate(([0], predictions, [0])))
478
+ starts = np.where(changes == 1)[0]
479
+ ends = np.where(changes == -1)[0]
480
+
481
+ for start, end in zip(starts, ends):
482
+ gene_seq = sequence[start:end]
483
+
484
+ # Enhanced codon detection
485
+ actual_start_codon = None
486
+ actual_stop_codon = None
487
+ start_codon_confidence = 0.0
488
+
489
+ if len(gene_seq) >= 3:
490
+ start_codon = gene_seq[:3]
491
+ if start_codon in self.processor.start_codons:
492
+ actual_start_codon = start_codon
493
+ start_codon_confidence = {
494
+ 'ATG': 1.0, 'GTG': 0.95, 'TTG': 0.9,
495
+ 'ATT': 0.8, 'ATA': 0.75, 'CTG': 0.7
496
+ }.get(start_codon, 0.5)
497
+
498
+ if len(gene_seq) >= 6:
499
+ for i in range(len(gene_seq) - 2, 2, -3):
500
+ codon = gene_seq[i:i+3]
501
+ if codon in self.processor.stop_codons:
502
+ actual_stop_codon = codon
503
+ break
504
+
505
+ # F gene specific analysis
506
+ f_gene_score = self._calculate_f_gene_score(gene_seq)
507
+
508
+ regions.append({
509
+ 'start': int(start),
510
+ 'end': int(end),
511
+ 'sequence': gene_seq,
512
+ 'length': int(end - start),
513
+ 'start_codon': actual_start_codon,
514
+ 'stop_codon': actual_stop_codon,
515
+ 'start_codon_confidence': start_codon_confidence,
516
+ 'in_frame': (end - start) % 3 == 0,
517
+ 'f_gene_score': f_gene_score,
518
+ 'likely_f_gene': f_gene_score > 0.7
519
+ })
520
+
521
+ return regions
522
+
523
+ def _calculate_f_gene_score(self, sequence: str) -> float:
524
+ """Calculate likelihood of sequence being an F gene."""
525
+ score = 0.0
526
+
527
+ # Length score (F genes are typically ~1662 bp)
528
+ length = len(sequence)
529
+ if 1500 <= length <= 1800:
530
+ score += 0.3
531
+ elif 1200 <= length <= 2000:
532
+ score += 0.2
533
+
534
+ # Pattern matching
535
+ pattern_matches = 0
536
+ for pattern in self.processor.viral_signals['f_gene_motifs']:
537
+ if pattern in sequence:
538
+ pattern_matches += 1
539
+ score += min(pattern_matches * 0.15, 0.3)
540
+
541
+ # GC content (viral F genes often have moderate GC content)
542
+ gc_count = sequence.count('G') + sequence.count('C')
543
+ gc_ratio = gc_count / len(sequence) if len(sequence) > 0 else 0
544
+ if 0.4 <= gc_ratio <= 0.6:
545
+ score += 0.2
546
+
547
+ # Codon usage bias (simplified)
548
+ if sequence.startswith('ATG'):
549
+ score += 0.2
550
+ elif sequence.startswith(('GTG', 'TTG')):
551
+ score += 0.15
552
+
553
+ return min(score, 1.0)
554
+
555
+ def compute_accuracy(self, predictions: np.ndarray, labels: List[int]) -> Dict:
556
+ """Compute accuracy metrics."""
557
+ min_len = min(len(predictions), len(labels))
558
+ predictions = predictions[:min_len]
559
+ labels = np.array(labels[:min_len])
560
+
561
+ accuracy = np.mean(predictions == labels)
562
+ true_pos = np.sum((predictions == 1) & (labels == 1))
563
+ false_neg = np.sum((predictions == 0) & (labels == 1))
564
+ false_pos = np.sum((predictions == 1) & (labels == 0))
565
+
566
+ precision = true_pos / (true_pos + false_pos) if (true_pos + false_pos) > 0 else 0.0
567
+ recall = true_pos / (true_pos + false_neg) if (true_pos + false_neg) > 0 else 0.0
568
+ f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0
569
+
570
+ return {
571
+ 'accuracy': accuracy,
572
+ 'precision': precision,
573
+ 'recall': recall,
574
+ 'f1': f1,
575
+ 'true_positives': int(true_pos),
576
+ 'false_positives': int(false_pos),
577
+ 'false_negatives': int(false_neg)
578
+ }
579
+
580
+ def labels_from_coordinates(self, seq_len: int, start: int, end: int) -> List[int]:
581
+ """Generate labels from coordinates."""
582
+ labels = [0] * seq_len
583
+ start = max(0, min(start, seq_len - 1))
584
+ end = max(start, min(end, seq_len))
585
+ for i in range(start, end):
586
+ labels[i] = 1
587
+ return labels