File size: 9,292 Bytes
3f0c5d2
 
 
 
 
 
 
 
 
 
 
 
 
b96f398
 
 
 
 
8468d1a
 
 
3f0c5d2
 
 
 
b96f398
 
 
d4e93cc
 
 
 
 
 
 
8468d1a
b96f398
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8468d1a
3f0c5d2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8468d1a
3f0c5d2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d4e93cc
3f0c5d2
8468d1a
 
 
 
 
 
d4e93cc
8468d1a
d4e93cc
 
 
 
 
 
 
 
 
 
 
 
 
8468d1a
 
d4e93cc
8468d1a
 
 
 
 
 
3f0c5d2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8468d1a
b96f398
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8468d1a
3f0c5d2
 
 
 
 
 
 
 
 
 
 
8468d1a
b96f398
3f0c5d2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d65ecb8
b96f398
 
3f0c5d2
b96f398
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
import gradio as gr
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms as transforms
from PIL import Image
import requests
import json
import os
from typing import List, Dict, Any
from io import BytesIO
import logging

# Add FastAPI imports
from fastapi import FastAPI
from pydantic import BaseModel
import uvicorn

# Add HuggingFace Hub import
from huggingface_hub import HfApi

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Create FastAPI app
fastapi_app = FastAPI()

# Set up authentication - the Space needs a token to upload to itself
HF_TOKEN = os.getenv("HF_TOKEN")  # Repository secret
if not HF_TOKEN:
    logger.warning("No HF_TOKEN found - model uploads may fail")

# Initialize HF API with token
hf_api = HfApi(token=HF_TOKEN) if HF_TOKEN else HfApi()

# Pydantic models for API
class TrainingData(BaseModel):
    imageUrl: str
    groundTruthText: str
    sourceScript: str
    imageType: str
    difficultyLevel: str

class TrainingConfig(BaseModel):
    epochs: int
    batchSize: int
    learningRate: float
    validationSplit: float
    saveInterval: int
    modelName: str
    sampleLimit: int

class TrainingRequest(BaseModel):
    trainingData: List[TrainingData]
    config: TrainingConfig
    sessionId: str

class TrainingMetrics(BaseModel):
    finalLoss: float
    finalAccuracy: float
    epochs: int
    sampleCount: int

class TrainingResponse(BaseModel):
    success: bool
    jobId: str = None
    modelUrl: str = None
    metrics: TrainingMetrics = None
    error: str = None

# Your existing classes
class OCRDataset(Dataset):
    def __init__(self, training_data: List[Dict], transform=None):
        self.data = training_data
        self.transform = transform or transforms.Compose([
            transforms.Grayscale(),
            transforms.Resize((64, 256)),
            transforms.ToTensor(),
            transforms.Normalize((0.5,), (0.5,))
        ])
    
    def __len__(self):
        return len(self.data)
    
    def __getitem__(self, idx):
        item = self.data[idx]
        
        try:
            response = requests.get(item['imageUrl'])
            image = Image.open(BytesIO(response.content))
            
            if self.transform:
                image = self.transform(image)
            
            text = item['groundTruthText']
            return image, text
        except Exception as e:
            logger.error(f"Error loading image {item['imageUrl']}: {e}")
            return torch.zeros((1, 64, 256)), ""

class SimpleCRNN(nn.Module):
    def __init__(self, vocab_size=200, hidden_size=256):
        super(SimpleCRNN, self).__init__()
        
        self.cnn = nn.Sequential(
            nn.Conv2d(1, 32, 3, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2),
            nn.Conv2d(32, 64, 3, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2),
            nn.Conv2d(64, 128, 3, padding=1),
            nn.ReLU(),
            nn.MaxPool2d((2, 1)),
        )
        
        self.rnn = nn.LSTM(128 * 8, hidden_size, bidirectional=True, batch_first=True)
        self.classifier = nn.Linear(hidden_size * 2, vocab_size)
        
    def forward(self, x):
        conv_out = self.cnn(x)
        b, c, h, w = conv_out.size()
        conv_out = conv_out.view(b, c * h, w).permute(0, 2, 1)
        rnn_out, _ = self.rnn(conv_out)
        output = self.classifier(rnn_out)
        return output

# Updated training function with proper HF Hub upload
def train_model(training_request: Dict[str, Any]) -> Dict[str, Any]:
    try:
        training_data = training_request['trainingData']
        config = training_request['config']
        session_id = training_request['sessionId']
        
        logger.info(f"Starting training session {session_id} with {len(training_data)} samples")
        
        dataset = OCRDataset(training_data)
        dataloader = DataLoader(dataset, batch_size=config['batchSize'], shuffle=True)
        
        model = SimpleCRNN()
        optimizer = torch.optim.Adam(model.parameters(), lr=config['learningRate'])
        
        model.train()
        total_loss = 0.5
        total_accuracy = 0.85
        
        for epoch in range(config['epochs']):
            epoch_loss = 0.5 - (epoch * 0.05)  # Mock improving loss
            epoch_accuracy = 0.7 + (epoch / config['epochs']) * 0.2  # Mock improving accuracy
            
            logger.info(f"Epoch {epoch+1}/{config['epochs']} - Loss: {epoch_loss:.4f}, Accuracy: {epoch_accuracy:.4f}")
            
            total_loss = epoch_loss
            total_accuracy = epoch_accuracy
        
        # FIXED: Use HF Hub API with proper authentication
        model_filename = f"model_{session_id}.pth"
        temp_model_path = f"/tmp/{model_filename}"
        
        # Save model to temp directory first
        torch.save(model.state_dict(), temp_model_path)
        logger.info(f"Model saved to temp path: {temp_model_path}")
        
        # Upload to HF Hub using API with authentication
        try:
            if HF_TOKEN:
                hf_api.upload_file(
                    path_or_fileobj=temp_model_path,
                    path_in_repo=model_filename,
                    repo_id="rysatech/rysa-ocr-trainer",
                    repo_type="space",
                    commit_message=f"Add trained model {session_id}",
                    token=HF_TOKEN  # Explicit token
                )
                logger.info(f"Model uploaded to HF Hub: {model_filename}")
            else:
                logger.error("No HF_TOKEN available for upload")
                raise Exception("Authentication required for model upload")
        except Exception as upload_error:
            logger.error(f"Failed to upload model to HF Hub: {upload_error}")
            raise upload_error  # Don't continue if upload fails
        
        # Clean up temp file
        try:
            os.remove(temp_model_path)
        except:
            pass
        
        model_url = f"https://huggingface.co/spaces/rysatech/rysa-ocr-trainer/resolve/main/{model_filename}"
        
        return {
            'success': True,
            'modelUrl': model_url,
            'metrics': {
                'finalLoss': total_loss,
                'finalAccuracy': total_accuracy,
                'epochs': config['epochs'],
                'sampleCount': len(training_data)
            }
        }
        
    except Exception as e:
        logger.error(f"Training failed: {e}")
        return {
            'success': False,
            'error': str(e)
        }

# REST API ENDPOINTS
@fastapi_app.get("/api/health")
async def health_endpoint():
    return {"status": "healthy", "gpu_available": torch.cuda.is_available()}

@fastapi_app.post("/api/train")
async def train_endpoint(request: TrainingRequest):
    try:
        # Convert Pydantic model to dict for your existing function
        training_request_dict = {
            'trainingData': [item.dict() for item in request.trainingData],
            'config': request.config.dict(),
            'sessionId': request.sessionId
        }
        
        result = train_model(training_request_dict)
        
        if result['success']:
            return TrainingResponse(
                success=True,
                jobId=request.sessionId,
                modelUrl=result['modelUrl'],
                metrics=TrainingMetrics(**result['metrics'])
            )
        else:
            return TrainingResponse(
                success=False,
                error=result['error']
            )
    except Exception as e:
        logger.error(f"API training failed: {e}")
        return TrainingResponse(
            success=False,
            error=str(e)
        )

# Gradio functions
def train_api(training_request_json: str) -> str:
    try:
        training_request = json.loads(training_request_json)
        result = train_model(training_request)
        return json.dumps(result)
    except Exception as e:
        return json.dumps({'success': False, 'error': str(e)})

def health_check() -> str:
    return json.dumps({'status': 'healthy', 'gpu_available': torch.cuda.is_available()})

# Gradio interface
with gr.Blocks(title="Rysa OCR Training Service") as gradio_app:
    gr.Markdown("# Rysa OCR Training Service")
    gr.Markdown("Multi-script OCR model training for Sanskrit and Indian scripts.")
    
    with gr.Tab("Training API"):
        training_input = gr.Textbox(
            label="Training Request JSON",
            placeholder="Paste training request JSON here",
            lines=10
        )
        train_button = gr.Button("Start Training")
        training_output = gr.Textbox(label="Training Result", lines=10)
        
        train_button.click(train_api, inputs=training_input, outputs=training_output)
    
    with gr.Tab("Health Check"):
        health_button = gr.Button("Check Health")
        health_output = gr.Textbox(label="Health Status")
        
        health_button.click(health_check, outputs=health_output)

# Mount Gradio app to FastAPI
app = gr.mount_gradio_app(fastapi_app, gradio_app, path="/")

if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=7860)