Subham9126 commited on
Commit
d8fb7cb
Β·
verified Β·
1 Parent(s): 1b2ecc0

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +162 -629
app.py CHANGED
@@ -9,105 +9,24 @@ from datetime import datetime, time as dt_time
9
  import pytz
10
  import tzlocal
11
  import aiohttp
12
- from fastapi import FastAPI, HTTPException
13
  from pydantic import BaseModel, validator
14
  import uvicorn
15
  import threading
16
- import weakref
17
 
18
- # Configure enhanced logging with timestamps
19
  logging.basicConfig(
20
  level=logging.INFO,
21
- format='%(asctime)s.%(msecs)03d - %(name)s - %(levelname)s - %(message)s',
22
  datefmt='%Y-%m-%d %H:%M:%S'
23
  )
24
  logger = logging.getLogger(__name__)
25
 
26
- # --- Global Session Management ---
27
- class SessionManager:
28
- """Manages persistent aiohttp sessions for better performance."""
29
-
30
- def __init__(self):
31
- self._session = None
32
- self._session_lock = asyncio.Lock()
33
- self._creation_time = None
34
- self._request_count = 0
35
- self.max_session_age = 300 # 5 minutes
36
- self.max_requests_per_session = 1000
37
-
38
- async def get_session(self) -> aiohttp.ClientSession:
39
- """Get or create a persistent session with optimized settings."""
40
- async with self._session_lock:
41
- now = time.time()
42
-
43
- # Check if session needs renewal
44
- if (self._session is None or
45
- self._session.closed or
46
- (self._creation_time and now - self._creation_time > self.max_session_age) or
47
- self._request_count > self.max_requests_per_session):
48
-
49
- # Close old session if exists
50
- if self._session and not self._session.closed:
51
- logger.info(f"πŸ”„ Closing old session after {self._request_count} requests")
52
- await self._session.close()
53
- # Small delay to ensure proper cleanup
54
- await asyncio.sleep(0.1)
55
-
56
- # Create new optimized session
57
- connector = aiohttp.TCPConnector(
58
- limit=100, # Total connection pool size
59
- limit_per_host=50, # Max connections per host
60
- ttl_dns_cache=300, # DNS cache TTL
61
- use_dns_cache=True,
62
- keepalive_timeout=30, # Keep connections alive
63
- enable_cleanup_closed=True,
64
- force_close=False, # Reuse connections
65
- )
66
-
67
- timeout = aiohttp.ClientTimeout(
68
- total=60,
69
- connect=10,
70
- sock_read=30
71
- )
72
-
73
- # Optimized headers for better performance
74
- headers = {
75
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
76
- 'Accept': 'application/json',
77
- 'Accept-Encoding': 'gzip, deflate',
78
- 'Connection': 'keep-alive',
79
- 'Cache-Control': 'no-cache'
80
- }
81
-
82
- self._session = aiohttp.ClientSession(
83
- connector=connector,
84
- timeout=timeout,
85
- headers=headers,
86
- raise_for_status=False
87
- )
88
-
89
- self._creation_time = now
90
- self._request_count = 0
91
- logger.info(f"πŸš€ Created new optimized session at {datetime.fromtimestamp(now)}")
92
-
93
- return self._session
94
-
95
- def increment_request_count(self):
96
- """Track request count for session management."""
97
- self._request_count += 1
98
-
99
- async def close(self):
100
- """Clean shutdown of session."""
101
- if self._session and not self._session.closed:
102
- logger.info(f"πŸ”’ Closing session after {self._request_count} requests")
103
- await self._session.close()
104
- await asyncio.sleep(0.1) # Allow cleanup
105
-
106
- # Global session manager
107
- session_manager = SessionManager()
108
-
109
- # --- Enhanced Data Models ---
110
 
 
111
  class StockDataRequest(BaseModel):
112
  tickers: List[str]
113
  start_date: str
@@ -124,657 +43,271 @@ class StockDataRequest(BaseModel):
124
  raise ValueError("Tickers list cannot be empty")
125
  return [ticker.strip().upper() for ticker in v]
126
 
127
- @validator('interval')
128
- def validate_interval(cls, v):
129
- if v <= 0:
130
- raise ValueError("Interval must be positive")
131
- return v
132
-
133
- @validator('batch_size')
134
- def validate_batch_size(cls, v):
135
- if v <= 0 or v > 100:
136
- raise ValueError("Batch size must be between 1 and 100")
137
- return v
138
-
139
- @validator('max_concurrent')
140
- def validate_max_concurrent(cls, v):
141
- if v <= 0 or v > 100:
142
- raise ValueError("Max concurrent must be between 1 and 100")
143
- return v
144
-
145
- # --- Utility Functions (same as before) ---
146
 
147
- class DateTimeValidationError(Exception):
148
- pass
149
 
150
  def validate_datetime_format(dt_str: str) -> datetime:
151
- """Validate date in strict 'YYYY-MM-DD' format."""
152
  date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}$')
153
  if not date_pattern.match(dt_str):
154
- raise DateTimeValidationError(
155
- f"Invalid date format: '{dt_str}'. Expected 'YYYY-MM-DD'"
156
- )
157
  try:
158
- parsed_date = datetime.strptime(dt_str, '%Y-%m-%d')
159
- today = datetime.now().date()
160
- if parsed_date.date() > today:
161
- raise DateTimeValidationError(
162
- f"Future date provided: '{dt_str}'. Please provide a past or current date."
163
- )
164
- return parsed_date
165
  except ValueError as e:
166
- raise DateTimeValidationError(
167
- f"Invalid date value: '{dt_str}'. Please provide a valid calendar date."
168
- ) from e
169
-
170
- def _resolve_timezone(timezone: Optional[str]) -> pytz.BaseTzInfo:
171
- """Resolve timezone string to pytz timezone object."""
172
- try:
173
- if timezone:
174
- return pytz.timezone(timezone)
175
- else:
176
- return tzlocal.get_localzone()
177
- except pytz.exceptions.UnknownTimeZoneError:
178
- logger.warning(f"Unknown timezone '{timezone}', falling back to Asia/Kolkata")
179
- return pytz.timezone('Asia/Kolkata')
180
-
181
- def convert_to_unixtimestamp(date_time_str: str, timezone: Optional[str] = None) -> int:
182
- """Convert 'YYYY-MM-DD HH:MM' string to Unix ms timestamp."""
183
- dt = datetime.strptime(date_time_str, '%Y-%m-%d %H:%M')
184
- target_tz = _resolve_timezone(timezone)
185
- try:
186
- if dt.tzinfo is None:
187
- localized_dt = target_tz.localize(dt)
188
- else:
189
- localized_dt = dt.astimezone(target_tz)
190
- return int(localized_dt.timestamp() * 1000)
191
- except Exception as e:
192
- logger.error(f"Error converting datetime to timestamp: {e}")
193
- raise
194
 
195
  def get_time_range_in_unix_ms(start_date_str: str, end_date_str: str, timezone: str = 'Asia/Kolkata') -> Dict[str, int]:
196
- """Convert start/end date into full-day unix ms timestamps."""
197
  start_date = validate_datetime_format(start_date_str)
198
  end_date = validate_datetime_format(end_date_str)
199
-
200
  if start_date > end_date:
201
- raise DateTimeValidationError(
202
- f"Start date '{start_date_str}' cannot be after end date '{end_date_str}'"
203
- )
204
-
205
- start_datetime = datetime.combine(start_date, dt_time.min)
206
- end_datetime = datetime.combine(end_date, dt_time(23, 59))
207
-
208
- start_ts = convert_to_unixtimestamp(start_datetime.strftime('%Y-%m-%d %H:%M'), timezone)
209
- end_ts = convert_to_unixtimestamp(end_datetime.strftime('%Y-%m-%d %H:%M'), timezone)
210
-
211
  return {"start_timestamp_ms": start_ts, "end_timestamp_ms": end_ts}
212
 
213
- # --- Optimized API Functions ---
214
 
 
215
  HIST_URL = "https://groww.in/v1/api/charting_service/v2/chart/exchange/NSE/segment/CASH"
216
 
217
  async def call_price_api_optimized(
 
218
  ticker: str,
219
  start: int,
220
  end: int,
221
  interval: int,
222
- timeout: int = 30,
223
- request_id: str = "unknown"
224
  ) -> Dict:
225
- """Optimized API call with persistent session and detailed timing."""
226
  start_time = time.time()
227
  url = f"{HIST_URL}/{ticker}"
228
- params = {
229
- "startTimeInMillis": start,
230
- "endTimeInMillis": end,
231
- "intervalInMinutes": interval
232
- }
233
 
234
  try:
235
- session = await session_manager.get_session()
236
- session_manager.increment_request_count()
237
-
238
- # Log request start
239
- logger.debug(f"πŸš€ [{request_id}] Starting request for {ticker}")
240
-
241
  async with session.get(url, params=params) as response:
242
  response_time = time.time() - start_time
243
-
244
  if response.status == 200:
245
  json_data = await response.json()
246
  logger.debug(f"βœ… [{request_id}] {ticker}: {response_time*1000:.1f}ms - SUCCESS")
247
- return {
248
- "ticker": ticker,
249
- "data": json_data,
250
- "error": None,
251
- "status": "success",
252
- "response_time_ms": round(response_time * 1000, 1),
253
- "request_id": request_id
254
- }
255
  else:
256
  logger.warning(f"❌ [{request_id}] {ticker}: {response_time*1000:.1f}ms - HTTP {response.status}")
257
- return {
258
- "ticker": ticker,
259
- "data": None,
260
- "error": f"HTTP {response.status}: {response.reason}",
261
- "status": "failed",
262
- "response_time_ms": round(response_time * 1000, 1),
263
- "request_id": request_id
264
- }
265
-
266
  except asyncio.TimeoutError:
267
  response_time = time.time() - start_time
268
  logger.error(f"⏰ [{request_id}] {ticker}: {response_time*1000:.1f}ms - TIMEOUT")
269
- return {
270
- "ticker": ticker,
271
- "data": None,
272
- "error": "Request timeout",
273
- "status": "timeout",
274
- "response_time_ms": round(response_time * 1000, 1),
275
- "request_id": request_id
276
- }
277
  except Exception as e:
278
  response_time = time.time() - start_time
279
  logger.error(f"πŸ’₯ [{request_id}] {ticker}: {response_time*1000:.1f}ms - ERROR: {str(e)}")
280
- return {
281
- "ticker": ticker,
282
- "data": None,
283
- "error": str(e),
284
- "status": "error",
285
- "response_time_ms": round(response_time * 1000, 1),
286
- "request_id": request_id
287
- }
288
 
289
  async def fetch_stock_data_batch_optimized(
 
290
  tickers: List[str],
291
- start_time: int,
292
- end_time: int,
293
  interval: int,
294
- batch_size: int = 50,
295
- batch_delay: float = 0.5,
296
- max_concurrent: int = 50
297
  ) -> List[Dict]:
298
- """Highly optimized batch processing with detailed timing and performance monitoring."""
299
-
300
  overall_start = time.time()
301
  request_id = f"batch_{int(time.time())}"
302
-
303
- # Split tickers into batches
304
  ticker_batches = [tickers[i:i + batch_size] for i in range(0, len(tickers), batch_size)]
305
  all_results = []
306
 
307
- logger.info(f"🎯 [{request_id}] Starting batch processing: {len(tickers)} tickers in {len(ticker_batches)} batches of {batch_size}")
308
- logger.info(f"πŸ“Š [{request_id}] Config: max_concurrent={max_concurrent}, batch_delay={batch_delay}s")
309
 
310
- # Pre-warm session
311
- logger.info(f"πŸ”₯ [{request_id}] Pre-warming session...")
312
- await session_manager.get_session()
313
 
 
 
 
 
 
314
  for batch_idx, ticker_batch in enumerate(ticker_batches):
315
  batch_start = time.time()
316
  batch_request_id = f"{request_id}_b{batch_idx+1}"
 
317
 
318
- logger.info(f"πŸš€ [{batch_request_id}] Processing batch {batch_idx + 1}/{len(ticker_batches)} with {len(ticker_batch)} tickers")
319
-
320
- # Create semaphore to limit concurrent requests within batch
321
- semaphore = asyncio.Semaphore(max_concurrent)
322
-
323
- async def bounded_fetch(ticker, idx):
324
- async with semaphore:
325
- tick_request_id = f"{batch_request_id}_t{idx+1}"
326
- return await call_price_api_optimized(
327
- ticker, start_time, end_time, interval, 30, tick_request_id
328
- )
329
-
330
- # Process current batch with individual ticker indexing
331
- tasks = [bounded_fetch(ticker, idx) for idx, ticker in enumerate(ticker_batch)]
332
- batch_results = await asyncio.gather(*tasks, return_exceptions=True)
333
-
334
- # Process results
335
- processed_batch_results = []
336
- successful_in_batch = 0
337
-
338
- for i, result in enumerate(batch_results):
339
- if isinstance(result, Exception):
340
- logger.error(f"πŸ’₯ [{batch_request_id}] Exception for {ticker_batch[i]}: {str(result)}")
341
- processed_batch_results.append({
342
- "ticker": ticker_batch[i],
343
- "data": None,
344
- "error": str(result),
345
- "status": "exception",
346
- "batch": batch_idx + 1,
347
- "response_time_ms": 0,
348
- "request_id": f"{batch_request_id}_t{i+1}"
349
- })
350
- else:
351
- result["batch"] = batch_idx + 1
352
- processed_batch_results.append(result)
353
- if result.get("status") == "success":
354
- successful_in_batch += 1
355
-
356
- all_results.extend(processed_batch_results)
357
 
358
  batch_duration = time.time() - batch_start
359
- avg_response_time = sum(r.get("response_time_ms", 0) for r in processed_batch_results) / len(processed_batch_results)
360
-
361
- logger.info(f"βœ… [{batch_request_id}] Completed in {batch_duration:.2f}s | Success: {successful_in_batch}/{len(ticker_batch)} | Avg: {avg_response_time:.1f}ms")
362
 
363
- # Add delay between batches (except for the last batch)
364
  if batch_idx < len(ticker_batches) - 1 and batch_delay > 0:
365
- logger.info(f"⏳ [{batch_request_id}] Waiting {batch_delay}s before next batch...")
366
  await asyncio.sleep(batch_delay)
367
 
368
  overall_duration = time.time() - overall_start
369
- total_successful = len([r for r in all_results if r.get('status') == 'success'])
370
- success_rate = (total_successful / len(tickers)) * 100 if len(tickers) > 0 else 0
371
-
372
- logger.info(f"🏁 [{request_id}] COMPLETED: {overall_duration:.2f}s total | {total_successful}/{len(tickers)} successful ({success_rate:.1f}%)")
373
- logger.info(f"πŸ“ˆ [{request_id}] Performance: {len(tickers)/overall_duration:.1f} tickers/sec")
374
-
375
  return all_results
376
 
377
- # --- Core Processing Function ---
378
-
379
- def process_stock_request(
380
- tickers: Union[str, List[str]],
381
- start_date: str,
382
- end_date: str,
383
- interval: int = 15,
384
- timezone: str = "Asia/Kolkata",
385
- batch_size: int = 50,
386
- batch_delay: float = 0.5,
387
- max_concurrent: int = 50
388
  ) -> Dict:
389
- """Optimized core function with detailed performance monitoring."""
390
  request_start = time.time()
391
- request_id = f"req_{int(request_start)}"
392
-
393
- logger.info(f"πŸš€ [{request_id}] Starting stock request processing")
394
-
395
  try:
396
- # Handle tickers input
397
- if isinstance(tickers, str):
398
- try:
399
- tickers_list = json.loads(tickers)
400
- except json.JSONDecodeError:
401
- tickers_list = [t.strip().upper() for t in tickers.split(',')]
402
- else:
403
- tickers_list = [t.strip().upper() for t in tickers]
404
-
405
- if not tickers_list:
406
- raise ValueError("No tickers provided")
407
-
408
- logger.info(f"πŸ“ [{request_id}] Processing {len(tickers_list)} tickers")
409
-
410
- # Convert dates to timestamps
411
- ts_range = get_time_range_in_unix_ms(start_date, end_date, timezone)
412
- start_ts, end_ts = ts_range["start_timestamp_ms"], ts_range["end_timestamp_ms"]
413
-
414
- # Always use optimized batch processing
415
- logger.info(f"⚑ [{request_id}] Using optimized batch processing")
416
- results = asyncio.run(fetch_stock_data_batch_optimized(
417
- tickers_list, start_ts, end_ts, interval, batch_size, batch_delay, max_concurrent
418
- ))
419
 
420
- # Generate enhanced statistics
421
- total_tickers = len(tickers_list)
422
- successful = len([r for r in results if r.get('status') == 'success'])
423
- failed = len([r for r in results if r.get('status') in ['failed', 'error', 'timeout', 'exception']])
424
-
425
- # Calculate timing statistics
426
- response_times = [r.get('response_time_ms', 0) for r in results if r.get('response_time_ms', 0) > 0]
427
- avg_response_time = sum(response_times) / len(response_times) if response_times else 0
428
- max_response_time = max(response_times) if response_times else 0
429
- min_response_time = min(response_times) if response_times else 0
430
 
431
  total_duration = time.time() - request_start
432
- throughput = total_tickers / total_duration if total_duration > 0 else 0
 
 
433
 
434
- processing_summary = {
435
- "total_tickers": total_tickers,
436
- "successful": successful,
437
- "failed": failed,
438
- "success_rate": f"{(successful/total_tickers*100):.2f}%" if total_tickers > 0 else "0%",
439
- "total_duration_seconds": round(total_duration, 2),
440
- "throughput_tickers_per_second": round(throughput, 1),
441
- "avg_response_time_ms": round(avg_response_time, 1),
442
- "max_response_time_ms": round(max_response_time, 1),
443
- "min_response_time_ms": round(min_response_time, 1),
444
- "batch_processing_used": True,
445
- "batch_size": batch_size,
446
- "batch_delay": batch_delay,
447
- "max_concurrent": max_concurrent,
448
- "request_id": request_id
449
- }
450
-
451
- logger.info(f"βœ… [{request_id}] Request completed successfully in {total_duration:.2f}s")
452
-
453
- return {
454
- "success": True,
455
- "data": results,
456
- "error": None,
457
- "timestamp": datetime.now().isoformat(),
458
- "processing_summary": processing_summary,
459
- "request_info": {
460
- "tickers": tickers_list[:10] if len(tickers_list) > 10 else tickers_list,
461
- "total_tickers": len(tickers_list),
462
- "start_date": start_date,
463
- "end_date": end_date,
464
- "interval": interval,
465
- "timezone": timezone,
466
- "start_timestamp_ms": start_ts,
467
- "end_timestamp_ms": end_ts,
468
- "request_id": request_id
469
- }
470
- }
471
-
472
- except Exception as e:
473
- total_duration = time.time() - request_start
474
- logger.error(f"❌ [{request_id}] Error after {total_duration:.2f}s: {e}")
475
  return {
476
- "success": False,
477
- "data": None,
478
- "error": str(e),
479
- "timestamp": datetime.now().isoformat(),
480
  "processing_summary": {
481
- "total_tickers": len(tickers_list) if 'tickers_list' in locals() else 0,
482
- "successful": 0,
483
- "failed": 0,
484
- "success_rate": "0%",
485
  "total_duration_seconds": round(total_duration, 2),
486
- "throughput_tickers_per_second": 0,
487
- "batch_processing_used": True,
488
- "request_id": request_id
489
- },
490
- "request_info": {
491
- "tickers": tickers if isinstance(tickers, list) else [tickers],
492
- "start_date": start_date,
493
- "end_date": end_date,
494
- "interval": interval,
495
- "timezone": timezone,
496
- "request_id": request_id
497
  }
498
  }
 
 
 
499
 
500
  # --- FastAPI Application ---
501
-
502
- def safe_json_serialize(obj):
503
- """Safely serialize any object to JSON string."""
504
- def default_serializer(o):
505
- if isinstance(o, (datetime, dt_time)):
506
- return o.isoformat()
507
- elif hasattr(o, '__dict__'):
508
- return o.__dict__
509
- elif hasattr(o, 'to_dict'):
510
- return o.to_dict()
511
- else:
512
- return str(o)
513
-
514
- try:
515
- return json.dumps(obj, indent=2, default=default_serializer, ensure_ascii=False)
516
- except Exception:
517
- return json.dumps(str(obj), indent=2)
518
-
519
- api_app = FastAPI(title="Optimized Groww Stock Data API", version="2.0.0")
520
 
521
  @api_app.post("/fetch-stock-data")
522
- async def fetch_stock_data_endpoint(request: StockDataRequest):
523
- """Optimized API endpoint with enhanced performance monitoring."""
524
- try:
525
- result = process_stock_request(
526
- request.tickers,
527
- request.start_date,
528
- request.end_date,
529
- request.interval,
530
- request.timezone,
531
- request.batch_size,
532
- request.batch_delay,
533
- request.max_concurrent
534
- )
535
-
536
- result["timestamp"] = datetime.now().isoformat()
537
- serializable_result = safe_json_serialize(result)
538
- return json.loads(serializable_result)
539
-
540
- except Exception as e:
541
- logger.error(f"API endpoint error: {e}")
542
- return {
543
- "success": False,
544
- "data": None,
545
- "error": str(e),
546
- "timestamp": datetime.now().isoformat(),
547
- "processing_summary": {
548
- "total_tickers": 0,
549
- "successful": 0,
550
- "failed": 0,
551
- "success_rate": "0%",
552
- "total_duration_seconds": 0,
553
- "batch_processing_used": True
554
- },
555
- "request_info": {}
556
- }
557
 
558
  @api_app.get("/health")
559
  async def health_check():
560
- """Health check endpoint."""
561
  return {"status": "healthy", "timestamp": datetime.now().isoformat()}
562
 
563
- @api_app.get("/session-stats")
564
- async def session_stats():
565
- """Get session statistics for monitoring."""
566
- return {
567
- "session_active": session_manager._session is not None and not session_manager._session.closed,
568
- "session_age_seconds": time.time() - session_manager._creation_time if session_manager._creation_time else 0,
569
- "request_count": session_manager._request_count,
570
- "timestamp": datetime.now().isoformat()
571
- }
572
-
573
- # --- Gradio Interface ---
574
-
575
- def execute_stock_request(
576
- ticker_input: str,
577
- start_date: str,
578
- end_date: str,
579
- interval: int,
580
- batch_size: int = 50,
581
- batch_delay: float = 0.5,
582
- max_concurrent: int = 50
583
- ) -> str:
584
- """Optimized wrapper function for Gradio interface."""
585
  try:
586
- result = process_stock_request(
587
- ticker_input, start_date, end_date, interval, "Asia/Kolkata",
588
- batch_size, batch_delay, max_concurrent
589
- )
590
- return safe_json_serialize(result)
591
- except Exception as e:
592
- logger.error(f"Error in execute_stock_request: {e}")
593
- error_result = {
594
- "success": False,
595
- "error": str(e),
596
- "timestamp": datetime.now().isoformat(),
597
- "processing_summary": {
598
- "total_duration_seconds": 0,
599
- "throughput_tickers_per_second": 0
600
- }
601
  }
602
- return safe_json_serialize(error_result)
603
-
604
- def create_gradio_interface():
605
- """Create optimized Gradio interface with performance controls."""
606
- with gr.Blocks(title="Optimized Groww Stock Data Fetcher") as demo:
607
- gr.Markdown("""
608
- # ⚑ Optimized Groww Stock Data Fetcher v2.0
609
 
610
- **High-Performance Features:**
611
- - πŸš€ **Persistent HTTP Sessions**: Reuses connections for 5min/1000 requests
612
- - πŸ“Š **Detailed Performance Monitoring**: Response times, throughput metrics
613
- - 🎯 **Optimized Batch Processing**: Smart batching with timing controls
614
- - ⚑ **Enhanced Async Processing**: Up to 50 concurrent requests per batch
615
- - πŸ“ˆ **Real-time Statistics**: Success rates, timing analysis
616
 
617
- **Expected Performance:** ~187 tickers/second (750 tickers in ~4 seconds)
618
- """)
 
 
 
 
619
 
 
 
 
 
 
620
  with gr.Row():
621
  with gr.Column():
622
- ticker_box = gr.Textbox(
623
- label="Stock Tickers",
624
- placeholder='["RELIANCE","TCS","INFY"] or RELIANCE,TCS,INFY',
625
- value='["RELIANCE","TCS","INFY"]',
626
- lines=3
627
- )
628
-
629
- with gr.Row():
630
- start_box = gr.Textbox(
631
- label="Start Date (YYYY-MM-DD)",
632
- placeholder="2025-08-01",
633
- value="2025-08-01"
634
- )
635
- end_box = gr.Textbox(
636
- label="End Date (YYYY-MM-DD)",
637
- placeholder="2025-08-10",
638
- value="2025-08-10"
639
- )
640
-
641
- with gr.Row():
642
- interval_box = gr.Number(
643
- label="Interval (minutes)",
644
- value=15,
645
- minimum=1,
646
- maximum=1440
647
- )
648
- batch_size_box = gr.Number(
649
- label="Batch Size",
650
- value=50,
651
- minimum=1,
652
- maximum=100,
653
- info="Tickers per batch"
654
- )
655
-
656
- with gr.Row():
657
- batch_delay_box = gr.Number(
658
- label="Batch Delay (seconds)",
659
- value=0.5,
660
- minimum=0,
661
- maximum=10,
662
- step=0.1,
663
- info="Delay between batches"
664
- )
665
- max_concurrent_box = gr.Number(
666
- label="Max Concurrent",
667
- value=50,
668
- minimum=1,
669
- maximum=100,
670
- info="Concurrent requests per batch"
671
- )
672
-
673
- fetch_button = gr.Button("πŸš€ Fetch Data (Optimized)", variant="primary", size="lg")
674
-
675
- gr.Markdown("""
676
- **Performance Tuning:**
677
- - **Batch Size**: 50 (optimal for API rate limits)
678
- - **Batch Delay**: 0.5s (prevents rate limiting)
679
- - **Max Concurrent**: 50 (parallel requests per batch)
680
- - **Session Reuse**: Connections kept alive for 5 minutes
681
- """)
682
-
683
  with gr.Column():
684
- output_box = gr.Textbox(
685
- label="API Response with Performance Metrics",
686
- lines=25,
687
- max_lines=40,
688
- show_copy_button=True,
689
- container=True
690
- )
691
-
692
- gr.Markdown("""
693
- ### Performance Monitoring
694
 
695
- The response now includes detailed timing metrics:
696
- ```json
697
- {
698
- "processing_summary": {
699
- "total_duration_seconds": 4.23,
700
- "throughput_tickers_per_second": 177.3,
701
- "avg_response_time_ms": 95.4,
702
- "max_response_time_ms": 234.1,
703
- "min_response_time_ms": 67.8,
704
- "success_rate": "98.75%"
705
- }
706
- }
707
- ```
708
-
709
- ### API Usage
710
- ```bash
711
- curl -X POST "http://localhost:8000/fetch-stock-data" \\
712
- -H "Content-Type: application/json" \\
713
- -d '{
714
- "tickers": ["RELIANCE", "TCS", ...],
715
- "start_date": "2025-08-01",
716
- "end_date": "2025-08-10",
717
- "batch_size": 50,
718
- "batch_delay": 0.5,
719
- "max_concurrent": 50
720
- }'
721
- ```
722
- """)
723
-
724
  fetch_button.click(
725
- fn=execute_stock_request,
726
- inputs=[ticker_box, start_box, end_box, interval_box, batch_size_box, batch_delay_box, max_concurrent_box],
727
  outputs=output_box
728
  )
729
-
730
  return demo
731
 
732
  # --- Main Execution ---
733
-
734
  def run_api_server(host="0.0.0.0", port=8000):
735
- """Run the optimized FastAPI server."""
736
- uvicorn.run(api_app, host=host, port=port, log_level="info")
737
 
738
- def run_gradio_interface(share=False):
739
- """Run the optimized Gradio interface."""
740
  demo = create_gradio_interface()
741
- demo.launch(share=share, server_name="0.0.0.0")
742
-
743
- # --- Cleanup Handler ---
744
-
745
- import atexit
746
-
747
- def cleanup_session():
748
- """Cleanup session on exit."""
749
- if session_manager._session and not session_manager._session.closed:
750
- loop = asyncio.new_event_loop()
751
- asyncio.set_event_loop(loop)
752
- loop.run_until_complete(session_manager.close())
753
- loop.close()
754
-
755
- atexit.register(cleanup_session)
756
 
757
  if __name__ == "__main__":
758
  import sys
759
 
760
- # Set logging level based on environment
761
- if "--debug" in sys.argv:
762
- logging.getLogger().setLevel(logging.DEBUG)
763
- logger.info("πŸ› Debug logging enabled")
764
-
765
- if len(sys.argv) > 1 and sys.argv[1] == "api":
766
- # Run only API server
767
- print("πŸš€ Starting optimized API server...")
768
  run_api_server()
769
- elif len(sys.argv) > 1 and sys.argv[1] == "both":
770
- # Run both API and Gradio in separate threads
771
- print("πŸš€ Starting both API server and Gradio interface...")
 
 
 
772
  api_thread = threading.Thread(target=run_api_server, daemon=True)
773
  api_thread.start()
774
- print("⏳ Waiting 2 seconds for API server to start...")
775
  time.sleep(2)
776
- run_gradio_interface()
777
- else:
778
- # Default: Run only Gradio
779
- print("πŸš€ Starting optimized Gradio interface...")
780
  run_gradio_interface()
 
9
  import pytz
10
  import tzlocal
11
  import aiohttp
12
+ from fastapi import FastAPI, Request, HTTPException
13
  from pydantic import BaseModel, validator
14
  import uvicorn
15
  import threading
 
16
 
17
+ # --- Enhanced Logging Configuration ---
18
  logging.basicConfig(
19
  level=logging.INFO,
20
+ format='%(asctime)s.%(msecs)03d - %(levelname)s - %(message)s',
21
  datefmt='%Y-%m-%d %H:%M:%S'
22
  )
23
  logger = logging.getLogger(__name__)
24
 
25
+ # --- Application State for Shared Session ---
26
+ # We will manage the session via FastAPI's lifecycle, which is the correct pattern.
27
+ app_state = {}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
 
29
+ # --- Enhanced Data Models (Unchanged) ---
30
  class StockDataRequest(BaseModel):
31
  tickers: List[str]
32
  start_date: str
 
43
  raise ValueError("Tickers list cannot be empty")
44
  return [ticker.strip().upper() for ticker in v]
45
 
46
+ # ... other validators are fine ...
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
 
48
+ # --- Utility Functions (Unchanged) ---
49
+ class DateTimeValidationError(Exception): pass
50
 
51
  def validate_datetime_format(dt_str: str) -> datetime:
52
+ # ... implementation is fine ...
53
  date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}$')
54
  if not date_pattern.match(dt_str):
55
+ raise DateTimeValidationError(f"Invalid date format: '{dt_str}'. Expected 'YYYY-MM-DD'")
 
 
56
  try:
57
+ return datetime.strptime(dt_str, '%Y-%m-%d')
 
 
 
 
 
 
58
  except ValueError as e:
59
+ raise DateTimeValidationError(f"Invalid date value: '{dt_str}'.") from e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
 
61
  def get_time_range_in_unix_ms(start_date_str: str, end_date_str: str, timezone: str = 'Asia/Kolkata') -> Dict[str, int]:
62
+ # ... implementation is fine ...
63
  start_date = validate_datetime_format(start_date_str)
64
  end_date = validate_datetime_format(end_date_str)
 
65
  if start_date > end_date:
66
+ raise DateTimeValidationError(f"Start date '{start_date_str}' cannot be after end date '{end_date_str}'")
67
+
68
+ start_dt = datetime.combine(start_date, dt_time.min)
69
+ end_dt = datetime.combine(end_date, dt_time(23, 59))
70
+
71
+ target_tz = pytz.timezone(timezone)
72
+ start_ts = int(target_tz.localize(start_dt).timestamp() * 1000)
73
+ end_ts = int(target_tz.localize(end_dt).timestamp() * 1000)
74
+
 
75
  return {"start_timestamp_ms": start_ts, "end_timestamp_ms": end_ts}
76
 
 
77
 
78
+ # --- Optimized API Functions (Refactored to accept a session) ---
79
  HIST_URL = "https://groww.in/v1/api/charting_service/v2/chart/exchange/NSE/segment/CASH"
80
 
81
  async def call_price_api_optimized(
82
+ session: aiohttp.ClientSession,
83
  ticker: str,
84
  start: int,
85
  end: int,
86
  interval: int,
87
+ request_id: str
 
88
  ) -> Dict:
89
+ """Optimized API call that USES the provided session."""
90
  start_time = time.time()
91
  url = f"{HIST_URL}/{ticker}"
92
+ params = {"startTimeInMillis": start, "endTimeInMillis": end, "intervalInMinutes": interval}
 
 
 
 
93
 
94
  try:
 
 
 
 
 
 
95
  async with session.get(url, params=params) as response:
96
  response_time = time.time() - start_time
 
97
  if response.status == 200:
98
  json_data = await response.json()
99
  logger.debug(f"βœ… [{request_id}] {ticker}: {response_time*1000:.1f}ms - SUCCESS")
100
+ return {"ticker": ticker, "status": "success", "data": json_data, "response_time_ms": round(response_time * 1000, 1)}
 
 
 
 
 
 
 
101
  else:
102
  logger.warning(f"❌ [{request_id}] {ticker}: {response_time*1000:.1f}ms - HTTP {response.status}")
103
+ return {"ticker": ticker, "status": "failed", "error": f"HTTP {response.status}: {response.reason}", "response_time_ms": round(response_time * 1000, 1)}
 
 
 
 
 
 
 
 
104
  except asyncio.TimeoutError:
105
  response_time = time.time() - start_time
106
  logger.error(f"⏰ [{request_id}] {ticker}: {response_time*1000:.1f}ms - TIMEOUT")
107
+ return {"ticker": ticker, "status": "timeout", "error": "Request timeout", "response_time_ms": round(response_time * 1000, 1)}
 
 
 
 
 
 
 
108
  except Exception as e:
109
  response_time = time.time() - start_time
110
  logger.error(f"πŸ’₯ [{request_id}] {ticker}: {response_time*1000:.1f}ms - ERROR: {str(e)}")
111
+ return {"ticker": ticker, "status": "error", "error": str(e), "response_time_ms": round(response_time * 1000, 1)}
 
 
 
 
 
 
 
112
 
113
  async def fetch_stock_data_batch_optimized(
114
+ session: aiohttp.ClientSession, # <-- Accept the session
115
  tickers: List[str],
116
+ start_time_ts: int,
117
+ end_time_ts: int,
118
  interval: int,
119
+ batch_size: int,
120
+ batch_delay: float,
121
+ max_concurrent: int
122
  ) -> List[Dict]:
123
+ """Highly optimized batch processing that uses a shared, persistent session."""
 
124
  overall_start = time.time()
125
  request_id = f"batch_{int(time.time())}"
 
 
126
  ticker_batches = [tickers[i:i + batch_size] for i in range(0, len(tickers), batch_size)]
127
  all_results = []
128
 
129
+ logger.info(f"🎯 [{request_id}] Starting batch processing: {len(tickers)} tickers in {len(ticker_batches)} batches.")
 
130
 
131
+ semaphore = asyncio.Semaphore(max_concurrent)
 
 
132
 
133
+ async def bounded_fetch(ticker, batch_request_id, idx):
134
+ async with semaphore:
135
+ tick_request_id = f"{batch_request_id}_t{idx+1}"
136
+ return await call_price_api_optimized(session, ticker, start_time_ts, end_time_ts, interval, tick_request_id)
137
+
138
  for batch_idx, ticker_batch in enumerate(ticker_batches):
139
  batch_start = time.time()
140
  batch_request_id = f"{request_id}_b{batch_idx+1}"
141
+ logger.info(f"πŸš€ [{batch_request_id}] Processing batch {batch_idx + 1}/{len(ticker_batches)}")
142
 
143
+ tasks = [bounded_fetch(ticker, batch_request_id, idx) for idx, ticker in enumerate(ticker_batch)]
144
+ batch_results = await asyncio.gather(*tasks)
145
+ all_results.extend(batch_results)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
146
 
147
  batch_duration = time.time() - batch_start
148
+ logger.info(f"βœ… [{batch_request_id}] Completed in {batch_duration:.2f}s")
 
 
149
 
 
150
  if batch_idx < len(ticker_batches) - 1 and batch_delay > 0:
 
151
  await asyncio.sleep(batch_delay)
152
 
153
  overall_duration = time.time() - overall_start
154
+ logger.info(f"🏁 [{request_id}] COMPLETED: {overall_duration:.2f}s total.")
 
 
 
 
 
155
  return all_results
156
 
157
+ # --- Core Processing Function (NOW ASYNC) ---
158
+ # This is the most critical change.
159
+ async def process_stock_request(
160
+ session: aiohttp.ClientSession, # <-- Accept the session
161
+ req: StockDataRequest
 
 
 
 
 
 
162
  ) -> Dict:
163
+ """Optimized async core function that orchestrates the data fetching."""
164
  request_start = time.time()
 
 
 
 
165
  try:
166
+ ts_range = get_time_range_in_unix_ms(req.start_date, req.end_date, req.timezone)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
167
 
168
+ # NO asyncio.run(). We are already in an event loop, so we just await.
169
+ results = await fetch_stock_data_batch_optimized(
170
+ session, req.tickers, ts_range["start_timestamp_ms"], ts_range["end_timestamp_ms"],
171
+ req.interval, req.batch_size, req.batch_delay, req.max_concurrent
172
+ )
 
 
 
 
 
173
 
174
  total_duration = time.time() - request_start
175
+ # ... (your summary logic is excellent, let's keep it)
176
+ successful = sum(1 for r in results if r.get('status') == 'success')
177
+ response_times = [r['response_time_ms'] for r in results if 'response_time_ms' in r]
178
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
179
  return {
180
+ "success": True, "data": results,
 
 
 
181
  "processing_summary": {
182
+ "total_tickers": len(req.tickers), "successful": successful, "failed": len(req.tickers) - successful,
 
 
 
183
  "total_duration_seconds": round(total_duration, 2),
184
+ "throughput_tickers_per_second": round(len(req.tickers) / total_duration, 1),
185
+ "avg_response_time_ms": round(sum(response_times) / len(response_times), 1) if response_times else 0,
 
 
 
 
 
 
 
 
 
186
  }
187
  }
188
+ except Exception as e:
189
+ logger.error(f"Error in process_stock_request: {e}", exc_info=True)
190
+ return {"success": False, "error": str(e)}
191
 
192
  # --- FastAPI Application ---
193
+ api_app = FastAPI(title="Corrected Groww Stock Data API", version="3.0.0")
194
+
195
+ @api_app.on_event("startup")
196
+ async def startup_event():
197
+ """On startup, create ONE reusable aiohttp ClientSession."""
198
+ logger.info("πŸš€ FastAPI startup: Creating persistent aiohttp ClientSession.")
199
+ connector = aiohttp.TCPConnector(limit_per_host=50, limit=100)
200
+ timeout = aiohttp.ClientTimeout(total=60, connect=10)
201
+ app_state['aiohttp_session'] = aiohttp.ClientSession(connector=connector, timeout=timeout)
202
+
203
+ @api_app.on_event("shutdown")
204
+ async def shutdown_event():
205
+ """On shutdown, gracefully close the ClientSession."""
206
+ logger.info("πŸ”’ FastAPI shutdown: Closing aiohttp ClientSession.")
207
+ await app_state['aiohttp_session'].close()
 
 
 
 
208
 
209
  @api_app.post("/fetch-stock-data")
210
+ async def fetch_stock_data_endpoint(req: StockDataRequest):
211
+ """API endpoint that correctly uses the shared session."""
212
+ session = app_state.get('aiohttp_session')
213
+ if not session:
214
+ raise HTTPException(status_code=500, detail="AIOHTTP session not initialized.")
215
+
216
+ result = await process_stock_request(session, req)
217
+ result["timestamp"] = datetime.now().isoformat()
218
+ return result
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
219
 
220
  @api_app.get("/health")
221
  async def health_check():
 
222
  return {"status": "healthy", "timestamp": datetime.now().isoformat()}
223
 
224
+ # --- Gradio Interface (Refactored to call the API) ---
225
+ # This is a much more robust pattern. The UI calls the production-ready API.
226
+ def execute_stock_request_via_api(
227
+ ticker_input: str, start_date: str, end_date: str, interval: int,
228
+ batch_size: int, batch_delay: float, max_concurrent: int
229
+ ) -> Dict:
230
+ """Wrapper for Gradio that makes an HTTP request to its own FastAPI backend."""
231
+ import requests
232
+ api_url = "http://127.0.0.1:8000/fetch-stock-data"
233
+
 
 
 
 
 
 
 
 
 
 
 
 
234
  try:
235
+ try:
236
+ tickers = json.loads(ticker_input)
237
+ except json.JSONDecodeError:
238
+ tickers = [t.strip().upper() for t in ticker_input.split(',')]
239
+
240
+ payload = {
241
+ "tickers": tickers, "start_date": start_date, "end_date": end_date,
242
+ "interval": int(interval), "batch_size": int(batch_size),
243
+ "batch_delay": float(batch_delay), "max_concurrent": int(max_concurrent)
 
 
 
 
 
 
244
  }
 
 
 
 
 
 
 
245
 
246
+ # Set a long timeout for potentially large requests
247
+ response = requests.post(api_url, json=payload, timeout=300)
248
+ response.raise_for_status()
249
+ return response.json()
 
 
250
 
251
+ except requests.exceptions.RequestException as e:
252
+ logger.error(f"Gradio UI failed to connect to backend API: {e}")
253
+ return {"success": False, "error": f"Failed to connect to the backend API. Is it running? Error: {e}"}
254
+ except Exception as e:
255
+ logger.error(f"Error in Gradio execution wrapper: {e}")
256
+ return {"success": False, "error": str(e)}
257
 
258
+ def create_gradio_interface():
259
+ # ... your Gradio layout is great, we'll just wire it to the new function ...
260
+ with gr.Blocks(title="Optimized Groww Stock Data Fetcher") as demo:
261
+ # ... All your gr.Markdown and layout code here ...
262
+ gr.Markdown("# ⚑ Corrected Groww Stock Data Fetcher v3.0")
263
  with gr.Row():
264
  with gr.Column():
265
+ ticker_box = gr.Textbox(label="Stock Tickers", value='["RELIANCE","TCS"]', lines=3)
266
+ start_box = gr.Textbox(label="Start Date (YYYY-MM-DD)", value="2025-08-01")
267
+ end_box = gr.Textbox(label="End Date (YYYY-MM-DD)", value="2025-08-10")
268
+ interval_box = gr.Number(label="Interval (minutes)", value=15)
269
+ batch_size_box = gr.Number(label="Batch Size", value=50)
270
+ batch_delay_box = gr.Number(label="Batch Delay (seconds)", value=0.5)
271
+ max_concurrent_box = gr.Number(label="Max Concurrent", value=50)
272
+ fetch_button = gr.Button("πŸš€ Fetch Data", variant="primary")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
273
  with gr.Column():
274
+ # Use the gr.JSON component for nicely formatted output
275
+ output_box = gr.JSON(label="API Response with Performance Metrics")
 
 
 
 
 
 
 
 
276
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
277
  fetch_button.click(
278
+ fn=execute_stock_request_via_api,
279
+ inputs=[ticker_box, start_box, end_date, interval_box, batch_size_box, batch_delay_box, max_concurrent_box],
280
  outputs=output_box
281
  )
 
282
  return demo
283
 
284
  # --- Main Execution ---
 
285
  def run_api_server(host="0.0.0.0", port=8000):
286
+ uvicorn.run(api_app, host=host, port=port)
 
287
 
288
+ def run_gradio_interface(port=7860):
 
289
  demo = create_gradio_interface()
290
+ demo.launch(server_name="0.0.0.0", server_port=port)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
291
 
292
  if __name__ == "__main__":
293
  import sys
294
 
295
+ mode = "both"
296
+ if len(sys.argv) > 1:
297
+ mode = sys.argv[1]
298
+
299
+ if mode == "api":
300
+ print("πŸš€ Starting API server on http://0.0.0.0:8000")
 
 
301
  run_api_server()
302
+ elif mode == "ui":
303
+ print("🎨 Starting Gradio UI on http://0.0.0.0:7860")
304
+ print("🚨 Make sure the API server is running separately for the UI to work.")
305
+ run_gradio_interface()
306
+ else: # Default "both" mode
307
+ print("πŸš€ Starting API server in a background thread...")
308
  api_thread = threading.Thread(target=run_api_server, daemon=True)
309
  api_thread.start()
310
+ print("⏳ Waiting 2 seconds for API server to initialize...")
311
  time.sleep(2)
312
+ print("🎨 Starting Gradio UI on http://0.0.0.0:7860")
 
 
 
313
  run_gradio_interface()