File size: 9,779 Bytes
22087d0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
#!/usr/bin/env python3
"""
Script to stop, archive, and delete sandboxes from the sandbox pool.

Usage:
    uv run python scripts/archive_pool_sandboxes.py <count> [--dry-run] [--batch-size=5]

Example:
    uv run python scripts/archive_pool_sandboxes.py 100 --dry-run
    uv run python scripts/archive_pool_sandboxes.py 100
    uv run python scripts/archive_pool_sandboxes.py 100 --batch-size=3
"""
import asyncio
import argparse
import sys
import os

# Add backend to path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from daytona_sdk import SandboxState
from core.utils.logger import logger
from core.sandbox.sandbox import daytona, delete_sandbox
from core.sandbox import pool_repo
from core.services.db import init_db, close_db


async def stop_archive_and_delete_sandbox(sandbox_id: str, dry_run: bool = False) -> bool:
    """Stop, archive, and delete a single sandbox from the pool."""
    try:
        if dry_run:
            logger.info(f"[DRY RUN] Would stop, archive, and delete sandbox: {sandbox_id}")
            return True
        
        # Get sandbox
        sandbox = await daytona.get(sandbox_id)
        logger.debug(f"Sandbox {sandbox_id} current state: {sandbox.state}")
        
        # If already archived, just delete it
        if sandbox.state == SandboxState.ARCHIVED:
            logger.info(f"Sandbox {sandbox_id} already archived, deleting...")
            await delete_sandbox(sandbox_id)
            logger.info(f"✓ Successfully deleted already-archived sandbox: {sandbox_id}")
            return True
        
        # If currently archiving, wait for it to finish
        if sandbox.state == SandboxState.ARCHIVING:
            logger.info(f"Sandbox {sandbox_id} is already archiving, waiting...")
            for i in range(60):
                await asyncio.sleep(2)
                sandbox = await daytona.get(sandbox_id)
                if sandbox.state == SandboxState.ARCHIVED:
                    logger.info(f"Sandbox {sandbox_id} archiving completed")
                    await delete_sandbox(sandbox_id)
                    logger.info(f"✓ Successfully deleted: {sandbox_id}")
                    return True
            logger.warning(f"Sandbox {sandbox_id} did not finish archiving within 120 seconds")
            return False
        
        # Stop sandbox if it's running
        if sandbox.state == SandboxState.STARTED:
            logger.info(f"Stopping sandbox: {sandbox_id}")
            await daytona.stop(sandbox)
            
            # Wait for sandbox to stop (increased timeout)
            for i in range(60):
                await asyncio.sleep(2)
                sandbox = await daytona.get(sandbox_id)
                if sandbox.state == SandboxState.STOPPED:
                    logger.debug(f"Sandbox {sandbox_id} stopped successfully")
                    break
                elif sandbox.state == SandboxState.STOPPING:
                    logger.debug(f"Sandbox {sandbox_id} still stopping... ({i+1}/60)")
                    continue
            else:
                logger.error(f"Sandbox {sandbox_id} did not stop within 120 seconds, current state: {sandbox.state}")
                return False
        
        # Make sure we're in STOPPED state before archiving
        if sandbox.state != SandboxState.STOPPED:
            logger.error(f"Sandbox {sandbox_id} is not in STOPPED state: {sandbox.state}")
            return False
        
        # Archive the sandbox
        logger.info(f"Archiving sandbox: {sandbox_id}")
        await sandbox.archive()
        
        # Wait for archive to complete
        for i in range(60):
            await asyncio.sleep(2)
            sandbox = await daytona.get(sandbox_id)
            if sandbox.state == SandboxState.ARCHIVED:
                logger.debug(f"Sandbox {sandbox_id} archived successfully")
                break
            elif sandbox.state == SandboxState.ARCHIVING:
                logger.debug(f"Sandbox {sandbox_id} still archiving... ({i+1}/60)")
                continue
        else:
            logger.warning(f"Sandbox {sandbox_id} did not finish archiving within 120 seconds")
            # Continue anyway to try deletion
        
        # Delete the sandbox from Daytona
        logger.info(f"Deleting sandbox from Daytona: {sandbox_id}")
        await delete_sandbox(sandbox_id)
        
        logger.info(f"✓ Successfully stopped, archived, and deleted: {sandbox_id}")
        return True
        
    except Exception as e:
        logger.error(f"✗ Failed to stop/archive/delete {sandbox_id}: {e}")
        return False


async def archive_pool_sandboxes(count: int, batch_size: int, dry_run: bool = False):
    """Archive and delete pooled sandboxes in batches."""
    
    logger.info(f"Starting pool sandbox archival and deletion: {count} sandboxes, batch_size={batch_size}, dry_run={dry_run}")
    
    # Initialize database
    await init_db()
    
    total_processed = 0
    total_success = 0
    total_failed = 0
    
    try:
        # First, check what sandboxes exist
        check_sql = """
        SELECT status, COUNT(*) as count
        FROM resources
        WHERE type = 'sandbox'
        GROUP BY status
        """
        from core.services.db import execute
        status_counts = await execute(check_sql, {})
        logger.info(f"Sandbox status breakdown: {status_counts}")
        
        # Get pooled sandboxes from database
        pooled = await pool_repo.get_pooled_sandboxes(limit=count)
        
        if not pooled:
            logger.warning("No pooled sandboxes found in database")
            logger.warning("Run the query above to see what statuses your sandboxes have")
            return total_success, total_failed
        
        actual_count = min(count, len(pooled))
        logger.info(f"Found {len(pooled)} pooled sandboxes, will process {actual_count}")
        
        # Process in batches
        batches = (actual_count + batch_size - 1) // batch_size
        
        for batch_num in range(batches):
            batch_start = batch_num * batch_size
            batch_end = min(batch_start + batch_size, actual_count)
            batch_items = pooled[batch_start:batch_end]
            
            logger.info(f"[Batch {batch_num + 1}/{batches}] Processing {len(batch_items)} sandboxes...")
            
            # Process batch in parallel
            tasks = []
            for item in batch_items:
                sandbox_id = item['external_id']
                tasks.append(stop_archive_and_delete_sandbox(sandbox_id, dry_run))
            
            results = await asyncio.gather(*tasks, return_exceptions=True)
            
            # Count results
            batch_success = 0
            batch_failed = 0
            for i, result in enumerate(results):
                total_processed += 1
                if isinstance(result, Exception):
                    logger.error(f"Exception processing sandbox: {result}")
                    batch_failed += 1
                    total_failed += 1
                elif result:
                    batch_success += 1
                    total_success += 1
                    
                    # Mark as deleted in database if not dry run
                    if not dry_run:
                        resource_id = batch_items[i]['id']
                        await pool_repo.mark_sandbox_deleted(resource_id)
                else:
                    batch_failed += 1
                    total_failed += 1
            
            logger.info(
                f"[Batch {batch_num + 1}/{batches}] "
                f"Success: {batch_success}, Failed: {batch_failed} "
                f"(Total: {total_success}/{total_processed})"
            )
            
            # Delay between batches to avoid rate limiting and let operations complete
            if batch_num < batches - 1:
                logger.info(f"Waiting 5 seconds before next batch...")
                await asyncio.sleep(5)
        
        logger.info(
            f"✅ Pool sandbox archival and deletion complete: "
            f"{total_success} succeeded, {total_failed} failed out of {total_processed} total"
        )
        
    except Exception as e:
        logger.error(f"Fatal error during pool archival and deletion: {e}")
        raise
    finally:
        await close_db()
    
    return total_success, total_failed


def main():
    parser = argparse.ArgumentParser(description="Stop, archive, and delete sandboxes from the pool")
    parser.add_argument("count", type=int, help="Number of sandboxes to archive and delete")
    parser.add_argument("--batch-size", type=int, default=5, help="Sandboxes per batch (default: 5)")
    parser.add_argument("--dry-run", action="store_true", help="Show what would be done without actually doing it")
    
    args = parser.parse_args()
    
    if args.count <= 0:
        print("Error: count must be positive")
        sys.exit(1)
    
    if args.batch_size <= 0:
        print("Error: batch-size must be positive")
        sys.exit(1)
    
    if args.dry_run:
        print("=== DRY RUN MODE ===")
        print("No changes will be made to sandboxes or pool")
        print()
    
    print(f"This will stop, archive, and DELETE {args.count} sandboxes from the pool")
    print()
    
    success, failed = asyncio.run(archive_pool_sandboxes(args.count, args.batch_size, args.dry_run))
    
    sys.exit(0 if failed == 0 else 1)


if __name__ == "__main__":
    main()


# Example usage:
# uv run python scripts/archive_pool_sandboxes.py 100 --dry-run
# uv run python scripts/archive_pool_sandboxes.py 100
# uv run python scripts/archive_pool_sandboxes.py 100 --batch-size=3  # Smaller batches for better stability