NOT-OMEGA commited on
Commit
a0864eb
Β·
verified Β·
1 Parent(s): fed1581

Update Dockerfile

Browse files
Files changed (1) hide show
  1. Dockerfile +38 -3
Dockerfile CHANGED
@@ -9,9 +9,24 @@ FROM ubuntu:22.04
9
  ENV DEBIAN_FRONTEND=noninteractive
10
  ENV PYTHONUNBUFFERED=1
11
  ENV HF_REPO_ID=NOT-OMEGA/NanoMind
12
- ENV N_ENGINES=2
 
 
 
 
 
 
 
13
  ENV OMP_NUM_THREADS=1
14
 
 
 
 
 
 
 
 
 
15
  RUN apt-get update && apt-get install -y \
16
  python3.11 \
17
  python3-pip \
@@ -34,7 +49,17 @@ COPY index.html .
34
  COPY model.bin .
35
  COPY tokenizer.bin .
36
 
37
- RUN g++ -O3 -march=native -fopenmp -ffast-math -std=c++17 \
 
 
 
 
 
 
 
 
 
 
38
  -o inference inference.cpp -lm && \
39
  chmod +x inference && \
40
  echo "βœ… inference binary compiled"
@@ -47,4 +72,14 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
47
  CMD curl -f http://localhost:7860/health || exit 1
48
 
49
  EXPOSE 7860
50
- CMD ["python", "main.py"]
 
 
 
 
 
 
 
 
 
 
 
9
  ENV DEBIAN_FRONTEND=noninteractive
10
  ENV PYTHONUNBUFFERED=1
11
  ENV HF_REPO_ID=NOT-OMEGA/NanoMind
12
+
13
+ # ── Concurrency tuning ────────────────────────────────────────
14
+ # HF free tier = 2 vCPU. Rule: N_ENGINES Γ— OMP_NUM_THREADS ≀ vCPU count.
15
+ # 4 engines Γ— 1 thread = 4 logical workers on 2 cores (light context switching,
16
+ # better queue saturation than 2 engines Γ— 1 thread).
17
+ # Do NOT set OMP_NUM_THREADS > 1 here β€” engines will fight for cores
18
+ # and tok/s will DROP. Keep it at 1.
19
+ ENV N_ENGINES=4
20
  ENV OMP_NUM_THREADS=1
21
 
22
+ # ── gomp thread pinning β€” prevents OS from migrating OMP threads ──
23
+ # Each engine stays on its assigned core β†’ fewer cache misses β†’ ~5-10% faster matmul
24
+ ENV GOMP_CPU_AFFINITY="0-3"
25
+ ENV OMP_PROC_BIND=close
26
+
27
+ # ── Python GIL / asyncio tuning ───────────────────────────────
28
+ ENV PYTHONOPTIMIZE=1
29
+
30
  RUN apt-get update && apt-get install -y \
31
  python3.11 \
32
  python3-pip \
 
49
  COPY model.bin .
50
  COPY tokenizer.bin .
51
 
52
+ # ── Compiler flags explanation ─────────────────────────────────
53
+ # -O3 : max optimization
54
+ # -march=native : AVX2 + FMA auto-detected at build time
55
+ # -fopenmp : OpenMP parallelism across attention heads
56
+ # -ffast-math : allows fused multiply-add reordering (safe for inference)
57
+ # -funroll-loops : unroll small inner loops (matmul hot path)
58
+ # -fno-stack-protector : remove stack canary overhead in tight loops
59
+ # -std=c++17 : required for structured bindings + std::optional
60
+ RUN g++ -O3 -march=native -fopenmp -ffast-math \
61
+ -funroll-loops -fno-stack-protector \
62
+ -std=c++17 \
63
  -o inference inference.cpp -lm && \
64
  chmod +x inference && \
65
  echo "βœ… inference binary compiled"
 
72
  CMD curl -f http://localhost:7860/health || exit 1
73
 
74
  EXPOSE 7860
75
+
76
+ # ── Use uvicorn directly β€” skips the main.py import overhead ──
77
+ # workers=1 is correct: parallelism comes from N_ENGINES async engines,
78
+ # NOT from multiple uvicorn workers (those would each spawn their own
79
+ # engine pool and OOM the container).
80
+ CMD ["python", "-m", "uvicorn", "main:app", \
81
+ "--host", "0.0.0.0", \
82
+ "--port", "7860", \
83
+ "--workers", "1", \
84
+ "--loop", "uvloop", \
85
+ "--http", "httptools"]