Download internal/syncer/syncer_test.go from a3216/gcli2api: direct link, hf CLI and curl.
- Browser
- Download file 13.9 kB
-
https://huggingface.co/spaces/a3216/gcli2api/resolve/main/internal/syncer/syncer_test.go
- Command line
-
hf download hf://spaces/a3216/gcli2api/internal/syncer/syncer_test.go
-
curl -L -o syncer_test.go https://huggingface.co/spaces/a3216/gcli2api/resolve/main/internal/syncer/syncer_test.go
13.9 kB
| package syncer | |
| import ( | |
| "context" | |
| "encoding/base64" | |
| "encoding/json" | |
| "fmt" | |
| "net/http" | |
| "net/http/httptest" | |
| "os" | |
| "path/filepath" | |
| "strings" | |
| "sync" | |
| "testing" | |
| "time" | |
| ) | |
| // fakeBucket 是一个内存版 HF Bucket:只实现同步用到的两个只读端点。 | |
| type fakeBucket struct { | |
| mu sync.Mutex | |
| files map[string][]byte | |
| mtime map[string]time.Time | |
| } | |
| func newFakeBucket() *fakeBucket { | |
| return &fakeBucket{files: map[string][]byte{}, mtime: map[string]time.Time{}} | |
| } | |
| func (f *fakeBucket) put(path string, content []byte, mtime time.Time) { | |
| f.mu.Lock() | |
| defer f.mu.Unlock() | |
| f.files[path] = content | |
| f.mtime[path] = mtime | |
| } | |
| func (f *fakeBucket) get(path string) ([]byte, bool) { | |
| f.mu.Lock() | |
| defer f.mu.Unlock() | |
| b, ok := f.files[path] | |
| return b, ok | |
| } | |
| func (f *fakeBucket) handler(bucket string) http.Handler { | |
| return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| switch { | |
| case strings.HasPrefix(r.URL.Path, "/api/buckets/"): | |
| f.mu.Lock() | |
| type entry struct { | |
| Type string `json:"type"` | |
| Path string `json:"path"` | |
| Size int64 `json:"size"` | |
| Mtime string `json:"mtime"` | |
| XetHash string `json:"xetHash"` | |
| } | |
| out := []entry{} | |
| for p, b := range f.files { | |
| out = append(out, entry{ | |
| Type: "file", Path: p, Size: int64(len(b)), | |
| Mtime: f.mtime[p].UTC().Format(time.RFC3339Nano), XetHash: "xet-" + p, | |
| }) | |
| } | |
| f.mu.Unlock() | |
| _ = json.NewEncoder(w).Encode(out) | |
| case strings.HasPrefix(r.URL.Path, "/buckets/"+bucket+"/resolve/"): | |
| // Go 已把 %2F 解码回 '/',直接剥前缀取路径。 | |
| p := strings.TrimPrefix(r.URL.Path, "/buckets/"+bucket+"/resolve/") | |
| b, ok := f.get(p) | |
| if !ok { | |
| http.Error(w, "not found", http.StatusNotFound) | |
| return | |
| } | |
| _, _ = w.Write(b) | |
| default: | |
| http.NotFound(w, r) | |
| } | |
| }) | |
| } | |
| // fakeWriter 记录写入,避免测试真的碰网络/子进程。 | |
| type fakeWriter struct { | |
| mu sync.Mutex | |
| put map[string][]byte | |
| batches int | |
| failOn string | |
| } | |
| func (f *fakeWriter) Name() string { return "fake" } | |
| func (f *fakeWriter) Put(_ context.Context, path string, content []byte) error { | |
| f.mu.Lock() | |
| defer f.mu.Unlock() | |
| if f.failOn == path { | |
| return fmt.Errorf("boom") | |
| } | |
| if f.put == nil { | |
| f.put = map[string][]byte{} | |
| } | |
| f.put[path] = content | |
| return nil | |
| } | |
| func (f *fakeWriter) PutBatch(_ context.Context, files []PushFile, _ bool) error { | |
| f.mu.Lock() | |
| defer f.mu.Unlock() | |
| f.batches++ | |
| if f.put == nil { | |
| f.put = map[string][]byte{} | |
| } | |
| for _, fl := range files { | |
| f.put[fl.Path] = fl.Content | |
| } | |
| return nil | |
| } | |
| func (f *fakeWriter) Delete(context.Context, string) error { return nil } | |
| func (f *fakeWriter) written(path string) ([]byte, bool) { | |
| f.mu.Lock() | |
| defer f.mu.Unlock() | |
| b, ok := f.put[path] | |
| return b, ok | |
| } | |
| // authDoc 造一个嵌套形态的凭证文件(网关 auth.Parse 认的形态)。 | |
| func authDoc(uid, nick, at, rt string, exp int64) []byte { | |
| return []byte(fmt.Sprintf( | |
| `{"account":{"uid":%q,"nickname":%q},"auth":{"accessToken":%q,"refreshToken":%q,"expiresAt":%d}}`, | |
| uid, nick, at, rt, exp)) | |
| } | |
| // newTestEngine 起一个假桶 + 假写入通道的 Engine。 | |
| func newTestEngine(t *testing.T, fb *fakeBucket, local map[string][]byte) (*Engine, string, string) { | |
| t.Helper() | |
| dir := t.TempDir() | |
| authDir := filepath.Join(dir, "auths") | |
| dataDir := filepath.Join(dir, "data") | |
| for rel, content := range local { | |
| fp := filepath.Join(dir, rel) | |
| if err := os.MkdirAll(filepath.Dir(fp), 0o755); err != nil { | |
| t.Fatal(err) | |
| } | |
| if err := os.WriteFile(fp, content, 0o600); err != nil { | |
| t.Fatal(err) | |
| } | |
| } | |
| ts := httptest.NewServer(fb.handler("u/b")) | |
| t.Cleanup(ts.Close) | |
| e, err := New(Config{ | |
| Bucket: "u/b", Endpoint: ts.URL, | |
| AuthDir: authDir, DataDir: dataDir, | |
| IncludeData: []string{"state.json"}, | |
| Writer: "relay", RelayURL: "http://127.0.0.1:1", // 占位,测试里换成 fakeWriter | |
| Timeout: "10s", | |
| }) | |
| if err != nil { | |
| t.Fatalf("New: %v", err) | |
| } | |
| return e, authDir, dataDir | |
| } | |
| // TestPullDownloadsRemoteOnly 本地首次启动:桶里独有的凭证要落到本地 auths/。 | |
| func TestPullDownloadsRemoteOnly(t *testing.T) { | |
| fb := newFakeBucket() | |
| fb.put("auths/workbuddy-u1.json", authDoc("u1", "n1", "at1", "rt1", 5000), time.Now()) | |
| e, authDir, _ := newTestEngine(t, fb, nil) | |
| fw := &fakeWriter{} | |
| e.writer = fw | |
| p, res, err := e.Sync(context.Background(), ModePull, false) | |
| if err != nil { | |
| t.Fatalf("Sync: %v", err) | |
| } | |
| if res.Downloaded != 1 || res.Failed != 0 { | |
| t.Fatalf("res=%+v plan=%+v", res, p.Actions) | |
| } | |
| raw, err := os.ReadFile(filepath.Join(authDir, "workbuddy-u1.json")) | |
| if err != nil { | |
| t.Fatalf("下载后本地文件不存在: %v", err) | |
| } | |
| if string(raw) != string(authDoc("u1", "n1", "at1", "rt1", 5000)) { | |
| t.Errorf("内容不一致: %s", raw) | |
| } | |
| } | |
| // TestPushUploadsLocalOnly 本地新增的账号要推到远端。 | |
| func TestPushUploadsLocalOnly(t *testing.T) { | |
| fb := newFakeBucket() | |
| e, _, _ := newTestEngine(t, fb, map[string][]byte{ | |
| "auths/workbuddy-u2.json": authDoc("u2", "n2", "at2", "rt2", 9000), | |
| }) | |
| fw := &fakeWriter{} | |
| e.writer = fw | |
| _, res, err := e.Sync(context.Background(), ModePush, false) | |
| if err != nil { | |
| t.Fatalf("Sync: %v", err) | |
| } | |
| if res.Uploaded != 1 || res.Failed != 0 { | |
| t.Fatalf("res=%+v", res) | |
| } | |
| got, ok := fw.written("auths/workbuddy-u2.json") | |
| if !ok { | |
| t.Fatal("远端没收到上传") | |
| } | |
| if string(got) != string(authDoc("u2", "n2", "at2", "rt2", 9000)) { | |
| t.Errorf("上传内容不一致: %s", got) | |
| } | |
| } | |
| // TestConflictNewerAccessTokenWins 两端都有同一账号且都刷新过 → 采用 expiresAt 更晚的一方, | |
| // 并把赢家回灌到输家。这是"防止凭证不同步"的核心行为。 | |
| func TestConflictNewerAccessTokenWins(t *testing.T) { | |
| fb := newFakeBucket() | |
| // 远端更新(expiresAt 9000),本地陈旧(expiresAt 1000)。 | |
| fb.put("auths/workbuddy-u3.json", authDoc("u3", "n3", "remote-at", "remote-rt", 9000), time.Now()) | |
| e, authDir, _ := newTestEngine(t, fb, map[string][]byte{ | |
| "auths/workbuddy-u3.json": authDoc("u3", "n3", "local-at", "local-rt", 1000), | |
| }) | |
| fw := &fakeWriter{} | |
| e.writer = fw | |
| p, res, err := e.Sync(context.Background(), ModeSync, false) | |
| if err != nil { | |
| t.Fatalf("Sync: %v", err) | |
| } | |
| if len(p.Conflicts()) != 1 { | |
| t.Fatalf("应识别出 1 个冲突: %+v", p.Actions) | |
| } | |
| if res.Downloaded != 1 || res.Uploaded != 0 { | |
| t.Fatalf("应由远端覆盖本地: res=%+v", res) | |
| } | |
| raw, _ := os.ReadFile(filepath.Join(authDir, "workbuddy-u3.json")) | |
| if !strings.Contains(string(raw), "remote-at") { | |
| t.Errorf("本地未被远端覆盖: %s", raw) | |
| } | |
| } | |
| // TestConflictLocalNewerPushes 反向:本地更新 → 推到远端。 | |
| func TestConflictLocalNewerPushes(t *testing.T) { | |
| fb := newFakeBucket() | |
| fb.put("auths/workbuddy-u4.json", authDoc("u4", "n4", "remote-at", "remote-rt", 1000), time.Now()) | |
| e, authDir, _ := newTestEngine(t, fb, map[string][]byte{ | |
| "auths/workbuddy-u4.json": authDoc("u4", "n4", "local-at", "local-rt", 9000), | |
| }) | |
| fw := &fakeWriter{} | |
| e.writer = fw | |
| _, res, err := e.Sync(context.Background(), ModeSync, false) | |
| if err != nil { | |
| t.Fatalf("Sync: %v", err) | |
| } | |
| if res.Uploaded != 1 || res.Downloaded != 0 { | |
| t.Fatalf("应由本地覆盖远端: res=%+v", res) | |
| } | |
| got, _ := fw.written("auths/workbuddy-u4.json") | |
| if !strings.Contains(string(got), "local-at") { | |
| t.Errorf("远端未被本地覆盖: %s", got) | |
| } | |
| raw, _ := os.ReadFile(filepath.Join(authDir, "workbuddy-u4.json")) | |
| if !strings.Contains(string(raw), "local-at") { | |
| t.Errorf("本地被误改: %s", raw) | |
| } | |
| } | |
| // TestIdenticalIsNoop 两端一致时不应产生任何读写。 | |
| func TestIdenticalIsNoop(t *testing.T) { | |
| doc := authDoc("u5", "n5", "at", "rt", 5000) | |
| fb := newFakeBucket() | |
| fb.put("auths/workbuddy-u5.json", doc, time.Now()) | |
| e, _, _ := newTestEngine(t, fb, map[string][]byte{"auths/workbuddy-u5.json": doc}) | |
| fw := &fakeWriter{} | |
| e.writer = fw | |
| p, res, err := e.Sync(context.Background(), ModeSync, false) | |
| if err != nil { | |
| t.Fatalf("Sync: %v", err) | |
| } | |
| if p.Changes() != 0 || res.Uploaded != 0 || res.Downloaded != 0 { | |
| t.Fatalf("一致时应无动作: plan=%+v res=%+v", p.Actions, res) | |
| } | |
| } | |
| // TestExcludedPathsNeverSynced config.json 与 request-logs 绝不能进同步范围。 | |
| func TestExcludedPathsNeverSynced(t *testing.T) { | |
| fb := newFakeBucket() | |
| fb.put("config.json", []byte(`{"listen":":7861","api_key":"relay-secret"}`), time.Now()) | |
| fb.put("data/request-logs/requests-2026-10-02.jsonl", []byte("{}\n"), time.Now()) | |
| fb.put("data/state.json", []byte(`{"accounts":{}}`), time.Now()) | |
| e, authDir, dataDir := newTestEngine(t, fb, nil) | |
| fw := &fakeWriter{} | |
| e.writer = fw | |
| p, res, err := e.Sync(context.Background(), ModeSync, false) | |
| if err != nil { | |
| t.Fatalf("Sync: %v", err) | |
| } | |
| if res.Downloaded != 1 { // 只有 data/state.json | |
| t.Fatalf("应只下载 state.json: res=%+v plan=%+v", res, p.Actions) | |
| } | |
| for _, a := range p.Actions { | |
| if a.Path == "config.json" || strings.HasPrefix(a.Path, "data/request-logs/") { | |
| t.Errorf("被排除的路径出现在计划里: %+v", a) | |
| } | |
| } | |
| if _, err := os.Stat(filepath.Join(authDir, "..", "config.json")); err == nil { | |
| t.Error("config.json 被写到了本地") | |
| } | |
| if _, err := os.Stat(filepath.Join(dataDir, "request-logs")); err == nil { | |
| t.Error("request-logs 被同步了") | |
| } | |
| } | |
| // TestDryRunNeverWrites dry-run(status)不得产生任何落盘或上传, | |
| // 但**必须**照常把差异算出来——漏算差异会让 status 骗人。 | |
| func TestDryRunNeverWrites(t *testing.T) { | |
| fb := newFakeBucket() | |
| fb.put("auths/workbuddy-u6.json", authDoc("u6", "n6", "at", "rt", 9000), time.Now()) | |
| e, authDir, _ := newTestEngine(t, fb, map[string][]byte{ | |
| "auths/workbuddy-u6.json": authDoc("u6", "n6", "old", "old", 1), | |
| }) | |
| fw := &fakeWriter{} | |
| e.writer = fw | |
| plan, res, err := e.Sync(context.Background(), ModeSync, true) | |
| if err != nil { | |
| t.Fatalf("Sync: %v", err) | |
| } | |
| if plan.Changes() != 1 { | |
| t.Fatalf("dry-run 也必须报出这 1 处差异: %+v", plan.Actions) | |
| } | |
| if res.Downloaded != 1 { // 计入"将会做",但没真做 | |
| t.Fatalf("dry-run 应计入计划动作: %+v", res) | |
| } | |
| if len(fw.put) != 0 || fw.batches != 0 { | |
| t.Errorf("dry-run 不应上传: %+v", fw.put) | |
| } | |
| raw, _ := os.ReadFile(filepath.Join(authDir, "workbuddy-u6.json")) | |
| if !strings.Contains(string(raw), "old") { | |
| t.Errorf("dry-run 不应改本地: %s", raw) | |
| } | |
| } | |
| // TestApplyNoWriterFailsLoudly 没有写入通道时必须明确失败,不能假装成功。 | |
| func TestApplyNoWriterFailsLoudly(t *testing.T) { | |
| fb := newFakeBucket() | |
| e, _, _ := newTestEngine(t, fb, map[string][]byte{ | |
| "auths/workbuddy-u7.json": authDoc("u7", "n7", "at", "rt", 9000), | |
| }) | |
| e.writer = nil | |
| _, res, err := e.Sync(context.Background(), ModePush, false) | |
| if err != nil { | |
| t.Fatalf("Sync 本身不应报错: %v", err) | |
| } | |
| if res.Failed != 1 || res.Uploaded != 0 { | |
| t.Fatalf("应报告 1 个失败: %+v", res) | |
| } | |
| } | |
| // TestNextLink 分页 Link 头解析(含相对 URL 与多段)。 | |
| func TestNextLink(t *testing.T) { | |
| const ep = "https://huggingface.co" | |
| cases := []struct{ in, want string }{ | |
| {"", ""}, | |
| {`<https://huggingface.co/api/buckets/u/b/tree?cursor=2>; rel="next"`, "https://huggingface.co/api/buckets/u/b/tree?cursor=2"}, | |
| {`</api/buckets/u/b/tree?cursor=3>; rel="next"`, ep + "/api/buckets/u/b/tree?cursor=3"}, | |
| {`<https://x/y>; rel="prev", </api/buckets/u/b/tree?c=9>; rel="next"`, ep + "/api/buckets/u/b/tree?c=9"}, | |
| {`<https://x/y>; rel="prev"`, ""}, | |
| {"garbage", ""}, | |
| } | |
| for _, tc := range cases { | |
| if got := nextLink(tc.in, ep); got != tc.want { | |
| t.Errorf("nextLink(%q) = %q, want %q", tc.in, got, tc.want) | |
| } | |
| } | |
| } | |
| // TestLocalPathRejectsTraversal localPath 必须挡住越界路径。 | |
| func TestLocalPathRejectsTraversal(t *testing.T) { | |
| e := &Engine{cfg: Config{AuthDir: "/tmp/a", DataDir: "/tmp/d", IncludeData: []string{"state.json"}}} | |
| for _, bad := range []string{"config.json", "../evil.json", "auths/../../evil.json", "other/x.json"} { | |
| if _, err := e.localPath(bad); err == nil { | |
| t.Errorf("localPath(%q) 应报错", bad) | |
| } | |
| } | |
| if fp, err := e.localPath("auths/workbuddy-1.json"); err != nil || !strings.HasSuffix(fp, "workbuddy-1.json") { | |
| t.Errorf("localPath 正常路径失败: %q %v", fp, err) | |
| } | |
| } | |
| // TestJSONOutputNeverLeaksCredentialMaterial 安全回归:FileState.Content 绝不能被序列化出去。 | |
| // | |
| // Action 会被 `wb2api-sync -json` 整个打印,Content 一旦进 JSON,`status -json` | |
| // 就会把 accessToken / refreshToken 明文(原始与 base64 两种形态)灌进终端、 | |
| // CI 日志或 issue 附件——等于自己把凭证泄露出去。实测早期版本正是这样泄露的。 | |
| func TestJSONOutputNeverLeaksCredentialMaterial(t *testing.T) { | |
| const secret = "SUPER-SECRET-REFRESH-TOKEN-VALUE" | |
| doc := authDoc("u1", "nick", "access-"+secret, "refresh-"+secret, 9000) | |
| st := newFileState("auths/workbuddy-u1.json", doc, time.Now(), catAuth) | |
| if st.Content == nil { | |
| t.Fatal("夹具前提不成立:Content 应为原始字节") | |
| } | |
| if !strings.Contains(string(st.Content), secret) { | |
| t.Fatal("夹具前提不成立:内容里应含 secret") | |
| } | |
| raw, err := json.Marshal(Action{Op: OpDownload, Path: st.Path, Category: catAuth, Local: st, Remote: st}) | |
| if err != nil { | |
| t.Fatal(err) | |
| } | |
| got := string(raw) | |
| if strings.Contains(got, secret) { | |
| t.Fatalf("JSON 里出现了凭证明文: %s", got) | |
| } | |
| if strings.Contains(strings.ToLower(got), `"content"`) { | |
| t.Errorf("JSON 里出现了 content 字段: %s", got) | |
| } | |
| b64 := base64.StdEncoding.EncodeToString(doc) | |
| if len(b64) > 24 && strings.Contains(got, b64[:24]) { | |
| t.Errorf("JSON 里出现了 content 的 base64 前缀: %s", got) | |
| } | |
| // 该有的诊断字段不能因为脱敏一起丢掉。 | |
| if !strings.Contains(got, st.SHA256) || !strings.Contains(got, "u1") { | |
| t.Errorf("SHA256/UID 等诊断字段应保留: %s", got) | |
| } | |
| } | |