chatyou commited on
Commit
f8fb305
·
verified ·
1 Parent(s): 17bb390

Update hf_bucket_provider.py

Browse files
Files changed (1) hide show
  1. hf_bucket_provider.py +55 -89
hf_bucket_provider.py CHANGED
@@ -1,76 +1,68 @@
1
  import os
2
  import tempfile
3
  import time
4
- from typing import List, Dict, Any
5
  from huggingface_hub import (
6
  batch_bucket_files,
7
  download_bucket_files,
8
  list_bucket_tree,
9
  )
10
- from wsgidav.dav_provider import DAVProvider, _DAVResource
11
  from wsgidav.dav_error import DAVError, HTTP_FORBIDDEN, HTTP_NOT_FOUND
12
 
13
 
14
- class HfBucketResource(_DAVResource):
15
- """表示 Hugging Face Bucket 中的文件或目录"""
16
 
17
- def __init__(self, path, environ, provider):
18
  super().__init__(path, environ)
19
  self.provider = provider
20
- self._info = None
21
-
22
- def _get_info(self):
23
- if self._info is None:
24
- self._info = self.provider._get_file_info(self.path)
25
- return self._info
26
-
27
- @property
28
- def exists(self):
29
- info = self._get_info()
30
- return info.get('exists', False)
31
 
32
  def get_content_length(self):
33
- info = self._get_info()
34
- if not info or info.get('is_dir', False):
35
- return None
36
- return info.get('size', 0)
37
 
38
  def get_content_type(self):
39
- return None
40
 
41
  def get_creation_date(self):
42
- info = self._get_info()
43
- return info.get('last_modified', 0) if info else 0
44
 
45
  def get_display_name(self):
46
  return self.name
47
 
48
  def get_etag(self):
49
- info = self._get_info()
50
- if info and info.get('last_modified'):
51
- return f'"{info["last_modified"]}"'
52
- return None
53
 
54
  def get_last_modified(self):
55
- info = self._get_info()
56
- return info.get('last_modified', 0) if info else 0
 
 
57
 
58
- def is_collection(self):
59
- info = self._get_info()
60
- return info.get('is_dir', False) if info else False
61
 
62
- def is_link(self):
63
- return False
64
 
65
- def support_etag(self):
66
- return True
 
 
 
 
 
 
 
 
 
 
 
67
 
68
- def support_ranges(self):
69
- return False
70
 
71
  def get_member_names(self):
72
- if not self.is_collection():
73
- return []
74
  return self.provider._list_directory(self.path)
75
 
76
  def get_member(self, name):
@@ -84,54 +76,18 @@ class HfBucketResource(_DAVResource):
84
 
85
  def create_empty_resource(self, name):
86
  new_path = self.util.join_uri(self.path, name)
 
87
  return self.provider.get_resource_inst(new_path, self.environ)
88
 
89
  def delete(self):
90
- if not self.exists:
91
- raise DAVError(HTTP_NOT_FOUND)
92
- if self.is_collection():
93
- for member_name in self.get_member_names():
94
- member = self.get_member(member_name)
95
- member.delete()
96
- else:
97
- self.provider._delete_file(self.path)
98
-
99
- def move_recursive(self, dest_path):
100
- self.copy_recursive(dest_path)
101
- self.delete()
102
-
103
- def copy_recursive(self, dest_path):
104
- if not self.exists:
105
- raise DAVError(HTTP_NOT_FOUND)
106
- if self.is_collection():
107
- dest_resource = self.provider.get_resource_inst(dest_path, self.environ)
108
- if not dest_resource.exists:
109
- self.provider._create_collection(dest_path)
110
- dest_resource = self.provider.get_resource_inst(dest_path, self.environ)
111
- for member_name in self.get_member_names():
112
- member = self.get_member(member_name)
113
- dest_member_path = self.util.join_uri(dest_path, member_name)
114
- member.copy_recursive(dest_member_path)
115
- else:
116
- self.provider._copy_file(self.path, dest_path)
117
-
118
- def get_content(self):
119
- if not self.exists:
120
- raise DAVError(HTTP_NOT_FOUND)
121
- if self.is_collection():
122
- raise DAVError(HTTP_FORBIDDEN)
123
- return self.provider._get_file_content(self.path)
124
-
125
- def begin_write(self, content_type=None):
126
- if not self.exists:
127
- # 新文件允许写入
128
- pass
129
- if self.is_collection():
130
- raise DAVError(HTTP_FORBIDDEN)
131
- return HfBucketWriteFile(self.provider, self.path)
132
 
133
 
134
  class HfBucketWriteFile:
 
135
  def __init__(self, provider, path):
136
  self.provider = provider
137
  self.path = path
@@ -147,6 +103,8 @@ class HfBucketWriteFile:
147
 
148
 
149
  class HfBucketProvider(DAVProvider):
 
 
150
  def __init__(self, bucket_id: str, token: str):
151
  super().__init__()
152
  self.bucket_id = bucket_id
@@ -159,8 +117,8 @@ class HfBucketProvider(DAVProvider):
159
  path = path[:-1]
160
  return path
161
 
162
- def _get_file_info(self, path: str) -> Dict[str, Any]:
163
- """返回字典,包含 exists, is_dir, size, last_modified"""
164
  path = self._normalize_path(path)
165
  if path == '/':
166
  return {'exists': True, 'is_dir': True, 'size': 0, 'last_modified': time.time()}
@@ -190,9 +148,9 @@ class HfBucketProvider(DAVProvider):
190
  }
191
  if items:
192
  return {'exists': True, 'is_dir': True, 'size': 0, 'last_modified': time.time()}
193
- return {'exists': False, 'is_dir': False, 'size': 0, 'last_modified': 0}
194
  except Exception:
195
- return {'exists': False, 'is_dir': False, 'size': 0, 'last_modified': 0}
196
 
197
  def _list_directory(self, path: str) -> List[str]:
198
  path = self._normalize_path(path)
@@ -279,8 +237,16 @@ class HfBucketProvider(DAVProvider):
279
  os.unlink(tmp.name)
280
 
281
  def get_resource_inst(self, path, environ):
282
- # 始终返回资源对象(即使不存在)
283
- return HfBucketResource(path, environ, self)
 
 
 
 
 
 
 
 
284
 
285
  def exists(self, path, environ):
286
- return self.get_resource_inst(path, environ).exists
 
1
  import os
2
  import tempfile
3
  import time
4
+ from typing import List, Dict, Any, Optional
5
  from huggingface_hub import (
6
  batch_bucket_files,
7
  download_bucket_files,
8
  list_bucket_tree,
9
  )
10
+ from wsgidav.dav_provider import DAVProvider, DAVNonCollection, DAVCollection
11
  from wsgidav.dav_error import DAVError, HTTP_FORBIDDEN, HTTP_NOT_FOUND
12
 
13
 
14
+ class HfBucketFile(DAVNonCollection):
15
+ """表示 Hugging Face Bucket 中的文件"""
16
 
17
+ def __init__(self, path, environ, provider, info):
18
  super().__init__(path, environ)
19
  self.provider = provider
20
+ self.info = info
 
 
 
 
 
 
 
 
 
 
21
 
22
  def get_content_length(self):
23
+ return self.info.get('size', 0)
 
 
 
24
 
25
  def get_content_type(self):
26
+ return None # 由 wsgidav 根据扩展名自动判断
27
 
28
  def get_creation_date(self):
29
+ return self.info.get('last_modified', 0)
 
30
 
31
  def get_display_name(self):
32
  return self.name
33
 
34
  def get_etag(self):
35
+ lm = self.info.get('last_modified', 0)
36
+ return f'"{lm}"' if lm else None
 
 
37
 
38
  def get_last_modified(self):
39
+ return self.info.get('last_modified', 0)
40
+
41
+ def get_content(self):
42
+ return self.provider._get_file_content(self.path)
43
 
44
+ def begin_write(self, content_type=None):
45
+ return HfBucketWriteFile(self.provider, self.path)
 
46
 
 
 
47
 
48
+ class HfBucketFolder(DAVCollection):
49
+ """表示 Hugging Face Bucket 中的目录"""
50
+
51
+ def __init__(self, path, environ, provider, info):
52
+ super().__init__(path, environ)
53
+ self.provider = provider
54
+ self.info = info
55
+
56
+ def get_creation_date(self):
57
+ return self.info.get('last_modified', 0)
58
+
59
+ def get_display_name(self):
60
+ return self.name
61
 
62
+ def get_last_modified(self):
63
+ return self.info.get('last_modified', 0)
64
 
65
  def get_member_names(self):
 
 
66
  return self.provider._list_directory(self.path)
67
 
68
  def get_member(self, name):
 
76
 
77
  def create_empty_resource(self, name):
78
  new_path = self.util.join_uri(self.path, name)
79
+ # 空文件将在写入时创建,这里只需返回资源对象
80
  return self.provider.get_resource_inst(new_path, self.environ)
81
 
82
  def delete(self):
83
+ # 递归删除所有子项
84
+ for member_name in self.get_member_names():
85
+ member = self.get_member(member_name)
86
+ member.delete()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
 
88
 
89
  class HfBucketWriteFile:
90
+ """辅助类:处理文件写入"""
91
  def __init__(self, provider, path):
92
  self.provider = provider
93
  self.path = path
 
103
 
104
 
105
  class HfBucketProvider(DAVProvider):
106
+ """将 Hugging Face Bucket 适配为 DAVProvider"""
107
+
108
  def __init__(self, bucket_id: str, token: str):
109
  super().__init__()
110
  self.bucket_id = bucket_id
 
117
  path = path[:-1]
118
  return path
119
 
120
+ def _get_file_info(self, path: str) -> Optional[Dict[str, Any]]:
121
+ """获取文件/目录信息,如果不存在则返回 None"""
122
  path = self._normalize_path(path)
123
  if path == '/':
124
  return {'exists': True, 'is_dir': True, 'size': 0, 'last_modified': time.time()}
 
148
  }
149
  if items:
150
  return {'exists': True, 'is_dir': True, 'size': 0, 'last_modified': time.time()}
151
+ return None
152
  except Exception:
153
+ return None
154
 
155
  def _list_directory(self, path: str) -> List[str]:
156
  path = self._normalize_path(path)
 
237
  os.unlink(tmp.name)
238
 
239
  def get_resource_inst(self, path, environ):
240
+ """返回资源实例(文件或目录)"""
241
+ info = self._get_file_info(path)
242
+ if info is None:
243
+ # 路径不存在,返回一个占位的文件资源(允许后续写入)
244
+ # 使用默认信息
245
+ info = {'exists': False, 'is_dir': False, 'size': 0, 'last_modified': 0}
246
+ if info['is_dir']:
247
+ return HfBucketFolder(path, environ, self, info)
248
+ else:
249
+ return HfBucketFile(path, environ, self, info)
250
 
251
  def exists(self, path, environ):
252
+ return self._get_file_info(path) is not None