Peiyuan Xu / oss

Download as
Browse Code »

Commit ad17753418f454ec63a00d2647e073967c54d587

Authored by Jianwei Han
0 parents
Exists in master

OSS from Aliyun

Showing 11 changed files with 4835 additions and 0 deletions Inline Diff

File was created 1 *.pyc
2 .venv
3 .idea
4 *.iml
5 *.egg-info
No preview for this file type
File was created 1 阿里云开放存储服务 Open Storage Service (OSS) Python SDK说明文档
2 ===============================================================
3 阿里云开放存储服务官方网站:
4 http://oss.aliyun.com
5
6 阿里云开放存储
7 ===============================================================
8 存储在OSS里的文件叫做"object". 所有的object都放在bucket里面。
9
10 简介
11 ===============================================================
12 这篇文档主要介绍如何使用Python来进行OSS API调用，并且介绍osscmd
13 的简单使用。
14 这篇文档假设你已经熟悉Python，熟悉OSS的相关概念，并且已经注册了
15 阿里云的OSS服务，且获得了相应的ID和KEY。
16 如果你还没有开通或者还不了解OSS，请移步OSS官方网站。
17
18 环境要求
19 ===============================================================
20 Python SDK需要：安装python 2.5（包括）以上且在3.0（不包括）以下
21 的版本。
22 可以在Windows平台和Linux平台使用。
23
24 如何获取
25 ===============================================================
26 1. 打开浏览器，输入oss.aliyun.com
27 2. 找到Python SDK链接：
28 3. 下载后可以得到类似OSS_Python_API_xxxxxxxx.tar.gz的包
29 4. 进入压缩包所在的目录，进行解压缩
30 5. 解压缩后得到，oss文件夹和osscmd文件
31
32 使用说明
33 ===============================================================
34 使用oss_api.py
35 ===============================================================
36 1. 创建bucket
37 def put_bucket(self, bucket, acl='', headers=None):
38 等同create_bucket函数
39 def create_bucket(self, bucket, acl='', headers=None):
40 参数说明：
41 bucket，类型：string
42 acl，类型：string，目前为private，public-read，
43 public-read-write中的一种
44 headers， 类型：dict，默认为空
45 返回值说明：
46 HTTP Response
47 参见http://docs.python.org/2/library/httplib.html
48 def put_bucket_with_location(self, bucket, acl='', \
49 location='', headers=None):
50 参数说明：
51 bucket，类型：string
52 acl，类型：string
53 location， 类型：string
54 headers， 类型：dict
55 返回值说明：
56 HTTP Response
57 2. 删除bucket
58 def delete_bucket(self, bucket, headers=None):
59 参数说明：
60 bucket，类型：string
61 headers， 类型：dict
62 返回值说明：
63 HTTP Response
64 3. 修改bucket访问权限
65 def put_bucket(self, bucket, acl='', headers=None):
66 def create_bucket(self, bucket, acl='', headers=None):
67 同1中的put_bucket和create_bucket
68 4. 获取bucket访问权限
69 def get_bucket_acl(self, bucket):
70 参数说明：
71 bucket，类型：string
72 返回值说明：
73 HTTP Response
74 5. 显示创建的bucket
75 def get_service(self, headers=None):
76 参数说明：
77 headers， 类型：dict
78 返回值说明：
79 HTTP Response
80 def list_all_my_buckets(self, headers=None):
81 参数说明：
82 headers， 类型：dict
83 返回值说明：
84 HTTP Response
85 6. 上传object
86 def put_object_from_string(self, bucket, object,\
87 input_content,\
88 content_type=DefaultContentType,\
89 headers=None, params=None):
90 参数说明：
91 bucket，类型：string
92 object，类型：string
93 input_content，类型：string
94 content_type，类型：string
95 headers，类型：dict
96 params，类型：dict
97 返回值说明：
98 HTTP Response
99 def put_object_from_file(self, bucket, object,\
100 filename, content_type='',\
101 headers=None, params=None):
102 参数说明：
103 bucket，类型：string
104 object，类型：string
105 filename，类型：string，本地需要上传的文件名
106 content_type，类型：string，object的类型
107 headers，类型：dict
108 params，类型：dict
109 返回值说明：
110 HTTP Response
111 7. 显示上传的object
112 def get_bucket(self, bucket, prefix='', marker='',\
113 delimiter='', maxkeys='', headers=None):
114 同list_bucket
115 def list_bucket(self, bucket, prefix='', marker='',\
116 delimiter='', maxkeys='', headers=None):
117 参数说明：
118 bucket，类型：string
119 prefix，类型：string
120 marker，类型：string
121 delimiter，类型：string
122 maxkeys，类型：string
123 headers，类型：dict
124 返回值说明：
125 HTTP Response
126 8. 删除object
127 def delete_object(self, bucket, object, headers=None):
128 参数说明：
129 bucket，类型：string
130 object，类型：string
131 headers，类型：dict
132 返回值说明：
133 HTTP Response
134 9. 下载object
135 def get_object_to_file(self, bucket, object,\
136 filename, headers=None):
137 参数说明：
138 bucket，类型：string
139 object，类型：string
140 filename，类型：string，
141 将object内容下载到本地文件的文件名
142 headers，类型：dict
143 返回值说明：
144 HTTP Response
145 10. 使用示例：
146 在解压的oss目录下，创建一个测试文件test.py内容如下，
147 并将ACCESS_ID和SECRET_ACCESS_KEY的内容填写正确，
148 并且将BUCKET填写一个唯一的名字。
149
150 #!/usr/bin/env python
151 #coding=utf8
152 import time
153 from oss_api import *
154 from oss_xml_handler import *
155 HOST="oss.aliyuncs.com"
156 ACCESS_ID = ""
157 SECRET_ACCESS_KEY = ""
158 #ACCESS_ID and SECRET_ACCESS_KEY 默认是空，
159 #请填入您申请的正确的ID和KEY.
160 BUCKET = ""
161 #bucket 默认是空，请填入唯一的bucket名称
162 #例如test-bucket-20130101等带唯一日期的bucket名字.
163
164 def check_not_empty(input, msg=""):
165 if not input:
166 print "Please make sure %s not empty!" % msg
167 exit(-1)
168 def check_res(res, msg=""):
169 if res.status / 100 == 2:
170 print "%s OK" % msg
171 else:
172 print "%s FAIL" % msg
173 print "ret:%s" % res.status
174 print "request-id:%s" % res.getheader("x-oss-request-id")
175 print "reason:%s" % res.read()
176 exit(-1)
177
178 if __name__ == "__main__":
179 #初始化
180 check_not_empty(ACCESS_ID, "ACCESS_ID")
181 check_not_empty(SECRET_ACCESS_KEY, "SECRET_ACCESS_KEY")
182 oss = OssAPI(HOST, ACCESS_ID, SECRET_ACCESS_KEY)
183 #创建属于自己的bucket
184 bucket = BUCKET
185 check_not_empty(bucket, "bucket")
186 acl = 'private'
187 headers = {}
188 res = oss.put_bucket(bucket, acl, headers)
189 check_res(res, "create bucket")
190
191 #列出创建的bucket
192 res = oss.get_service()
193 check_res(res, "list all buckets")
194 #把指定的字符串内容上传到bucket中,在bucket中的文件名叫object。
195 object = "object_test"
196 input_content = "hello, OSS"
197 content_type = "text/HTML"
198 headers = {}
199 res = oss.put_object_from_string(bucket, object, input_content, content_type, headers)
200 check_res(res, "upload from string")
201 #指定文件名, 把这个文件上传到bucket中,在bucket中的文件名叫object。
202 object = "object_test"
203 filename = __file__
204 content_type = "text/HTML"
205 headers = {}
206 res = oss.put_object_from_file(bucket, object, filename, content_type, headers)
207 check_res(res, "upload from localfile")
208 #下载bucket中的object，内容在body中
209 object = "object_test"
210 headers = {}
211 res = oss.get_object(bucket, object, headers)
212 check_res(res, "download object")
213 #下载bucket中的object，把内容写入到本地文件中
214 object = "object_test"
215 headers = {}
216 filename = "get_object_test_file"
217 res = oss.get_object_to_file(bucket, object, filename, headers)
218 if os.path.isfile(filename):
219 os.remove(filename)
220 check_res(res, "download object to localfile")
221 #查看bucket中所拥有的权限
222 res = oss.get_bucket_acl(bucket)
223 check_res(res, "get bucket acl")
224 #列出bucket中所拥有的object
225 prefix = ""
226 marker = ""
227 delimiter = "/"
228 maxkeys = "100"
229 headers = {}
230 res = oss.get_bucket(bucket, prefix, marker, delimiter, maxkeys, headers)
231 check_res(res, "list objects in bucket")
232 #删除bucket中的object
233 object = "object_test"
234 headers = {}
235 res = oss.delete_object(bucket, object, headers)
236 check_res(res, "delete object")
237 #删除bucket
238 res = oss.delete_bucket(bucket)
239 check_res(res, "delete bucket")
240
241 11. 示例的预期结果：
242 create bucket OK
243 list all buckets OK
244 upload from string OK
245 upload from localfile OK
246 download object OK
247 download object to localfile OK
248 get bucket acl OK
249 list objects in bucket OK
250 delete object OK
251 delete bucket OK
252
253 使用osscmd
254 ===============================================================
255 1. 在Linux上安装 osscmd.
256 sudo rm -r /usr/local/oss
257 sudo mkdir -p /usr/local/oss
258 sudo tar -zxf oss.tar.gz -C /usr/local/oss
259 sudo rm -f /usr/bin/osscmd
260 sudo ln -s /usr/local/oss/osscmd /usr/bin/osscmd
261 sudo chmod 755 /usr/local/oss/osscmd
262
263 在Windows上安装osscmd
264 解压缩包
265
266 2. 运行"python osscmd config --id=YOUR_ID --key=YOUR_KEY"
267 用来配置访问OSS所需要的认证码
268
269
270 3. 运行"python osscmd getallbucket" 列出创建的bucket.
271 如果是刚刚使用OSS的用户因为没有创建bucket，输出是空
272
273 4. 运行"python osscmd createbucket mybucket" 创建一个名字
274 叫mybucket的bucket，有可能不成功，这个时候需要换一个名字。
275 因为OSS中的bucket名字是全局唯一的。
276
277 5. 可以再次运行"python osscmd getallbucket" 查看是否创建成功。
278 如果没有成功请检查osscmd返回的错误信息。
279
280 6. 成功创建bucket后。运行"python osscmd list oss://mybucket/",
281 查看bucket中有哪些object。
282 由于bucket中还没有
283 object，输出是空的。
284
285 7. 向bucket中上传一个object.
286 $ md5sum my_local_file
287 7625e1adc3a4b129763d580ca0a78e44 my_local_file
288
289 $ python osscmd put my_local_file oss://mybucket/myobject
290
291 8. 如果创建成功，再次运行"python osscmd list oss://mybucket/"
292
293 9. 从bucket中下载object到本地文件，并比对下载的文件的md5值
294 $ python osscmd get oss://mybucket/myobject local_file
295
296 $ md5sum local_file
297 7625e1adc3a4b129763d580ca0a78e44 local_file
File was created 1 #!/usr/bin/env python
2 #coding=utf-8
3
4 # Copyright (c) 2011, Alibaba Cloud Computing
5 # All rights reserved.
6 #
7 # Permission is hereby granted, free of charge, to any person obtaining a
8 # copy of this software and associated documentation files (the
9 # "Software"), to deal in the Software without restriction, including
10 # without limitation the rights to use, copy, modify, merge, publish, dis-
11 # tribute, sublicense, and/or sell copies of the Software, and to permit
12 # persons to whom the Software is furnished to do so, subject to the fol-
13 # lowing conditions:
14 #
15 # The above copyright notice and this permission notice shall be included
16 # in all copies or substantial portions of the Software.
17 #
18 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
20 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
21 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
22 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 # IN THE SOFTWARE.
25
26 import httplib
27 import time
28 import base64
29 import urllib
30 import StringIO
31 import sys
32 try:
33 from oss.oss_util import *
34 except:
35 from oss_util import *
36 try:
37 from oss.oss_xml_handler import *
38 except:
39 from oss_xml_handler import *
40
41 class OssAPI:
42 '''
43 A simple OSS API
44 '''
45 DefaultContentType = 'application/octet-stream'
46 provider = PROVIDER
47 __version__ = '0.3.2'
48 Version = __version__
49 AGENT = 'oss-python%s (%s)' % (__version__, sys.platform)
50
51 def __init__(self, host, access_id, secret_access_key='', port=80, is_security=False):
52 self.SendBufferSize = 8192
53 self.RecvBufferSize = 1024*1024*10
54 self.host = get_second_level_domain(host)
55 self.port = port
56 self.access_id = access_id
57 self.secret_access_key = secret_access_key
58 self.show_bar = False
59 self.is_security = is_security
60 self.retry_times = 5
61 self.agent = self.AGENT
62 self.debug = False
63
64 def set_debug(self, is_debug):
65 if is_debug:
66 self.debug = True
67
68 def set_retry_times(self, retry_times=5):
69 self.retry_times = retry_times
70
71 def set_send_buf_size(self, buf_size):
72 try:
73 self.SendBufferSize = (int)(buf_size)
74 except ValueError:
75 pass
76
77 def set_recv_buf_size(self, buf_size):
78 try:
79 self.RecvBufferSize = (int)(buf_size)
80 except ValueError:
81 pass
82
83 def get_connection(self, tmp_host=None):
84 host = ''
85 port = 80
86 timeout = 10
87 if not tmp_host:
88 tmp_host = self.host
89 host_port_list = tmp_host.split(":")
90 if len(host_port_list) == 1:
91 host = host_port_list[0].strip()
92 elif len(host_port_list) == 2:
93 host = host_port_list[0].strip()
94 port = int(host_port_list[1].strip())
95 if self.is_security or port == 443:
96 self.is_security = True
97 if sys.version_info >= (2, 6):
98 return httplib.HTTPSConnection(host=host, port=port, timeout=timeout)
99 else:
100 return httplib.HTTPSConnection(host=host, port=port)
101 else:
102 if sys.version_info >= (2, 6):
103 return httplib.HTTPConnection(host=host, port=port, timeout=timeout)
104 else:
105 return httplib.HTTPConnection(host=host, port=port)
106
107 def sign_url_auth_with_expire_time(self, method, url, headers=None, resource="/", timeout=60, params=None):
108 '''
109 Create the authorization for OSS based on the input method, url, body and headers
110
111 :type method: string
112 :param method: one of PUT, GET, DELETE, HEAD
113
114 :type url: string
115 :param:HTTP address of bucket or object, eg: http://HOST/bucket/object
116
117 :type headers: dict
118 :param: HTTP header
119
120 :type resource: string
121 :param:path of bucket or object, eg: /bucket/ or /bucket/object
122
123 :type timeout: int
124 :param
125
126 Returns:
127 signature url.
128 '''
129 if not headers:
130 headers = {}
131 if not params:
132 params = {}
133 send_time = str(int(time.time()) + timeout)
134 headers['Date'] = send_time
135 auth_value = get_assign(self.secret_access_key, method, headers, resource, None, self.debug)
136 params["OSSAccessKeyId"] = self.access_id
137 params["Expires"] = str(send_time)
138 params["Signature"] = auth_value
139 sign_url = append_param(url, params)
140 return sign_url
141
142 def sign_url(self, method, bucket, object, timeout=60, headers=None, params=None):
143 '''
144 Create the authorization for OSS based on the input method, url, body and headers
145
146 :type method: string
147 :param method: one of PUT, GET, DELETE, HEAD
148
149 :type bucket: string
150 :param:
151
152 :type object: string
153 :param:
154
155 :type timeout: int
156 :param
157
158 :type headers: dict
159 :param: HTTP header
160
161 :type params: dict
162 :param: the parameters that put in the url address as query string
163
164 :type resource: string
165 :param:path of bucket or object, eg: /bucket/ or /bucket/object
166
167 Returns:
168 signature url.
169 '''
170 if not headers:
171 headers = {}
172 if not params:
173 params = {}
174 send_time = str(int(time.time()) + timeout)
175 headers['Date'] = send_time
176 if isinstance(object, unicode):
177 object = object.encode('utf-8')
178 resource = "/%s/%s%s" % (bucket, object, get_resource(params))
179 auth_value = get_assign(self.secret_access_key, method, headers, resource, None, self.debug)
180 params["OSSAccessKeyId"] = self.access_id
181 params["Expires"] = str(send_time)
182 params["Signature"] = auth_value
183 url = ''
184 if self.is_security:
185 if is_ip(self.host):
186 url = "https://%s/%s/%s" % (self.host, bucket, object)
187 else:
188 url = "https://%s.%s/%s" % (bucket, self.host, object)
189 else:
190 if is_ip(self.host):
191 url = "http://%s/%s/%s" % (self.host, bucket, object)
192 else:
193 url = "http://%s.%s/%s" % (bucket, self.host, object)
194 sign_url = append_param(url, params)
195 return sign_url
196
197 def _create_sign_for_normal_auth(self, method, headers=None, resource="/"):
198 '''
199 NOT public API
200 Create the authorization for OSS based on header input.
201 it should be put into "Authorization" parameter of header.
202
203 :type method: string
204 :param:one of PUT, GET, DELETE, HEAD
205
206 :type headers: dict
207 :param: HTTP header
208
209 :type resource: string
210 :param:path of bucket or object, eg: /bucket/ or /bucket/object
211
212 Returns:
213 signature string
214 '''
215 auth_value = "%s %s:%s" % (self.provider, self.access_id, get_assign(self.secret_access_key, method, headers, resource, None, self.debug))
216 return auth_value
217
218 def bucket_operation(self, method, bucket, headers=None, params=None):
219 return self.http_request(method, bucket, '', headers, '', params)
220
221 def object_operation(self, method, bucket, object, headers=None, body='', params=None):
222 return self.http_request(method, bucket, object, headers, body, params)
223
224 def http_request(self, method, bucket, object, headers=None, body='', params=None):
225 '''
226 Send http request of operation
227
228 :type method: string
229 :param method: one of PUT, GET, DELETE, HEAD, POST
230
231 :type bucket: string
232 :param
233
234 :type object: string
235 :param
236
237 :type headers: dict
238 :param: HTTP header
239
240 :type body: string
241 :param
242
243 Returns:
244 HTTP Response
245 '''
246 retry = 5
247 res = None
248 while retry > 0:
249 retry -= 1
250 tmp_bucket = bucket
251 tmp_object = object
252 tmp_headers = {}
253 if headers and isinstance(headers, dict):
254 tmp_headers = headers.copy()
255 tmp_params = {}
256 if params and isinstance(params, dict):
257 tmp_params = params.copy()
258
259 res = self.http_request_with_redirect(method, tmp_bucket, tmp_object, tmp_headers, body, tmp_params)
260 if res.status == 301 or res.status == 302:
261 self.host = helper_get_host_from_resp(res, bucket)
262 else:
263 return res
264 return res
265
266 def http_request_with_redirect(self, method, bucket, object, headers=None, body='', params=None):
267 '''
268 Send http request of operation
269
270 :type method: string
271 :param method: one of PUT, GET, DELETE, HEAD, POST
272
273 :type bucket: string
274 :param
275
276 :type object: string
277 :param
278
279 :type headers: dict
280 :param: HTTP header
281
282 :type body: string
283 :param
284
285 Returns:
286 HTTP Response
287 '''
288 if not params:
289 params = {}
290 if not headers:
291 headers = {}
292 if isinstance(object, unicode):
293 object = object.encode('utf-8')
294 if not bucket:
295 resource = "/"
296 headers['Host'] = self.host
297 else:
298 headers['Host'] = "%s.%s" % (bucket, self.host)
299 resource = "/%s/" % bucket
300 resource = "%s%s%s" % (resource.encode('utf-8'), object, get_resource(params))
301 object = urllib.quote(object)
302 url = "/%s" % object
303 if is_ip(self.host):
304 url = "/%s/%s" % (bucket, object)
305 if not bucket:
306 url = "/%s" % object
307 headers['Host'] = self.host
308 url = append_param(url, params)
309 date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
310 headers['Date'] = date
311 headers['Authorization'] = self._create_sign_for_normal_auth(method, headers, resource)
312 headers['User-Agent'] = self.agent
313 if check_bucket_valid(bucket) and not is_ip(self.host):
314 conn = self.get_connection(headers['Host'])
315 else:
316 conn = self.get_connection()
317 conn.request(method, url, body, headers)
318 return conn.getresponse()
319
320 def get_service(self, headers=None):
321 '''
322 List all buckets of user
323 '''
324 return self.list_all_my_buckets(headers)
325
326 def list_all_my_buckets(self, headers=None):
327 '''
328 List all buckets of user
329 type headers: dict
330 :param
331
332 Returns:
333 HTTP Response
334 '''
335 method = 'GET'
336 bucket = ''
337 object = ''
338 body = ''
339 params = {}
340 return self.http_request(method, bucket, object, headers, body, params)
341
342 def get_bucket_acl(self, bucket):
343 '''
344 Get Access Control Level of bucket
345
346 :type bucket: string
347 :param
348
349 Returns:
350 HTTP Response
351 '''
352 method = 'GET'
353 object = ''
354 headers = {}
355 body = ''
356 params = {}
357 params['acl'] = ''
358 return self.http_request(method, bucket, object, headers, body, params)
359
360 def get_bucket_location(self, bucket):
361 '''
362 Get Location of bucket
363 '''
364 method = 'GET'
365 object = ''
366 headers = {}
367 body = ''
368 params = {}
369 params['location'] = ''
370 return self.http_request(method, bucket, object, headers, body, params)
371
372 def get_bucket(self, bucket, prefix='', marker='', delimiter='', maxkeys='', headers=None):
373 '''
374 List object that in bucket
375 '''
376 return self.list_bucket(bucket, prefix, marker, delimiter, maxkeys, headers)
377
378 def list_bucket(self, bucket, prefix='', marker='', delimiter='', maxkeys='', headers=None):
379 '''
380 List object that in bucket
381
382 :type bucket: string
383 :param
384
385 :type prefix: string
386 :param
387
388 :type marker: string
389 :param
390
391 :type delimiter: string
392 :param
393
394 :type maxkeys: string
395 :param
396
397 :type headers: dict
398 :param: HTTP header
399
400 Returns:
401 HTTP Response
402 '''
403 method = 'GET'
404 object = ''
405 body = ''
406 params = {}
407 params['prefix'] = prefix
408 params['marker'] = marker
409 params['delimiter'] = delimiter
410 params['max-keys'] = maxkeys
411 return self.http_request(method, bucket, object, headers, body, params)
412
413 def create_bucket(self, bucket, acl='', headers=None):
414 '''
415 Create bucket
416 '''
417 return self.put_bucket(bucket, acl, headers)
418
419 def put_bucket(self, bucket, acl='', headers=None):
420 '''
421 Create bucket
422
423 :type bucket: string
424 :param
425
426 :type acl: string
427 :param: one of private public-read public-read-write
428
429 :type headers: dict
430 :param: HTTP header
431
432 Returns:
433 HTTP Response
434 '''
435 if not headers:
436 headers = {}
437 if acl != '':
438 if "AWS" == self.provider:
439 headers['x-amz-acl'] = acl
440 else:
441 headers['x-oss-acl'] = acl
442 method = 'PUT'
443 object = ''
444 body = ''
445 params = {}
446 return self.http_request(method, bucket, object, headers, body, params)
447
448 def put_bucket_with_location(self, bucket, acl='', location='', headers=None):
449 '''
450 Create bucket
451
452 :type bucket: string
453 :param
454
455 :type acl: string
456 :param: one of private public-read public-read-write
457
458 :type location: string
459 :param:
460
461 :type headers: dict
462 :param: HTTP header
463
464 Returns:
465 HTTP Response
466 '''
467 if not headers:
468 headers = {}
469 if acl != '':
470 if "AWS" == self.provider:
471 headers['x-amz-acl'] = acl
472 else:
473 headers['x-oss-acl'] = acl
474 params = {}
475 body = ''
476 if location != '':
477 body = r'<CreateBucketConfiguration>'
478 body += r'<LocationConstraint>'
479 body += location
480 body += r'</LocationConstraint>'
481 body += r'</CreateBucketConfiguration>'
482 method = 'PUT'
483 object = ''
484 return self.http_request(method, bucket, object, headers, body, params)
485
486 def delete_bucket(self, bucket, headers=None):
487 '''
488 Delete bucket
489
490 :type bucket: string
491 :param
492
493 Returns:
494 HTTP Response
495 '''
496 method = 'DELETE'
497 object = ''
498 body = ''
499 params = {}
500 return self.http_request(method, bucket, object, headers, body, params)
501
502 def put_object_with_data(self, bucket, object, input_content, content_type=DefaultContentType, headers=None, params=None):
503 '''
504 Put object into bucket, the content of object is from input_content
505 '''
506 return self.put_object_from_string(bucket, object, input_content, content_type, headers, params)
507
508 def put_object_from_string(self, bucket, object, input_content, content_type=DefaultContentType, headers=None, params=None):
509 '''
510 Put object into bucket, the content of object is from input_content
511
512 :type bucket: string
513 :param
514
515 :type object: string
516 :param
517
518 :type input_content: string
519 :param
520
521 :type content_type: string
522 :param: the object content type that supported by HTTP
523
524 :type headers: dict
525 :param: HTTP header
526
527 Returns:
528 HTTP Response
529 '''
530 if not headers:
531 headers = {}
532 headers['Content-Type'] = content_type
533 headers['Content-Length'] = str(len(input_content))
534 fp = StringIO.StringIO(input_content)
535 res = self.put_object_from_fp(bucket, object, fp, content_type, headers, params)
536 fp.close()
537 return res
538
539 def _open_conn_to_put_object(self, bucket, object, filesize, content_type=DefaultContentType, headers=None, params=None):
540 '''
541 NOT public API
542 Open a connectioon to put object
543
544 :type bucket: string
545 :param
546
547 :type filesize: int
548 :param
549
550 :type object: string
551 :param
552
553 :type input_content: string
554 :param
555
556 :type content_type: string
557 :param: the object content type that supported by HTTP
558
559 :type headers: dict
560 :param: HTTP header
561
562 Returns:
563 Initialized HTTPConnection
564 '''
565 if not params:
566 params = {}
567 if not headers:
568 headers = {}
569 method = 'PUT'
570 if isinstance(object, unicode):
571 object = object.encode('utf-8')
572 resource = "/%s/" % bucket
573 if not bucket:
574 resource = "/"
575 resource = "%s%s%s" % (resource.encode('utf-8'), object, get_resource(params))
576
577 object = urllib.quote(object)
578 url = "/%s" % object
579 if bucket:
580 headers['Host'] = "%s.%s" % (bucket, self.host)
581 else:
582 headers['Host'] = self.host
583 if is_ip(self.host):
584 url = "/%s/%s" % (bucket, object)
585 headers['Host'] = self.host
586 url = append_param(url, params)
587 date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
588
589 if check_bucket_valid(bucket) and not is_ip(self.host):
590 conn = self.get_connection(headers['Host'])
591 else:
592 conn = self.get_connection()
593 conn.putrequest(method, url)
594 if isinstance(content_type, unicode):
595 content_type = content_type.encode('utf-8')
596 headers["Content-Type"] = content_type
597 headers["Content-Length"] = filesize
598 headers["Date"] = date
599 headers["Expect"] = "100-Continue"
600 headers['User-Agent'] = self.agent
601 for k in headers.keys():
602 conn.putheader(str(k), str(headers[k]))
603 if '' != self.secret_access_key and '' != self.access_id:
604 auth = self._create_sign_for_normal_auth(method, headers, resource)
605 conn.putheader("Authorization", auth)
606 conn.endheaders()
607 return conn
608
609 def put_object_from_file(self, bucket, object, filename, content_type='', headers=None, params=None):
610 '''
611 put object into bucket, the content of object is read from file
612
613 :type bucket: string
614 :param
615
616 :type object: string
617 :param
618
619 :type fllename: string
620 :param: the name of the read file
621
622 :type content_type: string
623 :param: the object content type that supported by HTTP
624
625 :type headers: dict
626 :param: HTTP header
627
628 Returns:
629 HTTP Response
630 '''
631 fp = open(filename, 'rb')
632 if not content_type:
633 content_type = get_content_type_by_filename(filename)
634 res = self.put_object_from_fp(bucket, object, fp, content_type, headers, params)
635 fp.close()
636 return res
637
638 def view_bar(self, num=1, sum=100):
639 rate = float(num) / float(sum)
640 rate_num = int(rate * 100)
641 print '\r%d%% ' % (rate_num),
642 sys.stdout.flush()
643
644 def put_object_from_fp(self, bucket, object, fp, content_type=DefaultContentType, headers=None, params=None):
645 '''
646 Put object into bucket, the content of object is read from file pointer
647
648 :type bucket: string
649 :param
650
651 :type object: string
652 :param
653
654 :type fp: file
655 :param: the pointer of the read file
656
657 :type content_type: string
658 :param: the object content type that supported by HTTP
659
660 :type headers: dict
661 :param: HTTP header
662
663 Returns:
664 HTTP Response
665 '''
666 tmp_object = object
667 tmp_headers = {}
668 tmp_params = {}
669 if headers and isinstance(headers, dict):
670 tmp_headers = headers.copy()
671 if params and isinstance(params, dict):
672 tmp_params = params.copy()
673
674 fp.seek(os.SEEK_SET, os.SEEK_END)
675 filesize = fp.tell()
676 fp.seek(os.SEEK_SET)
677 conn = self._open_conn_to_put_object(bucket, object, filesize, content_type, headers, params)
678 totallen = 0
679 l = fp.read(self.SendBufferSize)
680 retry_times = 0
681 while len(l) > 0:
682 if retry_times > 100:
683 print "retry too many times"
684 raise
685 try:
686 conn.send(l)
687 retry_times = 0
688 except:
689 retry_times += 1
690 continue
691 totallen += len(l)
692 if self.show_bar:
693 self.view_bar(totallen, filesize)
694 l = fp.read(self.SendBufferSize)
695 res = conn.getresponse()
696 if res.status == 301 or res.status == 302:
697 self.host = helper_get_host_from_resp(res, bucket)
698 return self.put_object_from_fp(bucket, tmp_object, fp, content_type, tmp_headers, tmp_params)
699 return res
700
701 def get_object(self, bucket, object, headers=None, params=None):
702 '''
703 Get object
704
705 :type bucket: string
706 :param
707
708 :type object: string
709 :param
710
711 :type headers: dict
712 :param: HTTP header
713
714 Returns:
715 HTTP Response
716 '''
717 method = 'GET'
718 body = ''
719 return self.http_request(method, bucket, object, headers, body, params)
720
721 def get_object_to_file(self, bucket, object, filename, headers=None):
722 '''
723 Get object and write the content of object into a file
724
725 :type bucket: string
726 :param
727
728 :type object: string
729 :param
730
731 :type filename: string
732 :param
733
734 :type headers: dict
735 :param: HTTP header
736
737 Returns:
738 HTTP Response
739 '''
740 res = self.get_object(bucket, object, headers)
741 totalread = 0
742 if res.status / 100 == 2:
743 header = {}
744 header = convert_header2map(res.getheaders())
745 filesize = safe_get_element("content-length", header)
746 f = file(filename, 'wb')
747 data = ''
748 while True:
749 data = res.read(self.RecvBufferSize)
750 if data:
751 f.write(data)
752 totalread += len(data)
753 if self.show_bar:
754 self.view_bar(totalread, filesize)
755 else:
756 break
757 f.close()
758 # TODO: get object with flow
759 return res
760
761 def delete_object(self, bucket, object, headers=None):
762 '''
763 Delete object
764
765 :type bucket: string
766 :param
767
768 :type object: string
769 :param
770
771 :type headers: dict
772 :param: HTTP header
773
774 Returns:
775 HTTP Response
776 '''
777 method = 'DELETE'
778 body = ''
779 params = {}
780 return self.http_request(method, bucket, object, headers, body, params)