Peiyuan Xu / oss

Download as
Browse Code »

Commit ad17753418f454ec63a00d2647e073967c54d587

Authored by Jianwei Han
0 parents
Exists in master

OSS from Aliyun

Showing 11 changed files with 4835 additions and 0 deletions Side-by-side Diff

... ... @@ -0,0 +1,6 @@
  1 +*.pyc
  2 +.venv
  3 +.idea
  4 +*.iml
  5 +*.egg-info
  6 +
No preview for this file type
... ... @@ -0,0 +1,326 @@
  1 +阿里云开放存储服务 Open Storage Service (OSS) Python SDK说明文档
  2 +===============================================================
  3 +阿里云开放存储服务官方网站:
  4 + http://oss.aliyun.com
  5 +
  6 +阿里云开放存储
  7 +===============================================================
  8 +存储在OSS里的文件叫做"object". 所有的object都放在bucket里面。
  9 +
  10 +简介
  11 +===============================================================
  12 +这篇文档主要介绍如何使用Python来进行OSS API调用，并且介绍osscmd
  13 +的简单使用。
  14 +这篇文档假设你已经熟悉Python，熟悉OSS的相关概念，并且已经注册了
  15 +阿里云的OSS服务，且获得了相应的ID和KEY。
  16 +如果你还没有开通或者还不了解OSS，请移步OSS官方网站。
  17 +
  18 +环境要求
  19 +===============================================================
  20 +Python SDK需要：安装python 2.5（包括）以上且在3.0（不包括）以下
  21 +的版本。
  22 +可以在Windows平台和Linux平台使用。
  23 +
  24 +如何获取
  25 +===============================================================
  26 +1. 打开浏览器，输入oss.aliyun.com
  27 +2. 找到Python SDK链接：
  28 +3. 下载后可以得到类似OSS_Python_API_xxxxxxxx.tar.gz的包
  29 +4. 进入压缩包所在的目录，进行解压缩
  30 +5. 解压缩后得到，oss文件夹和osscmd文件
  31 +
  32 +使用说明
  33 +===============================================================
  34 +使用oss_api.py
  35 +===============================================================
  36 +1. 创建bucket
  37 + def put_bucket(self, bucket, acl='', headers=None):
  38 + 等同create_bucket函数
  39 + def create_bucket(self, bucket, acl='', headers=None):
  40 + 参数说明：
  41 + bucket，类型：string
  42 + acl，类型：string，目前为private，public-read，
  43 + public-read-write中的一种
  44 + headers， 类型：dict，默认为空
  45 + 返回值说明：
  46 + HTTP Response
  47 + 参见http://docs.python.org/2/library/httplib.html
  48 + def put_bucket_with_location(self, bucket, acl='', \
  49 + location='', headers=None):
  50 + 参数说明：
  51 + bucket，类型：string
  52 + acl，类型：string
  53 + location， 类型：string
  54 + headers， 类型：dict
  55 + 返回值说明：
  56 + HTTP Response
  57 +2. 删除bucket
  58 + def delete_bucket(self, bucket, headers=None):
  59 + 参数说明：
  60 + bucket，类型：string
  61 + headers， 类型：dict
  62 + 返回值说明：
  63 + HTTP Response
  64 +3. 修改bucket访问权限
  65 + def put_bucket(self, bucket, acl='', headers=None):
  66 + def create_bucket(self, bucket, acl='', headers=None):
  67 + 同1中的put_bucket和create_bucket
  68 +4. 获取bucket访问权限
  69 + def get_bucket_acl(self, bucket):
  70 + 参数说明：
  71 + bucket，类型：string
  72 + 返回值说明：
  73 + HTTP Response
  74 +5. 显示创建的bucket
  75 + def get_service(self, headers=None):
  76 + 参数说明：
  77 + headers， 类型：dict
  78 + 返回值说明：
  79 + HTTP Response
  80 + def list_all_my_buckets(self, headers=None):
  81 + 参数说明：
  82 + headers， 类型：dict
  83 + 返回值说明：
  84 + HTTP Response
  85 +6. 上传object
  86 + def put_object_from_string(self, bucket, object,\
  87 + input_content,\
  88 + content_type=DefaultContentType,\
  89 + headers=None, params=None):
  90 + 参数说明：
  91 + bucket，类型：string
  92 + object，类型：string
  93 + input_content，类型：string
  94 + content_type，类型：string
  95 + headers，类型：dict
  96 + params，类型：dict
  97 + 返回值说明：
  98 + HTTP Response
  99 + def put_object_from_file(self, bucket, object,\
  100 + filename, content_type='',\
  101 + headers=None, params=None):
  102 + 参数说明：
  103 + bucket，类型：string
  104 + object，类型：string
  105 + filename，类型：string，本地需要上传的文件名
  106 + content_type，类型：string，object的类型
  107 + headers，类型：dict
  108 + params，类型：dict
  109 + 返回值说明：
  110 + HTTP Response
  111 +7. 显示上传的object
  112 + def get_bucket(self, bucket, prefix='', marker='',\
  113 + delimiter='', maxkeys='', headers=None):
  114 + 同list_bucket
  115 + def list_bucket(self, bucket, prefix='', marker='',\
  116 + delimiter='', maxkeys='', headers=None):
  117 + 参数说明：
  118 + bucket，类型：string
  119 + prefix，类型：string
  120 + marker，类型：string
  121 + delimiter，类型：string
  122 + maxkeys，类型：string
  123 + headers，类型：dict
  124 + 返回值说明：
  125 + HTTP Response
  126 +8. 删除object
  127 + def delete_object(self, bucket, object, headers=None):
  128 + 参数说明：
  129 + bucket，类型：string
  130 + object，类型：string
  131 + headers，类型：dict
  132 + 返回值说明：
  133 + HTTP Response
  134 +9. 下载object
  135 + def get_object_to_file(self, bucket, object,\
  136 + filename, headers=None):
  137 + 参数说明：
  138 + bucket，类型：string
  139 + object，类型：string
  140 + filename，类型：string，
  141 + 将object内容下载到本地文件的文件名
  142 + headers，类型：dict
  143 + 返回值说明：
  144 + HTTP Response
  145 +10. 使用示例：
  146 +在解压的oss目录下，创建一个测试文件test.py内容如下，
  147 +并将ACCESS_ID和SECRET_ACCESS_KEY的内容填写正确，
  148 +并且将BUCKET填写一个唯一的名字。
  149 +
  150 +#!/usr/bin/env python
  151 +#coding=utf8
  152 +import time
  153 +from oss_api import *
  154 +from oss_xml_handler import *
  155 +HOST="oss.aliyuncs.com"
  156 +ACCESS_ID = ""
  157 +SECRET_ACCESS_KEY = ""
  158 +#ACCESS_ID and SECRET_ACCESS_KEY 默认是空，
  159 +#请填入您申请的正确的ID和KEY.
  160 +BUCKET = ""
  161 +#bucket 默认是空，请填入唯一的bucket名称
  162 +#例如test-bucket-20130101等带唯一日期的bucket名字.
  163 +
  164 +def check_not_empty(input, msg=""):
  165 + if not input:
  166 + print "Please make sure %s not empty!" % msg
  167 + exit(-1)
  168 +def check_res(res, msg=""):
  169 + if res.status / 100 == 2:
  170 + print "%s OK" % msg
  171 + else:
  172 + print "%s FAIL" % msg
  173 + print "ret:%s" % res.status
  174 + print "request-id:%s" % res.getheader("x-oss-request-id")
  175 + print "reason:%s" % res.read()
  176 + exit(-1)
  177 +
  178 +if __name__ == "__main__":
  179 + #初始化
  180 + check_not_empty(ACCESS_ID, "ACCESS_ID")
  181 + check_not_empty(SECRET_ACCESS_KEY, "SECRET_ACCESS_KEY")
  182 + oss = OssAPI(HOST, ACCESS_ID, SECRET_ACCESS_KEY)
  183 + #创建属于自己的bucket
  184 + bucket = BUCKET
  185 + check_not_empty(bucket, "bucket")
  186 + acl = 'private'
  187 + headers = {}
  188 + res = oss.put_bucket(bucket, acl, headers)
  189 + check_res(res, "create bucket")
  190 +
  191 + #列出创建的bucket
  192 + res = oss.get_service()
  193 + check_res(res, "list all buckets")
  194 + #把指定的字符串内容上传到bucket中,在bucket中的文件名叫object。
  195 + object = "object_test"
  196 + input_content = "hello, OSS"
  197 + content_type = "text/HTML"
  198 + headers = {}
  199 + res = oss.put_object_from_string(bucket, object, input_content, content_type, headers)
  200 + check_res(res, "upload from string")
  201 + #指定文件名, 把这个文件上传到bucket中,在bucket中的文件名叫object。
  202 + object = "object_test"
  203 + filename = __file__
  204 + content_type = "text/HTML"
  205 + headers = {}
  206 + res = oss.put_object_from_file(bucket, object, filename, content_type, headers)
  207 + check_res(res, "upload from localfile")
  208 + #下载bucket中的object，内容在body中
  209 + object = "object_test"
  210 + headers = {}
  211 + res = oss.get_object(bucket, object, headers)
  212 + check_res(res, "download object")
  213 + #下载bucket中的object，把内容写入到本地文件中
  214 + object = "object_test"
  215 + headers = {}
  216 + filename = "get_object_test_file"
  217 + res = oss.get_object_to_file(bucket, object, filename, headers)
  218 + if os.path.isfile(filename):
  219 + os.remove(filename)
  220 + check_res(res, "download object to localfile")
  221 + #查看bucket中所拥有的权限
  222 + res = oss.get_bucket_acl(bucket)
  223 + check_res(res, "get bucket acl")
  224 + #列出bucket中所拥有的object
  225 + prefix = ""
  226 + marker = ""
  227 + delimiter = "/"
  228 + maxkeys = "100"
  229 + headers = {}
  230 + res = oss.get_bucket(bucket, prefix, marker, delimiter, maxkeys, headers)
  231 + check_res(res, "list objects in bucket")
  232 + #删除bucket中的object
  233 + object = "object_test"
  234 + headers = {}
  235 + res = oss.delete_object(bucket, object, headers)
  236 + check_res(res, "delete object")
  237 + #删除bucket
  238 + res = oss.delete_bucket(bucket)
  239 + check_res(res, "delete bucket")
  240 +
  241 +11. 示例的预期结果：
  242 + create bucket OK
  243 + list all buckets OK
  244 + upload from string OK
  245 + upload from localfile OK
  246 + download object OK
  247 + download object to localfile OK
  248 + get bucket acl OK
  249 + list objects in bucket OK
  250 + delete object OK
  251 + delete bucket OK
  252 +
  253 +使用osscmd
  254 +===============================================================
  255 +1. 在Linux上安装 osscmd.
  256 + sudo rm -r /usr/local/oss
  257 + sudo mkdir -p /usr/local/oss
  258 + sudo tar -zxf oss.tar.gz -C /usr/local/oss
  259 + sudo rm -f /usr/bin/osscmd
  260 + sudo ln -s /usr/local/oss/osscmd /usr/bin/osscmd
  261 + sudo chmod 755 /usr/local/oss/osscmd
  262 +
  263 + 在Windows上安装osscmd
  264 + 解压缩包
  265 +
  266 +2. 运行"python osscmd config --id=YOUR_ID --key=YOUR_KEY"
  267 + 用来配置访问OSS所需要的认证码
  268 +
  269 +
  270 +3. 运行"python osscmd getallbucket" 列出创建的bucket.
  271 + 如果是刚刚使用OSS的用户因为没有创建bucket，输出是空
  272 +
  273 +4. 运行"python osscmd createbucket mybucket" 创建一个名字
  274 + 叫mybucket的bucket，有可能不成功，这个时候需要换一个名字。
  275 + 因为OSS中的bucket名字是全局唯一的。
  276 +
  277 +5. 可以再次运行"python osscmd getallbucket" 查看是否创建成功。
  278 + 如果没有成功请检查osscmd返回的错误信息。
  279 +
  280 +6. 成功创建bucket后。运行"python osscmd list oss://mybucket/",
  281 + 查看bucket中有哪些object。
  282 + 由于bucket中还没有
  283 + object，输出是空的。
  284 +
  285 +7. 向bucket中上传一个object.
  286 + $ md5sum my_local_file
  287 + 7625e1adc3a4b129763d580ca0a78e44 my_local_file
  288 +
  289 + $ python osscmd put my_local_file oss://mybucket/myobject
  290 +
  291 +8. 如果创建成功，再次运行"python osscmd list oss://mybucket/"
  292 +
  293 +9. 从bucket中下载object到本地文件，并比对下载的文件的md5值
  294 + $ python osscmd get oss://mybucket/myobject local_file
  295 +
  296 + $ md5sum local_file
  297 + 7625e1adc3a4b129763d580ca0a78e44 local_file
  298 +
  299 +10. 删除bucket中的object
  300 + $ python osscmd delete oss://mybucket/myobject
  301 +
  302 +11. 删除bucket, 注意，如果bucket中有object或者parts
  303 + 则这个bucket不能被删除
  304 + $ python osscmd deletebucket test-oss-aliyun-com
  305 +
  306 +ChangeHistory
  307 +===============================================================
  308 +0.3.2 - 2013-10-20
  309 +* 修复osscmd的uploadfromdir命令中，在中文目录下，上传的object名字被截断的问题。
  310 +* 修复oss_util.py中DEBUG被设置成True的情况下无法打印log的问题。
  311 +* oss_api.py中增加了设置每次上传和下载大小的接口。
  312 +* osscmd增加了--debug=true来打印日志。
  313 +* oss_api.py中增加了上传失败时候的重试，最大重试100次。
  314 +
  315 +0.3.1 - 2013-08-02
  316 +* 支持跳转。
  317 +* 给oss_api.py部分函数增加说明。
  318 +
  319 +0.1.3 - 2013-07-12
  320 +* 支持multipart相关操作，osscmd增加multipart相关的接口。
  321 + osscmd支持本地目录上传。
  322 + osscmd支持将bucket的某一个prefix的object下载到本地的目录。
  323 +
  324 +0.1.0 - 2011-11-15
  325 +* 第一次操作支持基本的创建，删除和显示bucket。
  326 + 支持创建，删除和显示object。
Diff suppressed. Click to show
... ... @@ -0,0 +1,1497 @@
  1 +#!/usr/bin/env python
  2 +#coding=utf-8
  3 +
  4 +# Copyright (c) 2011, Alibaba Cloud Computing
  5 +# All rights reserved.
  6 +#
  7 +# Permission is hereby granted, free of charge, to any person obtaining a
  8 +# copy of this software and associated documentation files (the
  9 +# "Software"), to deal in the Software without restriction, including
  10 +# without limitation the rights to use, copy, modify, merge, publish, dis-
  11 +# tribute, sublicense, and/or sell copies of the Software, and to permit
  12 +# persons to whom the Software is furnished to do so, subject to the fol-
  13 +# lowing conditions:
  14 +#
  15 +# The above copyright notice and this permission notice shall be included
  16 +# in all copies or substantial portions of the Software.
  17 +#
  18 +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19 +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
  20 +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
  21 +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  22 +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23 +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  24 +# IN THE SOFTWARE.
  25 +
  26 +import httplib
  27 +import time
  28 +import base64
  29 +import urllib
  30 +import StringIO
  31 +import sys
  32 +try:
  33 + from oss.oss_util import *
  34 +except:
  35 + from oss_util import *
  36 +try:
  37 + from oss.oss_xml_handler import *
  38 +except:
  39 + from oss_xml_handler import *
  40 +
  41 +class OssAPI:
  42 + '''
  43 + A simple OSS API
  44 + '''
  45 + DefaultContentType = 'application/octet-stream'
  46 + provider = PROVIDER
  47 + __version__ = '0.3.2'
  48 + Version = __version__
  49 + AGENT = 'oss-python%s (%s)' % (__version__, sys.platform)
  50 +
  51 + def __init__(self, host, access_id, secret_access_key='', port=80, is_security=False):
  52 + self.SendBufferSize = 8192
  53 + self.RecvBufferSize = 1024*1024*10
  54 + self.host = get_second_level_domain(host)
  55 + self.port = port
  56 + self.access_id = access_id
  57 + self.secret_access_key = secret_access_key
  58 + self.show_bar = False
  59 + self.is_security = is_security
  60 + self.retry_times = 5
  61 + self.agent = self.AGENT
  62 + self.debug = False
  63 +
  64 + def set_debug(self, is_debug):
  65 + if is_debug:
  66 + self.debug = True
  67 +
  68 + def set_retry_times(self, retry_times=5):
  69 + self.retry_times = retry_times
  70 +
  71 + def set_send_buf_size(self, buf_size):
  72 + try:
  73 + self.SendBufferSize = (int)(buf_size)
  74 + except ValueError:
  75 + pass
  76 +
  77 + def set_recv_buf_size(self, buf_size):
  78 + try:
  79 + self.RecvBufferSize = (int)(buf_size)
  80 + except ValueError:
  81 + pass
  82 +
  83 + def get_connection(self, tmp_host=None):
  84 + host = ''
  85 + port = 80
  86 + timeout = 10
  87 + if not tmp_host:
  88 + tmp_host = self.host
  89 + host_port_list = tmp_host.split(":")
  90 + if len(host_port_list) == 1:
  91 + host = host_port_list[0].strip()
  92 + elif len(host_port_list) == 2:
  93 + host = host_port_list[0].strip()
  94 + port = int(host_port_list[1].strip())
  95 + if self.is_security or port == 443:
  96 + self.is_security = True
  97 + if sys.version_info >= (2, 6):
  98 + return httplib.HTTPSConnection(host=host, port=port, timeout=timeout)
  99 + else:
  100 + return httplib.HTTPSConnection(host=host, port=port)
  101 + else:
  102 + if sys.version_info >= (2, 6):
  103 + return httplib.HTTPConnection(host=host, port=port, timeout=timeout)
  104 + else:
  105 + return httplib.HTTPConnection(host=host, port=port)
  106 +
  107 + def sign_url_auth_with_expire_time(self, method, url, headers=None, resource="/", timeout=60, params=None):
  108 + '''
  109 + Create the authorization for OSS based on the input method, url, body and headers
  110 +
  111 + :type method: string
  112 + :param method: one of PUT, GET, DELETE, HEAD
  113 +
  114 + :type url: string
  115 + :param:HTTP address of bucket or object, eg: http://HOST/bucket/object
  116 +
  117 + :type headers: dict
  118 + :param: HTTP header
  119 +
  120 + :type resource: string
  121 + :param:path of bucket or object, eg: /bucket/ or /bucket/object
  122 +
  123 + :type timeout: int
  124 + :param
  125 +
  126 + Returns:
  127 + signature url.
  128 + '''
  129 + if not headers:
  130 + headers = {}
  131 + if not params:
  132 + params = {}
  133 + send_time = str(int(time.time()) + timeout)
  134 + headers['Date'] = send_time
  135 + auth_value = get_assign(self.secret_access_key, method, headers, resource, None, self.debug)
  136 + params["OSSAccessKeyId"] = self.access_id
  137 + params["Expires"] = str(send_time)
  138 + params["Signature"] = auth_value
  139 + sign_url = append_param(url, params)
  140 + return sign_url
  141 +
  142 + def sign_url(self, method, bucket, object, timeout=60, headers=None, params=None):
  143 + '''
  144 + Create the authorization for OSS based on the input method, url, body and headers
  145 +
  146 + :type method: string
  147 + :param method: one of PUT, GET, DELETE, HEAD
  148 +
  149 + :type bucket: string
  150 + :param:
  151 +
  152 + :type object: string
  153 + :param:
  154 +
  155 + :type timeout: int
  156 + :param
  157 +
  158 + :type headers: dict
  159 + :param: HTTP header
  160 +
  161 + :type params: dict
  162 + :param: the parameters that put in the url address as query string
  163 +
  164 + :type resource: string
  165 + :param:path of bucket or object, eg: /bucket/ or /bucket/object
  166 +
  167 + Returns:
  168 + signature url.
  169 + '''
  170 + if not headers: