#!/usr/bin/env python #coding=utf-8 from oss.oss_api import * from oss.oss_util import * from oss.oss_xml_handler import * import signal import ConfigParser from optparse import OptionParser import os import re import time import Queue import sys reload(sys) sys.setdefaultencoding("utf-8") CMD_LIST = {} ACL_LIST = ['private', 'public-read', 'public-read-write'] OSS_PREFIX = 'oss://' CONFIGFILE = os.path.expanduser('~') + '/.osscredentials' CONFIGSECTION = 'OSSCredentials' DEFAUL_HOST = "oss.aliyuncs.com" OSS_HOST = DEFAUL_HOST ID = "" KEY = "" PUT_OK = 0 GET_OK = 0 DELETE_OK = 0 SEND_BUF_SIZE = 8192 RECV_BUF_SIZE = 1024*1024*10 MAX_OBJECT_SIZE = 5*1024*1024*1024 IS_DEBUG = False HELP = \ '''osscmd: getallbucket(gs) createbucket(cb,mb,pb) oss://bucket --acl=[acl] --location=[location] deletebucket(db) oss://bucket deletewholebucket oss://bucket getbucketlocation(gl) oss://bucket getacl oss://bucket setacl oss://bucket --acl=[acl] allow private, public-read, public-read-write ls(list) oss://bucket/[prefix] [marker] [delimiter] [maxkeys] ls(list) oss://bucket/[prefix] --marker=xxx --delimiter=xxx --maxkeys=xxx mkdir oss://bucket/dirname listallobject oss://bucket/[prefix] deleteallobject oss://bucket/[prefix] downloadallobject oss://bucket/[prefix] localdir --replace=false downloadtodir oss://bucket/[prefix] localdir --replace=false uploadfromdir localdir oss://bucket/[prefix] --check_point=check_point_file put localfile oss://bucket/object --content-type=[content_type] --headers=\"key1:value1, key2:value2\" upload localfile oss://bucket/object --content-type=[content_type] get oss://bucket/object localfile multiget(multi_get) oss://bucket/object localfile cat oss://bucket/object meta oss://bucket/object info oss://bucket/object copy oss://source_bucket/source_object oss://target_bucket/target_object --headers=\"key1:value1, key2:value2\" rm(delete,del) oss://bucket/object signurl(sign) oss://bucket/object --timeout=[timeout_seconds] init oss://bucket/object listpart oss://bucket/object --upload_id=xxx listparts oss://bucket getallpartsize oss://bucket cancel oss://bucket/object --upload_id=xxx multiupload(multi_upload,mp) localfile oss://bucket/object multiupload(multi_upload,mp) localfile oss://bucket/object --upload_id=xxx --thread_num=10 --max_part_num=1000 uploadpartfromfile (upff) localfile oss://bucket/object --upload_id=xxx --part_number=xxx uploadpartfromstring(upfs) oss://bucket/object --upload_id=xxx --part_number=xxx --data=xxx config --host=[oss.aliyuncs.com] --id=[accessid] --key=[accesskey] ''' def print_result(cmd, res): ''' Print HTTP Response if failedd. ''' try: if res.status / 100 == 2: pass else: print "Error Headers:\n" print res.getheaders() print "Error Body:\n" print res.read(1024) print "Error Status:\n" print res.status print cmd, "Failed!" except AttributeError: pass def format_datetime(osstimestamp): date = re.compile("(\.\d*)?Z").sub(".000Z", osstimestamp) ts = time.strptime(date, "%Y-%m-%dT%H:%M:%S.000Z") return time.strftime("%Y-%m-%d %H:%M", ts) def format_unixtime(osstimestamp): date = re.compile("(\.\d*)?Z").sub(".000Z", osstimestamp) ts = time.strptime(date, "%Y-%m-%dT%H:%M:%S.000Z") return (int)(time.mktime(ts)) def format_size(size): size = int(size) coeffs = ['K', 'M', 'G', 'T'] coeff = "" while size > 2048: size /= 1024 coeff = coeffs.pop(0) return str(size) + coeff + "B" def format_utf8(string): string = smart_code(string) if isinstance(string, unicode): string = string.encode('utf-8') return string def split_path(path): if not path.lower().startswith(OSS_PREFIX): print "%s parameter %s invalid, " \ "must be start with %s" % \ (args[0], args[1], OSS_PREFIX) sys.exit(1) pather = path[len(OSS_PREFIX):].split('/') return pather 'split an object path' def split_object_path(command, path): pather = split_path(path) if len(pather) <= 1: print "%s parameters invalid, must be object format" % command sys.exit(1) return pather def check_bucket(bucket): if len(bucket) == 0: print "Bucket should not be empty!" print "Please input oss://bucket" sys.exit(1) def check_object(object): if len(object) == 0: print "Object should not be empty!" print "Please input oss://bucket/object" sys.exit(1) if object.startswith("/"): print "object name should not begin with / " sys.exit(-1) def check_localfile(localfile): if not os.path.isfile(localfile): print "%s is not existed!" % localfile sys.exit(1) def check_args(argv, args=None): if not args: args = [] if len(args) < argv: print "%s miss parameters" % args[0] sys.exit(1) def check_bucket_object(bucket, object): check_bucket(bucket) check_object(object) def parse_bucket_object(path): pather = split_path(path) bucket = "" object = "" if len(pather) > 0: bucket = pather[0] if len(pather) > 1: object += '/'.join(pather[1:]) object = smart_code(object) if object.startswith("/"): print "object name SHOULD NOT begin with /" sys.exit(1) return (bucket, object) def parse_bucket(path): bucket = path if bucket.startswith(OSS_PREFIX): bucket = bucket[len(OSS_PREFIX):] tmp_list = bucket.split("/") if len(tmp_list) > 0: bucket = tmp_list[0] return bucket def cmd_listing(args, options): if len(args) == 1: return cmd_getallbucket(args, options) (bucket, object) = parse_bucket_object(args[1]) if len(bucket) == 0: return cmd_getallbucket(args, options) prefix = object marker = '' delimiter = '' maxkeys = 1000 if options.marker: marker = options.marker if options.delimiter: delimiter = options.delimiter if options.maxkeys: maxkeys = options.maxkeys if len(args) == 3: marker = args[2] elif len(args) == 4: marker = args[2] delimiter = args[3] elif len(args) >= 5: marker = args[2] delimiter = args[3] maxkeys = args[4] prefix = smart_code(prefix) marker = smart_code(marker) delimiter = smart_code(delimiter) maxkeys = smart_code(maxkeys) res = get_oss().get_bucket(bucket, prefix, marker, delimiter, maxkeys) if (res.status / 100) == 2: body = res.read() hh = GetBucketXml(body) (fl, pl) = hh.list() print "prefix list is: " for i in pl: print i print "object list is: " for i in fl: if len(i) == 7: try: print "%16s %6s %s/%s " % (format_datetime(i[1]), format_size((int)(i[3])), OSS_PREFIX + bucket, i[0]) except: print "Exception when print :", i print "\nprefix list number is: %s " % len(pl) print "object list number is: %s " % len(fl) return res def cmd_listparts(args, options): if len(args) == 1: return cmd_getallbucket(args, options) (bucket, object) = parse_bucket_object(args[1]) if len(bucket) == 0: return cmd_getallbucket(args, options) print "%20s %20s" % ("UploadId", "Path") for i in get_all_upload_id_list(get_oss(), bucket): print "%20s oss://%s/%s" % (i[1], bucket, i[0]) def cmd_getallpartsize(args, options): if len(args) == 1: return cmd_getallbucket(args, options) (bucket, object) = parse_bucket_object(args[1]) if len(bucket) == 0: return cmd_getallbucket(args, options) total_part_size = 0 print "%5s %20s %20s %s" % ("Number", "UploadId", "Size", "Path") for i in get_all_upload_id_list(get_oss(), bucket): upload_id = i[1] object = i[0] for i in get_part_list(get_oss(), bucket, object, upload_id): part_size = (int)(i[2]) total_part_size += part_size print "%5s %20s %10s oss://%s/%s" % (i[0], upload_id, format_size(part_size), bucket, object) print "totalsize is: real:%s, format:%s " % (total_part_size, format_size(total_part_size)) def cmd_init_upload(args, options): check_args(2, args) path = args[1] (bucket, object) = parse_bucket_object(path) check_bucket_object(bucket, object) upload_id = get_upload_id(get_oss(), bucket, object) print upload_id def cmd_listpart(args, options): if len(args) == 1: return cmd_getallbucket(args, options) path = args[1] (bucket, object) = parse_bucket_object(path) if len(bucket) == 0: return cmd_getallbucket(args, options) if options.upload_id is None: print "upload_id invalid, please set with --upload_id=xxx" sys.exit(1) print "%5s %32s %20s %20s" % ("PartNumber".ljust(10), "ETag".ljust(34), "Size".ljust(20), "LastModifyTime".ljust(32)) for i in get_part_list(get_oss(), bucket, object, options.upload_id): if len(i) >= 4: print "%s %s %s %s" % (str(i[0]).ljust(10), str(i[1]).ljust(34), str(i[2]).ljust(20), str(i[3]).ljust(32)) def cmd_upload_part_from_file(args, options): check_args(3, args) localfile = args[1] check_localfile(localfile) path = args[2] (bucket, object) = parse_bucket_object(path) check_bucket_object(bucket, object) if options.upload_id is None: print "upload_id invalid, please set with --upload_id=xxx" sys.exit(1) if options.part_number is None: print "part_number invalid, please set with --part_number=xxx" sys.exit(1) res = get_oss().upload_part(bucket, object, localfile, options.upload_id, options.part_number) return res def cmd_upload_part_from_string(args, options): check_args(2, args) path = args[1] (bucket, object) = parse_bucket_object(path) check_bucket_object(bucket, object) if options.upload_id is None: print "upload_id invalid, please set with --upload_id=xxx" sys.exit(1) if options.part_number is None: print "part_number invalid, please set with --part_number=xxx" sys.exit(1) if options.data is None: print "data invalid, please set with --data=xxx" sys.exit(1) res = get_oss().upload_part_from_string(bucket, object, options.data, options.upload_id, options.part_number) return res def cmd_listallobject(args, options): if len(args) == 1: return cmd_getallbucket(args, options) path = args[1] (bucket, object) = parse_bucket_object(path) if len(bucket) == 0: return cmd_getallbucket(args, options) prefix = object marker = "" total_object_num = 0 totalsize = 0 totaltimes = 0 delimiter = '' maxkeys = '1000' if options.out: f = open(options.out, "w") while 1: res = get_oss().get_bucket(bucket, prefix, marker, delimiter, maxkeys) if res.status != 200: return res body = res.read() (tmp_object_list, marker) = get_object_list_marker_from_xml(body) for i in tmp_object_list: object = i[0] length = i[1] last_modify_time = i[2] total_object_num += 1 totalsize += (int)(length) msg = "%s%s/%s" % (OSS_PREFIX, bucket, object) print "%16s %6s %s/%s " % (format_datetime(last_modify_time), format_size(length), OSS_PREFIX + bucket, object) if options.out: f.write(msg) f.write("\n") totaltimes += 1 if len(marker) == 0: break if options.out: f.close() print "the object list result is saved into %s" % options.out print "object list number is: %s " % total_object_num print "totalsize is: real:%s, format:%s " % (totalsize, format_size(totalsize)) print "request times is: %s" % totaltimes return res def get_object(bucket, object, object_prefix, local_path, length, last_modify_time, replace, retry_times = 5): show_bar = False object = smart_code(object) tmp_object = object if object_prefix == object[:len(object_prefix)]: tmp_object = object[len(object_prefix):] while 1: if not tmp_object.startswith("/"): break tmp_object = tmp_object[1:] localfile = os.path.join(local_path, tmp_object) localfile = smart_code(localfile) global GET_OK for i in xrange(retry_times): try: if os.path.isfile(localfile): t1 = format_unixtime(last_modify_time) t2 = (int)(os.path.getmtime(localfile)) if not replace and (int)(length) == os.path.getsize(localfile) and t1 < t2: #skip download this object these conditions match print "no need to get %s/%s to %s" % (bucket, object, localfile) GET_OK += 1 return True else: try: dirname = os.path.dirname(localfile) if not os.path.isdir(dirname): os.makedirs(dirname) except: pass ret = False oss = get_oss(show_bar) res = oss.get_object_to_file(bucket, object, localfile) if 200 == res.status: ret = True if ret and (int)(length) == os.path.getsize(localfile): GET_OK += 1 print "get %s/%s to %s OK" % (bucket, object, localfile) return ret else: print "get %s/%s to %s FAIL" % (bucket, object, localfile) except: print "get %s/%s to %s exception" % (bucket, object, localfile) return False class DownloadObjectWorker(threading.Thread): def __init__(self, retry_times, queue): threading.Thread.__init__(self) self.queue = queue self.retry_times = retry_times def run(self): while 1: try: (get_object, bucket, object, object_prefix, local_path, length, last_modify_time, replace) = self.queue.get(block=False) get_object(bucket, object, object_prefix, local_path, length, last_modify_time, replace, self.retry_times) self.queue.task_done() except Queue.Empty: break except: self.queue.task_done() def cmd_downloadallobject(args, options): check_args(3, args) path = args[1] (bucket, object) = parse_bucket_object(path) check_bucket(bucket) local_path = args[2] if os.path.isfile(local_path): print "%s is not dir, please input localdir" % local_path return replace = False if options.replace is not None and options.replace.lower() == "true": replace = True prefix = object thread_num = 5 if options.thread_num: thread_num = (int)(options.thread_num) retry_times = 5 if options.retry_times: retry_times = (int)(options.retry_times) marker = "" delimiter = '' maxkeys = '1000' handled_obj_num = 0 while 1: queue = Queue.Queue(0) res = get_oss().get_bucket(bucket, prefix, marker, delimiter, maxkeys) if res.status != 200: return res body = res.read() (tmp_object_list, marker) = get_object_list_marker_from_xml(body) for i in tmp_object_list: object = i[0] length = i[1] last_modify_time = i[2] if str(length) == "0" and object.endswith("/"): continue handled_obj_num += 1 queue.put((get_object, bucket, object, prefix, local_path, length, last_modify_time, replace)) thread_pool = [] for i in xrange(thread_num): current = DownloadObjectWorker(retry_times, queue) thread_pool.append(current) current.start() queue.join() for item in thread_pool: item.join() if len(marker) == 0: break print "Total being downloaded objects num: %s, they are downloaded into %s" % (handled_obj_num, local_path) global GET_OK print "OK num:%s" % GET_OK print "FAIL num:%s" % (handled_obj_num - GET_OK) def put_object(bucket, object, local_file, retry_times=5): show_bar = False for i in xrange(retry_times): try: oss = get_oss(show_bar) object = smart_code(object) local_file = smart_code(local_file) local_file_size = os.path.getsize(local_file) if local_file_size > 10*1024*1024: upload_id = "" thread_num = 5 max_part_num = 10000 headers = {} res = oss.multi_upload_file(bucket, object, local_file, upload_id, thread_num, max_part_num, headers) else: res = oss.put_object_from_file(bucket, object, local_file) if 200 == res.status: global PUT_OK PUT_OK += 1 print "upload %s OK" % (local_file) return True else: print "upload %s FAIL, status:%s, request-id:%s" % (local_file, res.status, res.getheader("x-oss-request-id")) except: print "put %s/%s from %s exception" % (bucket, object, local_file) return False class UploadObjectWorker(threading.Thread): def __init__(self, check_point_file, retry_times, queue): threading.Thread.__init__(self) self.check_point_file = check_point_file self.queue = queue self.file_time_map = {} self.retry_times = retry_times def run(self): while 1: try: (put_object, bucket, object, local_file) = self.queue.get(block=False) ret = put_object(bucket, object, local_file, self.retry_times) if ret: local_file_full_path = os.path.abspath(local_file) local_file_full_path = format_utf8(local_file_full_path) self.file_time_map[local_file_full_path] = (int)(os.path.getmtime(local_file)) self.queue.task_done() except Queue.Empty: break except: self.queue.task_done() if len(self.file_time_map) != 0: dump_check_point(self.check_point_file, self.file_time_map) def load_check_point(check_point_file): file_time_map = {} if os.path.isfile(check_point_file): f = open(check_point_file) for line in f: line = line.strip() tmp_list = line.split('#') if len(tmp_list) > 1: time_stamp = (float)(tmp_list[0]) time_stamp = (int)(time_stamp) file_name = "".join(tmp_list[1:]) file_name = format_utf8(file_name) if file_time_map.has_key(file_name) and file_time_map[file_name] > time_stamp: continue file_time_map[file_name] = time_stamp f.close() return file_time_map lock = threading.Lock() def dump_check_point(check_point_file, result_map=None): if len(check_point_file) == 0 or len(result_map) == 0: return lock.acquire() old_file_time_map = {} if os.path.isfile(check_point_file): old_file_time_map = load_check_point(check_point_file) try: f = open(check_point_file,"w") for k, v in result_map.items(): if old_file_time_map.has_key(k) and old_file_time_map[k] < v: del old_file_time_map[k] line = "%s#%s\n" % (v, k) line = format_utf8(line) f.write(line) for k, v in old_file_time_map.items(): line = "%s#%s\n" % (v, k) line = format_utf8(line) f.write(line) except: pass try: f.close() except: pass lock.release() def format_object(object): tmp_list = object.split(os.sep) object = "/".join(x for x in tmp_list if x.strip() and x != "/") while 1: if object.find('//') == -1: break object = object.replace('//', '/') return object def cmd_upload_object_from_localdir(args, options): check_args(3, args) path = args[2] (bucket, object) = parse_bucket_object(path) check_bucket(bucket) local_path = args[1] if not os.path.isdir(local_path): print "%s is not dir, please input localdir" % local_path return is_check_point = False check_point_file = "" file_time_map = {} if options.check_point: is_check_point = True check_point_file = options.check_point file_time_map = load_check_point(check_point_file) prefix = object thread_num = 5 if options.thread_num: thread_num = (int)(options.thread_num) retry_times = 5 if options.retry_times: retry_times = (int)(options.retry_times) topdown = True queue = Queue.Queue(0) local_path = smart_code(local_path) def process_localfile(items): for item in items: local_file = os.path.join(root, item) local_file = smart_code(local_file) if os.path.isfile(local_file): local_file_full_path = os.path.abspath(local_file) local_file_full_path = format_utf8(local_file_full_path) if is_check_point and file_time_map.has_key(local_file_full_path): t1 = (int)(os.path.getmtime(local_file)) t2 = file_time_map[local_file_full_path] if t1 <= t2: continue if prefix != "": object = prefix + "/" + local_file[len(local_path) + 1:] else: object = local_file[len(local_path) + 1:] object = format_object(object) queue.put((put_object, bucket, object, local_file)) for root, dirs, files in os.walk(local_path, topdown): process_localfile(files) process_localfile(dirs) qsize = queue.qsize() thread_pool = [] for i in xrange(thread_num): current = UploadObjectWorker(check_point_file, retry_times, queue) thread_pool.append(current) current.start() queue.join() for item in thread_pool: item.join() print "Total being uploaded localfiles num: %s" % qsize global PUT_OK print "OK num:%s" % PUT_OK print "FAIL num:%s" % (qsize - PUT_OK) def get_object_list_marker_from_xml(body): #return ([(object, object_length, last_modify_time)...], marker) object_meta_list = [] next_marker = "" hh = GetBucketXml(body) (fl, pl) = hh.list() if len(fl) != 0: for i in fl: if isinstance(i[0], unicode): object = i[0].encode('utf-8') else: object = i[0] last_modify_time = i[1] length = i[3] object_meta_list.append((object, length, last_modify_time)) if hh.is_truncated: next_marker = hh.nextmarker return (object_meta_list, next_marker) def cmd_deleteallobject(args, options): if len(args) == 1: return cmd_getallbucket(args, options) path = args[1] (bucket, object) = parse_bucket_object(path) if len(bucket) == 0: return cmd_getallbucket(args, options) prefix = object marker = '' delimiter = '' maxkeys = '1000' if options.marker: marker = options.marker if options.delimiter: delimiter = options.delimiter if options.maxkeys: maxkeys = options.maxkeys debug = True delete_all_objects(get_oss(), bucket, prefix, delimiter, marker, maxkeys, debug) def cmd_getallbucket(args, options): res = get_oss().get_service() width = 20 if (res.status / 100) == 2: body = res.read() h = GetServiceXml(body) is_init = False for i in h.list(): if len(i) >= 3 and i[2].strip(): if not is_init: print "%s %s %s" % ("CreateTime".ljust(width), "BucketLocation".ljust(width), "BucketName".ljust(width)) is_init = True print "%s %s %s" % (str(format_datetime(i[1])).ljust(width), i[2].ljust(width), i[0]) elif len(i) >= 2: if not is_init: print "%s %s" % ("CreateTime".ljust(width), "BucketName".ljust(width)) is_init = True print "%s %s" % (str(format_datetime(i[1])).ljust(width), i[0]) print "\nBucket Number is: ", len(h.list()) return res def cmd_createbucket(args, options): check_args(2, args) if options.acl is not None and options.acl not in ACL_LIST: print "acl invalid, SHOULD be one of %s" % (ACL_LIST) sys.exit(1) acl = '' if options.acl: acl = options.acl bucket = parse_bucket(args[1]) if options.location is not None: location = options.location return get_oss().put_bucket_with_location(bucket, acl, location) return get_oss().put_bucket(bucket, acl) def cmd_getbucketlocation(args, options): check_args(2, args) bucket = parse_bucket(args[1]) res = get_oss().get_bucket_location(bucket) if res.status / 100 == 2: body = res.read() h = GetBucketLocationXml(body) print h.location return res def cmd_deletebucket(args, options): check_args(2, args) bucket = parse_bucket(args[1]) return get_oss().delete_bucket(bucket) def cmd_deletewholebucket(args, options): check_args(2, args) bucket = parse_bucket(args[1]) debug = True delete_marker = "" delete_upload_id_marker = "" if options.marker: delete_marker = options.marker if options.upload_id: delete_upload_id_marker = options.upload_id return clear_all_objects_in_bucket(get_oss(), bucket, delete_marker, delete_upload_id_marker, debug) def delete_object(bucket, object, retry_times=5): object = smart_code(object) global DELETE_OK ret = False for i in xrange(retry_times): try: oss = get_oss() res = oss.delete_object(bucket, object) if 2 == res.status / 100: ret = True if ret: DELETE_OK += 1 print "delete %s/%s OK" % (bucket, object) return ret else: print "delete %s/%s FAIL, status:%s, request-id:%s" % (bucket, object, res.status, res.getheader("x-oss-request-id")) except: print "delete %s/%s exception" % (bucket, object) return False class DeleteObjectWorker(threading.Thread): def __init__(self, retry_times, queue): threading.Thread.__init__(self) self.queue = queue self.retry_times = retry_times def run(self): while 1: try: (delete_object, bucket, object) = self.queue.get(block=False) delete_object(bucket, object, self.retry_times) self.queue.task_done() except Queue.Empty: break except: self.queue.task_done() def cmd_deletebyfile(args, options): check_args(2, args) localfile = args[1] check_localfile(localfile) queue = Queue.Queue(0) f = open(localfile) for line in f: line = line.strip() (bucket, object) = parse_bucket_object(line) if len(bucket) != 0 and len(object) != 0: queue.put((delete_object, bucket, object)) f.close() thread_num = 5 if options.thread_num: thread_num = (int)(options.thread_num) retry_times = 5 if options.retry_times: retry_times = (int)(options.retry_times) thread_pool = [] for i in xrange(thread_num): current = DeleteObjectWorker(retry_times, queue) thread_pool.append(current) current.start() queue.join() for item in thread_pool: item.join() def cmd_setacl(args, options): check_args(2, args) if options.acl is None or options.acl not in ACL_LIST: print "acl invalid, SHOULD be one of %s" % (ACL_LIST) sys.exit(1) bucket = parse_bucket(args[1]) return get_oss().put_bucket(bucket, options.acl) def cmd_getacl(args, options): check_args(2, args) bucket = parse_bucket(args[1]) res = get_oss().get_bucket_acl(bucket) if (res.status / 100) == 2: body = res.read() h = GetBucketAclXml(body) print h.grant return res def to_http_headers(string): headers_map = {} for i in string.split(','): key_value_list = i.strip().split(':') if len(key_value_list) == 2: headers_map[key_value_list[0]] = key_value_list[1] return headers_map def cmd_mkdir(args, options): check_args(2, args) if not args[1].endswith('/'): args[1] += '/' (bucket, object) = parse_bucket_object(args[1]) res = get_oss().put_object_from_string(bucket, object, "") return res def handler(signum, frame): print 'Signal handler called with signal', signum raise Exception("timeout") try: signal.signal(signal.SIGALRM, handler) except: pass def cmd_put(args, options): check_args(3, args) localfile = args[1] check_localfile(localfile) if os.path.getsize(localfile) > MAX_OBJECT_SIZE: print "locafile:%s is bigger than %s, it is not support by put, please use multiupload instead." % (localfile, MAX_OBJECT_SIZE) return #user specified objectname oss://bucket/[path]/object (bucket, object) = parse_bucket_object(args[2]) if len(object) == 0: # e.g. upload to oss://bucket/ object = os.path.basename(localfile) elif object.endswith("/"): #e.g. uplod to oss://bucket/a/b/ object += os.path.basename(localfile) content_type = "" headers = {} if options.content_type: content_type = options.content_type if options.headers: headers = to_http_headers(options.headers) timeout = 0 if options.timeout: timeout = (int)(options.timeout) print "timeout", timeout try: signal.alarm(timeout) except: pass res = get_oss().put_object_from_file(bucket, object, localfile, content_type, headers) try: signal.alarm(0) # Disable the signal except: pass if res.status == 200: second_level_domain = get_second_level_domain(OSS_HOST) if check_bucket_valid(bucket) and not is_ip(second_level_domain): print "Object URL is: http://%s.%s/%s" % (bucket, second_level_domain, object) else: print "Object URL is: http://%s/%s/%s" % (second_level_domain, bucket, object) print "Object abstract path is: oss://%s/%s" % (bucket, object) header_map = convert_header2map(res.getheaders()) print "ETag is %s " % safe_get_element("etag", header_map) return res def cmd_upload(args, options): check_args(3, args) localfile = args[1] check_localfile(localfile) #user specified objectname oss://bucket/[path]/object (bucket, object) = parse_bucket_object(args[2]) if len(object) == 0: # e.g. upload to oss://bucket/ object = os.path.basename(localfile) elif object.endswith("/"): #e.g. uplod to oss://bucket/a/b/ object += os.path.basename(localfile) headers = {} content_type = '' if options.headers: headers = to_http_headers(options.headers) if options.content_type: content_type = options.content_type headers['Content-Type'] = content_type thread_num = 10 if options.thread_num: thread_num = (int)(options.thread_num) max_part_num = 1000 if options.max_part_num: max_part_num = (int)(options.max_part_num) retry_times = 5 if options.retry_times: retry_times = (int)(options.retry_times) oss = get_oss() oss.set_retry_times(retry_times) res = oss.upload_large_file(bucket, object, localfile, thread_num, max_part_num, headers) if res.status == 200: print "Object URL is: http://%s/%s/%s" % (OSS_HOST, bucket, object) print "Object abstract path is: oss://%s/%s" % (bucket, object) header_map = convert_header2map(res.getheaders()) print "ETag is %s " % safe_get_element("etag", header_map) return res def cmd_multi_upload(args, options): check_args(3, args) localfile = args[1] check_localfile(localfile) #user specified objectname oss://bucket/[path]/object (bucket, object) = parse_bucket_object(args[2]) if len(object) == 0: # e.g. upload to oss://bucket/ object = os.path.basename(localfile) elif object.endswith("/"): #e.g. uplod to oss://bucket/a/b/ object += os.path.basename(localfile) headers = {} if options.headers: headers = to_http_headers(options.headers) thread_num = 10 if options.thread_num: thread_num = (int)(options.thread_num) max_part_num = 1000 if options.max_part_num: max_part_num = options.max_part_num upload_id = "" if options.upload_id: upload_id = options.upload_id retry_times = 5 if options.retry_times: retry_times = (int)(options.retry_times) oss = get_oss() oss.set_retry_times(retry_times) res = oss.multi_upload_file(bucket, object, localfile, upload_id, thread_num, max_part_num, headers) if res.status == 200: print "Object URL is: http://%s/%s/%s" % (OSS_HOST, bucket, object) print "Object abstract path is: oss://%s/%s" % (bucket, object) header_map = convert_header2map(res.getheaders()) print "ETag is %s " % safe_get_element("etag", header_map) return res def cmd_copy(args, options): check_args(3, args) (bucket_source, object_source) = parse_bucket_object(args[1]) check_bucket_object(bucket_source, object_source) (bucket, object) = parse_bucket_object(args[2]) check_bucket_object(bucket, object) content_type = "" headers = {} if options.headers: headers = to_http_headers(options.headers) if options.content_type: content_type = options.content_type headers['Content-Type'] = content_type res = get_oss().copy_object(bucket_source, object_source, bucket, object, headers) if res.status == 200: print "Object URL is: http://%s/%s/%s" % (OSS_HOST, bucket, object) print "Object abstract path is: oss://%s/%s" % (bucket, object) header_map = convert_header2map(res.getheaders()) print "ETag is %s " % safe_get_element("etag", header_map) return res def cmd_get(args, options): check_args(3, args) (bucket, object) = parse_bucket_object(args[1]) check_bucket_object(bucket, object) localfile = args[2] localfile = smart_code(localfile) res = get_oss().get_object_to_file(bucket, object, localfile) if res.status == 200: print "The object %s is downloaded to %s, please check." % (object, localfile) return res def multi_get(bucket, object, localfile, thread_num, retry_times): length = 0 try: res = get_oss().head_object(bucket, object) if 200 == res.status: length = (int)(res.getheader('content-length')) else: print "can not get the length of object:", object return False ranges = [] ranges.append(0) size = length // thread_num for i in xrange(thread_num - 1): ranges.append((i + 1) * size) ranges.append(length) threadpool = [] show_bar = False for i in xrange(len(ranges) - 1): exec("file_%s = open(localfile, 'wb+')" % i) oss = get_oss(show_bar) exec("current = MultiGetWorker(oss, bucket, object, file_%s, ranges[i], ranges[i + 1] - 1, %s)" % (i, retry_times)) threadpool.append(current) current.start() for item in threadpool: item.join() except Exception: return False if not os.path.isfile(localfile) or length != os.path.getsize(localfile): print "localfile:%s size:%s is not equal with object:%s size:%s " % (localfile, os.path.getsize(localfile), object, length) return False return True def cmd_multi_get(args, options): check_args(3, args) (bucket, object) = parse_bucket_object(args[1]) check_bucket_object(bucket, object) localfile = args[2] localfile = smart_code(localfile) thread_num = 5 if options.thread_num: thread_num = (int)(options.thread_num) retry_times = 5 if options.retry_times: retry_times = (int)(options.retry_times) ret = multi_get(bucket, object, localfile, thread_num, retry_times) if ret: print "The object %s is downloaded to %s, please check." % (object, localfile) else: print "Download object:%s failed!" % (object) def cmd_cat(args, options): check_args(2, args) (bucket, object) = parse_bucket_object(args[1]) check_bucket_object(bucket, object) res = get_oss().get_object(bucket, object) if res.status == 200: data = "" while 1: data = res.read(10240) if len(data) != 0: print data else: break return res def cmd_meta(args, options): check_args(2, args) (bucket, object) = parse_bucket_object(args[1]) check_bucket_object(bucket, object) res = get_oss().head_object(bucket, object) if res.status == 200: header_map = convert_header2map(res.getheaders()) width = 16 print "%s: %s" % ("objectname".ljust(width), object) for key, value in header_map.items(): print "%s: %s" % (key.ljust(width), value) return res def cmd_info(args, options): check_args(2, args) (bucket, object) = parse_bucket_object(args[1]) check_bucket_object(bucket, object) res = get_oss().get_object_info(bucket, object) if res.status == 200: print res.read() return res def cmd_delete(args, options): check_args(2, args) (bucket, object) = parse_bucket_object(args[1]) check_bucket_object(bucket, object) return get_oss().delete_object(bucket, object) def cmd_cancel(args, options): check_args(2, args) (bucket, object) = parse_bucket_object(args[1]) if options.upload_id is None: print "upload_id invalid, please set with --upload_id=xxx" sys.exit(1) (bucket, object) = parse_bucket_object(args[1]) check_bucket_object(bucket, object) return get_oss().cancel_upload(bucket, object, options.upload_id) def cmd_sign_url(args, options): check_args(2, args) (bucket, object) = parse_bucket_object(args[1]) if options.timeout: timeout = options.timeout else: timeout = "600" print "timeout is %s seconds." % timeout (bucket, object) = parse_bucket_object(args[1]) check_bucket_object(bucket, object) method = 'GET' print get_oss().sign_url(method, bucket, object, int(timeout)) def cmd_configure(args, options): if options.accessid is None or options.accesskey is None: print "%s miss parameters, use --id=[accessid] --key=[accesskey] to specify id/key pair" % args[0] sys.exit(1) config = ConfigParser.RawConfigParser() config.add_section(CONFIGSECTION) if options.host is not None: config.set(CONFIGSECTION, 'host', options.host) config.set(CONFIGSECTION, 'accessid', options.accessid) config.set(CONFIGSECTION, 'accesskey', options.accesskey) cfgfile = open(CONFIGFILE, 'w+') config.write(cfgfile) print "Your configuration is saved into %s ." % CONFIGFILE cfgfile.close() def cmd_help(args, options): print HELP def get_oss(show_bar = True): oss = OssAPI(OSS_HOST, ID, KEY) oss.show_bar = show_bar oss.set_send_buf_size(SEND_BUF_SIZE) oss.set_recv_buf_size(RECV_BUF_SIZE) oss.set_debug(IS_DEBUG) return oss def setup_crenditials(): config = ConfigParser.ConfigParser() try: config.read(CONFIGFILE) global OSS_HOST global ID global KEY try: OSS_HOST = config.get(CONFIGSECTION, 'host') except Exception: OSS_HOST = DEFAUL_HOST ID = config.get(CONFIGSECTION, 'accessid') KEY = config.get(CONFIGSECTION, 'accesskey') if options.accessid is not None: ID = options.accessid if options.accesskey is not None: KEY = options.accesskey if options.host is not None: OSS_HOST = options.host except Exception: if options.accessid is not None: ID = options.accessid if options.accesskey is not None: KEY = options.accesskey if options.host is not None: OSS_HOST = options.host if len(ID) == 0 or len(KEY) == 0: print "can't get accessid/accesskey, setup use : config --id=accessid --key=accesskey" sys.exit(1) def setup_cmdlist(): CMD_LIST['getallbucket'] = cmd_getallbucket CMD_LIST['gs'] = cmd_getallbucket CMD_LIST['createbucket'] = cmd_createbucket CMD_LIST['cb'] = cmd_createbucket CMD_LIST['mb'] = cmd_createbucket CMD_LIST['pb'] = cmd_createbucket CMD_LIST['deletebucket'] = cmd_deletebucket CMD_LIST['deletewholebucket'] = cmd_deletewholebucket CMD_LIST['deletebyfile'] = cmd_deletebyfile CMD_LIST['db'] = cmd_deletebucket CMD_LIST['getbucketlocation'] = cmd_getbucketlocation CMD_LIST['gl'] = cmd_getbucketlocation CMD_LIST['getacl'] = cmd_getacl CMD_LIST['setacl'] = cmd_setacl CMD_LIST['ls'] = cmd_listing CMD_LIST['list'] = cmd_listing CMD_LIST['mkdir'] = cmd_mkdir CMD_LIST['init'] = cmd_init_upload CMD_LIST['uploadpartfromstring'] = cmd_upload_part_from_string CMD_LIST['upfs'] = cmd_upload_part_from_string CMD_LIST['upff'] = cmd_upload_part_from_file CMD_LIST['uploadpartfromfile'] = cmd_upload_part_from_file CMD_LIST['listpart'] = cmd_listpart CMD_LIST['listparts'] = cmd_listparts CMD_LIST['getallpartsize'] = cmd_getallpartsize CMD_LIST['listallobject'] = cmd_listallobject CMD_LIST['downloadallobject'] = cmd_downloadallobject CMD_LIST['downloadtodir'] = cmd_downloadallobject CMD_LIST['uploadfromdir'] = cmd_upload_object_from_localdir CMD_LIST['deleteallobject'] = cmd_deleteallobject CMD_LIST['put'] = cmd_put CMD_LIST['copy'] = cmd_copy CMD_LIST['upload'] = cmd_upload CMD_LIST['multiupload'] = cmd_multi_upload CMD_LIST['multi_upload'] = cmd_multi_upload CMD_LIST['mp'] = cmd_multi_upload CMD_LIST['get'] = cmd_get CMD_LIST['multiget'] = cmd_multi_get CMD_LIST['multi_get'] = cmd_multi_get CMD_LIST['cat'] = cmd_cat CMD_LIST['meta'] = cmd_meta CMD_LIST['info'] = cmd_info CMD_LIST['rm'] = cmd_delete CMD_LIST['delete'] = cmd_delete CMD_LIST['del'] = cmd_delete CMD_LIST['cancel'] = cmd_cancel CMD_LIST['signurl'] = cmd_sign_url CMD_LIST['config'] = cmd_configure CMD_LIST['help'] = cmd_help CMD_LIST['sign'] = cmd_sign_url if __name__ == '__main__': parser = OptionParser() parser.add_option("-H", "--host", dest="host", help="specify ") parser.add_option("-i", "--id", dest="accessid", help="specify access id") parser.add_option("-k", "--key", dest="accesskey", help="specify access key") parser.add_option("-t", "--timeout", dest="timeout", help="timeout for sign url") parser.add_option("-a", "--acl", dest="acl", help="when createbucket/setacl use this option to specify acl") parser.add_option("", "--upload_id", dest="upload_id", help="get parts to specify upload_id") parser.add_option("", "--part_number", dest="part_number", help="get parts to specify upload_id") parser.add_option("", "--data", dest="data", help="get parts to specify upload_id") parser.add_option("--headers", dest="headers", help="HTTP headers for put object, input format SHOULE like --headers=\"key1:value1,key2:value2\"") parser.add_option("-c", "--content-type", dest="content_type", help="content type for the file, will return in GET") parser.add_option("", "--thread_num", dest="thread_num", help="object group upload thread num") parser.add_option("", "--max_part_num", dest="max_part_num", help="object group max part num") parser.add_option("", "--retry_times", dest="retry_times", help="max retry times when fail") parser.add_option("", "--out", dest="out", help="output the result when listallobject") parser.add_option("", "--config_file", dest="config_file", help="the file which stores id-key pair") parser.add_option("", "--replace", dest="replace", help="replace the localfile if it is true") parser.add_option("", "--check_point", dest="check_point", help="the file which records the localfile with name and time") parser.add_option("", "--marker", dest="marker", help="get bucket(list objects) parameter") parser.add_option("", "--delimiter", dest="delimiter", help="get bucket(list objects) parameter") parser.add_option("", "--maxkeys", dest="maxkeys", help="get bucket(list objects) parameter") parser.add_option("", "--location", dest="location", help="bucket location") parser.add_option("", "--send_buf_size", dest="send_buf_size", help="buffer size that client sends each time") parser.add_option("", "--recv_buf_size", dest="recv_buf_size", help="buffer size that client receives each time") parser.add_option("", "--debug", dest="debug", help="set log") setup_cmdlist() #when not specify a option, the value is None (options, args) = parser.parse_args() if options.config_file is not None: CONFIGFILE = options.config_file if options.debug is not None: debug = options.debug if debug.lower() == "true": IS_DEBUG = True else: IS_DEBUG = False if options.send_buf_size is not None: try: SEND_BUF_SIZE = (int)(options.send_buf_size) except ValueError: pass if options.recv_buf_size is not None: try: RECV_BUF_SIZE = (int)(options.recv_buf_size) except ValueError: pass if len(args) < 1: print HELP sys.exit(1) if args[0] != 'config': setup_crenditials() else: CMD_LIST['config'](args, options) sys.exit(1) if args[0] not in CMD_LIST.keys(): print "unsupported command : %s " % args[0] print "use --help for more information" sys.exit(1) cmd = args[0] begin = time.time() res = CMD_LIST[cmd](args, options) print_result(cmd, res) end = time.time() sys.stderr.write("%.3f(s) elapsed\n" % (end - begin))