import boto3 from boto3.s3.transfer import TransferConfig from botocore import UNSIGNED from botocore.config import Config import os import threading import sys soracom_beam_endpoint = 'http://beam.soracom.io:18080' s3_resource = boto3.resource('s3', use_ssl=False, endpoint_url=soracom_beam_endpoint, config=Config(signature_version=UNSIGNED)) config = TransferConfig(multipart_threshold=100 * 1024 * 1024, max_concurrency=10, multipart_chunksize=10 * 1024 * 1024, use_threads=True) class Progress(object): def __init__(self, filename): self._filename = filename self._processed_bytes = 0 self._lock = threading.Lock() def __call__(self, bytes_amount): with self._lock: self._processed_bytes += bytes_amount displayed_processed_bytes = self._processed_bytes unit = 'bytes' if self._processed_bytes > 1024 * 1024 * 1024: displayed_processed_bytes = displayed_processed_bytes / 1024 / 1024/ 1024 unit = 'GB' elif self._processed_bytes > 1024 * 1024: displayed_processed_bytes = displayed_processed_bytes / 1024 / 1024 unit = 'MB' elif self._processed_bytes > 1024: displayed_processed_bytes = displayed_processed_bytes / 1024 unit = 'KB' sys.stdout.write( "\r%s %s %s " % ( self._filename, f'{displayed_processed_bytes:,}', unit)) sys.stdout.flush() def upload(bucket_name, rel_file_path, key, content_type): file_path = os.path.dirname(__file__) + '/' + rel_file_path s3_resource.Object(bucket_name, key).upload_file(file_path, ExtraArgs={'ContentType': content_type}, Config=config, Callback=Progress(file_path) ) def download(bucket_name, key, rel_output_file_path): output_file_path = os.path.dirname(__file__) + '/' + rel_output_file_path s3_resource.Object(bucket_name, key).download_file(output_file_path, Config=config, Callback=Progress(output_file_path) ) # python -c "import file_upload_resource; file_upload_resource.upload(bucket_name='beam-amazon-s3-bucket', rel_file_path='./bigfile.zip', key='bigfile.zip', content_type='application/zip')" # python -c "import file_upload_resource; file_upload_resource.download(bucket_name='beam-amazon-s3-bucket', key='bigfile.zip', rel_output_file_path='downloaded_bigfile.zip')"