BOS客户端加密

概览

客户端加密，是指由用户在本地自行完成文件的加密和解密过程，百度智能云对象存储不参与加密和解密过程，只负责文件的上传、存储、下载过程，明文密钥由用户自行保管在本地。客户端加密增强了文件安全性，即使文件意外泄露，别人也无法解密得到原始数据。本文档提供一种客户端加密方案。

注意事项：

• 由于用户需要自行负责保管密钥，如果明文密钥丢失，您将无法获得原文件内容。

需求场景

客户自己希望在本地自行完成文件的加密和解密过程，切明文密钥由用户自行在本地保管。

方案概述

1. 本地使用RSA算法生成非对称密钥(private_rsa_key和public_rsa_key)；
2. 上传文件时，使用AES 256的CTR模式生成对称密钥（aes_key）；
3. 对于普通文件，使用对称密钥aes_key加密原始数据；使用公钥public_rsa_key加密aes_key生成encrypted_aes_key，作为object的meta，并上传文件到Bos。对于大文件，使用Bos的分块上传，每次取一个分块加密并上传，分块大小为16字节的整数倍，同样使用公钥public _rsa_key加密aes_key生成encrypted_aes_key，作为object的meta。
4. 下载文件时，首先获取文件的meta信息，得到encrypted_aes_key，利用本地私钥private_rsa_key解密encrypted_aes_key得到aes_key。对于普通文件，下载加密后文件并使用aes_key解密得到原始数据。对于大文件，可以使用分块下载，使用Python SDK的函数get_object(bucket_name,object_key,[range_start,range_end])下载大文件的指定区间[range_start,range_end]中的字节,包含结尾位置range_end处的字节，其中0 ≤range_start≤range_end≤文件大小。

注意：使用AES 256方式加密，AES算法加密时的块大小是128bits=16byte,因此在大文件分块上传时，每个分块大小应该是16字节的整数倍。

实践步骤

以下以 Python 代码进行示例。

准备工作

1.用户需要安装python SDk，请参考安装python SDK工具包。

2.执行下列命令，安装PyCrypto库。

pip install pycrypto

3.修改配置。修改示例代码中的HOST和AK、SK,作为本示例的配置项。

示例代码如下

#coding=utf-8

'''
The file aims to help client to encrypt data on python sdk with RSA algorithm  and symmetric encryption algorithm 
'''

import os
import shutil
import base64
import random
import string

#引入配置文件和对象存储模块
from baidubce.bce_client_configuration import BceClientConfiguration
from baidubce.auth.bce_credentials import BceCredentials
from baidubce import exception
from baidubce.services import bos
from baidubce.services.bos import canned_acl
from baidubce.services.bos.bos_client import BosClient

#引入Crypto加密模块
from Crypto import Random
from Crypto.Cipher  import AES
from Crypto.Cipher  import PKCS1_OAEP
from Crypto.PublicKey import RSA
from Crypto.Util    import Counter

#设置对称密钥长度为128bits
_AES256_KEY_SIZE = 32
#设置AES CTR模式的计数器Counter长度
_COUNTER_BITS_LENGTH_AES_CTR = 8*16

class CipherWithAES:
    # start为CTR计数器初始值
    def __init__(self, key= None, start= None):
        if not key:
            key = Random.new().read(_AES256_KEY_SIZE)
        if not start:
            start = random.randint(1,100)
        self.key = key
        self.start = start
        #生成计数器
        my_counter = Counter.new(_COUNTER_BITS_LENGTH_AES_CTR, initial_value=self.start)
        #生成AES对象
        self.cipher = AES.new(self.key, AES.MODE_CTR, counter = my_counter)

    #加密数据
    def encrypt(self, plaintext):
        return self.cipher.encrypt(plaintext)

    #解密数据
    def decrypt(self, ciphertext):
        return self.cipher.decrypt(ciphertext)

class CipherWithRSA:
    # 输入参数为公钥文件名和私钥文件名
    def __init__(self, public_key_file_name = None, private_key_file_name = None):
        self.public_key_file_name=public_key_file_name
        self.private_key_file_name = private_key_file_name
        if not self.public_key_file_name:
            self.public_key_file_name = "rsa_public_key.pem"
        if not self.private_key_file_name:
            self.private_key_file_name = "rsa_private_key.pem"
        #如果没有输入公钥和私钥文件，则本地产生RSA密钥对
        if not (os.path.isfile(self.public_key_file_name) and os.path.isfile(self.private_key_file_name)):
            self._generate_rsa_key()
            return
        #从文件读取公钥和私钥，并产生RSA对象
        with open(self.public_key_file_name) as file_key:
            public_key_obj_rsa = RSA.importKey(file_key.read())
            self.encrypt_obj = PKCS1_OAEP.new(public_key_obj_rsa)
        with open(self.private_key_file_name) as file_key:
            private_key_obj_rsa = RSA.importKey(file_key.read())
            self.decrypt_obj = PKCS1_OAEP.new(private_key_obj_rsa)

    #本地生成RSA密钥对
    def _generate_rsa_key(self):
        private_key_obj_rsa = RSA.generate(2048)
        public_key_obj_rsa = private_key_obj_rsa.publickey()
        self.encrypt_obj = PKCS1_OAEP.new(public_key_obj_rsa)
        self.decrypt_obj = PKCS1_OAEP.new(private_key_obj_rsa)
        #将生成的密钥对存入本地
        with open(self.public_key_file_name,"w") as file_export:
            file_export.write(public_key_obj_rsa.exportKey())
        with open(self.private_key_file_name,"w") as file_export:
            file_export.write(private_key_obj_rsa.exportKey())

    #加密数据
    def encrypt(self,plaintext_key):
        return self.encrypt_obj.encrypt(plaintext_key)

    #解密数据
    def decrypt(self,ciphertext_key):
        return self.decrypt_obj.decrypt(ciphertext_key)
#产生随机文件名
def _random_string(length):
    return ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(length))

################put super file#################
#分块上传大文件到百度对象存储，"part_size"为分块大小，必须为16字节的整数倍
def put_super_file(bos_client, bucket_name, super_object_key, super_file, part_size,cipher_rsa):
    """
    Put super file to baidu object storage by multipart uploading."part_size" must be multiple of 16 bytes

    :param bos_client:bos client
    :type bos_client:baidubce.services.bos.bos_client.BosClient

    :param bucket_name:None
    :type bucket_name:string

    :param super_object_key: destion object key of super file
    :type super_object_key: string

    :param super_file: super file name
    :type super_file:string

    :param part_size: size of part to upload once,"part_size" must be multiple of 16 bytes and more than 5MB
    :type: int

    :param cipher_rsa: encrypt symmetric key
    :type cipher_rsa: CipherWithRSA

    :return :**Http Response**

    """
    #1.initial
    try:
        if not isinstance(bos_client,BosClient):
            raise Exception("bos client is None!")
        if not (bucket_name and super_object_key):
            raise Exception("bucket or object is invalid!")
        if not os.path.isfile(super_file):
            raise Exception("source file is invalid!")
        if not isinstance(cipher_rsa,CipherWithRSA):
            raise Exception("cipher_rsa is invalid!")
    except Exception as e:
        print e
        exit()
    if not part_size:
        part_size = 10*1024*1024
    #temp_file加密后的文件分块
    temp_file = _random_string(20)
    cipher_aes = CipherWithAES()
    #2.分块上传
    #分块上传分三步，第一步初始化，获取upload_id
    upload_id = bos_client.initiate_multipart_upload(
        bucket_name = bucket_name,
        key = super_object_key,
        ).upload_id
    left_size = os.path.getsize(super_file)
    offset = 0
    part_number = 1
    part_list = []
    fs = open(super_file,"r")
    while left_size > 0:
        if left_size < part_size:
            part_size = left_size
        # 读取分块并加密 
        part_content = fs.read(part_size)
        encrypted_part_content = cipher_aes.encrypt(part_content)
        #分块加密后重新写入temp_file
        with open(temp_file,"w") as ft:
            ft.write(encrypted_part_content)
        #分块上传第二步，上传分块
        response = bos_client.upload_part_from_file(
                bucket_name, super_object_key, upload_id, part_number, part_size, temp_file, offset)
        left_size -= part_size
        #保存part number和etag用于调用complete_multipart_upload()
        part_list.append({
            "partNumber": part_number,
            "eTag": response.metadata.etag
            })
        part_number += 1
    os.remove(temp_file)
    fs.close()
    #使用公钥加密AES密钥，并作为object的meta
    user_metadata = {
        "key": base64.b64encode(cipher_rsa.encrypt(str(cipher_aes.key))),
        "start":base64.b64encode(cipher_rsa.encrypt(str(cipher_aes.start)))
    }
    #分块上传第三步，完成分块上传
    return bos_client.complete_multipart_upload(bucket_name, super_object_key, upload_id, part_list,user_metadata = user_metadata)

####################### Get super file#######################
#获取大文件的分块，其中range_start, range_end分别为欲获取分块在文件中的起止位置,包含技术位置的字节
def get_part_super_file(bos_client, bucket_name, super_object_key, range_start, range_end,cipher_rsa):

    """
    Get part of  super file from baidu object storage

    :param bos_client:bos client
    :type bos_client:baidubce.services.bos.bos_client.BosClient

    :param bucket_name:None
    :type bucket_name:string

    :param super_object_key: destion object key of super file
    :type super_object_key: string

    :param range_start: index of first bytes of part,minimum  value is 0
    :type super_file:int

    :param range_end: index of last bytes of part 
    :type: int

    :param cipher_rsa: dencrypt symmetric key
    :type cipher_rsa: CipherWithRSA

    :return :**Http Response**

    """

    try:
        if not isinstance(bos_client,BosClient):
            raise Exception("bos client is None!")
        if not (bucket_name and super_object_key):
            raise Exception("bucket or object is invalid!")
        if not (range_start and range_end):
            raise Exception("range is invalid!")
        if not isinstance(cipher_rsa,CipherWithRSA):
            raise Exception("cipher_rsa is invalid!")
    except Exception as e:
        print e
        exit()
    #1.对齐分块起始位置到16字节的整数倍
    left_offset = range_start%16
    right_offset = 15 -range_end%16
    test_range = [range_start-left_offset,range_end+right_offset]
    #2.获取object的meta
    response = bos_client.get_object_meta_data(bucket_name, super_object_key)
    #使用本地公钥来解密AES密钥密文，得到AES密钥明文
    download_aes_key = base64.b64decode(getattr(response.metadata,"bce_meta_key"))
    download_aes_start = base64.b64decode(str(getattr(response.metadata,"bce_meta_start")))
    aes_key = cipher_rsa.decrypt(download_aes_key)
    aes_start = cipher_rsa.decrypt(download_aes_start)
    #根据欲获取分块的起始位置调整计数器初始值
    offset_start = int(aes_start)+range_start/16
    cipher_aes = CipherWithAES(aes_key,int(offset_start))
    #3. 下载分块密文数据并使用AES密钥解密
    response = bos_client.get_object(bucket_name, super_object_key,test_range)
    download_content = response.data.read()
    plaintext_content = cipher_aes.decrypt(download_content)
    #截取用户指定区段的明文数据并返回
    return plaintext_content[left_offset:range_end-range_start+left_offset+1]

#############  put common file ####################
#加密上传普通文件
def put_common_file(bos_client, bucket_name, object_key, file_name, cipher_rsa):
    """
    Put file to baidu object storage

    :param bos_client:bos client
    :type bos_client:baidubce.services.bos.bos_client.BosClient

    :param bucket_name:None
    :type bucket_name:string

    :param object_key: destion object key of file
    :type object_key: string

    :param file_name: source file name
    :type file_name:string

    :param cipher_rsa: encrypt symmetric key
    :type cipher_rsa: CipherWithRSA

    :return :**Http Response**

    """

    try:
        if not isinstance(bos_client,BosClient):
            raise Exception("bos client is None!")
        if not (bucket_name and object_key):
            raise Exception("bucket or object is invalid!")
        if not os.path.isfile(file_name):
            raise Exception("file name  is invalid!")
        if not isinstance(cipher_rsa,CipherWithRSA):
            raise Exception("cipher_rsa is invalid!")
    except Exception as e:
        print e
        exit()
    temp_file = _random_string(20)
    #读取欲上传文件数据
    content=""
    with open(file_name,"r") as fp:
        content = fp.read()
    cipher_aes = CipherWithAES()
    #加密数据并写到temp_file
    encrypt_content = cipher_aes.encrypt(content)
    with  open(temp_file,"w") as ft:
        ft.write(encrypt_content)
    cipher_rsa = CipherWithRSA()
    #加密AES密钥作为Object的meta
    user_metadata = {
            "key": base64.b64encode(cipher_rsa.encrypt(str(cipher_aes.key))),
            "start":base64.b64encode(cipher_rsa.encrypt(str(cipher_aes.start)))
            }
    #上传加密后文件
    response = bos_client.put_object_from_file(bucket = bucket_name,
            key = object_key,
            file_name = temp_file,
            user_metadata = user_metadata)
    os.remove(temp_file)
    return response

################get common file#####################
#下载加密的普通文件
def get_common_file(bos_client, bucket_name, object_key, des_file, cipher_rsa):

    """
    Put file to baidu object storage

    :param bos_client:bos client
    :type bos_client:baidubce.services.bos.bos_client.BosClient

    :param bucket_name:None
    :type bucket_name:string

    :param object_key: destion object key of file
    :type object_key: string

    :param des_file: destination file name
    :type des_file:string

    :param cipher_rsa: dencrypt symmetric key
    :type cipher_rsa: CipherWithRSA

    :return :**Http Response**

    """
    try:
        if not isinstance(bos_client,BosClient):
            raise Exception("bos client is None!")
        if not (bucket_name and object_key):
            raise Exception("bucket or object is invalid!")
        if not des_file:
            raise Exception("destination file is invalid!")
        if not isinstance(cipher_rsa,CipherWithRSA):
            raise Exception("cipher_rsa is invalid!")
    except Exception as e:
        print e
        exit()
    #下载获取meta
    response = bos_client.get_object_meta_data(bucket_name, object_key)
    download_aes_key = base64.b64decode(getattr(response.metadata,"bce_meta_key"))
    download_aes_start = base64.b64decode(str(getattr(response.metadata,"bce_meta_start")))
    #下载加密数据到本地
    download_content = bos_client.get_object_as_string(bucket_name, object_key)
    #解密得到AES密钥明文
    aes_key = cipher_rsa.decrypt(download_aes_key)
    aes_start = cipher_rsa.decrypt(download_aes_start)

    cipher_aes = CipherWithAES(aes_key,int(aes_start))
    plaintext_content = cipher_aes.decrypt(download_content)
    with open(des_file,"w") as fd:
        fd.write(plaintext_content)

if __name__ == "__main__":
    #以北京地区对象存储为例，替换AK、SK、bucket_name为用户的数据
    HOST = 'bj.bcebos.com'
    AK = 'Your_Access_Key'
    SK = 'Your_Secret_Access_Key'
    bucket_name = "Your-Bucket-Name"
    super_file= "super_file"
    super_object_key = "my-super-object"
    #获取bos client
    config = BceClientConfiguration(credentials=BceCredentials(AK, SK), endpoint=HOST)
    bos_client = BosClient(config)

    #1.1上传大文件
    #设置分块大小
    part_size = 10*1024*1024
    cipher_rsa = CipherWithRSA()
    #分块上传大文件
    put_super_file(bos_client, bucket_name, super_object_key, super_file, part_size,cipher_rsa)
    #1.2 获取大文件分块
    #设置欲获取的分块起始位置，建议设置成16字节的整数倍
    range_start = 16*10
    range_end = 16*11-1
    #获取明文数据块
    result = get_part_super_file(bos_client, bucket_name, super_object_key, range_start, range_end,cipher_rsa)
    print "#"*20
    print result
    print "#"*20
    print "length:",len(result)

    #2.1上传普通文件
    object_key = "myobject"
    source_file = "myobject.txt"
    put_common_file(bos_client,bucket_name,object_key,source_file,cipher_rsa)
    #2.2 下载普通文件
    des_file = "des_myobject.txt"
    get_common_file(bos_client,bucket_name,object_key,des_file,cipher_rsa)

相关产品

BOS服务端加密、密钥管理服务KMS