客户端加密实践
更新时间:2022-12-01
概览
客户端加密,是指由用户在本地自行完成文件的加密和解密过程,百度智能云对象存储不参与加密和解密过程,只负责文件的上传、存储、下载过程,明文密钥由用户自行保管在本地。客户端加密增强了文件安全性,即使文件意外泄露,别人也无法解密得到原始数据。本文档提供一种客户端加密方案。
注意事项:
- 由于用户需要自行负责保管密钥,如果明文密钥丢失,您将无法获得原文件内容。
需求场景
客户自己希望在本地自行完成文件的加密和解密过程,切明文密钥由用户自行在本地保管。
方案概述
- 本地使用RSA算法生成非对称密钥(
private_rsa_key
和public_rsa_key
); - 上传文件时,使用AES 256的CTR模式生成对称密钥(
aes_key
); - 对于普通文件,使用对称密钥
aes_key
加密原始数据;使用公钥public_rsa_key
加密aes_key
生成encrypted_aes_key
,作为object的meta,并上传文件到Bos。对于大文件,使用Bos的分块上传,每次取一个分块加密并上传,分块大小为16字节的整数倍,同样使用公钥public _rsa_key
加密aes_key
生成encrypted_aes_key
,作为object的meta。 - 下载文件时,首先获取文件的meta信息,得到
encrypted_aes_key
,利用本地私钥private_rsa_key
解密encrypted_aes_key
得到aes_key
。对于普通文件,下载加密后文件并使用aes_key解密得到原始数据。对于大文件,可以使用分块下载,使用Python SDK的函数get_object(bucket_name,object_key,[range_start,range_end])
下载大文件的指定区间[range_start,range_end]中的字节,包含结尾位置range_end处的字节,其中0 ≤range_start≤range_end≤文件大小。
注意:使用AES 256方式加密,AES算法加密时的块大小是128bits=16byte,因此在大文件分块上传时,每个分块大小应该是16字节的整数倍。
实践步骤
以下以 Python 代码进行示例。
准备工作
1.用户需要安装python SDk,请参考安装python SDK工具包。
2.执行下列命令,安装PyCrypto库。
Python
1pip install pycrypto
3.修改配置。修改示例代码中的HOST和AK、SK,作为本示例的配置项。
示例代码如下
Plain Text
1#coding=utf-8
2
3'''
4The file aims to help client to encrypt data on python sdk with RSA algorithm and symmetric encryption algorithm
5'''
6
7import os
8import shutil
9import base64
10import random
11import string
12
13#引入配置文件和对象存储模块
14from baidubce.bce_client_configuration import BceClientConfiguration
15from baidubce.auth.bce_credentials import BceCredentials
16from baidubce import exception
17from baidubce.services import bos
18from baidubce.services.bos import canned_acl
19from baidubce.services.bos.bos_client import BosClient
20
21#引入Crypto加密模块
22from Crypto import Random
23from Crypto.Cipher import AES
24from Crypto.Cipher import PKCS1_OAEP
25from Crypto.PublicKey import RSA
26from Crypto.Util import Counter
27
28#设置对称密钥长度为128bits
29_AES256_KEY_SIZE = 32
30#设置AES CTR模式的计数器Counter长度
31_COUNTER_BITS_LENGTH_AES_CTR = 8*16
32
33class CipherWithAES:
34 # start为CTR计数器初始值
35 def __init__(self, key= None, start= None):
36 if not key:
37 key = Random.new().read(_AES256_KEY_SIZE)
38 if not start:
39 start = random.randint(1,100)
40 self.key = key
41 self.start = start
42 #生成计数器
43 my_counter = Counter.new(_COUNTER_BITS_LENGTH_AES_CTR, initial_value=self.start)
44 #生成AES对象
45 self.cipher = AES.new(self.key, AES.MODE_CTR, counter = my_counter)
46
47 #加密数据
48 def encrypt(self, plaintext):
49 return self.cipher.encrypt(plaintext)
50
51 #解密数据
52 def decrypt(self, ciphertext):
53 return self.cipher.decrypt(ciphertext)
54
55class CipherWithRSA:
56 # 输入参数为公钥文件名和私钥文件名
57 def __init__(self, public_key_file_name = None, private_key_file_name = None):
58 self.public_key_file_name=public_key_file_name
59 self.private_key_file_name = private_key_file_name
60 if not self.public_key_file_name:
61 self.public_key_file_name = "rsa_public_key.pem"
62 if not self.private_key_file_name:
63 self.private_key_file_name = "rsa_private_key.pem"
64 #如果没有输入公钥和私钥文件,则本地产生RSA密钥对
65 if not (os.path.isfile(self.public_key_file_name) and os.path.isfile(self.private_key_file_name)):
66 self._generate_rsa_key()
67 return
68 #从文件读取公钥和私钥,并产生RSA对象
69 with open(self.public_key_file_name) as file_key:
70 public_key_obj_rsa = RSA.importKey(file_key.read())
71 self.encrypt_obj = PKCS1_OAEP.new(public_key_obj_rsa)
72 with open(self.private_key_file_name) as file_key:
73 private_key_obj_rsa = RSA.importKey(file_key.read())
74 self.decrypt_obj = PKCS1_OAEP.new(private_key_obj_rsa)
75
76 #本地生成RSA密钥对
77 def _generate_rsa_key(self):
78 private_key_obj_rsa = RSA.generate(2048)
79 public_key_obj_rsa = private_key_obj_rsa.publickey()
80 self.encrypt_obj = PKCS1_OAEP.new(public_key_obj_rsa)
81 self.decrypt_obj = PKCS1_OAEP.new(private_key_obj_rsa)
82 #将生成的密钥对存入本地
83 with open(self.public_key_file_name,"w") as file_export:
84 file_export.write(public_key_obj_rsa.exportKey())
85 with open(self.private_key_file_name,"w") as file_export:
86 file_export.write(private_key_obj_rsa.exportKey())
87
88 #加密数据
89 def encrypt(self,plaintext_key):
90 return self.encrypt_obj.encrypt(plaintext_key)
91
92 #解密数据
93 def decrypt(self,ciphertext_key):
94 return self.decrypt_obj.decrypt(ciphertext_key)
95#产生随机文件名
96def _random_string(length):
97 return ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(length))
98
99################put super file#################
100#分块上传大文件到百度对象存储,"part_size"为分块大小,必须为16字节的整数倍
101def put_super_file(bos_client, bucket_name, super_object_key, super_file, part_size,cipher_rsa):
102 """
103 Put super file to baidu object storage by multipart uploading."part_size" must be multiple of 16 bytes
104
105 :param bos_client:bos client
106 :type bos_client:baidubce.services.bos.bos_client.BosClient
107
108 :param bucket_name:None
109 :type bucket_name:string
110
111 :param super_object_key: destion object key of super file
112 :type super_object_key: string
113
114 :param super_file: super file name
115 :type super_file:string
116
117 :param part_size: size of part to upload once,"part_size" must be multiple of 16 bytes and more than 5MB
118 :type: int
119
120 :param cipher_rsa: encrypt symmetric key
121 :type cipher_rsa: CipherWithRSA
122
123 :return :**Http Response**
124
125 """
126 #1.initial
127 try:
128 if not isinstance(bos_client,BosClient):
129 raise Exception("bos client is None!")
130 if not (bucket_name and super_object_key):
131 raise Exception("bucket or object is invalid!")
132 if not os.path.isfile(super_file):
133 raise Exception("source file is invalid!")
134 if not isinstance(cipher_rsa,CipherWithRSA):
135 raise Exception("cipher_rsa is invalid!")
136 except Exception as e:
137 print e
138 exit()
139 if not part_size:
140 part_size = 10*1024*1024
141 #temp_file加密后的文件分块
142 temp_file = _random_string(20)
143 cipher_aes = CipherWithAES()
144 #2.分块上传
145 #分块上传分三步,第一步初始化,获取upload_id
146 upload_id = bos_client.initiate_multipart_upload(
147 bucket_name = bucket_name,
148 key = super_object_key,
149 ).upload_id
150 left_size = os.path.getsize(super_file)
151 offset = 0
152 part_number = 1
153 part_list = []
154 fs = open(super_file,"r")
155 while left_size > 0:
156 if left_size < part_size:
157 part_size = left_size
158 # 读取分块并加密
159 part_content = fs.read(part_size)
160 encrypted_part_content = cipher_aes.encrypt(part_content)
161 #分块加密后重新写入temp_file
162 with open(temp_file,"w") as ft:
163 ft.write(encrypted_part_content)
164 #分块上传第二步,上传分块
165 response = bos_client.upload_part_from_file(
166 bucket_name, super_object_key, upload_id, part_number, part_size, temp_file, offset)
167 left_size -= part_size
168 #保存part number和etag用于调用complete_multipart_upload()
169 part_list.append({
170 "partNumber": part_number,
171 "eTag": response.metadata.etag
172 })
173 part_number += 1
174 os.remove(temp_file)
175 fs.close()
176 #使用公钥加密AES密钥,并作为object的meta
177 user_metadata = {
178 "key": base64.b64encode(cipher_rsa.encrypt(str(cipher_aes.key))),
179 "start":base64.b64encode(cipher_rsa.encrypt(str(cipher_aes.start)))
180 }
181 #分块上传第三步,完成分块上传
182 return bos_client.complete_multipart_upload(bucket_name, super_object_key, upload_id, part_list,user_metadata = user_metadata)
183
184####################### Get super file#######################
185#获取大文件的分块,其中range_start, range_end分别为欲获取分块在文件中的起止位置,包含技术位置的字节
186def get_part_super_file(bos_client, bucket_name, super_object_key, range_start, range_end,cipher_rsa):
187
188 """
189 Get part of super file from baidu object storage
190
191 :param bos_client:bos client
192 :type bos_client:baidubce.services.bos.bos_client.BosClient
193
194 :param bucket_name:None
195 :type bucket_name:string
196
197 :param super_object_key: destion object key of super file
198 :type super_object_key: string
199
200 :param range_start: index of first bytes of part,minimum value is 0
201 :type super_file:int
202
203 :param range_end: index of last bytes of part
204 :type: int
205
206 :param cipher_rsa: dencrypt symmetric key
207 :type cipher_rsa: CipherWithRSA
208
209 :return :**Http Response**
210
211 """
212
213 try:
214 if not isinstance(bos_client,BosClient):
215 raise Exception("bos client is None!")
216 if not (bucket_name and super_object_key):
217 raise Exception("bucket or object is invalid!")
218 if not (range_start and range_end):
219 raise Exception("range is invalid!")
220 if not isinstance(cipher_rsa,CipherWithRSA):
221 raise Exception("cipher_rsa is invalid!")
222 except Exception as e:
223 print e
224 exit()
225 #1.对齐分块起始位置到16字节的整数倍
226 left_offset = range_start%16
227 right_offset = 15 -range_end%16
228 test_range = [range_start-left_offset,range_end+right_offset]
229 #2.获取object的meta
230 response = bos_client.get_object_meta_data(bucket_name, super_object_key)
231 #使用本地公钥来解密AES密钥密文,得到AES密钥明文
232 download_aes_key = base64.b64decode(getattr(response.metadata,"bce_meta_key"))
233 download_aes_start = base64.b64decode(str(getattr(response.metadata,"bce_meta_start")))
234 aes_key = cipher_rsa.decrypt(download_aes_key)
235 aes_start = cipher_rsa.decrypt(download_aes_start)
236 #根据欲获取分块的起始位置调整计数器初始值
237 offset_start = int(aes_start)+range_start/16
238 cipher_aes = CipherWithAES(aes_key,int(offset_start))
239 #3. 下载分块密文数据并使用AES密钥解密
240 response = bos_client.get_object(bucket_name, super_object_key,test_range)
241 download_content = response.data.read()
242 plaintext_content = cipher_aes.decrypt(download_content)
243 #截取用户指定区段的明文数据并返回
244 return plaintext_content[left_offset:range_end-range_start+left_offset+1]
245
246############# put common file ####################
247#加密上传普通文件
248def put_common_file(bos_client, bucket_name, object_key, file_name, cipher_rsa):
249 """
250 Put file to baidu object storage
251
252 :param bos_client:bos client
253 :type bos_client:baidubce.services.bos.bos_client.BosClient
254
255 :param bucket_name:None
256 :type bucket_name:string
257
258 :param object_key: destion object key of file
259 :type object_key: string
260
261 :param file_name: source file name
262 :type file_name:string
263
264 :param cipher_rsa: encrypt symmetric key
265 :type cipher_rsa: CipherWithRSA
266
267 :return :**Http Response**
268
269 """
270
271 try:
272 if not isinstance(bos_client,BosClient):
273 raise Exception("bos client is None!")
274 if not (bucket_name and object_key):
275 raise Exception("bucket or object is invalid!")
276 if not os.path.isfile(file_name):
277 raise Exception("file name is invalid!")
278 if not isinstance(cipher_rsa,CipherWithRSA):
279 raise Exception("cipher_rsa is invalid!")
280 except Exception as e:
281 print e
282 exit()
283 temp_file = _random_string(20)
284 #读取欲上传文件数据
285 content=""
286 with open(file_name,"r") as fp:
287 content = fp.read()
288 cipher_aes = CipherWithAES()
289 #加密数据并写到temp_file
290 encrypt_content = cipher_aes.encrypt(content)
291 with open(temp_file,"w") as ft:
292 ft.write(encrypt_content)
293 cipher_rsa = CipherWithRSA()
294 #加密AES密钥作为Object的meta
295 user_metadata = {
296 "key": base64.b64encode(cipher_rsa.encrypt(str(cipher_aes.key))),
297 "start":base64.b64encode(cipher_rsa.encrypt(str(cipher_aes.start)))
298 }
299 #上传加密后文件
300 response = bos_client.put_object_from_file(bucket = bucket_name,
301 key = object_key,
302 file_name = temp_file,
303 user_metadata = user_metadata)
304 os.remove(temp_file)
305 return response
306
307################get common file#####################
308#下载加密的普通文件
309def get_common_file(bos_client, bucket_name, object_key, des_file, cipher_rsa):
310
311 """
312 Put file to baidu object storage
313
314 :param bos_client:bos client
315 :type bos_client:baidubce.services.bos.bos_client.BosClient
316
317 :param bucket_name:None
318 :type bucket_name:string
319
320 :param object_key: destion object key of file
321 :type object_key: string
322
323 :param des_file: destination file name
324 :type des_file:string
325
326 :param cipher_rsa: dencrypt symmetric key
327 :type cipher_rsa: CipherWithRSA
328
329 :return :**Http Response**
330
331 """
332 try:
333 if not isinstance(bos_client,BosClient):
334 raise Exception("bos client is None!")
335 if not (bucket_name and object_key):
336 raise Exception("bucket or object is invalid!")
337 if not des_file:
338 raise Exception("destination file is invalid!")
339 if not isinstance(cipher_rsa,CipherWithRSA):
340 raise Exception("cipher_rsa is invalid!")
341 except Exception as e:
342 print e
343 exit()
344 #下载获取meta
345 response = bos_client.get_object_meta_data(bucket_name, object_key)
346 download_aes_key = base64.b64decode(getattr(response.metadata,"bce_meta_key"))
347 download_aes_start = base64.b64decode(str(getattr(response.metadata,"bce_meta_start")))
348 #下载加密数据到本地
349 download_content = bos_client.get_object_as_string(bucket_name, object_key)
350 #解密得到AES密钥明文
351 aes_key = cipher_rsa.decrypt(download_aes_key)
352 aes_start = cipher_rsa.decrypt(download_aes_start)
353
354 cipher_aes = CipherWithAES(aes_key,int(aes_start))
355 plaintext_content = cipher_aes.decrypt(download_content)
356 with open(des_file,"w") as fd:
357 fd.write(plaintext_content)
358
359if __name__ == "__main__":
360 #以北京地区对象存储为例,替换AK、SK、bucket_name为用户的数据
361 HOST = 'bj.bcebos.com'
362 AK = 'Your_Access_Key'
363 SK = 'Your_Secret_Access_Key'
364 bucket_name = "Your-Bucket-Name"
365 super_file= "super_file"
366 super_object_key = "my-super-object"
367 #获取bos client
368 config = BceClientConfiguration(credentials=BceCredentials(AK, SK), endpoint=HOST)
369 bos_client = BosClient(config)
370
371 #1.1上传大文件
372 #设置分块大小
373 part_size = 10*1024*1024
374 cipher_rsa = CipherWithRSA()
375 #分块上传大文件
376 put_super_file(bos_client, bucket_name, super_object_key, super_file, part_size,cipher_rsa)
377 #1.2 获取大文件分块
378 #设置欲获取的分块起始位置,建议设置成16字节的整数倍
379 range_start = 16*10
380 range_end = 16*11-1
381 #获取明文数据块
382 result = get_part_super_file(bos_client, bucket_name, super_object_key, range_start, range_end,cipher_rsa)
383 print "#"*20
384 print result
385 print "#"*20
386 print "length:",len(result)
387
388 #2.1上传普通文件
389 object_key = "myobject"
390 source_file = "myobject.txt"
391 put_common_file(bos_client,bucket_name,object_key,source_file,cipher_rsa)
392 #2.2 下载普通文件
393 des_file = "des_myobject.txt"
394 get_common_file(bos_client,bucket_name,object_key,des_file,cipher_rsa)