forked from yidao620c/python3-cookbook
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathprocesspool.py
More file actions
46 lines (37 loc) · 980 Bytes
/
processpool.py
File metadata and controls
46 lines (37 loc) · 980 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
Topic: 进程池
Desc :
"""
import os
import multiprocessing
import hashlib
__author__ = 'Xiong Neng'
BUFSIZE = 8192
POOLSIZE = 2
def compute_digest(filename):
try:
f = open(filename, 'rb')
except IOError:
return None
digest = hashlib.sha512()
while True:
chunk = f.read(BUFSIZE)
if not chunk: break
digest.update(chunk)
f.close()
return filename, digest.digest()
def build_digest_map(topdir):
digest_pool = multiprocessing.Pool(POOLSIZE)
allfiles = (os.path.join(path, name)
for path, dirs, files in os.walk(topdir)
for name in files)
digest_map = dict(digest_pool.imap_unordered(compute_digest, allfiles, 20))
digest_pool.close()
return digest_map
def demo():
digest_map = build_digest_map(r'D:\work\projects\core-python')
print(len(digest_map))
if __name__ == '__main__':
demo()