forked from TheMetaphysicalCrook/learning-python
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy paththreading_queue.py
More file actions
81 lines (64 loc) · 1.95 KB
/
threading_queue.py
File metadata and controls
81 lines (64 loc) · 1.95 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
#!/usr/bin/env python
import Queue
import threading
import urllib2
import time
from BeautifulSoup import BeautifulSoup
hosts = ["http://yahoo.com", "http://google.com", "http://amazon.com", "http://ibm.com", "http://apple.com"]
queue = Queue.Queue()
out_queue = Queue.Queue()
class ThreadUrl(threading.Thread):
"""
Threaded Url Grab
"""
def __init__(self, queue, out_queue):
threading.Thread.__init__(self)
self.queue = queue
self.out_queue = out_queue
def run(self):
while True:
# grabs host from queue
host = self.queue.get()
# grabs urls of hosts and then grabs chunk of webpage
url = urllib2.urlopen(host)
chunk = url.read()
# place chunk into out queue
self.out_queue.put(chunk)
# signals to queue job is done
self.queue.task_done()
class DatamineThread(threading.Thread):
"""
Thread Url Grab
"""
def __init__(self, out_queue):
threading.Thread.__init__(self)
self.out_queue = out_queue
def run(self):
while True:
# grabs hosts from queue
chunk = self.out_queue.get()
# parse the chunk
soup = BeautifulSoup(chunk)
print(soup.findAll(['title']))
# signals to queue job is done
self.out_queue.task_done()
def main():
# spawn a pool of threads, and pass them queue instance
for i in range(5):
t = ThreadUrl(queue, out_queue)
t.setDaemon(True)
t.start()
# populate queue with data
for host in hosts:
queue.put(host)
for i in range(5):
dt = DatamineThread(out_queue)
dt.setDaemon(True)
dt.start()
# wait on the queue until everything has been processed
queue.join()
out_queue.join()
if __name__ == "__main__":
start = time.time()
main()
print("Elapsed Time: %s" % (time.time() - start))