cd ..
/home/john/Development/python/rpyc_docker
import logging
logger = logging.getLogger("rpyc_docker")
logger.setLevel(logging.INFO)
from docker import Client
docker = Client(base_url='unix://var/run/docker.sock')
import rpyc_docker.worker
reload(rpyc_docker.worker)
import rpyc_docker.rpyc_browser_worker
reload(rpyc_docker.rpyc_browser_worker)
import rpyc_docker.manager
reload(rpyc_docker.manager)
from rpyc_docker import BrowserRpycWorker
from rpyc_docker import Manager
cd examples/
/home/john/Development/python/rpyc_docker/examples
import time
from reddit_page import RedditPage
class RedditWorker(BrowserRpycWorker):
def __init__(self,docker,subReddit,redditResultQueue):
BrowserRpycWorker.__init__(self,docker)
self.subReddit = subReddit
self.redditResultQueue = redditResultQueue
def work(self):
self.create_container()
time.sleep(2)
self.connect_rpyc()
self.setup_browser("firefox")
redditPage = RedditPage(self.browser)
redditPage.goto_subreddit(self.subReddit)
results = redditPage.get_reddit_titles()
self.redditResultQueue.put([self.subReddit,results])
return True
import time
from duckduckgo_page import DuckDuckGoPage
class DuckDuckGoWorker(BrowserRpycWorker):
def __init__(self,docker,searchTerm,resultQueue):
BrowserRpycWorker.__init__(self,docker)
self.searchTerm = searchTerm
self.resultQueue = resultQueue
def work(self):
self.create_container()
time.sleep(2)
self.connect_rpyc()
self.setup_browser("firefox")
duckDuckGoPage = DuckDuckGoPage(self.browser)
duckDuckGoPage.search(self.searchTerm)
results = duckDuckGoPage.get_results()
self.resultQueue.put([self.searchTerm,results])
return True
cd ..
/home/john/Development/python/rpyc_docker
import Queue
redditResultQueue = Queue.Queue()
workQueue = Queue.Queue()
redditQueries = ["/r/python","/r/clojure","r/programming","r/javascript"]
for redditQuery in redditQueries :
workQueue.put([RedditWorker,[docker,redditQuery,redditResultQueue],{}])
manager = Manager(workQueue,1)
manager.start()
INFO:rpyc_docker:<class '__main__.RedditWorker'>,[<docker.client.Client object at 0x7fa328704d10>, '/r/python', <Queue.Queue instance at 0x7fa31aed9a28>],{}
manager.report()
'WorkerManager Report\nRunning workers 0 \nResults 4 \nErrors 0 \nQueue Size 0'
result = redditResultQueue.get_nowait()
import Queue
duckDuckGoResultQueue = Queue.Queue()
duckDuckGoQueries = ["python","clojure","programming","javascript"]
workQueue = Queue.Queue()
for duckDuckGoQuery in duckDuckGoQueries :
workQueue.put([DuckDuckGoWorker,[docker,duckDuckGoQuery,duckDuckGoResultQueue],{}])
manager = Manager(workQueue,1)
manager.start()
INFO:rpyc_docker:<class '__main__.DuckDuckGoWorker'>,[<docker.client.Client object at 0x7fd9340b5c50>, 'python', <Queue.Queue instance at 0x7fd92d096290>],{}
manager.report()
'WorkerManager Report\nRunning workers 0 \nResults 2 \nErrors 2 \nQueue Size 0'
manager.errors[0][0].splitlines()
[u'Traceback (most recent call last):', u' File "rpyc_docker/worker.py", line 38, in run', u' self.result = self.work()', u' File "<ipython-input-8-b699811c2d6e>", line 17, in work', u' results = duckDuckGoPage.get_results()', u' File "duckduckgo_page.py", line 38, in get_results', u' File "/usr/local/lib/python2.7/dist-packages/rpyc/core/netref.py", line 196, in __call__', u' return syncreq(_self, consts.HANDLE_CALL, args, kwargs)', u' File "/usr/local/lib/python2.7/dist-packages/rpyc/core/netref.py", line 71, in syncreq', u' return conn.sync_request(handler, oid, *args)', u' File "/usr/local/lib/python2.7/dist-packages/rpyc/core/protocol.py", line 441, in sync_request', u' raise obj', u'WebDriverException: Message: resultElm.querySelector(...) is null', u'Stacktrace:', u'[', u'u', u"'", u' ', u' ', u' ', u' ', u'a', u't', u' ', u'a', u'n', u'o', u'n', u'y', u'm', u'o', u'u', u's', u'/', u'<', u' ', u'(', u'h', u't', u't', u'p', u's', u':', u'/', u'/', u'd', u'u', u'c', u'k', u'd', u'u', u'c', u'k', u'g', u'o', u'.', u'c', u'o', u'm', u'/', u'?', u'q', u'=', u'c', u'l', u'o', u'j', u'u', u'r', u'e', u'&', u'i', u'a', u'=', u'a', u'b', u'o', u'u', u't', u' ', u'l', u'i', u'n', u'e', u' ', u'6', u'9', u' ', u'>', u' ', u'F', u'u', u'n', u'c', u't', u'i', u'o', u'n', u':', u'8', u')', u"'", u',', u' ', u'u', u"'", u' ', u' ', u' ', u' ', u'a', u't', u' ', u'a', u'n', u'o', u'n', u'y', u'm', u'o', u'u', u's', u' ', u'(', u'h', u't', u't', u'p', u's', u':', u'/', u'/', u'd', u'u', u'c', u'k', u'd', u'u', u'c', u'k', u'g', u'o', u'.', u'c', u'o', u'm', u'/', u'?', u'q', u'=', u'c', u'l', u'o', u'j', u'u', u'r', u'e', u'&', u'i', u'a', u'=', u'a', u'b', u'o', u'u', u't', u' ', u'l', u'i', u'n', u'e', u' ', u'6', u'9', u' ', u'>', u' ', u'F', u'u', u'n', u'c', u't', u'i', u'o', u'n', u':', u'3', u')', u"'", u',', u' ', u'u', u"'", u' ', u' ', u' ', u' ', u'a', u't', u' ', u'h', u'a', u'n', u'd', u'l', u'e', u'E', u'v', u'a', u'l', u'u', u'a', u't', u'e', u'E', u'v', u'e', u'n', u't', u' ', u'(', u'h', u't', u't', u'p', u's', u':', u'/', u'/', u'd', u'u', u'c', u'k', u'd', u'u', u'c', u'k', u'g', u'o', u'.', u'c', u'o', u'm', u'/', u'?', u'q', u'=', u'c', u'l', u'o', u'j', u'u', u'r', u'e', u'&', u'i', u'a', u'=', u'a', u'b', u'o', u'u', u't', u':', u'6', u'9', u')', u"'", u']', u'', u'========= Remote Traceback (1) =========', u'Traceback (most recent call last):', u' File "/usr/local/lib/python2.7/dist-packages/rpyc/core/protocol.py", line 305, in _dispatch_request', u' res = self._HANDLERS[handler](self, *args)', u' File "/usr/local/lib/python2.7/dist-packages/rpyc/core/protocol.py", line 535, in _handle_call', u' return self._local_objects[oid](*args, **dict(kwargs))', u' File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webdriver.py", line 403, in execute_script', u" {'script': script, 'args':converted_args})['value']", u' File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webdriver.py", line 175, in execute', u' self.error_handler.check_response(response)', u' File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/errorhandler.py", line 166, in check_response', u' raise exception_class(message, screen, stacktrace)', u'WebDriverException: Message: resultElm.querySelector(...) is null', u'Stacktrace:', u' at anonymous/< (https://duckduckgo.com/?q=clojure&ia=about line 69 > Function:8)', u' at anonymous (https://duckduckgo.com/?q=clojure&ia=about line 69 > Function:3)', u' at handleEvaluateEvent (https://duckduckgo.com/?q=clojure&ia=about:69)', u'']
duckDuckGoResultQueue = Queue.Queue()
redditResultQueue = Queue.Queue()
workQueue = Queue.Queue()
for duckDuckGoQuery,redditQuery in zip(duckDuckGoQueries,redditQueries):
workQueue.put([DuckDuckGoWorker,[docker,duckDuckGoQuery,duckDuckGoResultQueue],{}])
workQueue.put([RedditWorker,[docker,redditQuery,redditResultQueue],{}])
manager = Manager(workQueue,2)
manager.start()
INFO:rpyc_docker:<class '__main__.DuckDuckGoWorker'>,[<docker.client.Client object at 0x7fd9340b5c50>, 'python', <Queue.Queue instance at 0x7fd917f251b8>],{} INFO:rpyc_docker:<class '__main__.RedditWorker'>,[<docker.client.Client object at 0x7fd9340b5c50>, '/r/python', <Queue.Queue instance at 0x7fd917f25128>],{} WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: localhost
manager.report()
'WorkerManager Report\nRunning workers 0 \nResults 6 \nErrors 2 \nQueue Size 0'