From b57552d009e6d8af08a66a88e01897faed5ba945 Mon Sep 17 00:00:00 2001 From: Arun Pandian G Date: Tue, 8 Apr 2014 03:12:12 +0530 Subject: [PATCH 1/4] Store tor exit router's ip and name with every request --- ooni/templates/httpt.py | 46 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 403e2224c..c9e42bf34 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -2,10 +2,14 @@ import random import struct +from zope.interface import implements + from twisted.plugin import IPlugin from twisted.internet import protocol, defer from twisted.internet.ssl import ClientContextFactory +from txtorcon.interface import IStreamListener + from twisted.internet import reactor from twisted.internet.error import ConnectionRefusedError, DNSLookupError, TCPTimedOutError from twisted.internet.endpoints import TCP4ClientEndpoint @@ -18,7 +22,6 @@ from ooni.settings import config from ooni.utils.net import BodyReceiver, StringProducer, userAgents - from ooni.utils.trueheaders import TrueHeaders from ooni.errors import handleAllFailures @@ -26,6 +29,33 @@ class InvalidSocksProxyOption(Exception): pass +class StreamListener(object): + implements(IStreamListener) + + def __init__(self, request): + self.request = request + + def stream_new(self, stream): + pass + + def stream_succeeded(self, stream): + host=self.request['url'].split('/')[2] + try: + if stream.target_host == host and len(self.request['tor']) == 1: + self.request['tor']['exit_ip'] = stream.circuit.path[-1].ip + self.request['tor']['exit_name'] = stream.circuit.path[-1].name + except: + log.err("Tor Exit ip detection failed") + + def stream_attach(self, stream, circuit): + pass + + def stream_closed(self, stream,**k): + pass + + def stream_failed(self, stream, reason, remote_reason): + pass + class HTTPTest(NetTestCase): """ A utility class for dealing with HTTP based testing. It provides methods to @@ -228,7 +258,7 @@ def _cbResponse(self, response, request, content_length = int(response.headers.getRawHeaders('content-length')[0]) except Exception: content_length = None - + finished = defer.Deferred() response.deliverBody(BodyReceiver(finished, content_length)) finished.addCallback(self._processResponseBody, request, @@ -285,9 +315,11 @@ def doRequest(self, url, method="GET", request['url'] = url request['headers'] = headers request['body'] = body - request['tor'] = False + request['tor'] = {} if use_tor: - request['tor'] = True + request['tor']['is_tor'] = True + else: + request['tor']['is_tor'] = False if self.randomizeUA: log.debug("Randomizing user agent") @@ -306,7 +338,7 @@ def doRequest(self, url, method="GET", headers = TrueHeaders(request['headers']) def errback(failure, request): - if request['tor']: + if request['tor']['is_tor']: log.err("Error performing torified request: %s" % request['url']) else: log.err("Error performing request: %s" % request['url']) @@ -314,6 +346,10 @@ def errback(failure, request): self.addToReport(request, failure_string=failure_string) return failure + if use_tor: + state = config.tor_state + state.add_stream_listener(StreamListener(request)) + d = agent.request(request['method'], request['url'], headers, body_producer) d.addErrback(errback, request) From f8162d8f4a32f0e70625ef35042609bfb06fbb7d Mon Sep 17 00:00:00 2001 From: Arun Pandian G Date: Tue, 8 Apr 2014 03:20:49 +0530 Subject: [PATCH 2/4] removed unnecessary imports --- ooni/templates/httpt.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index c9e42bf34..44d01be82 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -1,20 +1,13 @@ -import copy import random -import struct from zope.interface import implements -from twisted.plugin import IPlugin -from twisted.internet import protocol, defer -from twisted.internet.ssl import ClientContextFactory +from twisted.internet import defer from txtorcon.interface import IStreamListener from twisted.internet import reactor -from twisted.internet.error import ConnectionRefusedError, DNSLookupError, TCPTimedOutError from twisted.internet.endpoints import TCP4ClientEndpoint -from twisted.web._newclient import Request, Response, ResponseNeverReceived -from twisted.web.client import Agent from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent from ooni.nettest import NetTestCase From 53f207416383b7782639d0ee2177580ba23bc5e7 Mon Sep 17 00:00:00 2001 From: Arun Pandian G Date: Tue, 8 Apr 2014 04:12:03 +0530 Subject: [PATCH 3/4] Refactorings --- ooni/templates/httpt.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 44d01be82..1263261d7 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -1,10 +1,8 @@ import random -from zope.interface import implements - from twisted.internet import defer -from txtorcon.interface import IStreamListener +from txtorcon.interface import StreamListenerMixin from twisted.internet import reactor from twisted.internet.endpoints import TCP4ClientEndpoint @@ -22,33 +20,21 @@ class InvalidSocksProxyOption(Exception): pass -class StreamListener(object): - implements(IStreamListener) +class StreamListener(StreamListenerMixin): def __init__(self, request): self.request = request - def stream_new(self, stream): - pass - def stream_succeeded(self, stream): host=self.request['url'].split('/')[2] try: if stream.target_host == host and len(self.request['tor']) == 1: self.request['tor']['exit_ip'] = stream.circuit.path[-1].ip self.request['tor']['exit_name'] = stream.circuit.path[-1].name + config.tor_state.stream_listeners.remove(self) except: log.err("Tor Exit ip detection failed") - def stream_attach(self, stream, circuit): - pass - - def stream_closed(self, stream,**k): - pass - - def stream_failed(self, stream, reason, remote_reason): - pass - class HTTPTest(NetTestCase): """ A utility class for dealing with HTTP based testing. It provides methods to From 8452924ad680658ff4f0cebfe5b44311aa7f2db3 Mon Sep 17 00:00:00 2001 From: Arun Pandian G Date: Fri, 2 May 2014 22:06:04 +0530 Subject: [PATCH 4/4] state null check --- ooni/templates/httpt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 1263261d7..59b550e65 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -327,7 +327,8 @@ def errback(failure, request): if use_tor: state = config.tor_state - state.add_stream_listener(StreamListener(request)) + if state: + state.add_stream_listener(StreamListener(request)) d = agent.request(request['method'], request['url'], headers, body_producer)