Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enhancement/http_req #299

Merged
merged 4 commits into from
May 5, 2014
Merged

Enhancement/http_req #299

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b57552d
Store tor exit router's ip and name with every request
arunpandianp Apr 7, 2014
f8162d8
removed unnecessary imports
arunpandianp Apr 7, 2014
53f2074
Refactorings
arunpandianp Apr 7, 2014
8452924
state null check
arunpandianp May 2, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 29 additions & 13 deletions ooni/templates/httpt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,31 +1,40 @@
import copy
import random
import struct

from twisted.plugin import IPlugin
from twisted.internet import protocol, defer
from twisted.internet.ssl import ClientContextFactory
from twisted.internet import defer

from txtorcon.interface import StreamListenerMixin

from twisted.internet import reactor
from twisted.internet.error import ConnectionRefusedError, DNSLookupError, TCPTimedOutError
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.web._newclient import Request, Response, ResponseNeverReceived
from twisted.web.client import Agent
from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent

from ooni.nettest import NetTestCase
from ooni.utils import log
from ooni.settings import config

from ooni.utils.net import BodyReceiver, StringProducer, userAgents

from ooni.utils.trueheaders import TrueHeaders
from ooni.errors import handleAllFailures


class InvalidSocksProxyOption(Exception):
pass

class StreamListener(StreamListenerMixin):

def __init__(self, request):
self.request = request

def stream_succeeded(self, stream):
host=self.request['url'].split('/')[2]
try:
if stream.target_host == host and len(self.request['tor']) == 1:
self.request['tor']['exit_ip'] = stream.circuit.path[-1].ip
self.request['tor']['exit_name'] = stream.circuit.path[-1].name
config.tor_state.stream_listeners.remove(self)
except:
log.err("Tor Exit ip detection failed")

class HTTPTest(NetTestCase):
"""
A utility class for dealing with HTTP based testing. It provides methods to
Expand Down Expand Up @@ -228,7 +237,7 @@ def _cbResponse(self, response, request,
content_length = int(response.headers.getRawHeaders('content-length')[0])
except Exception:
content_length = None

finished = defer.Deferred()
response.deliverBody(BodyReceiver(finished, content_length))
finished.addCallback(self._processResponseBody, request,
Expand Down Expand Up @@ -285,9 +294,11 @@ def doRequest(self, url, method="GET",
request['url'] = url
request['headers'] = headers
request['body'] = body
request['tor'] = False
request['tor'] = {}
if use_tor:
request['tor'] = True
request['tor']['is_tor'] = True
else:
request['tor']['is_tor'] = False

if self.randomizeUA:
log.debug("Randomizing user agent")
Expand All @@ -306,14 +317,19 @@ def doRequest(self, url, method="GET",
headers = TrueHeaders(request['headers'])

def errback(failure, request):
if request['tor']:
if request['tor']['is_tor']:
log.err("Error performing torified request: %s" % request['url'])
else:
log.err("Error performing request: %s" % request['url'])
failure_string = handleAllFailures(failure)
self.addToReport(request, failure_string=failure_string)
return failure

if use_tor:
state = config.tor_state
if state:
state.add_stream_listener(StreamListener(request))
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Condition added


d = agent.request(request['method'], request['url'], headers,
body_producer)
d.addErrback(errback, request)
Expand Down