diff --git a/README.md b/README.md index ec4cbe14..82574c99 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,23 @@ -#nogotofail +# nogotofail -Nogotofail is a network security testing tool designed to help developers and -security researchers spot and fix weak TLS/SSL connections and sensitive -cleartext traffic on devices and applications in a flexible, scalable, powerful way. -It includes testing for common SSL certificate verification issues, HTTPS and TLS/SSL -library bugs, SSL and STARTTLS stripping issues, cleartext issues, and more. +Nogotofail is a network security testing tool designed to help developers and security researchers spot and fix weak TLS/SSL connections and sensitive cleartext traffic on devices and applications in a flexible, scalable, powerful way. +It includes testing for common SSL certificate verification issues, HTTPS and TLS/SSL library bugs, SSL and STARTTLS stripping issues, cleartext issues, personally identifiable information (PII) disclosure issues and more. -##Design +See [docs/pii_analysis.md](docs/pii_analysis.md) for an overview of PII detection features. + +## Design Nogotofail is composed of an on-path network MiTM and optional clients for the devices being tested. See [docs/design.md](docs/design.md) for the overview and design goals of nogotofail. -##Dependencies +## Dependencies Nogotofail depends only on Python 2.7 and pyOpenSSL>=0.13. The MiTM is designed to work on Linux machines and the transparent traffic capture modes are Linux specific and require iptables as well. Additionally the Linux client depends on [psutil](https://pypi.python.org/pypi/psutil). -##Getting started +## Getting started See [docs/getting_started.md](docs/getting_started.md) for setup and a walkthrough of nogotofail. -##Discussion +## Discussion For discussion please use our [nogotofail Google Group](https://groups.google.com/forum/#!forum/nogotofail). diff --git a/docs/create_tls_proxy_cert.md b/docs/create_tls_proxy_cert.md new file mode 100644 index 00000000..958765b0 --- /dev/null +++ b/docs/create_tls_proxy_cert.md @@ -0,0 +1,169 @@ +# Creating a Certificate to performing MitM TLS Proxying + +nogotofail-pii can be configured to operate as a man-in-the-middle (MitM) TLS proxy and inspect encrypted (HTTPS) traffic for PII. The method described here is using a self-signed certificate and requires two certificate chain files (PEM format) to be created: + +- **ca-chain-cleartext.key.cert.pem** certificate chain file contains the two certificate public key files (root and intermediate) and the intermediate certificate private key (the private key is unencrypted). +- **ca-chain.cert.pem** certificate chain file contains the two certificate public key files (for the root and intermediate certificates). + +The recommended procedure below and is based on the method used here: https://jamielinux.com/docs/openssl-certificate-authority/create-the-root-pair.html + +## 1. Setting up the Certificate Authority + +### a. Preparation + +Create a folder to store the Certificate Authority (CA) files. + +``` mkdir /root/ca ``` + +Text files index.txt and serial are setup to act as a kind of flat file database to keep track of signed certificates. +``` +cd /root/ca +mkdir certs crl newcerts private +chmod 700 private +touch index.txt +echo 1000 > serial +``` +An OpenSSL configuration file openssl.cnf needs to be created for the CA. The format used is based on the following instructions: https://jamielinux.com/docs/openssl-certificate-authority/create-the-root-pair.html#prepare-the-configuration-file + +### b. Creating the root key + +The root key is encrypted using AES 256-bit encryption and a strong password should be used. +``` +cd /root/ca +openssl genrsa -aes256 -out private/ca.key.pem 4096 +``` +Enter pass phrase for ca.key.pem: secretpassword +Verifying - Enter pass phrase for ca.key.pem: secretpassword + +```chmod 400 private/ca.key.pem``` + +### c. Create the root certificate + +The root certficate (ca.cert.pem) is created using the root key (ca.key.pem). The expiry date of the root certificate was set to approx 20 years (7300) days. +``` +cd /root/ca +openssl req -config openssl.cnf -key private/ca.key.pem -new -x509 -days 7300 -sha256 -extensions v3_ca -out certs/ca.cert.pem + +Enter pass phrase for ca.key.pem: secretpassword +You are about to be asked to enter information that will be incorporated +into your certificate request. + +Country Name (2 letter code) [XX]:AU +State or Province Name []:Australia +Locality Name []: +Organization Name []:PII MitM Ltd +Organizational Unit Name []:PII MitM Ltd Certificate Authority +Common Name []:pii.mitm.ca +Email Address []: + +chmod 444 certs/ca.cert.pem +``` +The root certificate should be verified using the instructions at: https://jamielinux.com/docs/openssl-certificate-authority/create-the-root-pair.html#verify-the-root-certificate + +## 2. Create the TLS man-in-the-middle certificate key pair + +A new certificate will be created to perform the TLS man-in-the-middle (MitM) inspection between the mobile device and server. The certificate keys will be generated from the root CA. + +### a. Preparation + +The new certificate files will be stored in a different directory. The suggested folder name is tlsmitm and should be created under the CA folder: + +```mkdir /root/ca/tlsmitm``` + +Create the folders needed for this certificate using: +``` +cd /root/ca/tlsmitm +mkdir certs crl csr newcerts private +chmod 700 private +touch index.txt +echo 1000 > serial +``` +Add a crlnumber file to the intermediate CA directory tree to keep track of certificate revocation lists. + +```echo 1000 > /root/ca/intermediate/crlnumber``` + +Copy the intermediate CA configuration file to /root/ca/mitm/openssl.cnf. The following five options need to be changed for this certificate: +``` +[ CA_default ] +dir = /root/ca/tlsmitm +private_key = $dir/private/tlsmitm.key.pem +certificate = $dir/certs/tlsmitm.cert.pem +crl = $dir/crl/tlsmitm.crl.pem +policy = policy_loose +``` + +### b. Create the certificate key + +Create the tls mitm key tls.pii.mitm.ca. The intermediate key is encrypted using AES 256-bit encryption and a strong password. +``` +cd /root/ca +openssl genrsa -aes256 -out tlsmitm/private/tlsmitm.key.pem 4096 + +Enter pass phrase for tlsmitm.key.pem: secretpassword +Verifying - Enter pass phrase for tlsmitm.key.pem: secretpassword + +chmod 400 tlsmitm/private/tlsmitm.key.pem +``` + +### c. Create the TLS MitM certificate + +The TLS MitM key is used to create a certificate signing request (CSR). The details should generally match the root CA, except the Common Name which must be different. +``` +cd /root/ca +openssl req -config tlsmitm/openssl.cnf -new -sha256 -key tlsmitm/private/tlsmitm.key.pem -out tlsmitm/csr/tlsmitm.csr.pem + +Enter pass phrase for tlsmitm.key.pem: secretpassword +You are about to be asked to enter information that will be incorporated +into your certificate request. +----- +Country Name (2 letter code) [XX]:AU +State or Province Name []:Australia +Locality Name []: +Organization Name []:PII MitM Ltd +Organizational Unit Name []:PII MitM Ltd Certificate Authority +Common Name []:tls.pii.mitm.ca +Email Address []: +``` +To create the TLS MitM certificate, use the root CA with the v3_intermediate_ca extension to sign the intermediate CSR. +``` +cd /root/ca +openssl ca -config openssl.cnf -extensions v3_intermediate_ca -days 3650 -notext -md sha256 -in tlsmitm/csr/tlsmitm.csr.pem -out tlsmitm/certs/tlsmitm.cert.pem + +Enter pass phrase for ca.key.pem: secretpassword +Sign the certificate? [y/n]: y + +chmod 444 tlsmitm/certs/tlsmitm.cert.pem +``` +To verify the details of this certificate are correct use the instructions at: https://jamielinux.com/docs/openssl-certificate-authority/create-the-intermediate-pair.html#verify-the-intermediate-certificate + +## 3. Setting up the TLS MitM certificates + +### a. Creating the certificate chain file + +To create the certificate chain file ca-chain.cert.pem containing the two certificate public key files (root and TLS MitM) the two files are concatinated: +``` +cat tlsmitm/certs/tlsmitm.cert.pem certs/ca.cert.pem > tlsmitm/certs/ca-chain.cert.pem +chmod 444 tlsmitm/certs/ca-chain.cert.pem +``` + +### b. Creating the certificate chain file with TLS MitM private key + +Firstly, an unencrypted version of the TLS MitM private key needs to be created by removing the passphrase: +``` +openssl rsa -in tlsmitm/private/tlsmitm.key.pem -out tlsmitm/private/tlsmitm.unencrypted.key.pem +``` +Note. You will prompted to enter the passphrase. + +To create the certificate chain file ca-chain-cleartext.key.cert.pem containing the two certificate public key files (root and TLS MitM) and the intermediate certificate private key (private key unencrypted), the private key and certificate chain file (form part a.) need to be concatinated: +``` +cat tlsmitm/private/tlsmitm.unencrypted.key.pem tlsmitm/certs/ca-chain.cert.pem > tlsmitm/certs/ca-chain-cleartext.cert.pem +chmod 444 tlsmitm/certs/ca-chain-cleartext.cert.pem +``` + +### c. Installing the TLS MitM certificates + +The two PEM files need to be installed before TLS MitM functionality can be enabled. + +The file containing the two public keys ca-chain.cert.pem needs to be installed in the Android device's certificate key store (under the Settings > Security > Trusted Credentials option). + +The file containing the two public keys and private key ca-chain-cleartext.cert.pem must be copied onto the server in the /opt/nogotofail folder. diff --git a/docs/gce/_update_dev.sh b/docs/gce/_update_dev.sh new file mode 100644 index 00000000..849c7bb7 --- /dev/null +++ b/docs/gce/_update_dev.sh @@ -0,0 +1,66 @@ +#!/bin/sh + +set -e + +# Directory paths used for nogotofail. +INSTALL_DIR=/opt/nogotofail +CONFIG_DIR=/etc/nogotofail +LOG_DIR=/var/log/nogotofail + +# Stop the nogotofail-mitm and other associated services if they're running. +if (ps ax | grep -v grep | grep nogotofail-mitm > /dev/null) then +sudo /etc/init.d/nogotofail-mitm stop +fi +if (ps ax | grep -v grep | grep dnsmasq > /dev/null) then +sudo /etc/init.d/dnsmasq stop +fi +if (ps ax | grep -v grep | grep openvpn > /dev/null) then +sudo /etc/init.d/openvpn stop +fi +# Remove Python files and compiled versions i.e. *.py and *.pyc files. +# TODO: Find a more elegant method for uninstalling a Python program. +#rm -rf $INSTALL_DIR +#rm -rf $CONFIG_DIR +#rm -rf $LOG_DIR +find $INSTALL_DIR -type f -name '*.py' -delete +find $INSTALL_DIR -type f -name '*.pyc' -delete + +# Install toolchain dependencies +sudo apt-get update +sudo apt-get -y upgrade +#sudo apt-get -y install patch make gcc libssl-dev python-openssl liblzo2-dev libpam-dev + +# Install OpenVPN and dnsmasq +#sudo apt-get -y install openvpn dnsmasq + +# Build and install a patched version of OpenVPN. +# This is needed because the OpenVPN 2.3.x still does not properly handle +# floating clients (those whose source IP address as seen by the server changes +# from time to time) which is a regular occurrence in the mobile world. +# OpenVPN 2.4 might ship with proper support out of the box. In that case, this +# kludge can be removed. +#./build_openvpn.sh + +# Build and install a patched version of dnsmasq. +# This is needed because GCE does not support IPv6. We thus blackhole IPv6 +# traffic from clients so that they are forced to use IPv4. However, default +# DNS servers will still resolve hostnames to IPv6 addresses causing clients to +# attempt IPv6. To avoid clients attempting IPv6, we run a patched dnsmasq DNS +# server which empties AAAA records thus causing clients to go for A records +# which provide IPv4 addresses. +#./build_dnsmasq.sh + +# Set up OpenVPN server +#sudo ./setup_openvpn.sh + +# Set up the MiTM daemons +sudo ./setup_mitm.sh + +# Move dev mitm.conf file into /etc/nogotofail directory +sudo cp /home/michael/noseyp_setup/mitm.conf /etc/nogotofail/mitm.conf + +# Restart all the relevant daemons +sudo /etc/init.d/dnsmasq start +sudo /etc/init.d/openvpn start +#sudo /etc/init.d/nogotofail-mitm stop || true +sudo /etc/init.d/nogotofail-mitm start diff --git a/docs/gce/mitm.conf b/docs/gce/mitm.conf index f766798a..108271fa 100644 --- a/docs/gce/mitm.conf +++ b/docs/gce/mitm.conf @@ -8,11 +8,22 @@ #verbose=True #port=8080 #attacks=selfsigned invalidhostname +attacks=httpspii #data=httpdetection httpauthdetection +data=httppii -probability=0.5 +probability=0.2 +debug=True serverssl=/etc/nogotofail/mitm_controller_cert_and_key.pem logfile=/var/log/nogotofail/mitm.log eventlogfile=/var/log/nogotofail/mitm.event trafficfile=/var/log/nogotofail/mitm.traffic + +[nogotofail.pii] +facebook_id=abc@facebook.com +ip_address=55.66.77.88 +email = joe.blogs@gmail.com +first_name = joe +last_name = blogs +postal_address = "1 Long Road, Towns-ville" diff --git a/docs/gce/update.sh b/docs/gce/update.sh new file mode 100644 index 00000000..f3874e7f --- /dev/null +++ b/docs/gce/update.sh @@ -0,0 +1,63 @@ +#!/bin/sh + +set -e + +# Directory paths used for nogotofail. +INSTALL_DIR=/opt/nogotofail +CONFIG_DIR=/etc/nogotofail +LOG_DIR=/var/log/nogotofail + +# Stop the nogotofail-mitm and other associated services if they're running. +if (ps ax | grep -v grep | grep nogotofail-mitm > /dev/null) then +sudo /etc/init.d/nogotofail-mitm stop +fi +if (ps ax | grep -v grep | grep dnsmasq > /dev/null) then +sudo /etc/init.d/dnsmasq stop +fi +if (ps ax | grep -v grep | grep openvpn > /dev/null) then +sudo /etc/init.d/openvpn stop +fi +# Remove Python files and compiled versions i.e. *.py and *.pyc files. +# TODO: Find a more elegant method for uninstalling a Python program. +#rm -rf $INSTALL_DIR +#rm -rf $CONFIG_DIR +#rm -rf $LOG_DIR +find $INSTALL_DIR -type f -name '*.py' -delete +find $INSTALL_DIR -type f -name '*.pyc' -delete + +# Install toolchain dependencies +sudo apt-get update +sudo apt-get -y upgrade +#sudo apt-get -y install patch make gcc libssl-dev python-openssl liblzo2-dev libpam-dev + +# Install OpenVPN and dnsmasq +#sudo apt-get -y install openvpn dnsmasq + +# Build and install a patched version of OpenVPN. +# This is needed because the OpenVPN 2.3.x still does not properly handle +# floating clients (those whose source IP address as seen by the server changes +# from time to time) which is a regular occurrence in the mobile world. +# OpenVPN 2.4 might ship with proper support out of the box. In that case, this +# kludge can be removed. +#./build_openvpn.sh + +# Build and install a patched version of dnsmasq. +# This is needed because GCE does not support IPv6. We thus blackhole IPv6 +# traffic from clients so that they are forced to use IPv4. However, default +# DNS servers will still resolve hostnames to IPv6 addresses causing clients to +# attempt IPv6. To avoid clients attempting IPv6, we run a patched dnsmasq DNS +# server which empties AAAA records thus causing clients to go for A records +# which provide IPv4 addresses. +#./build_dnsmasq.sh + +# Set up OpenVPN server +#sudo ./setup_openvpn.sh + +# Set up the MiTM daemon +sudo ./setup_mitm.sh + +# Restart all the relevant daemons +sudo /etc/init.d/dnsmasq start +sudo /etc/init.d/openvpn start +#sudo /etc/init.d/nogotofail-mitm stop || true +sudo /etc/init.d/nogotofail-mitm start diff --git a/docs/pii_analysis.md b/docs/pii_analysis.md new file mode 100644 index 00000000..d2984e79 --- /dev/null +++ b/docs/pii_analysis.md @@ -0,0 +1,94 @@ +# PII Analysis + +The PII (personally identifiable information) features in nogotofail can detect PII sent in traffic between Android mobile applications and online services. + +Key features include: +- Detection of PII in encrypted (HTTPS) and unencrypted (HTTP) traffic +- Auto-collection of PII data from the client's device +- Ability to define custom PII test data (using the server's configuration file) + +This functionality is designed to assist in assessing two privacy risks - the disclosure of personal information in unencrypted traffic, and the excessive disclosure of personal information to application services and third parties i.e. advertising and analytics services. + +TODO: Summary reporting of PII issues showing accumulated PII sent to application services over an application session. + +## PII Detection Handlers + +There are two handlers available that inspect mobile application traffic for PII: +- **httppii** - parses unencrypted (HTTP) traffic +- **httpspii** - parses encrypted (HTTPS) traffic + +The **httpspii** handler acts as a man-in-the-middle (MitM) TLS proxy, intercepting and terminating requests between the client and MitM daemon, and later handling encryption of traffic between the MitM daemon and online service. + +For the **httpspii** handler to perform a man-in-the-middle attack a certificate is required that is trusted by the client. There are two options available: + +**a**. (Recommended) purchasing a TLS certificate from a trusted commercial CA, or + +**b**. See [create_tls_proxy_cert.md](create_tls_proxy_cert.md) for instructions on generating your own CA and trusted certificate. + +### a. Specifying PII Detection Handlers + +nogotofail-pii can be configured to use the PII detection handlers by adding the handler arguments in the configuration (`*.mitm`) file. An example [nogotofail.mitm] configuration file section is: +``` +[nogotofail.mitm] +attacks=httpspii +data=httppii + +probability=0.2 +debug=True + +serverssl=/etc/nogotofail/mitm_controller_cert_and_key.pem +logfile=/var/log/nogotofail/mitm.log +eventlogfile=/var/log/nogotofail/mitm.event +trafficfile=/var/log/nogotofail/mitm.traffic +``` + +The **httppii** handler is a "data" handler and analyses the http data stream for PII information. The **httpspii** is an "attack" handler and manipulates the TLS connection. + +Note. Tampering of the TLS connection by the **httpspii** handler adds latency to requests and it is recommended that you choose an attack "probability" value which minimizes the chance of request timeouts. Trial and error is required to find a suitable probability for your setup. + +### b. Specifying PII Items + +nogotofail has two types of PII that are detected in mobile mobile application traffic: + +1. Information manually specified in the server configuration file discussed in the [Server PII](#server_pii) section. +2. Device information collected by the client app presented in the [Client PII](#client_pii) section. + + +#### Server PII + +The server configuration file (*.mitm) has a section named "[nogotofail.pii]" reserved for personal information that can be specified for detection. An example [nogotofail.pii] configuration file section is: + +``` +[nogotofail.pii] +# PII identifiers +facebook_id=abc@facebook.com +ip_address=55.66.77.88 +email = joe.blogs@gmail.com +# PII details +first_name = joe +last_name = blogs +postal_address = "1 Long Road, Towns-ville" +``` + +To assist assessing the impact of PII disclosure in this example two arbitrary categories of PII were specified using the comment lines "PII identifiers" and "PII details". + +**PII identifiers** are identifiers that uniquely identify a device or user. Examples include phone number, Facebook user ID, email. + +**PII Details** describe data about the individual that may not by themselves uniquely identify them, but could identify the individual if combined with other data. Examples include first name, last name, postal address. + + +#### Client PII + +A number of PII items are automatically collected by the client from the device. The PII items the client collects are: + +| Reserved PII | Description | +|--------------|---| +| android_id | The Android ID used by the device | +| imei | The devices IMEI number (for SIM devices only) | +| mac_address | The devices MAC address | +| google_ad_id | The Google Advertising ID currently assigned to the device | +| ip_address | The devices IP address | + +Note. These PII labels are reserved and cannot be used in the server configuration file. + +In terms of the arbitrary PII categories discussed earlier client PII information is typically considered to be **PII identifiers** as they uniquely identify a individual user of device. diff --git a/nogotofail/mitm/__main__.py b/nogotofail/mitm/__main__.py index b6550331..682b02e5 100644 --- a/nogotofail/mitm/__main__.py +++ b/nogotofail/mitm/__main__.py @@ -28,6 +28,7 @@ import ConfigParser import collections import sys +import copy from nogotofail.mitm.blame import Server as AppBlameServer from nogotofail.mitm.connection import Server, RedirectConnection, SocksConnection, TproxyConnection @@ -42,6 +43,9 @@ event_logger = logging.getLogger("event") traffic_logger = logging.getLogger("traffic") +# PII configuration file section names +SECTION_MITM = "nogotofail.mitm" +SECTION_PII_ITEMS = "nogotofail.pii" def build_selector(MITM_all=False): def handler_selector(connection, app_blame): @@ -102,8 +106,43 @@ def data_selector(connection, app_blame): return internal + passive + active return data_selector - -def build_server(port, blame, selector, ssl_selector, data_selector, block, ipv6, cls): +def get_client_pii_items(): + + def client_pii_items(connection, app_blame): + pii_items = {} + try: + if (not app_blame.client_available(connection.client_addr)): + return internal + [] + # Figure out our possible handlers + client_info = app_blame.clients.get(connection.client_addr) + client_info = client_info.info if client_info else None + if client_info: + pii_items = client_info.pii_store.pii_items_plaintext + return pii_items + except Exception as e: + logger.exception(str(e)) + return {} + +def get_client_pii_location(): + + def client_pii_location(connection, app_blame): + pii_location = {} + try: + if (not app_blame.client_available(connection.client_addr)): + return internal + [] + # Figure out our possible handlers + client_info = app_blame.clients.get(connection.client_addr) + client_info = client_info.info if client_info else None + if client_info: + pii_location = client_info.pii_store.pii_location + # client_info["PII-Location"] + return pii_location + except Exception as e: + logger.exception(str(e)) + return {} + +def build_server(port, blame, selector, ssl_selector, data_selector, block, + ipv6, cls): return Server(port, blame, handler_selector=selector, ssl_handler_selector=ssl_selector, data_handler_selector=data_selector, @@ -112,8 +151,9 @@ def build_server(port, blame, selector, ssl_selector, data_selector, block, ipv6 connection_class=cls) -def build_blame(port, cert, probability, attacks, data_attacks): - return AppBlameServer(port, cert, probability, attacks, data_attacks) +def build_blame(port, cert, probability, attacks, data_attacks, config_pii): + return AppBlameServer(port, cert, probability, attacks, data_attacks, + config_pii) def set_redirect_rules(args): port = args.port @@ -155,14 +195,27 @@ def parse_args(): parser = argparse.ArgumentParser(add_help=False) parser.add_argument("-c", "--config") args, argv = parser.parse_known_args() + if args.config: - config = ConfigParser.SafeConfigParser() - config.read(args.config) - config = dict(config.items("nogotofail.mitm")) - if "attacks" in config: - config["attacks"] = config["attacks"].split(" ") - if "data" in config: - config["data"] = config["data"].split(" ") + config = {} + configBase = ConfigParser.SafeConfigParser() + configBase.read(args.config) + has_mitm_section = configBase.has_section(SECTION_MITM) + has_pii_items_section = configBase.has_section(SECTION_PII_ITEMS) + + config_copy = copy.copy(configBase) + # Check if the "nogotofail.mitm" section exists. + if has_mitm_section: + config_mitm = dict(config_copy.items(SECTION_MITM)) + if "attacks" in config_mitm: + config_mitm["attacks"] = config_mitm["attacks"].split(" ") + if "data" in config_mitm: + config_mitm["data"] = config_mitm["data"].split(" ") + config = config_mitm + # Check if the "nogotofail.pii" section exists. + if has_pii_items_section: + config_pii_items = dict(config_copy.items(SECTION_PII_ITEMS)) + config["pii_items"] = config_pii_items else: config = {} @@ -216,6 +269,10 @@ def parse_args(): help=("Route IPv6 traffic. " "Requires support for ip6tables NAT redirect when in redirect mode (iptables > 1.4.17)"), default=False, action="store_true") + + parser.add_argument( + "--fns1", help="***** Nogotofail functions *****************************", + action="store_true", default=False) parser.add_argument( "-A", "--attacks", help="Connection attacks to run. Supported attacks are " + @@ -242,6 +299,7 @@ def parse_args(): parser.set_defaults(**config) return parser.parse_args(argv) + def sigterm_handler(num, frame): """Gracefully exit on a SIGTERM. atexit isn't called on a SIGTERM, causing our cleanup code not to be called. @@ -290,7 +348,6 @@ def setup_logging(args): logger.setLevel(logging.DEBUG) def run(): - args = parse_args() setup_logging(args) extras.extras_dir = args.extrasdir @@ -302,9 +359,14 @@ def run(): data_cls = [handlers.data.handlers.map[name] for name in args.data] data_cls = preconditions.filter_preconditions(data_cls, logger) ssl_selector = build_ssl_selector(attack_cls, args.probability, args.all) - data_selector = build_data_selector(data_cls, args.all, prob_attack=args.probability) + data_selector = build_data_selector(data_cls, args.all, + prob_attack=args.probability) + # Build PII collection. + server_config_pii = {} + server_config_pii["items"] = args.pii_items logger.info("Starting...") + try: signal.signal(signal.SIGTERM, sigterm_handler) mode = modes[args.mode] @@ -313,7 +375,7 @@ def run(): blame = ( build_blame( args.cport, args.serverssl, args.probability, attack_cls, - data_cls)) + data_cls, server_config_pii)) server = ( build_server( args.port, blame, selector, ssl_selector, diff --git a/nogotofail/mitm/blame/app_blame.py b/nogotofail/mitm/blame/app_blame.py index 87d3c2dd..f7fecf49 100644 --- a/nogotofail/mitm/blame/app_blame.py +++ b/nogotofail/mitm/blame/app_blame.py @@ -16,15 +16,14 @@ from collections import namedtuple import logging import socket -import select -import sys import ssl import time import urllib +import ast from nogotofail.mitm.connection import handlers from nogotofail.mitm.connection.handlers import preconditions -from nogotofail.mitm.util import close_quietly +from nogotofail.mitm.util import close_quietly, truncate, PiiStore Application = namedtuple("Application", ["package", "version"]) @@ -56,6 +55,7 @@ def __init__(self, socket, server, now=None): self.address = self.socket.getpeername()[0] self.logger = logging.getLogger("nogotofail.mitm") self._handshake_completed = False + self.pii_store = None @property def available(self): @@ -236,6 +236,7 @@ def _send_headers(self): self.socket.sendall("\n") def _parse_headers(self, lines): + # try: raw_headers = [line.split(":", 1) for line in lines[1:]] headers = {entry.strip(): header.strip() for entry, header in raw_headers} @@ -257,17 +258,37 @@ def _parse_headers(self, lines): attacks = headers["Attacks"].split(",") attacks = map(str.strip, attacks) client_info["Attacks"] = preconditions.filter_preconditions([ - handlers.connection.handlers.map[attack] for attack in attacks - if attack in handlers.connection.handlers.map]) + handlers.connection.handlers.map[attack] + for attack in attacks + if attack in handlers.connection.handlers.map]) if "Data-Attacks" in headers: attacks = headers["Data-Attacks"].split(",") attacks = map(str.strip, attacks) client_info["Data-Attacks"] = preconditions.filter_preconditions( - [handlers.data.handlers.map[attack] - for attack in attacks - if attack in - handlers.data.handlers.map]) + [handlers.data.handlers.map[attack] + for attack in attacks + if attack in handlers.data.handlers.map]) + + if ("PII-Items" in headers): + client_pii_items = ast.literal_eval(headers["PII-Items"]) + + # TODO: Think if HTML encoding is needed for PII information. + # e.g. ',",&,<,> characters. + if ("PII-Location" in headers): + client_pii_location = {} + # Convert personal location string to a dictionary + personal_location = ast.literal_eval(headers["PII-Location"]) + if (personal_location): + longitude = personal_location.get("longitude", "0.00000") + latitude = personal_location.get("latitude", "0.00000") + else: + longitude = "0.00000" + latitude = "0.00000" + client_pii_location["longitude"] = \ + truncate(float(longitude), 2) + client_pii_location["latitude"] = \ + truncate(float(latitude), 2) # Store the raw headers as well in case a handler needs something the # client sent in an additional header @@ -275,11 +296,18 @@ def _parse_headers(self, lines): self.info = client_info + # Merge client and server pii items. + server_pii_items = self.server.pii["items"] + merge_pii_ids = client_pii_items.copy() + merge_pii_ids.update(server_pii_items) + # Create pii_store attribute which holds pii items. + self.pii_store = PiiStore(merge_pii_ids, client_pii_location) + def _response_select_fn(self): try: data = self.socket.recv(8192) except socket.error: - self.logger.info("Blame: Erorr reading from client %s.", self.address) + self.logger.info("Blame: Error reading from client %s.", self.address) return False if not data: @@ -311,7 +339,8 @@ class Server: port = None clients = None - def __init__(self, port, cert, default_prob, default_attacks, default_data): + def __init__(self, port, cert, default_prob, default_attacks, default_data, + config_pii): self.txid = 0 self.kill = False self.port = port @@ -323,6 +352,8 @@ def __init__(self, port, cert, default_prob, default_attacks, default_data): self.fd_map = {} self.logger = logging.getLogger("nogotofail.mitm") self.server_socket = None + # Server config pii parameters + self.pii = config_pii def start_listening(self): self.server_socket = socket.socket() @@ -455,3 +486,11 @@ def shutdown(self): client.close() except: pass + + def get_pii(self): + """ Function return server config pii values. + """ + if (self.pii): + return self.pii + else: + return {} diff --git a/nogotofail/mitm/connection/connection.py b/nogotofail/mitm/connection/connection.py index 16100483..43ff19ef 100644 --- a/nogotofail/mitm/connection/connection.py +++ b/nogotofail/mitm/connection/connection.py @@ -661,6 +661,7 @@ def inject_response(self, response): break self.client_socket.sendall(response) + class RedirectConnection(BaseConnection): """Connection based on getting traffic from iptables redirect rules""" @@ -833,5 +834,3 @@ def _get_original_dest(self, sock): sock.sendall(self._build_error_response(SocksConnection.RESP_GENERAL_ERROR)) raise ValueError("Unknown ATYP") return addr, port - - diff --git a/nogotofail/mitm/connection/handlers/connection/__init__.py b/nogotofail/mitm/connection/handlers/connection/__init__.py index e52130a8..5534c2b9 100644 --- a/nogotofail/mitm/connection/handlers/connection/__init__.py +++ b/nogotofail/mitm/connection/handlers/connection/__init__.py @@ -26,3 +26,4 @@ from droptls import DropTLS from ccs import EarlyCCS from serverkeyreplace import ServerKeyReplacementMITM +from httpspii import * diff --git a/nogotofail/mitm/connection/handlers/connection/httpspii.py b/nogotofail/mitm/connection/handlers/connection/httpspii.py new file mode 100644 index 00000000..50e105c8 --- /dev/null +++ b/nogotofail/mitm/connection/handlers/connection/httpspii.py @@ -0,0 +1,245 @@ +r''' +Copyright 2014 Google Inc. All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +import logging +from nogotofail.mitm.connection.handlers import preconditions +from nogotofail.mitm.connection.handlers.connection import handlers +from nogotofail.mitm.connection.handlers.connection import LoggingHandler +from nogotofail.mitm.connection.handlers.store import handler +from nogotofail.mitm.event import connection +from nogotofail.mitm import util +import nogotofail.mitm.util.pii as piiutil + + +class HttpsPiiContentHandler(LoggingHandler): + + name = "piidetection" + description = "Detect HTTPS requests and responses and allow \ + classes that inherit from this to process content" + + ssl = False + + def __init__(self, connection): + super(HttpsPiiContentHandler, self).__init__(connection) + self.client = \ + self.connection.app_blame.clients.get(connection.client_addr) + + def on_ssl(self, client_hello): + self.client_session_id = client_hello.session_id + return True + + def on_ssl_establish(self): + self.ssl = True + + def on_request(self, request): + http = util.http.parse_request(request) + if http and not http.error_code: + host = http.headers.get("host", self.connection.server_addr) + if not self.connection.hostname: + self.connection.hostname = host + # Call the specific http request handler based on the use of TLS + if self.ssl: + http_request = util.httppii.HTTPPiiRequestWrapper(http) + self.on_https_request(http_request) + return request + + def on_https_request(self, http_request): + comment = "Code to be added in class inheriting this." + + def on_response(self, response): + http = util.http.parse_response(response) + if http: + try: + headers = dict(http.getheaders()) + host = headers.get("host", self.connection.server_addr) + except AttributeError: + host = self.connection.server_addr + if not self.connection.hostname: + self.connection.hostname = host + # Call the specific http response handler based on the use of TLS + if self.ssl: + http_response = util.httppii.HTTPPiiResponseWrapper(http) + self.on_https_response(http_response) + return response + + def on_https_response(self, http_response): + comment = "Code to be added in class inheriting this." + + +@handler(handlers, default=True) +@preconditions.requires_files(files=["mitm_key_cert_chain.pem"]) +class HttpsPiiDetection(HttpsPiiContentHandler): + + name = "httpspii" + description = ( + "Testing to see if encrypted PII is present in HTTPS content.") + # Location of trusted MitM certificate. + MITM_CA = "./mitm_key_cert_chain.pem" + ca = util.CertificateAuthority(MITM_CA) + certificate = None + + def on_certificate(self, server_cert): + """ Terminate on_certificate interaction between server and client & + insert a trusted certificate in traffic to server to initiate a + MitM attack + """ + subject = server_cert.get_subject() + for k, v in subject.get_components(): + if k == "CN": + cn = v + debug_message = ["Generating MitM TLS certificate with CN - ", cn] + self.log(logging.DEBUG, "".join(debug_message)) + extensions = [server_cert.get_extension(i) + for i in range(server_cert.get_extension_count())] + altnames = [extension for extension in extensions + if extension.get_short_name() == "subjectAltName"] + san = altnames[0] if len(altnames) > 0 else None + self.certificate = self.ca.get_cert(cn, san) + return self.certificate + + def on_https_request(self, http_request): + """ Parse HTTPS requests for PII paramters + """ + if (self.client and http_request): + headers = http_request.headers_dict + host = headers.get("host", self.connection.server_addr) + url = host + http_request.path + # Extract query string from request url. + query_string = http_request.query_string + # Check for PII in HTTP query string + if (query_string): + self._alert_on_pii_query_string(query_string, url) + # Check for PII in HTTP headers + valid_header_text = "" + # Remove headers which won't contain PII + valid_headers = http_request.pii_headers_dict + if (valid_headers): + valid_header_text = \ + str(valid_headers.values()).translate(None, "[']") + self._alert_on_pii_headers(valid_header_text, url) + # Check for PII in HTTP message body + msg_content = http_request.pii_message_body + if msg_content: + self._alert_on_pii_request_message_body(msg_content, url) + + def on_https_response(self, http_response): + """ Parse HTTPS responses for PII paramters + """ + if (self.client and http_response): + url = "" + msg_content = http_response.message_body + # Check for PII in HTTP message body + self._alert_on_pii_response_message_body(msg_content, url) + + def _alert_on_pii_query_string(self, query_string, url): + """ Test and alert on instances of PII found in query string + """ + pii_items_found = [] + pii_location_found = [] + # Check if PII found in query string + pii_items_found = \ + self.client.pii_store.detect_pii_items(query_string) + pii_location_found = \ + self.client.pii_store.detect_pii_location(query_string) + # If PII is found in query string raise INFO message in + # message and event logs + if (pii_items_found): + error_message = [piiutil.CAVEAT_PII_QRY_STRING, + ": Personal IDs found in request query string ", + str(pii_items_found)] + self.log(logging.INFO, "".join(error_message)) + self.log_event(logging.INFO, connection.AttackEvent( + self.connection, self.name, True, url)) + if (pii_location_found): + error_message = [piiutil.CAVEAT_PII_QRY_STRING, + ": Location found in request query string ", + "(longitude, latitude) - ", str(pii_location_found)] + self.log(logging.INFO, "".join(error_message)) + self.log_event(logging.INFO, connection.AttackEvent( + self.connection, self.name, True, url)) + + def _alert_on_pii_headers(self, header_text, url): + """ Test and alert on instances of PII found in HTTP headers + """ + pii_items_found = [] + pii_location_found = [] + # Check if PII found in header + pii_items_found = \ + self.client.pii_store.detect_pii_items(header_text) + pii_location_found = \ + self.client.pii_store.detect_pii_location(header_text) + if (pii_items_found): + error_message = [piiutil.CAVEAT_PII_HEADER, + ": Personal IDs found in request headers ", + str(pii_items_found)] + self.log(logging.INFO, "".join(error_message)) + self.log_event(logging.INFO, connection.AttackEvent( + self.connection, self.name, True, url)) + if (pii_location_found): + error_message = [piiutil.CAVEAT_PII_HEADER, + ": Location found in request headers ", + "(longitude, latitude) - ", str(pii_location_found)] + self.log(logging.INFO, "".join(error_message)) + self.log_event(logging.INFO, connection.AttackEvent( + self.connection, self.name, True, url)) + + def _alert_on_pii_request_message_body(self, msg_content, url): + """ Test and alert on instances of PII found in HTTP message body + """ + pii_items_found = [] + pii_location_found = [] + # Check if PII found in message body + pii_items_found = \ + self.client.pii_store.detect_pii_items(msg_content) + pii_location_found = \ + self.client.pii_store.detect_pii_location(msg_content) + # If PII is found in message body raise INFO message in + # message and event logs + if (pii_items_found): + error_message = [piiutil.CAVEAT_PII_MSG_BODY, + ": Personal IDs found in request message body ", + str(pii_items_found)] + self.log(logging.INFO, "".join(error_message)) + self.log_event(logging.INFO, connection.AttackEvent( + self.connection, self.name, True, url)) + if (pii_location_found): + error_message = [piiutil.CAVEAT_PII_MSG_BODY, + ": Location found in request message body ", + "(longitude, latitude) - ", str(pii_location_found)] + self.log(logging.INFO, "".join(error_message)) + self.log_event(logging.INFO, connection.AttackEvent( + self.connection, self.name, True, url)) + + def _alert_on_pii_response_message_body(self, msg_content, url): + """ Test and alert on instances of PII found in HTTP message body + """ + pii_items_found = [] + pii_location_found = [] + # Check if PII found in query string + pii_items_found = \ + self.client.pii_store.detect_pii_items(msg_content) + pii_location_found = \ + self.client.pii_store.detect_pii_location(msg_content) + if (pii_items_found): + error_message = [piiutil.CAVEAT_PII_MSG_BODY, + ": Personal IDs found in response message body ", + str(pii_items_found)] + self.log(logging.INFO, "".join(error_message)) + self.log_event(logging.INFO, connection.AttackEvent( + self.connection, self.name, True, url)) + if (pii_location_found): + error_message = [piiutil.CAVEAT_PII_MSG_BODY, + ": Location found in response message body ", + "(longitude, latitude) - ", str(pii_location_found)] + self.log(logging.INFO, "".join(error_message)) + self.log_event(logging.INFO, connection.AttackEvent( + self.connection, self.name, True, url)) diff --git a/nogotofail/mitm/connection/handlers/data/__init__.py b/nogotofail/mitm/connection/handlers/data/__init__.py index 1ecd73d5..608c84cb 100644 --- a/nogotofail/mitm/connection/handlers/data/__init__.py +++ b/nogotofail/mitm/connection/handlers/data/__init__.py @@ -22,6 +22,7 @@ from report import ClientReportDetection from log import RawTrafficLogger from http import * +from httppii import * from imap import * from smtp import * from xmpp import * diff --git a/nogotofail/mitm/connection/handlers/data/http.py b/nogotofail/mitm/connection/handlers/data/http.py index 6f20700c..ad1e58d6 100644 --- a/nogotofail/mitm/connection/handlers/data/http.py +++ b/nogotofail/mitm/connection/handlers/data/http.py @@ -156,6 +156,45 @@ def on_http(self, http): self.connection.vuln_notify(util.vuln.VULN_CLEARTEXT_AUTH) +class HttpContentHandler(DataHandler): + """ Provides methods for parsing the content of plaintext HTTP request and + response objects. """ + + ssl = False + + def on_ssl(self, client_hello): + self.ssl = True + return True + + def on_request(self, request): + http = util.http.parse_request(request) + if http and not self.ssl and not http.error_code: + host = http.headers.get("host", self.connection.server_addr) + if not self.connection.hostname: + self.connection.hostname = host + http_request = util.http.HTTPRequestWrapper(http) + self.on_http_request(http_request) + return request + + def on_http_request(self, http_request): + comment = "Code to be added in class inheriting this." + + def on_response(self, response): + http = util.http.parse_response(response) + if http: + headers = dict(http.getheaders()) + host = headers.get("host", self.connection.server_addr) + if not self.connection.hostname: + self.connection.hostname = host + if not self.connection.ssl: + http_response = util.http.HTTPResponseWrapper(http) + self.on_http_response(http_response) + return response + + def on_http_response(self, http_response): + comment = "Code to be added in class inheriting this." + + class _HttpReqReplacement(DataHandler): """Basic class for replacing the conents of a HTTP Request """ diff --git a/nogotofail/mitm/connection/handlers/data/httppii.py b/nogotofail/mitm/connection/handlers/data/httppii.py new file mode 100644 index 00000000..81a28857 --- /dev/null +++ b/nogotofail/mitm/connection/handlers/data/httppii.py @@ -0,0 +1,210 @@ +r''' +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +from nogotofail.mitm import util +from nogotofail.mitm.connection.handlers.data import handlers +from nogotofail.mitm.connection.handlers.data import HttpContentHandler +from nogotofail.mitm.connection.handlers.store import handler +from nogotofail.mitm.event import connection +import nogotofail.mitm.util.pii as piiutil +import logging + + +class HttpPiiContentHandler(HttpContentHandler): + """ Provides methods for parsing the content of plaintext HTTP request and + response objects for PII. """ + + def on_request(self, request): + http = util.http.parse_request(request) + if http and not self.ssl and not http.error_code: + host = http.headers.get("host", self.connection.server_addr) + if not self.connection.hostname: + self.connection.hostname = host + http_request = util.httppii.HTTPPiiRequestWrapper(http) + self.on_http_request(http_request) + return request + + def on_http_request(self, http_request): + comment = "Code to be added in class inheriting this." + + def on_response(self, response): + http = util.http.parse_response(response) + if http: + headers = dict(http.getheaders()) + host = headers.get("host", self.connection.server_addr) + if not self.connection.hostname: + self.connection.hostname = host + if not self.connection.ssl: + http_response = util.httppii.HTTPPiiResponseWrapper(http) + self.on_http_response(http_response) + return response + + def on_http_response(self, http_response): + comment = "Code to be added in class inheriting this." + + +@handler.passive(handlers) +class HttpPiiDetection(HttpPiiContentHandler): + """ Detects PII appearing in plaintext HTTP request and response + content. """ + + name = "httppii" + description = "Detect PII in clear text http requests and responses" + + def __init__(self, connection): + super(HttpPiiDetection, self).__init__(connection) + self.client = \ + self.connection.app_blame.clients.get(connection.client_addr) + + def on_http_request(self, http_request): + if (self.client and http_request): + headers = http_request.headers_dict + host = headers.get("host", self.connection.server_addr) + url = host + http_request.path + # Extract query string from request url. + query_string = http_request.query_string + # Check for PII in HTTP query string + if (query_string): + self._alert_on_pii_query_string(query_string, url) + # Check for PII in HTTP headers + valid_header_text = "" + # Fetch a dictionary of headers which could contain PII. + valid_headers = http_request.pii_headers_dict + if (valid_headers): + valid_header_text = \ + str(valid_headers.values()).translate(None, "[']") + self._alert_on_pii_headers(valid_header_text, url) + # Check for PII in HTTP message body + msg_content = http_request.pii_message_body + if msg_content: + self._alert_on_pii_request_message_body(msg_content, url) + + def on_http_response(self, http_response): + """ Method processes unencrypted (non-HTTPS) HTTP response message bodies + """ + if (self.client and http_response): + url = "" + msg_content = http_response.message_body + # Check for PII in HTTP message body + self._alert_on_pii_response_message_body(msg_content, url) + + def _alert_on_pii_query_string(self, query_string, url): + """ Test and alert on instances of PII found in query string + """ + pii_items_found = [] + pii_location_found = [] + error_message = "" + # Check if PII found in query string + pii_items_found = \ + self.client.pii_store.detect_pii_items(query_string) + pii_location_found = \ + self.client.pii_store.detect_pii_location(query_string) + if (pii_items_found): + error_message = [piiutil.CAVEAT_PII_QRY_STRING, + ": Personal items found in request query string ", + str(pii_items_found)] + self.log(logging.ERROR, "".join(error_message)) + self.log_event(logging.ERROR, connection.AttackEvent( + self.connection, self.name, True, url)) + self.connection.vuln_notify(util.vuln.VULN_CLEARTEXT_HTTP_PII) + if (pii_location_found): + error_message = [piiutil.CAVEAT_PII_QRY_STRING, + ": Location found in request query string ", + "(longitude, latitude) - ", str(pii_location_found)] + self.log(logging.ERROR, "".join(error_message)) + self.log_event(logging.ERROR, connection.AttackEvent( + self.connection, self.name, True, url)) + self.connection.vuln_notify(util.vuln.VULN_CLEARTEXT_HTTP_PII) + + def _alert_on_pii_headers(self, header_text, url): + """ Test and alert on instances of PII found in HTTP headers + """ + pii_items_found = [] + pii_location_found = [] + # Check if PII found in message body + pii_items_found = \ + self.client.pii_store.detect_pii_items(header_text) + pii_location_found = \ + self.client.pii_store.detect_pii_location(header_text) + if (pii_items_found): + error_message = [piiutil.CAVEAT_PII_HEADER, + ": Personal items found in request headers ", + str(pii_items_found)] + self.log(logging.ERROR, "".join(error_message)) + self.log_event(logging.ERROR, connection.AttackEvent( + self.connection, self.name, True, url)) + self.connection.vuln_notify(util.vuln.VULN_CLEARTEXT_HTTP_PII) + if (pii_location_found): + error_message = [piiutil.CAVEAT_PII_HEADER, + ": Location found in request headers ", + "(longitude, latitude) - ", str(pii_location_found)] + self.log(logging.ERROR, "".join(error_message)) + self.log_event(logging.ERROR, connection.AttackEvent( + self.connection, self.name, True, url)) + self.connection.vuln_notify(util.vuln.VULN_CLEARTEXT_HTTP_PII) + + def _alert_on_pii_request_message_body(self, msg_content, url): + """ Test and alert on instances of PII found in HTTP message body + """ + pii_items_found = [] + pii_location_found = [] + # Check if PII found in message body + pii_items_found = \ + self.client.pii_store.detect_pii_items(msg_content) + pii_location_found = \ + self.client.pii_store.detect_pii_location(msg_content) + if (pii_items_found): + error_message = [piiutil.CAVEAT_PII_MSG_BODY, + ": Personal items found in request message body ", + str(pii_items_found)] + self.log(logging.ERROR, "".join(error_message)) + self.log_event(logging.ERROR, connection.AttackEvent( + self.connection, self.name, True, url)) + self.connection.vuln_notify(util.vuln.VULN_CLEARTEXT_HTTP_PII) + if (pii_location_found): + error_message = [piiutil.CAVEAT_PII_MSG_BODY, + ": Location found in request message body ", + "(longitude, latitude) - ", str(pii_location_found)] + self.log(logging.ERROR, "".join(error_message)) + self.log_event(logging.ERROR, connection.AttackEvent( + self.connection, self.name, True, url)) + self.connection.vuln_notify(util.vuln.VULN_CLEARTEXT_HTTP_PII) + + def _alert_on_pii_response_message_body(self, msg_content, url): + """ Test and alert on instances of PII found in HTTP message body + """ + pii_items_found = [] + pii_location_found = [] + # Check if PII found in message body + pii_items_found = \ + self.client.pii_store.detect_pii_items(msg_content) + pii_location_found = \ + self.client.pii_store.detect_pii_location(msg_content) + if (pii_items_found): + error_message = [piiutil.CAVEAT_PII_MSG_BODY, + ": Personal items found in response message body ", + str(pii_items_found)] + self.log(logging.ERROR, "".join(error_message)) + self.log_event(logging.ERROR, connection.AttackEvent( + self.connection, self.name, True, url)) + self.connection.vuln_notify(util.vuln.VULN_CLEARTEXT_HTTP_PII) + if (pii_location_found): + error_message = [piiutil.CAVEAT_PII_MSG_BODY, + ": Location found in response message body ", + "(longitude, latitude) - ", str(pii_location_found)] + self.log(logging.ERROR, "".join(error_message)) + self.log_event(logging.ERROR, connection.AttackEvent( + self.connection, self.name, True, url)) + self.connection.vuln_notify(util.vuln.VULN_CLEARTEXT_HTTP_PII) diff --git a/nogotofail/mitm/event/application.py b/nogotofail/mitm/event/application.py index 5d2e2e55..937bbef6 100644 --- a/nogotofail/mitm/event/application.py +++ b/nogotofail/mitm/event/application.py @@ -15,6 +15,7 @@ ''' from nogotofail.mitm.event.base import Event +import logging class ApplicationEvent(Event): """Event relating to nogotofail.mitm itself. diff --git a/nogotofail/mitm/util/__init__.py b/nogotofail/mitm/util/__init__.py index 9f0a9039..1f190e60 100644 --- a/nogotofail/mitm/util/__init__.py +++ b/nogotofail/mitm/util/__init__.py @@ -20,4 +20,7 @@ from socket import close_quietly import vuln import http +from miscellaneous import * +from pii import * +from httppii import * import extras diff --git a/nogotofail/mitm/util/http.py b/nogotofail/mitm/util/http.py index 771e9089..d86b2106 100644 --- a/nogotofail/mitm/util/http.py +++ b/nogotofail/mitm/util/http.py @@ -17,16 +17,17 @@ from StringIO import StringIO import httplib import re +import urlparse +import zlib class HTTPRequest(BaseHTTPRequestHandler): - """Basic RequestHandler to try and parse a given request_text as an HTTP request. - - """ + """ Basic RequestHandler to try and parse a given request_text as an HTTP + request. """ def __init__(self, request_text): - # sometimes path and headers don't get set in the object, set some dummy - # ones so we don't have to check for them elsewhere. + # sometimes path and headers don't get set in the object, set some + # dummy ones so we don't have to check for them elsewhere. self.path = "" self.headers = {} self.rfile = StringIO(request_text) @@ -37,6 +38,122 @@ def __init__(self, request_text): def send_error(self, code, message): self.error_code = code self.error_message = message + + +class HTTPRequestWrapper(object): + """ Wrapper class for the HTTPRequest object providing properties to access + common request attributes. + Note. The HTTPRequest object could not be readily extended as the + parent class (BaseHTTPRequestHandler) doesn't inherit from 'object' """ + + http_request = None + + def __init__(self, a_http_request): + self.http_request = a_http_request + + @property + def headers_dict(self): + """ Returns the request headers as a dictionary with each name/value + pair as header-name/header-value. """ + return dict(self.http_request.headers) + + @property + def path(self): + """ Returns the path for the HTTPRequest object.""" + return self.http_request.path + + @property + def query_string(self): + """ Returns the request query string as a string """ + # Extract query string from request url. + url_parts = urlparse.urlparse(self.http_request.path) + return url_parts[4] + + @property + def query_string_dict(self): + """ Returns the request query string as a dictionary with each + key/value pair as parameter-name/parameter-value.""" + qs_dict = urlparse.parse_qs(self.query_string) + return qs_dict + + @property + def message_body(self): + """ Returns the HTTP request message body content. Compressed content + is uncompressed.""" + http_content = "" + headers = dict(self.http_request.headers) + content_len = int(headers.get("content-length", 0)) + # Retrieve content from HTTP request message body + if (content_len > 0): + http_content = self.http_request.rfile.read(content_len) + return http_content + + +class HTTPResponseWrapper(object): + """ Wrapper class for the HTTPResponse object providing properties to access + common response attributes. + Note. The HTTPResponse object could not be readily extended as it + doesn't inherit from 'object' """ + + http_response = None + + def __init__(self, a_http_response): + self.http_response = a_http_response + + @property + def response(self): + """ Returns the HTTPResponse object.""" + return self.http_response + + @property + def headers_dict(self): + """ Returns the request headers as a dictionary with each name/value + pair as header-name/header-value. """ + return dict(self.http_response.getheaders()) + + @property + def message_body(self): + """ Returns the HTTP request message body content. Compressed content + is uncompressed.""" + CHUNK_SIZE = 1024 + http_content = "" + headers = dict(self.http_response.getheaders()) + content_type = headers.get("content-type", "") + content_encoding = headers.get("content-encoding", "") + content_chunk_list = [] + number_of_chunks = 0 + try: + while True: + content_chunk = self.http_response.read(CHUNK_SIZE) + content_chunk_list.append(content_chunk) + number_of_chunks += 1 + """ Stop reading HTTP content after all chunks are read """ + if not content_chunk: + break + """ Stop reading HTTP content after 2 chunks """ + elif ((content_type == "text/html" or + content_type == "text/plain") and + number_of_chunks == 2): + break + except httplib.IncompleteRead, e: + content_chunk = e.partial + content_chunk_list.append(content_chunk) + http_content = ''.join(content_chunk_list) + # self.log(logging.DEBUG, "HTTP response headers: " + \ + # "content-type - %s; content-encoding - %s" \ + try: + """ Decompress compressed content """ + if ("deflate" in content_encoding or "gzip" in content_encoding): + http_content = \ + zlib.decompress(http_content, zlib.MAX_WBITS | 32) + # self.log(logging.DEBUG, "HTTP Content - %s." + # % http_content) + except zlib.error, e: + """ Handling decompression of a truncated or partial file + is read """ + zlib_partial = zlib.decompressobj(zlib.MAX_WBITS | 32) + http_content = zlib_partial.decompress(http_content) + return http_content class _FakeSocket(StringIO): diff --git a/nogotofail/mitm/util/httppii.py b/nogotofail/mitm/util/httppii.py new file mode 100644 index 00000000..a1d8ea0f --- /dev/null +++ b/nogotofail/mitm/util/httppii.py @@ -0,0 +1,73 @@ +r''' +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +from nogotofail.mitm.util.http import HTTPRequestWrapper, HTTPResponseWrapper + +# HTTP request and response valid "content-type" header values +VALID_CONTENT_TYPES = ["text/html", "application/json", + "text/plain", "text/xml", "application/xml"] +# HTTP headers to ignore not containing PII +IGNORE_HEADERS = ["host", "connection", "content-length", "accept", + "user-agent", "content-type", "accept-encoding", + "accept-language", "accept-charset"] + + +class HTTPPiiRequestWrapper(HTTPRequestWrapper): + """ Wrapper for theHTTPRequestWrapper class to provide PII specific + properties for HTTP request object. """ + + def __init__(self, a_http_request): + super(HTTPPiiRequestWrapper, self).__init__(a_http_request) + + @property + def pii_headers_dict(self): + """ Returns the request headers as a dictionary of types which can + hold PII.""" + # Remove headers which won't contain PII + valid_pii_headers = {k: v for k, v in self.headers_dict.iteritems() + if k not in IGNORE_HEADERS} + return valid_pii_headers + + @property + def pii_message_body(self): + """ Returns the HTTP request message body content for content types + which could contain PII. Compressed content is uncompressed.""" + http_content = "" + headers = self.headers_dict + content_len = int(headers.get("content-length", 0)) + content_type = headers.get("content-type", "") + # Retrieve content from HTTP request message body + if (content_len > 0 and content_type in VALID_CONTENT_TYPES): + http_content = self.http_request.rfile.read(content_len) + return http_content + + +class HTTPPiiResponseWrapper(HTTPResponseWrapper): + """ Wrapper for theHTTPResponseWrapper class to provide PII specific + properties for HTTP response object. """ + + def __init__(self, a_http_response): + super(HTTPPiiResponseWrapper, self).__init__(a_http_response) + + @property + def pii_message_body(self): + """ Returns the HTTP response message body content for content types + which could contain PII. Compressed content is uncompressed.""" + headers = self.headers_dict + content_type = headers.get("content-type", "") + # Retrieve content from HTTP request message body + if (content_type in VALID_CONTENT_TYPES): + http_content = self.message_body + return http_content diff --git a/nogotofail/mitm/util/miscellaneous.py b/nogotofail/mitm/util/miscellaneous.py new file mode 100644 index 00000000..e322796b --- /dev/null +++ b/nogotofail/mitm/util/miscellaneous.py @@ -0,0 +1,28 @@ +r''' +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +""" Module contains miscellaneous utility functions. +""" + + +def truncate(f, n): + """ Truncates/pads a float f to n decimal places without rounding and + returns a string """ + s = '{}'.format(f) + if 'e' in s or 'E' in s: + return '{0:.{1}f}'.format(f, n) + i, p, d = s.partition('.') + return '.'.join([i, (d+'0'*n)[:n]]) diff --git a/nogotofail/mitm/util/pii.py b/nogotofail/mitm/util/pii.py new file mode 100644 index 00000000..b452ee22 --- /dev/null +++ b/nogotofail/mitm/util/pii.py @@ -0,0 +1,97 @@ +r''' +Copyright 2016 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import base64 +import urllib + +# PII log entry caveats +CAVEAT_PII_QRY_STRING = "PII-QueryString" +CAVEAT_PII_HEADER = "PII-Header" +CAVEAT_PII_MSG_BODY = "PII-Message-Body" + + +class PiiStore(object): + """ Holds PII items supplied and methods for detecting these in + HTTP content + """ + + # Dictionary contains specified pii items and base64 and url-encoded + # variations of these. + _pii_items = {} + # Dictionary holds plain text version of pii items. + _pii_items_plaintext = {} + # Dictionary containing the device's location. + _pii_location = {} + + def __init__(self, pii_items, pii_location): + self._pii_items_plaintext = pii_items + pii_items_plaintext = pii_items + pii_items_base64 = {} + pii_items_urlencoded = {} + # Create base64 dictionary of PII items + for id_key, id_value in pii_items_plaintext.iteritems(): + # Add a base64 version of ID to dictionary + pii_items_base64[id_key + " (base64)"] = base64.b64encode(id_value) + # Create url encoded dictionary of PII identifiers + for id_key, id_value in pii_items_plaintext.iteritems(): + # Add a url encoded version of ID to dictionary if its different + # from the plain text version + id_value_urln = urllib.quote_plus(id_value) + if (id_value != id_value_urln): + pii_items_urlencoded[id_key + " (url encoded)"] = id_value_urln + # Combine PII items and variations into a single dictionary. + self._pii_items = {k: v for d in + (pii_items_plaintext, pii_items_base64, pii_items_urlencoded) + for k, v in d.iteritems()} + # Assign device location to dictionary. + self._pii_location["longitude"] = pii_location["longitude"] + self._pii_location["latitude"] = pii_location["latitude"] + + @property + def pii_items(self): + return self._pii_items + + @property + def pii_items_plaintext(self): + return self._pii_items_plaintext + + @property + def pii_location(self): + return self._pii_location + + def detect_pii_items(self, http_string): + """ Method searches for PII items within a HTTP string + i.e. query string, headers, message body + """ + pii_items_found = [] + # Search query string for pii items. + if self._pii_items: + pii_items_found = [k for k, v in + self._pii_items.iteritems() if v in http_string] + return pii_items_found + + def detect_pii_location(self, http_string): + """ Method searches for location (longitude/latitude) within a + HTTP string i.e. query string, headers, message body + """ + pii_location_found = [] + if self._pii_location: + longitude = self._pii_location["longitude"] + latitude = self._pii_location["latitude"] + if (longitude in http_string and latitude in http_string): + pii_location_found.append(longitude) + pii_location_found.append(latitude) + return pii_location_found diff --git a/nogotofail/mitm/util/vuln.py b/nogotofail/mitm/util/vuln.py index 2de2f0f4..e0f0bc9e 100644 --- a/nogotofail/mitm/util/vuln.py +++ b/nogotofail/mitm/util/vuln.py @@ -30,3 +30,5 @@ VULN_WEAK_TLS_VERSION = "weaktlsversion" VULN_TLS_SERVER_KEY_REPLACEMENT = "serverkeyreplace" VULN_TLS_SUPERFISH_TRUSTED = "superfishca" + +VULN_CLEARTEXT_HTTP_PII = "httppii"