Changeset 40


Ignore:
Timestamp:
03/17/13 22:26:27 (11 years ago)
Author:
tmorgan
Message:

Switched to python3 only, since supporting both 2.x and 3.x is such a pain

Added preliminary python-requests support to http2py

Location:
trunk
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/SConstruct

    r26 r40  
    6565#   env.AddPostAction(libinstall, 'ldconfig')
    6666
    67 if sys.version_info[0] == 2:
    68    install_items.append('bletchley-python2.log')
    69    env.Command('bletchley-python2.log', Glob('lib/bletchley/*.py')+Glob('lib/bletchley/CBC/*.py'),
    70                "python bletchley-distutils install --root=/%s | tee bletchley-python2.log" % destdir)
     67#if sys.version_info[0] == 2:
     68#   install_items.append('bletchley-python2.log')
     69#   env.Command('bletchley-python2.log', Glob('lib/bletchley/*.py')+Glob('lib/bletchley/CBC/*.py'),
     70#               "python bletchley-distutils install --root=/%s | tee bletchley-python2.log" % destdir)
    7171
    7272python_path = os.popen('which python3').read()

  • trunk/bin/bletchley-analyze

    r35 r40  
    1 #!/usr/bin/env python
     1#!/usr/bin/env python3
    22
    3 # Requires Python 2.7+
     3# Requires Python 3+
    44
    55'''
    … …  
    5454    sys.exit(0)
    5555
    56 input_file = sys.stdin
     56input_file = sys.stdin.buffer
    5757if options.input_file is not None:
    58     input_file = file(options.input_file, 'rb')
     58    input_file = open(options.input_file, 'rb')
    5959
    60 blobs = input_file.read().rstrip('\n').replace('\r', '').split('\n')
     60blobs = input_file.read().rstrip(b'\n').replace(b'\r', b'').split(b'\n')
    6161#print(repr(blobs))
    6262
    … …  
    8181    line_size = 64
    8282    group_size *= 2 # two hex digits per byte
    83     hex_blobs = map(binascii.b2a_hex, blobs)
     83    hex_blobs = [binascii.b2a_hex(b) for b in blobs]
    8484    color_map = buffertools.blockWiseColorMap(group_size, hex_blobs)
    8585
    8686    for k in range(0,len(hex_blobs)):
    87         hex = hex_blobs[k]
     87        hex = hex_blobs[k].decode()
    8888       
    8989        for i in range(0,len(hex),line_size):
    … …  
    9898       
    9999            line += '| '
    100             line += repr(blobs[k][i/2:(i+line_size)/2])
     100            line += repr(blobs[k][i//2:(i+line_size)//2])
    101101       
    102102            print(line)
    … …  
    136136        encoding = specified_encodings.pop(0)
    137137
    138 
    139138    blobs_to_print = blobs[:options.output_lines]
    140139    print('First %d Values:' % len(blobs_to_print))

  • trunk/bin/bletchley-decode

    r28 r40  
    1 #!/usr/bin/env python
     1#!/usr/bin/env python3
    22
    3 # Requires Python 2.7+
     3# Requires Python 3+
    44
    55'''
    … …  
    4545    sys.exit(0)
    4646
    47 input_file = sys.stdin
     47input_file = sys.stdin.buffer
    4848if options.input_file is not None:
    49     input_file = file(options.input_file, 'rb')
     49    input_file = open(options.input_file, 'rb')
    5050
    51 blob = input_file.read()
     51blob = input_file.read().rstrip(b'\r\n')
    5252
    5353specified_encodings = options.encoding_chain.split(',')
    54 sys.stdout.write(blobtools.decodeChain(specified_encodings, blob))
     54sys.stdout.buffer.write(blobtools.decodeChain(specified_encodings, blob))

  • trunk/bin/bletchley-encode

    r28 r40  
    1 #!/usr/bin/env python
     1#!/usr/bin/env python3
    22
    3 # Requires Python 2.7+
     3# Requires Python 3+
    44
    55'''
    … …  
    4646    sys.exit(0)
    4747
    48 input_file = sys.stdin
     48input_file = sys.stdin.buffer
    4949if options.input_file is not None:
    5050    input_file = file(options.input_file, 'rb')
    … …  
    5555specified_encodings.reverse()
    5656# XXX: report invalid encodings
    57 sys.stdout.write(blobtools.encodeChain(specified_encodings, blob))
    58 sys.stdout.write('\n')
     57sys.stdout.buffer.write(blobtools.encodeChain(specified_encodings, blob))
     58sys.stdout.buffer.write(b'\n')

  • trunk/bin/bletchley-http2py

    r39 r40  
    1 #!/usr/bin/env python
    2 
    3 # Requires Python 2.7+
     1#!/usr/bin/env python3
     2
     3# Requires Python 3+
    44
    55'''
    6 This script reads a raw HTTP request from stdin and writes to stdout
    7 a Python script.  The generated script sends the same (or a very similar)
    8 request using the httplib/http.client libraries.
     6This script reads a raw HTTP request and writes to stdout a Python
     7script.  The generated script sends the same (or a very similar)
     8request using the standard httplib/http.client library, or optionally
     9using the more user friendly python-requests library.
    910
    1011Certainly if you have a raw request, you could simply send it via TCP
    … …  
    3435import sys
    3536import argparse
    36 try:
    37     from lxml import etree
    38 except:
    39     sys.stderr.write('ERROR: Could not import lxml module.  Ensure it is installed.\n')
    40     sys.stderr.write('       Under Debian, the package name is "python-lxml"\n.')
    41     sys.exit(1)
    4237
    4338parser = argparse.ArgumentParser(
    … …  
    4843    ' For more information, see: http://code.google.com/p/bletchley/wiki/Overview')
    4944parser.add_argument(
    50     'requestfile', type=file, nargs='?', default=sys.stdin,
     45    'requestfile', type=open, nargs='?', default=sys.stdin,
    5146    help='A file containing an HTTP request.  Defaults to stdin if omitted.')
    5247parser.add_argument(
    53     '--burp', action='store_true', help='Input file is a XML export from Burp.'
    54     ' (First request in file is used.)')
     48    '--requests', action='store_true', help='Generate a script that uses the'
     49    ' python-requests module rather than httplib/http.client (experimental).')
     50
    5551args = parser.parse_args()
    56 
    57 if args.burp:
    58     safe_parser = etree.ETCompatXMLParser(resolve_entities=False)
    59     root = etree.parse(args.requestfile, parser=safe_parser)
    60     input_req = root.xpath('/items/item/request')[0].text
    61     root = None
    62 else:
    63     input_req = args.requestfile.read()
    64 
     52input_req = args.requestfile.read()
    6553
    6654
    … …  
    7967port = 80
    8068use_ssl = False
     69protocol = 'http'
    8170
    8271headers = []
    … …  
    8574        break
    8675    # Handle header line continuations
    87     if l[0] == '\t':
     76    if l[0] in ' \t':
    8877        if len(headers) == 0:
    8978            continue
    … …  
    10897            if port == 443:
    10998                use_ssl = True
     99                protocol = 'https'
    110100        else:
    111101            host = value
    112102
    113103
    114 print('''#!/usr/bin/env python
     104formatted_body = '\n            '.join([repr(body[i:i+40].encode()) for i in range(0,len(body),40)])
     105if formatted_body == '':
     106    formatted_body = "b''"
     107
     108
     109if args.requests:
     110    print('''#!/usr/bin/env python3
    115111
    116112import sys
    117 # function with either Python 2.7 or 3.x
    118113try:
    119     import http.client as httpc
     114    import requests
    120115except:
    121     import httplib as httpc
     116    sys.stderr.write('ERROR: Could not import requests module.  Ensure it is installed.\\n')
     117    sys.stderr.write('       Under Debian, the package name is "python3-requests"\\n.')
     118    sys.exit(1)
     119
     120
     121# TODO: ensure the host, port, and SSL settings are correct.
     122host = %s
     123port = %s
     124protocol = %s
     125''' % (repr(host),repr(port),repr(protocol)))
     126
     127    headers = dict(headers)
     128    # XXX: We don't currently support exactly formatted header
     129    #      continuations with python requests, but this should be
     130    #      semantically equivalent.
     131    for h in headers.keys():
     132        headers[h] = ' '.join(headers[h])
     133
     134    print('''
     135session = requests.Session()
     136# TODO: use "data" to supply any parameters to be included in the request
     137def sendRequest(session, data=None):
     138    method = %s
     139    path = %s
     140    headers = %s
     141    url = "%%s://%%s:%%d%%s" %% (protocol,host,port,path)
     142    body = (%s)
     143
     144    return session.request(method, url, headers=headers, data=body)
     145    ''' % (repr(method), repr(path), repr(headers), formatted_body))
     146
     147    print('''   
     148
     149def fetch(data):
     150    global session
     151    ret_val = None
     152
     153    # TODO: customize code here to retrieve what you need from the response(s)
     154    # For information on the response object's interface, see:
     155    #   http://docs.python-requests.org/en/latest/api/#requests.Response
     156    response = sendRequest(session, data)
     157    print(response.headers)
     158    print(repr(response.content))
     159
     160    return ret_val
     161
     162data = ''
     163fetch(data)
    122164''')
    123165
    124166
    125 print('''
     167
     168else:
     169    print('''#!/usr/bin/env python3
     170
     171import sys
     172import http.client as httpc
     173
     174
    126175# TODO: ensure the host, port, and SSL settings are correct.
    127176host = %s
    … …  
    130179''' % (repr(host),repr(port),repr(use_ssl)))
    131180
    132 chunked_body = '\n            '.join([repr(body[i:i+40]) for i in range(0,len(body),40)])
    133 if chunked_body == '':
    134     chunked_body = "''"
    135 
    136 print('''
     181    print('''
    137182# TODO: use "data" to supply any parameters to be included in the request
    138183def sendRequest(connection, data=None):
    … …  
    142187   
    143188    connection.putrequest(method, path)
    144     ''' % (repr(method), repr(path), chunked_body))
    145 
    146 for name,values in headers:
    147     if len(values) > 1:
    148         continuations = ','.join([repr(v) for v in values[1:]])
    149         print('''    connection.putheader(%s, %s, %s)''' % (repr(name),repr(values[0]),continuations))
    150     else:
    151         print('''    connection.putheader(%s, %s)''' % (repr(name),repr(values[0])))
    152 
    153 print('''   
     189    ''' % (repr(method), repr(path), formatted_body))
     190
     191    for name,values in headers:
     192        if len(values) > 1:
     193            continuations = ','.join([repr(v) for v in values[1:]])
     194            print('''    connection.putheader(%s, %s, %s)''' % (repr(name),repr(values[0]),continuations))
     195        else:
     196            print('''    connection.putheader(%s, %s)''' % (repr(name),repr(values[0])))
     197
     198    print('''   
    154199    if len(body) > 0:
    155200        connection.putheader('Content-Length', len(body))
    … …  
    161206
    162207def newConnection():
    163     connection = None
    164208    if use_ssl:
    165209        return httpc.HTTPSConnection(host, port)

  • trunk/lib/bletchley/blobtools.py

    r35 r40  
    2222import base64
    2323import binascii
    24 import urllib
    2524import fractions
    2625import operator
    2726import functools
    2827import itertools
    29 import buffertools
     28from . import buffertools
     29
     30
     31# urllib.parse's functions are not well suited for encoding/decoding
     32# bytes or managing encoded case
     33def _percentEncode(binary, plus=False, upper=True):
     34    fmt = "%%%.2X"
     35    if upper:
     36        fmt = "%%%.2x"
     37
     38    ret_val = b''
     39    for c in binary:
     40        if c not in b'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789':
     41            ret_val += (fmt % c).encode('ascii')
     42        elif plus and (c == 20):
     43            ret_val += b'+'
     44        else:
     45            ret_val += c
     46   
     47    return ret_val
     48
     49
     50def _percentDecode(binary, plus=False):
     51    ret_val = b''
     52    if plus:
     53        binary = binary.replace(b'+', b' ')
     54    if binary == b'':
     55        return b''
     56    chunks = binary.split(b'%')
     57    if binary[0] == 0x25:
     58        chunks = chunks[1:]
     59
     60    for chunk in chunks:
     61        if len(chunk) < 2:
     62            return None
     63        try:
     64            ret_val += bytes([int(chunk[0:2], 16)]) + chunk[2:]
     65        except:
     66            print(repr(chunk))
     67            return None
     68           
     69    return ret_val
     70
    3071
    3172# abstract class
    3273class DataEncoding(object):
    33     charset = frozenset('')
    34     extraneous_chars = ''
     74    charset = frozenset(b'')
     75    extraneous_chars = b''
    3576    dialect = None
    3677    name = None
    … …  
    65106        super(base64Encoding, self).__init__(dialect)
    66107        if dialect.startswith('rfc3548'):
    67             self.c62 = '+'
    68             self.c63 = '/'
    69             self.pad = '='
     108            self.c62 = b'+'
     109            self.c63 = b'/'
     110            self.pad = b'='
    70111        elif dialect.startswith('filename'):
    71             self.c62 = '+'
    72             self.c63 = '-'
    73             self.pad = '='
     112            self.c62 = b'+'
     113            self.c63 = b'-'
     114            self.pad = b'='
    74115        elif dialect.startswith('url1'):
    75             self.c62 = '-'
    76             self.c63 = '_'
    77             self.pad = '='
     116            self.c62 = b'-'
     117            self.c63 = b'_'
     118            self.pad = b'='
    78119        elif dialect.startswith('url2'):
    79             self.c62 = '-'
    80             self.c63 = '_'
    81             self.pad = '.'
     120            self.c62 = b'-'
     121            self.c63 = b'_'
     122            self.pad = b'.'
    82123        elif dialect.startswith('url3'):
    83             self.c62 = '_'
    84             self.c63 = '-'
    85             self.pad = '.'
     124            self.c62 = b'_'
     125            self.c63 = b'-'
     126            self.pad = b'.'
    86127        elif dialect.startswith('url4'):
    87             self.c62 = '-'
    88             self.c63 = '_'
    89             self.pad = '!'
     128            self.c62 = b'-'
     129            self.c63 = b'_'
     130            self.pad = b'!'
    90131        elif dialect.startswith('url5'):
    91             self.c62 = '+'
    92             self.c63 = '/'
    93             self.pad = '$'
     132            self.c62 = b'+'
     133            self.c63 = b'/'
     134            self.pad = b'$'
    94135        elif dialect.startswith('otkurl'):
    95             self.c62 = '-'
    96             self.c63 = '_'
    97             self.pad = '*'
     136            self.c62 = b'-'
     137            self.c63 = b'_'
     138            self.pad = b'*'
    98139        elif dialect.startswith('xmlnmtoken'):
    99             self.c62 = '.'
    100             self.c63 = '-'
    101             self.pad = '='
     140            self.c62 = b'.'
     141            self.c63 = b'-'
     142            self.pad = b'='
    102143        elif dialect.startswith('xmlname'):
    103             self.c62 = '_'
    104             self.c63 = ':'
    105             self.pad = '='
     144            self.c62 = b'_'
     145            self.c63 = b':'
     146            self.pad = b'='
    106147       
    107148        if 'newline' in dialect:
    108             self.extraneous_chars = '\r\n'
    109 
    110         self.charset = frozenset('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
    111                                  +'abcdefghijklmnopqrstuvwxyz0123456789'
     149            self.extraneous_chars = b'\r\n'
     150
     151        self.charset = frozenset(b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
     152                                 +b'abcdefghijklmnopqrstuvwxyz0123456789'
    112153                                 +self.c62+self.c63+self.pad+self.extraneous_chars)
    113154
    … …  
    120161    def extraTests(self, blob):
    121162        for c in self.extraneous_chars:
    122             blob = blob.replace(c, '')
     163            blob = blob.replace(bytes([c]), b'')
    123164
    124165        nopad = blob.rstrip(self.pad)
    … …  
    137178    def decode(self, blob):
    138179        for c in self.extraneous_chars:
    139             blob = blob.replace(c, '')
     180            blob = blob.replace(bytes(c), b'')
    140181
    141182        if self.dialect.endswith('nopad'):
    … …  
    150191
    151192        if not self.dialect.startswith('rfc3548'):
    152             table = string.maketrans(self.c62+self.c63+self.pad, '+/=')
     193            table = string.maketrans(self.c62+self.c63+self.pad, b'+/=')
    153194            blob = blob.translate(table)
    154195
    … …  
    160201
    161202        if not self.dialect.startswith('rfc3548'):
    162             table = string.maketrans('+/=', self.c62+self.c63+self.pad)
     203            table = string.maketrans(b'+/=', self.c62+self.c63+self.pad)
    163204            ret_val = ret_val.translate(table)
    164205
    … …  
    174215        super(base32Encoding, self).__init__(dialect)
    175216        if dialect.startswith('rfc3548upper'):
    176             self.pad = '='
    177             self.charset = frozenset('ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'+self.pad)
     217            self.pad = b'='
     218            self.charset = frozenset(b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'+self.pad)
    178219
    179220        elif dialect.startswith('rfc3548lower'):
    180             self.pad = '='
    181             self.charset = frozenset('abcdefghijklmnopqrstuvwxyz234567'+self.pad)
     221            self.pad = b'='
     222            self.charset = frozenset(b'abcdefghijklmnopqrstuvwxyz234567'+self.pad)
    182223
    183224    def _guessPadLength(self, nopad_len):
    … …  
    203244        if self.dialect.endswith('nopad'):
    204245            if self.pad in blob:
    205                 raise Exception("Unpadded base64 string contains pad character")
     246                raise Exception("Unpadded base32 string contains pad character")
    206247
    207248            padlen = self._guessPadLength(len(blob))
    … …  
    233274        super(hexEncoding, self).__init__(dialect)
    234275        if 'mixed' in dialect:
    235             self.charset = frozenset('ABCDEFabcdef0123456789')
     276            self.charset = frozenset(b'ABCDEFabcdef0123456789')
    236277        elif 'upper' in dialect:
    237             self.charset = frozenset('ABCDEF0123456789')           
     278            self.charset = frozenset(b'ABCDEF0123456789')           
    238279        elif 'lower' in dialect:
    239             self.charset = frozenset('abcdef0123456789')
     280            self.charset = frozenset(b'abcdef0123456789')
    240281
    241282
    … …  
    261302        self.charset = None
    262303        if 'mixed' in dialect:
    263             self.hexchars = frozenset('ABCDEFabcdef0123456789')
     304            self.hexchars = frozenset(b'ABCDEFabcdef0123456789')
    264305        elif 'upper' in dialect:
    265             self.hexchars = frozenset('ABCDEF0123456789')           
     306            self.hexchars = frozenset(b'ABCDEF0123456789')           
    266307        elif 'lower' in dialect:
    267             self.hexchars = frozenset('abcdef0123456789')
     308            self.hexchars = frozenset(b'abcdef0123456789')
    268309
    269310    def extraTests(self, blob):
    270         chunks = blob.split('%')
     311        chunks = blob.split(b'%')
    271312        if len(chunks) < 2:
    272313            return None
    … …  
    279320
    280321    def decode(self, blob):
     322        plus = False
    281323        if 'plus' in self.dialect:
    282             return urllib.unquote(blob)
    283         else:
    284             return urllib.unquote_plus(blob)
    285 
    286     # XXX: should technically produce quoted digits in same upper/lower case
     324            plus = True
     325        return _percentDecode(blob, plus=plus)
     326
    287327    def encode(self, blob):
     328        upper = True
     329        plus = False
    288330        if 'plus' in self.dialect:
    289             return urllib.quote(blob, '')
    290         else:
    291             return urllib.quote_plus(blob, '')
     331            plus = True
     332        if 'lower' in self.dialect:
     333            upper = False
     334
     335        return _percentEncode(blob, plus=plus, upper=upper)
    292336
    293337
    … …  
    336380
    337381def supportedEncodings():
    338     e = encodings.keys()
     382    e = list(encodings.keys())
    339383    e.sort()
    340384    return e
    … …  
    380424
    381425def decodeAll(encoding, blobs):
    382     return map(encodings[encoding].decode, blobs)
     426    return [encodings[encoding].decode(b) for b in blobs]
    383427
    384428def encodeAll(encoding, blobs):
    385     return map(encodings[encoding].encode, blobs)
     429    return [encodings[encoding].encode(b) for b in blobs]
    386430
    387431def decodeChain(decoding_chain, blob):
    … …  
    412456
    413457
    414 allTrue = functools.partial(reduce, (lambda x,y: x and y))
     458allTrue = functools.partial(functools.reduce, (lambda x,y: x and y))
    415459
    416460def checkCommonBlocksizes(lengths):

  • trunk/lib/bletchley/buffertools.py

    r36 r40  
    7575
    7676    colors = {}
    77     for block,count in block_counts.iteritems():
     77    for block,count in block_counts.items():
    7878        if count == 1:
    7979            # mask needed for portability
Note: See TracChangeset for help on using the changeset viewer.