~mcfletch/+junk/restclient

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
#!/usr/local/bin/python
# 2005/08/04
#v1.0.1 

# upload_test.py
# A set of functions to test file uploading using urllib2.

# Copyright Michael Foord, 2004 & 2005.
# Released subject to the BSD License
# Please see http://www.voidspace.org.uk/documents/BSD-LICENSE.txt

# For information about bugfixes, updates and support, please join the
# Pythonutils mailing list.
# http://voidspace.org.uk/mailman/listinfo/pythonutils_voidspace.org.uk
# Comments, suggestions and bug reports welcome.
# Scripts maintained at http://www.voidspace.org.uk/python/index.shtml
# E-mail fuzzyman@voidspace.org.uk

########################################################################

"""
A function to encode files *and* normal fields into a multipart/form-data body for urllib2 to use.
This allows multiple file uploads via urllib2.

Based on http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
With inspiration from urllib2_file.py by Fabien Seisen, http://fabien.seisen.org/python/ (without his twiddly bits)

It actually uses my upload script located at http://www.voidspace.xennos.com
It has a maximum upload size - so you shouldn't be able to overflow my webspace !!
"""

import urllib2
import mimetypes
import mimetools

def encode_multipart_formdata(fields, files, BOUNDARY = '-----'+mimetools.choose_boundary()+'-----'):
    """ Encodes fields and files for uploading.
    fields is a sequence of (name, value) elements for regular form fields - or a dictionary.
    files is a sequence of (name, filename, value) elements for data to be uploaded as files.
    Return (content_type, body) ready for urllib2.Request instance
    You can optionally pass in a boundary string to use or we'll let mimetools provide one.
    """    
    CRLF = '\r\n'
    L = []
    if isinstance(fields, dict):
        fields = fields.items()
    for (key, value) in fields:
        L.append('--' + BOUNDARY)
        L.append('Content-Disposition: form-data; name="%s"' % key)
        L.append('')
        L.append(value)
    for (key, filename, value) in files:
        filetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
        L.append('--' + BOUNDARY)
        L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
#        if not filetype.startswith('text'):
#            L.append('Content-Transfer-Encoding: binary')
        L.append('Content-Type: %s' % filetype)
#        L.append('Content-Length: %s\r\n' % str(len(value)))
        L.append('')
        L.append(value)
    L.append('--' + BOUNDARY + '--')
    L.append('')
    body = CRLF.join(L)
    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY        # XXX what if no files are encoded
    return content_type, body

def build_request(theurl, fields, files, txheaders=None):
    """Given the fields to set and the files to encode it returns a fully formed urllib2.Request object.
    You can optionally pass in additional headers to encode into the opject. (Content-type and Content-length will be overridden if they are set).
    fields is a sequence of (name, value) elements for regular form fields - or a dictionary.
    files is a sequence of (name, filename, value) elements for data to be uploaded as files.    
    """
    content_type, body = encode_multipart_formdata(fields, files)
    if not txheaders: txheaders = {}
    txheaders['Content-type'] = content_type
    txheaders['Content-length'] = str(len(body))
    return urllib2.Request(theurl, body, txheaders)

########################################################################
# The test below assumes you have a script 'upload.py' located at 'theurl'
# It should be a CGI taking file.1, file.2 and file.3
# and uploading them into the 'uploadDir' which is relative to 'theurl'
# upload.py from http://www.voidspace.org.uk/atlantibots/recipebook.html should do this...

if __name__ == '__main__':
    theurl = 'http://www.voidspace.xennos.com/cgi-bin/upload.py'
    uploadDir = '../upload/'                                       # must end in '/' for urljoin to work
    fields = {}
    file1 = ('file.1', 'file1.txt', 'test'*100)
    file2 = ('file.2', 'file2.txt', 'PADDING'*100)
    file3 = ('file.3', 'file3.dat', ''.join([chr(num) for num in range(100)]))
    files = (file1, file2, file3)

    print urllib2.urlopen(build_request(theurl, fields, files)).read()

    from urlparse import urljoin
    uploadDirpath = urljoin(theurl, uploadDir)
    file1path = urljoin(uploadDirpath, file1[1])
    file2path = urljoin(uploadDirpath, file2[1])
    file3path = urljoin(uploadDirpath, file3[1])

    print
    print "Testing 'file1' == uploaded file : "
    filehandle = urllib2.urlopen(file1path)
    print filehandle.read() == file1[2]
    print
    print "Testing 'file2' == uploaded file : "
    filehandle = urllib2.urlopen(file2path)
    print filehandle.read() == file2[2]
    print
    print "Testing 'file3' == uploaded file : "
    filehandle = urllib2.urlopen(file3path)
    print filehandle.read() == file3[2]
    

    

"""
TODO/ISSUES
Will incorrectly set content-type as multipart/form-data even if no files are sent.
This should work but isn't the best way to do it.
Will this work with binary data or does it need base64 encoding ? - shouldn't do application/octet-stream can include binary. It doesn't work with Xitmai as localhost though, fails on chr(26) ??

CHANGELOG
Version 1.0.1       2005/08/04
Added theurl parameter to build_request.
    (Fix by Andrew Dalke)

Version 1.0.0       02-10-04
Works fine with the Linux host at www.xennos.com and my upload.py
"""