~ubuntu-core-dev/eucalyptus/ubuntu

444.1.88 by Neil
updated license headers.
1
/*******************************************************************************
444.1.110 by Neil
fixed POST uploads (chunked and unchunked).
2
 *Copyright (c) 2009  Eucalyptus Systems, Inc.
3
 * 
4
 *  This program is free software: you can redistribute it and/or modify
5
 *  it under the terms of the GNU General Public License as published by
6
 *  the Free Software Foundation, only version 3 of the License.
7
 * 
8
 * 
9
 *  This file is distributed in the hope that it will be useful, but WITHOUT
10
 *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11
 *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12
 *  for more details.
13
 * 
14
 *  You should have received a copy of the GNU General Public License along
15
 *  with this program.  If not, see <http://www.gnu.org/licenses/>.
16
 * 
17
 *  Please contact Eucalyptus Systems, Inc., 130 Castilian
18
 *  Dr., Goleta, CA 93101 USA or visit <http://www.eucalyptus.com/licenses/>
19
 *  if you need additional information or have any questions.
20
 * 
21
 *  This file may incorporate work covered under the following copyright and
22
 *  permission notice:
23
 * 
24
 *    Software License Agreement (BSD License)
25
 * 
26
 *    Copyright (c) 2008, Regents of the University of California
27
 *    All rights reserved.
28
 * 
29
 *    Redistribution and use of this software in source and binary forms, with
30
 *    or without modification, are permitted provided that the following
31
 *    conditions are met:
32
 * 
33
 *      Redistributions of source code must retain the above copyright notice,
34
 *      this list of conditions and the following disclaimer.
35
 * 
36
 *      Redistributions in binary form must reproduce the above copyright
37
 *      notice, this list of conditions and the following disclaimer in the
38
 *      documentation and/or other materials provided with the distribution.
39
 * 
40
 *    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
41
 *    IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
42
 *    TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
43
 *    PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
44
 *    OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
45
 *    EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
46
 *    PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
47
 *    PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
48
 *    LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
49
 *    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
50
 *    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. USERS OF
51
 *    THIS SOFTWARE ACKNOWLEDGE THE POSSIBLE PRESENCE OF OTHER OPEN SOURCE
52
 *    LICENSED MATERIAL, COPYRIGHTED MATERIAL OR PATENTED MATERIAL IN THIS
53
 *    SOFTWARE, AND IF ANY SUCH MATERIAL IS DISCOVERED THE PARTY DISCOVERING
54
 *    IT MAY INFORM DR. RICH WOLSKI AT THE UNIVERSITY OF CALIFORNIA, SANTA
55
 *    BARBARA WHO WILL THEN ASCERTAIN THE MOST APPROPRIATE REMEDY, WHICH IN
56
 *    THE REGENTS’ DISCRETION MAY INCLUDE, WITHOUT LIMITATION, REPLACEMENT
57
 *    OF THE CODE SO IDENTIFIED, LICENSING OF THE CODE SO IDENTIFIED, OR
58
 *    WITHDRAWAL OF THE CODE CAPABILITY TO THE EXTENT NEEDED TO COMPLY WITH
59
 *    ANY SUCH LICENSES OR RIGHTS.
60
 *******************************************************************************/
286.1.2 by decker
adding more stuff
61
package com.eucalyptus.ws.handlers;
62
63
import java.io.ByteArrayOutputStream;
444.47.13 by Neil
correctly URL decode object key.
64
import java.io.UnsupportedEncodingException;
286.1.2 by decker
adding more stuff
65
import java.lang.reflect.Field;
66
import java.lang.reflect.Modifier;
67
import java.lang.reflect.ParameterizedType;
444.16.154 by Neil
fixes #490623
68
import java.text.ParseException;
286.1.2 by decker
adding more stuff
69
import java.util.ArrayList;
70
import java.util.Date;
71
import java.util.HashMap;
72
import java.util.Iterator;
73
import java.util.List;
74
import java.util.Map;
75
import java.util.Set;
444.27.9 by Neil
update to logging.
76
import java.util.UUID;
286.1.2 by decker
adding more stuff
77
78
import net.sf.json.JSONArray;
79
import net.sf.json.JSONObject;
80
81
import org.apache.axiom.om.OMElement;
444.46.6 by Neil
fixed concurrent puts for the same key.
82
import org.apache.commons.httpclient.util.DateUtil;
321 by Neil
fixed get.
83
import org.apache.log4j.Logger;
84
import org.apache.tools.ant.util.DateUtils;
85
import org.apache.xml.dtm.ref.DTMNodeList;
286.1.2 by decker
adding more stuff
86
import org.jboss.netty.buffer.ChannelBuffer;
87
import org.jboss.netty.buffer.ChannelBuffers;
444.27.11 by Neil
fixed state cleanup on interrupted puts (can cause memory leak over an extended period of time).
88
import org.jboss.netty.channel.ChannelEvent;
286.1.2 by decker
adding more stuff
89
import org.jboss.netty.channel.ChannelHandlerContext;
444.27.11 by Neil
fixed state cleanup on interrupted puts (can cause memory leak over an extended period of time).
90
import org.jboss.netty.channel.Channels;
444.1.50 by Neil
fix for clients that expect 100-continue.
91
import org.jboss.netty.channel.DownstreamMessageEvent;
286.1.2 by decker
adding more stuff
92
import org.jboss.netty.channel.MessageEvent;
444.1.50 by Neil
fix for clients that expect 100-continue.
93
import org.jboss.netty.handler.codec.http.DefaultHttpResponse;
302 by Neil
saving changes.
94
import org.jboss.netty.handler.codec.http.HttpChunk;
286.1.2 by decker
adding more stuff
95
import org.jboss.netty.handler.codec.http.HttpHeaders;
444.1.50 by Neil
fix for clients that expect 100-continue.
96
import org.jboss.netty.handler.codec.http.HttpResponse;
325 by Neil
fixed delete headers, post outbound headers.
97
import org.jboss.netty.handler.codec.http.HttpResponseStatus;
444.1.50 by Neil
fix for clients that expect 100-continue.
98
import org.jboss.netty.handler.codec.http.HttpVersion;
286.1.2 by decker
adding more stuff
99
444.1.50 by Neil
fix for clients that expect 100-continue.
100
import com.eucalyptus.auth.User;
444.1.144 by decker
fix database security, fix walrus url in local mode, quick fix for node cert checking, clean up entities to inherit common parent
101
import com.eucalyptus.auth.util.Hashes;
444.1.329 by Neil
added import script for 1.5.0, changed lock.
102
import com.eucalyptus.util.HoldMe;
444.27.11 by Neil
fixed state cleanup on interrupted puts (can cause memory leak over an extended period of time).
103
import com.eucalyptus.util.LogUtil;
444.1.50 by Neil
fix for clients that expect 100-continue.
104
import com.eucalyptus.util.StorageProperties;
105
import com.eucalyptus.util.WalrusProperties;
444.47.13 by Neil
correctly URL decode object key.
106
import com.eucalyptus.util.WalrusUtil;
286.1.2 by decker
adding more stuff
107
import com.eucalyptus.ws.BindingException;
444.1.34 by Neil
fixed form policy verification.
108
import com.eucalyptus.ws.InvalidOperationException;
286.1.2 by decker
adding more stuff
109
import com.eucalyptus.ws.MappingHttpRequest;
110
import com.eucalyptus.ws.MappingHttpResponse;
111
import com.eucalyptus.ws.binding.Binding;
112
import com.eucalyptus.ws.binding.BindingManager;
444.27.9 by Neil
update to logging.
113
import com.eucalyptus.ws.util.WalrusBucketLogger;
286.1.2 by decker
adding more stuff
114
import com.eucalyptus.ws.util.XMLParser;
115
import com.google.common.collect.Lists;
116
117
import edu.ucsb.eucalyptus.annotation.HttpEmbedded;
118
import edu.ucsb.eucalyptus.annotation.HttpParameterMapping;
444.27.9 by Neil
update to logging.
119
import edu.ucsb.eucalyptus.cloud.BucketLogData;
286.1.2 by decker
adding more stuff
120
import edu.ucsb.eucalyptus.msgs.AccessControlListType;
121
import edu.ucsb.eucalyptus.msgs.AccessControlPolicyType;
122
import edu.ucsb.eucalyptus.msgs.CanonicalUserType;
319 by Neil
saving.
123
import edu.ucsb.eucalyptus.msgs.EucalyptusErrorMessageType;
286.1.2 by decker
adding more stuff
124
import edu.ucsb.eucalyptus.msgs.EucalyptusMessage;
125
import edu.ucsb.eucalyptus.msgs.Grant;
126
import edu.ucsb.eucalyptus.msgs.Grantee;
127
import edu.ucsb.eucalyptus.msgs.Group;
444.27.3 by Neil
few updates to logging and fixes to group perms.
128
import edu.ucsb.eucalyptus.msgs.LoggingEnabled;
286.1.2 by decker
adding more stuff
129
import edu.ucsb.eucalyptus.msgs.MetaDataEntry;
444.27.3 by Neil
few updates to logging and fixes to group perms.
130
import edu.ucsb.eucalyptus.msgs.TargetGrants;
323 by Neil
added support for compressed transfers.
131
import edu.ucsb.eucalyptus.msgs.WalrusDataGetRequestType;
444.1.50 by Neil
fix for clients that expect 100-continue.
132
import edu.ucsb.eucalyptus.msgs.WalrusDataRequestType;
444.27.9 by Neil
update to logging.
133
import edu.ucsb.eucalyptus.msgs.WalrusRequestType;
444.46.6 by Neil
fixed concurrent puts for the same key.
134
import edu.ucsb.eucalyptus.msgs.PutObjectResponseType;
321 by Neil
fixed get.
135
import edu.ucsb.eucalyptus.util.WalrusDataMessage;
136
import edu.ucsb.eucalyptus.util.WalrusDataMessenger;
444.46.6 by Neil
fixed concurrent puts for the same key.
137
import edu.ucsb.eucalyptus.util.WalrusDataQueue;
286.1.2 by decker
adding more stuff
138
import groovy.lang.GroovyObject;
139
140
public class WalrusRESTBinding extends RestfulMarshallingHandler {
141
	private static Logger LOG = Logger.getLogger( WalrusRESTBinding.class );
142
	private static final String SERVICE = "service";
143
	private static final String BUCKET = "bucket";
144
	private static final String OBJECT = "object";
145
	private static final Map<String, String> operationMap = populateOperationMap();
302 by Neil
saving changes.
146
	private static WalrusDataMessenger putMessenger;
286.1.2 by decker
adding more stuff
147
	public static final int DATA_MESSAGE_SIZE = 102400;
444.27.11 by Neil
fixed state cleanup on interrupted puts (can cause memory leak over an extended period of time).
148
	private String key;
149
	private String randomKey;
444.46.6 by Neil
fixed concurrent puts for the same key.
150
	private WalrusDataQueue<WalrusDataMessage> putQueue;
319 by Neil
saving.
151
769 by Dustin Kirkland
clc/modules/wsstack/src/main/java/com/eucalyptus/ws/handlers/WalrusRESTBinding.java:
152
	@Override
444.27.11 by Neil
fixed state cleanup on interrupted puts (can cause memory leak over an extended period of time).
153
	public void handleUpstream( final ChannelHandlerContext channelHandlerContext, final ChannelEvent channelEvent ) throws Exception {
154
		LOG.trace( LogUtil.dumpObject( channelEvent ) );
155
		if ( channelEvent instanceof MessageEvent ) {
156
			final MessageEvent msgEvent = ( MessageEvent ) channelEvent;
157
			try {
158
				this.incomingMessage( channelHandlerContext, msgEvent );
159
			} catch ( Throwable e ) {
160
				LOG.error( e, e );
161
				Channels.fireExceptionCaught( channelHandlerContext, e );
162
				return;
163
			} 
164
		} else if (channelEvent.toString().contains("DISCONNECTED") || 
165
				channelEvent.toString().contains("CLOSED")) {
166
			if(key != null && randomKey != null) {
167
				putMessenger.removeQueue(key, randomKey);
168
				putQueue = null;
169
			}
170
		}
171
		channelHandlerContext.sendUpstream( channelEvent );
172
	}
173
174
	@Override
286.1.2 by decker
adding more stuff
175
	public void incomingMessage( ChannelHandlerContext ctx, MessageEvent event ) throws Exception {
176
		if ( event.getMessage( ) instanceof MappingHttpRequest ) {
177
			MappingHttpRequest httpRequest = ( MappingHttpRequest ) event.getMessage( );
178
			namespace = "http://s3.amazonaws.com/doc/" + WalrusProperties.NAMESPACE_VERSION;
179
			// TODO: get real user data here too
304 by Neil
saving changes.
180
			EucalyptusMessage msg = (EucalyptusMessage) this.bind( "admin", true, httpRequest );
181
			httpRequest.setMessage( msg );
323 by Neil
added support for compressed transfers.
182
			if(msg instanceof WalrusDataGetRequestType) {
183
				WalrusDataGetRequestType getObject = (WalrusDataGetRequestType) msg;
321 by Neil
fixed get.
184
				getObject.setChannel(ctx.getChannel());
317 by Neil
saving changes.
185
			}
444.1.50 by Neil
fix for clients that expect 100-continue.
186
			if(msg instanceof WalrusDataRequestType) {
187
				String expect = httpRequest.getHeader(HttpHeaders.Names.EXPECT);
188
				if(expect != null) {
189
					if(expect.equals("100-continue")) {
190
						HttpResponse response = new DefaultHttpResponse( HttpVersion.HTTP_1_1, HttpResponseStatus.CONTINUE );
191
						DownstreamMessageEvent newEvent = new DownstreamMessageEvent( ctx.getChannel( ), event.getFuture(), response, null );
192
						ctx.sendDownstream( newEvent );
193
					}
194
				}
195
			}
302 by Neil
saving changes.
196
		} else if(event.getMessage() instanceof HttpChunk) {
444.1.110 by Neil
fixed POST uploads (chunked and unchunked).
197
			if(putQueue != null) {
198
				HttpChunk httpChunk = (HttpChunk) event.getMessage();
199
				handleHttpChunk(httpChunk);
200
			}
286.1.2 by decker
adding more stuff
201
		}
202
	}
203
204
	@Override
205
	public void outgoingMessage( ChannelHandlerContext ctx, MessageEvent event ) throws Exception {
206
		if ( event.getMessage( ) instanceof MappingHttpResponse ) {
207
			MappingHttpResponse httpResponse = ( MappingHttpResponse ) event.getMessage( );
304 by Neil
saving changes.
208
			EucalyptusMessage msg = (EucalyptusMessage) httpResponse.getMessage( );
319 by Neil
saving.
209
			Binding binding;
210
			if(!(msg instanceof EucalyptusErrorMessageType)) {
444.46.6 by Neil
fixed concurrent puts for the same key.
211
				if(msg instanceof PutObjectResponseType) {
212
					if(putQueue != null) {
213
						putQueue = null;
214
					}
215
				}
319 by Neil
saving.
216
				binding = BindingManager.getBinding( BindingManager.sanitizeNamespace( namespace ) );
304 by Neil
saving changes.
217
			} else {
319 by Neil
saving.
218
				binding = BindingManager.getBinding( BindingManager.sanitizeNamespace( "http://msgs.eucalyptus.ucsb.edu" ) );
444.1.284 by Neil
better handling of interrupted/failed puts.
219
				if(putQueue != null) {
220
					putQueue = null;
221
				}
304 by Neil
saving changes.
222
			}
321 by Neil
fixed get.
223
			if(msg != null) {
333 by Neil
more fixes to post. chris plz make EucalyptusQueryPipeline idempotent k kthx
224
				OMElement omMsg = binding.toOM( msg );
321 by Neil
fixed get.
225
				ByteArrayOutputStream byteOut = new ByteArrayOutputStream();
444.1.329 by Neil
added import script for 1.5.0, changed lock.
226
				HoldMe.canHas.lock(); 
227
				try {
228
					omMsg.serialize( byteOut );
229
				} finally {
230
					HoldMe.canHas.unlock();
444.1.136 by Neil
disable snapshots if not on the same host and walrus config is empty.
231
				}
321 by Neil
fixed get.
232
				byte[] req = byteOut.toByteArray();
233
				ChannelBuffer buffer = ChannelBuffers.copiedBuffer( req );
333 by Neil
more fixes to post. chris plz make EucalyptusQueryPipeline idempotent k kthx
234
				httpResponse.addHeader( HttpHeaders.Names.CONTENT_LENGTH, String.valueOf(buffer.readableBytes() ) );
235
				httpResponse.addHeader( HttpHeaders.Names.CONTENT_TYPE, "application/xml" );
236
				httpResponse.setContent( buffer );
321 by Neil
fixed get.
237
			}
286.1.2 by decker
adding more stuff
238
		}
239
	}
240
241
	private static Map<String, String> populateOperationMap() {
242
		Map<String, String> newMap = new HashMap<String, String>();
243
		//Service operations
244
		newMap.put(SERVICE + WalrusProperties.HTTPVerb.GET.toString(), "ListAllMyBuckets");
245
246
		//Bucket operations
247
		newMap.put(BUCKET + WalrusProperties.HTTPVerb.GET.toString() + WalrusProperties.OperationParameter.acl.toString(), "GetBucketAccessControlPolicy");
355 by Neil
access control list or access control policy? umm i dunno. pick one randomly gaiz. amidoingitrite? sigh.
248
		newMap.put(BUCKET + WalrusProperties.HTTPVerb.PUT.toString() + WalrusProperties.OperationParameter.acl.toString(), "SetRESTBucketAccessControlPolicy");
286.1.2 by decker
adding more stuff
249
250
		newMap.put(BUCKET + WalrusProperties.HTTPVerb.GET.toString(), "ListBucket");
251
		newMap.put(BUCKET + WalrusProperties.HTTPVerb.GET.toString() + WalrusProperties.OperationParameter.prefix.toString(), "ListBucket");
252
		newMap.put(BUCKET + WalrusProperties.HTTPVerb.GET.toString() + WalrusProperties.OperationParameter.maxkeys.toString(), "ListBucket");
253
		newMap.put(BUCKET + WalrusProperties.HTTPVerb.GET.toString() + WalrusProperties.OperationParameter.marker.toString(), "ListBucket");
254
		newMap.put(BUCKET + WalrusProperties.HTTPVerb.GET.toString() + WalrusProperties.OperationParameter.delimiter.toString(), "ListBucket");
255
		newMap.put(BUCKET + WalrusProperties.HTTPVerb.PUT.toString(), "CreateBucket");
256
		newMap.put(BUCKET + WalrusProperties.HTTPVerb.DELETE.toString(), "DeleteBucket");
257
258
		newMap.put(BUCKET + WalrusProperties.HTTPVerb.GET.toString() + WalrusProperties.OperationParameter.location.toString(), "GetBucketLocation");
259
		newMap.put(BUCKET + WalrusProperties.HTTPVerb.GET.toString() + WalrusProperties.OperationParameter.logging.toString(), "GetBucketLoggingStatus");
260
		newMap.put(BUCKET + WalrusProperties.HTTPVerb.PUT.toString() + WalrusProperties.OperationParameter.logging.toString(), "SetBucketLoggingStatus");
261
262
		//Object operations
263
		newMap.put(OBJECT + WalrusProperties.HTTPVerb.GET.toString() + WalrusProperties.OperationParameter.acl.toString(), "GetObjectAccessControlPolicy");
355 by Neil
access control list or access control policy? umm i dunno. pick one randomly gaiz. amidoingitrite? sigh.
264
		newMap.put(OBJECT + WalrusProperties.HTTPVerb.PUT.toString() + WalrusProperties.OperationParameter.acl.toString(), "SetRESTObjectAccessControlPolicy");
286.1.2 by decker
adding more stuff
265
266
		newMap.put(BUCKET + WalrusProperties.HTTPVerb.POST.toString(), "PostObject");
267
268
		newMap.put(OBJECT + WalrusProperties.HTTPVerb.PUT.toString(), "PutObject");
269
		newMap.put(OBJECT + WalrusProperties.HTTPVerb.PUT.toString() + WalrusProperties.COPY_SOURCE.toString(), "CopyObject");
270
		newMap.put(OBJECT + WalrusProperties.HTTPVerb.GET.toString(), "GetObject");
271
		newMap.put(OBJECT + WalrusProperties.HTTPVerb.GET.toString() + WalrusProperties.OperationParameter.torrent.toString(), "GetObject");
272
		newMap.put(OBJECT + WalrusProperties.HTTPVerb.DELETE.toString(), "DeleteObject");
273
274
		newMap.put(OBJECT + WalrusProperties.HTTPVerb.HEAD.toString(), "GetObject");
275
		newMap.put(OBJECT + WalrusProperties.HTTPVerb.GET.toString() + "extended", "GetObjectExtended");
276
277
		return newMap;
278
	}
279
280
281
	@Override
444.1.34 by Neil
fixed form policy verification.
282
	public Object bind( final String userId, final boolean admin, final MappingHttpRequest httpRequest ) throws Exception {
286.1.2 by decker
adding more stuff
283
		String servicePath = httpRequest.getServicePath();
284
		Map bindingArguments = new HashMap();
285
		final String operationName = getOperation(httpRequest, bindingArguments);
333 by Neil
more fixes to post. chris plz make EucalyptusQueryPipeline idempotent k kthx
286
		if(operationName == null)
444.1.34 by Neil
fixed form policy verification.
287
			throw new InvalidOperationException("Could not determine operation name for " + servicePath);
367 by Neil
fixes for virtual hosting.
288
286.1.2 by decker
adding more stuff
289
		Map<String, String> params = httpRequest.getParameters();
290
291
		OMElement msg;
292
293
		EucalyptusMessage eucaMsg;
294
		Map<String, String> fieldMap;
295
		Class targetType;
296
		try
297
		{
298
			//:: try to create the target class :://
299
			targetType = Class.forName( "edu.ucsb.eucalyptus.msgs.".concat( operationName ).concat( "Type" ) );
300
			//:: get the map of parameters to fields :://
301
			fieldMap = this.buildFieldMap( targetType );
302
			//:: get an instance of the message :://
444.27.9 by Neil
update to logging.
303
			eucaMsg =  (EucalyptusMessage) targetType.newInstance();
286.1.2 by decker
adding more stuff
304
		}
305
		catch ( Exception e )
306
		{
307
			throw new BindingException( "Failed to construct message of type " + operationName );
308
		}
309
444.27.9 by Neil
update to logging.
310
		addLogData(eucaMsg, bindingArguments);
444.27.11 by Neil
fixed state cleanup on interrupted puts (can cause memory leak over an extended period of time).
311
286.1.2 by decker
adding more stuff
312
		//TODO: Refactor this to be more general
313
		List<String> failedMappings = populateObject( eucaMsg, fieldMap, params);
314
		populateObjectFromBindingMap(eucaMsg, fieldMap, httpRequest, bindingArguments);
315
365 by Neil
fixed walrus security to use new user stuff.
316
		User user = httpRequest.getUser();
302 by Neil
saving changes.
317
		setRequiredParams (eucaMsg, user);
286.1.2 by decker
adding more stuff
318
319
		if ( !failedMappings.isEmpty() || !params.isEmpty() )
320
		{
321
			StringBuilder errMsg = new StringBuilder( "Failed to bind the following fields:\n" );
322
			for ( String f : failedMappings )
323
				errMsg.append( f ).append( '\n' );
324
			for ( Map.Entry<String, String> f : params.entrySet() )
325
				errMsg.append( f.getKey() ).append( " = " ).append( f.getValue() ).append( '\n' );
326
			throw new BindingException( errMsg.toString() );
327
		}
328
329
		LOG.info(eucaMsg.toString());
330
		try
331
		{
332
			Binding binding = BindingManager.getBinding( BindingManager.sanitizeNamespace( "http://msgs.eucalyptus.ucsb.edu" ) );
333
			msg = binding.toOM( eucaMsg );
334
		}
335
		catch ( RuntimeException e )
336
		{
337
			throw new BindingException( "Failed to build a valid message: " + e.getMessage() );
338
		}
339
340
		return eucaMsg;
341
342
	}
343
444.27.9 by Neil
update to logging.
344
	private void addLogData(EucalyptusMessage eucaMsg,
345
			Map bindingArguments) {
346
		if(eucaMsg instanceof WalrusRequestType) {
347
			String operation = (String) bindingArguments.remove("Operation");
348
			if(operation != null) {
349
				WalrusRequestType request = (WalrusRequestType) eucaMsg;
350
				BucketLogData logData = WalrusBucketLogger.getInstance().makeLogEntry(UUID.randomUUID().toString());
351
				logData.setOperation("REST." + operation);
352
				request.setLogData(logData);
353
			}
354
		}
355
	}
356
365 by Neil
fixed walrus security to use new user stuff.
357
	private void setRequiredParams(final GroovyObject msg, User user) {
302 by Neil
saving changes.
358
		if(user != null) {
365 by Neil
fixed walrus security to use new user stuff.
359
			msg.setProperty("accessKeyID", user.getQueryId());
302 by Neil
saving changes.
360
		}
361
		msg.setProperty("timeStamp", new Date());
362
	}
363
286.1.2 by decker
adding more stuff
364
	private static String[] getTarget(String operationPath) {
365
		operationPath = operationPath.replaceAll("/{2,}", "/");
366
		if(operationPath.startsWith("/"))
367
			operationPath = operationPath.substring(1);
368
		return operationPath.split("/");
369
	}
370
371
	private String getOperation(MappingHttpRequest httpRequest, Map operationParams) throws BindingException {
372
		String[] target = null;
373
		String path = getOperationPath(httpRequest);
374
		boolean walrusInternalOperation = false;
367 by Neil
fixes for virtual hosting.
375
376
		String targetHost = httpRequest.getHeader(HttpHeaders.Names.HOST);
377
		if(targetHost.contains(".walrus")) {
378
			String bucket = targetHost.substring(0, targetHost.indexOf(".walrus"));
444.1.249 by Neil
fix bucket resolution for virtual hosting.
379
			path = "/" + bucket + path;
367 by Neil
fixes for virtual hosting.
380
		}
381
286.1.2 by decker
adding more stuff
382
		if(path.length() > 0) {
383
			target = getTarget(path);
384
		}
385
386
		String verb = httpRequest.getMethod().getName();
387
		String operationKey = "";
388
		Map<String, String> params = httpRequest.getParameters();
389
		String operationName = null;
390
		long contentLength = 0;
391
		String contentLengthString = httpRequest.getHeader(WalrusProperties.CONTENT_LEN);
392
		if(contentLengthString != null) {
393
			contentLength = Long.parseLong(contentLengthString);
394
		}
395
		if(httpRequest.containsHeader(StorageProperties.EUCALYPTUS_OPERATION)) {
396
			String value = httpRequest.getHeader(StorageProperties.EUCALYPTUS_OPERATION);
397
			for(WalrusProperties.WalrusInternalOperations operation: WalrusProperties.WalrusInternalOperations.values()) {
398
				if(value.toLowerCase().equals(operation.toString().toLowerCase())) {
399
					operationName = operation.toString();
400
					walrusInternalOperation = true;
401
					break;
402
				}
403
			}
404
405
			if(!walrusInternalOperation) {
406
				for(WalrusProperties.StorageOperations operation: WalrusProperties.StorageOperations.values()) {
407
					if(value.toLowerCase().equals(operation.toString().toLowerCase())) {
408
						operationName = operation.toString();
409
						walrusInternalOperation = true;
444.1.71 by Neil
updated storesnap to walrus, get snapshot.
410
						if(httpRequest.containsHeader(StorageProperties.StorageParameters.EucaSnapSize.toString())) {
411
							operationParams.put("SnapshotSize", httpRequest.getAndRemoveHeader(StorageProperties.StorageParameters.EucaSnapSize.toString()));
412
						}
286.1.2 by decker
adding more stuff
413
						break;
414
					}
415
				}
416
			}
417
418
		}
419
420
		if(target == null) {
421
			//target = service
422
			operationKey = SERVICE + verb;
423
		} else if(target.length < 2) {
424
			//target = bucket
425
			if(!target[0].equals("")) {
426
				operationKey = BUCKET + verb;
427
				operationParams.put("Bucket", target[0]);
444.27.9 by Neil
update to logging.
428
				operationParams.put("Operation", verb.toUpperCase() + "." + "BUCKET");
286.1.2 by decker
adding more stuff
429
				if(verb.equals(WalrusProperties.HTTPVerb.POST.toString())) {
430
					//TODO: handle POST.
444.1.110 by Neil
fixed POST uploads (chunked and unchunked).
431
					Map formFields = httpRequest.getFormFields();
286.1.2 by decker
adding more stuff
432
433
					String objectKey = null;
444.1.110 by Neil
fixed POST uploads (chunked and unchunked).
434
					String file = (String) formFields.get(WalrusProperties.FormField.file.toString());
286.1.2 by decker
adding more stuff
435
					String authenticationHeader = "";
436
					if(formFields.containsKey(WalrusProperties.FormField.key.toString())) {
444.1.110 by Neil
fixed POST uploads (chunked and unchunked).
437
						objectKey = (String) formFields.get(WalrusProperties.FormField.key.toString());
286.1.2 by decker
adding more stuff
438
						objectKey = objectKey.replaceAll("\\$\\{filename\\}", file);
333 by Neil
more fixes to post. chris plz make EucalyptusQueryPipeline idempotent k kthx
439
						operationParams.put("Key", objectKey);
286.1.2 by decker
adding more stuff
440
					}
441
					if(formFields.containsKey(WalrusProperties.FormField.acl.toString())) {
444.1.110 by Neil
fixed POST uploads (chunked and unchunked).
442
						String acl = (String) formFields.get(WalrusProperties.FormField.acl.toString());
286.1.2 by decker
adding more stuff
443
						httpRequest.addHeader(WalrusProperties.AMZ_ACL, acl);
444
					}
444.1.226 by Neil
another fix for form uploads.
445
					if(formFields.containsKey(WalrusProperties.FormField.redirect.toString())) {
446
						String successActionRedirect = (String) formFields.get(WalrusProperties.FormField.redirect.toString());
447
						operationParams.put("SuccessActionRedirect", successActionRedirect);
448
					}
286.1.2 by decker
adding more stuff
449
					if(formFields.containsKey(WalrusProperties.FormField.success_action_redirect.toString())) {
444.1.110 by Neil
fixed POST uploads (chunked and unchunked).
450
						String successActionRedirect = (String) formFields.get(WalrusProperties.FormField.success_action_redirect.toString());
286.1.2 by decker
adding more stuff
451
						operationParams.put("SuccessActionRedirect", successActionRedirect);
452
					}
453
					if(formFields.containsKey(WalrusProperties.FormField.success_action_status.toString())) {
444.1.110 by Neil
fixed POST uploads (chunked and unchunked).
454
						Integer successActionStatus = Integer.parseInt((String)formFields.get(WalrusProperties.FormField.success_action_status.toString()));
286.1.2 by decker
adding more stuff
455
						if(successActionStatus == 200 || successActionStatus == 201)
456
							operationParams.put("SuccessActionStatus", successActionStatus);
457
						else
458
							operationParams.put("SuccessActionStatus", 204);
459
					} else {
460
						operationParams.put("SuccessActionStatus", 204);
461
					}
444.1.37 by Neil
fixed (unchunked) form post.
462
					if(formFields.containsKey(WalrusProperties.CONTENT_TYPE)) {
463
						operationParams.put("ContentType", formFields.get(WalrusProperties.CONTENT_TYPE));
464
					}
444.27.11 by Neil
fixed state cleanup on interrupted puts (can cause memory leak over an extended period of time).
465
					key = target[0] + "." + objectKey;
466
					randomKey = key + "." + Hashes.getRandom(10);
444.1.77 by Neil
fixed binding error in storesnap.
467
					if(contentLengthString != null)
468
						operationParams.put("ContentLength", (new Long(contentLength).toString()));
321 by Neil
fixed get.
469
					operationParams.put(WalrusProperties.Headers.RandomKey.toString(), randomKey);
470
					putQueue = getWriteMessenger().interruptAllAndGetQueue(key, randomKey);
444.1.110 by Neil
fixed POST uploads (chunked and unchunked).
471
					handleFirstChunk(httpRequest, (ChannelBuffer)formFields.get(WalrusProperties.IGNORE_PREFIX + "FirstDataChunk"), contentLength);
444.27.3 by Neil
few updates to logging and fixes to group perms.
472
				} else if(WalrusProperties.HTTPVerb.PUT.toString().equals(verb) && 
473
						params.containsKey(WalrusProperties.OperationParameter.logging.toString())) {
474
					//read logging params
475
					getTargetBucketParams(operationParams, httpRequest);
286.1.2 by decker
adding more stuff
476
				}
477
			} else {
478
				operationKey = SERVICE + verb;
479
			}
480
		} else {
481
			//target = object
482
			operationKey = OBJECT + verb;
483
			String objectKey="";
484
			String splitOn = "";
485
			for(int i = 1; i < target.length; ++i) {
486
				objectKey += splitOn + target[i];
487
				splitOn = "/";
488
			}
444.47.13 by Neil
correctly URL decode object key.
489
			try {
490
				objectKey = WalrusUtil.URLdecode(objectKey);
491
			} catch (UnsupportedEncodingException e) {
492
				throw new BindingException("Unable to get key: " + e.getMessage());
493
			}
286.1.2 by decker
adding more stuff
494
			operationParams.put("Bucket", target[0]);
495
			operationParams.put("Key", objectKey);
444.27.9 by Neil
update to logging.
496
			operationParams.put("Operation", verb.toUpperCase() + "." + "OBJECT");
286.1.2 by decker
adding more stuff
497
498
			if(!params.containsKey(WalrusProperties.OperationParameter.acl.toString())) {
499
				if (verb.equals(WalrusProperties.HTTPVerb.PUT.toString())) {
500
					if(httpRequest.containsHeader(WalrusProperties.COPY_SOURCE.toString())) {
501
						String copySource = httpRequest.getHeader(WalrusProperties.COPY_SOURCE.toString());
502
						String[] sourceTarget = getTarget(copySource);
503
						String sourceObjectKey = "";
504
						String sourceSplitOn = "";
505
						if(sourceTarget.length > 1) {
506
							for(int i = 1; i < sourceTarget.length; ++i) {
507
								sourceObjectKey += sourceSplitOn + sourceTarget[i];
508
								sourceSplitOn = "/";
509
							}
444.47.13 by Neil
correctly URL decode object key.
510
							try {
511
								sourceObjectKey = WalrusUtil.URLdecode(sourceObjectKey);
512
							} catch (UnsupportedEncodingException e) {
513
								throw new BindingException("Unable to get source key: " + e.getMessage());
514
							}
286.1.2 by decker
adding more stuff
515
							operationParams.put("SourceBucket", sourceTarget[0]);
516
							operationParams.put("SourceObject", sourceObjectKey);
517
							operationParams.put("DestinationBucket", operationParams.remove("Bucket"));
518
							operationParams.put("DestinationObject", operationParams.remove("Key"));
519
520
							String metaDataDirective = httpRequest.getHeader(WalrusProperties.METADATA_DIRECTIVE.toString());
521
							if(metaDataDirective != null) {
522
								operationParams.put("MetadataDirective", metaDataDirective);
523
							}
524
							AccessControlListType accessControlList;
525
							if(contentLength > 0) {
526
								accessControlList = null;
527
								accessControlList = getAccessControlList(httpRequest);
528
							} else {
529
								accessControlList = new AccessControlListType();
530
							}
531
							operationParams.put("AccessControlList", accessControlList);
532
							operationKey += WalrusProperties.COPY_SOURCE.toString();
533
							Set<String> headerNames = httpRequest.getHeaderNames();
534
							for(String key : headerNames) {
535
								for(WalrusProperties.CopyHeaders header: WalrusProperties.CopyHeaders.values()) {
536
									if(key.replaceAll("-", "").equals(header.toString().toLowerCase())) {
537
										String value = httpRequest.getHeader(key);
538
										parseExtendedHeaders(operationParams, header.toString(), value);
539
									}
540
								}
541
							}
542
						} else {
543
							throw new BindingException("Malformed COPY request");
544
						}
545
546
					} else {
547
						//handle PUTs
444.27.11 by Neil
fixed state cleanup on interrupted puts (can cause memory leak over an extended period of time).
548
						key = target[0] + "." + objectKey;
549
						randomKey = key + "." + Hashes.getRandom(10);
286.1.2 by decker
adding more stuff
550
						String contentType = httpRequest.getHeader(WalrusProperties.CONTENT_TYPE);
551
						if(contentType != null)
552
							operationParams.put("ContentType", contentType);
553
						String contentDisposition = httpRequest.getHeader("Content-Disposition");
554
						if(contentDisposition != null)
555
							operationParams.put("ContentDisposition", contentDisposition);
444.1.77 by Neil
fixed binding error in storesnap.
556
						if(contentLengthString != null)
557
							operationParams.put("ContentLength", (new Long(contentLength).toString()));
286.1.2 by decker
adding more stuff
558
						operationParams.put(WalrusProperties.Headers.RandomKey.toString(), randomKey);
304 by Neil
saving changes.
559
						putQueue = getWriteMessenger().interruptAllAndGetQueue(key, randomKey);
560
						handleFirstChunk(httpRequest, contentLength);
286.1.2 by decker
adding more stuff
561
					}
562
				} else if(verb.equals(WalrusProperties.HTTPVerb.GET.toString())) {
563
					if(!walrusInternalOperation) {
564
565
						if(params.containsKey("torrent")) {
566
							operationParams.put("GetTorrent", Boolean.TRUE);
567
						} else {
568
							operationParams.put("GetData", Boolean.TRUE);
569
							operationParams.put("InlineData", Boolean.FALSE);
570
							operationParams.put("GetMetaData", Boolean.TRUE);
571
						}
572
304 by Neil
saving changes.
573
						Set<String> headerNames = httpRequest.getHeaderNames();
286.1.2 by decker
adding more stuff
574
						boolean isExtendedGet = false;
304 by Neil
saving changes.
575
						for(String key : headerNames) {
286.1.2 by decker
adding more stuff
576
							for(WalrusProperties.ExtendedGetHeaders header: WalrusProperties.ExtendedGetHeaders.values()) {
356 by Neil
fixes for extended get.
577
								if(key.replaceAll("-", "").equals(header.toString())) {
286.1.2 by decker
adding more stuff
578
									String value = httpRequest.getHeader(key);
579
									isExtendedGet = true;
580
									parseExtendedHeaders(operationParams, header.toString(), value);
581
								}
582
							}
583
584
						}
585
						if(isExtendedGet) {
586
							operationKey += "extended";
587
							//only supported through SOAP
588
							operationParams.put("ReturnCompleteObjectOnConditionFailure", Boolean.FALSE);
589
						}
321 by Neil
fixed get.
590
					} 
286.1.2 by decker
adding more stuff
591
					if(params.containsKey(WalrusProperties.GetOptionalParameters.IsCompressed.toString())) {
592
						Boolean isCompressed = Boolean.parseBoolean(params.remove(WalrusProperties.GetOptionalParameters.IsCompressed.toString()));
593
						operationParams.put("IsCompressed", isCompressed);
304 by Neil
saving changes.
594
					}
286.1.2 by decker
adding more stuff
595
596
				} else if(verb.equals(WalrusProperties.HTTPVerb.HEAD.toString())) {
597
					if(!walrusInternalOperation) {
598
						operationParams.put("GetData", Boolean.FALSE);
599
						operationParams.put("InlineData", Boolean.FALSE);
600
						operationParams.put("GetMetaData", Boolean.TRUE);
601
					}
602
				}
603
			}
604
605
		}
606
607
608
		if (verb.equals(WalrusProperties.HTTPVerb.PUT.toString()) && params.containsKey(WalrusProperties.OperationParameter.acl.toString())) {
609
			operationParams.put("AccessControlPolicy", getAccessControlPolicy(httpRequest));
610
		}
611
612
		ArrayList paramsToRemove = new ArrayList();
613
614
		boolean addMore = true;
615
		Iterator iterator = params.keySet().iterator();
616
		while(iterator.hasNext()) {
617
			Object key = iterator.next();
618
			String keyString = key.toString().toLowerCase();
619
			boolean dontIncludeParam = false;
620
			for(WalrusAuthenticationHandler.SecurityParameter securityParam : WalrusAuthenticationHandler.SecurityParameter.values()) {
621
				if(keyString.equals(securityParam.toString().toLowerCase())) {
622
					dontIncludeParam = true;
623
					break;
624
				}
625
			}
626
			if(dontIncludeParam)
627
				continue;
628
			String value = params.get(key);
629
			if(value != null) {
630
				String[] keyStringParts = keyString.split("-");
631
				if(keyStringParts.length > 1) {
632
					keyString = "";
633
					for(int i=0; i < keyStringParts.length; ++i) {
634
						keyString += toUpperFirst(keyStringParts[i]);
635
					}
636
				} else {
637
					keyString = toUpperFirst(keyString);
638
				}
639
				operationParams.put(keyString, value);
640
			}
641
			if(addMore) {
642
				//just add the first one to the key
643
				operationKey += keyString.toLowerCase();
644
				addMore = false;
645
			}
646
			paramsToRemove.add(key);
647
		}
648
649
		for(Object key : paramsToRemove) {
650
			params.remove(key);
651
		}
652
653
		if(!walrusInternalOperation) {
654
			operationName = operationMap.get(operationKey);
655
		}
656
657
		if("CreateBucket".equals(operationName)) {
658
			String locationConstraint = getLocationConstraint(httpRequest);
659
			if(locationConstraint != null)
660
				operationParams.put("LocationConstraint", locationConstraint);
661
		}
662
		return operationName;	
663
	}
664
444.27.3 by Neil
few updates to logging and fixes to group perms.
665
	private void getTargetBucketParams(Map operationParams,
666
			MappingHttpRequest httpRequest) throws BindingException {
667
		String message = getMessageString(httpRequest);
668
		if(message.length() > 0) {
669
			try {
670
				XMLParser xmlParser = new XMLParser(message);
671
				String targetBucket = xmlParser.getValue("//TargetBucket");
444.27.25 by Neil
fixes to bucket logging: correctly handle disabling logging, return logging disabled status correctly.
672
				if(targetBucket == null || targetBucket.length() == 0) 
673
					return;
444.27.3 by Neil
few updates to logging and fixes to group perms.
674
				String targetPrefix = xmlParser.getValue("//TargetPrefix");
675
				ArrayList<Grant> grants = new ArrayList<Grant>();
676
677
				List<String> permissions = xmlParser.getValues("//TargetGrants/Grant/Permission");
678
				if(permissions == null)
679
					throw new BindingException("malformed access control list");
680
681
				DTMNodeList grantees = xmlParser.getNodes("//TargetGrants/Grant/Grantee");
682
				if(grantees == null)
683
					throw new BindingException("malformed access control list");
684
685
				for(int i = 0 ; i < grantees.getLength() ; ++i) {
686
					String id = xmlParser.getValue(grantees.item(i), "ID");
687
					if(id.length() > 0) {
688
						String canonicalUserName = xmlParser.getValue(grantees.item(i), "DisplayName");
689
						Grant grant = new Grant();
690
						Grantee grantee = new Grantee();
691
						grantee.setCanonicalUser(new CanonicalUserType(id, canonicalUserName));
692
						grant.setGrantee(grantee);
693
						grant.setPermission(permissions.get(i));
694
						grants.add(grant);
695
					} else {
696
						String groupUri = xmlParser.getValue(grantees.item(i), "URI");
697
						if(groupUri.length() == 0)
698
							throw new BindingException("malformed access control list");
699
						Grant grant = new Grant();
700
						Grantee grantee = new Grantee();
701
						grantee.setGroup(new Group(groupUri));
702
						grant.setGrantee(grantee);
703
						grant.setPermission(permissions.get(i));
704
						grants.add(grant);
705
					}
706
				}
707
				TargetGrants targetGrants = new TargetGrants(grants);
708
				LoggingEnabled loggingEnabled = new LoggingEnabled(targetBucket, targetPrefix, new TargetGrants(grants));
709
				operationParams.put("LoggingEnabled", loggingEnabled);
710
			} catch(Exception ex) {
711
				LOG.warn(ex);
712
				throw new BindingException("Unable to parse access control policy " + ex.getMessage());
713
			}
714
		}
715
	}
716
444.16.154 by Neil
fixes #490623
717
	private void parseExtendedHeaders(Map operationParams, String headerString, String value) throws BindingException {
286.1.2 by decker
adding more stuff
718
		if(headerString.equals(WalrusProperties.ExtendedGetHeaders.Range.toString())) {
719
			String prefix = "bytes=";
720
			assert(value.startsWith(prefix));
721
			value = value.substring(prefix.length());
722
			String[]values = value.split("-");
723
			assert(values.length == 2);
724
			if(values[0].equals("")) {
725
				operationParams.put(WalrusProperties.ExtendedHeaderRangeTypes.ByteRangeStart.toString(), new Long(0));
726
			} else {
727
				operationParams.put(WalrusProperties.ExtendedHeaderRangeTypes.ByteRangeStart.toString(), Long.parseLong(values[0]));
728
			}
729
			assert(!values[1].equals(""));
730
			operationParams.put(WalrusProperties.ExtendedHeaderRangeTypes.ByteRangeEnd.toString(), Long.parseLong(values[1]));
731
		} else if(WalrusProperties.ExtendedHeaderDateTypes.contains(headerString)) {
732
			try {
444.16.154 by Neil
fixes #490623
733
				List<String> dateFormats = new ArrayList<String>();
734
				dateFormats.add(DateUtil.PATTERN_RFC1123);
735
				operationParams.put(headerString, DateUtil.parseDate(value, dateFormats));
286.1.2 by decker
adding more stuff
736
			} catch(Exception ex) {
444.16.154 by Neil
fixes #490623
737
				try {
738
					operationParams.put(headerString, DateUtils.parseIso8601DateTime(value));
739
				} catch (ParseException e) {
740
					LOG.error(e);
741
					throw new BindingException(e);
742
				}
286.1.2 by decker
adding more stuff
743
			}
744
		} else {
745
			operationParams.put(headerString, value);
746
		}
747
	}
748
749
	private AccessControlPolicyType getAccessControlPolicy(MappingHttpRequest httpRequest) throws BindingException {
750
		AccessControlPolicyType accessControlPolicy = new AccessControlPolicyType();
444.52.1 by Neil
fixes LP: #517769
751
		AccessControlListType accessControlList = new AccessControlListType();
752
		ArrayList<Grant> grants = new ArrayList<Grant>();
286.1.2 by decker
adding more stuff
753
		try {
754
			String aclString = getMessageString(httpRequest);
755
			if(aclString.length() > 0) {
756
				XMLParser xmlParser = new XMLParser(aclString);
757
				String ownerId = xmlParser.getValue("//Owner/ID");
758
				String displayName = xmlParser.getValue("//Owner/DisplayName");
759
760
				CanonicalUserType canonicalUser = new CanonicalUserType(ownerId, displayName);
761
				accessControlPolicy.setOwner(canonicalUser);
762
763
				List<String> permissions = xmlParser.getValues("//AccessControlList/Grant/Permission");
444.18.1 by Neil
fixed a number of warnings.
764
				if(permissions == null)
765
					throw new BindingException("malformed access control list");
286.1.2 by decker
adding more stuff
766
767
				DTMNodeList grantees = xmlParser.getNodes("//AccessControlList/Grant/Grantee");
444.18.1 by Neil
fixed a number of warnings.
768
				if(grantees == null)
769
					throw new BindingException("malformed access control list");
286.1.2 by decker
adding more stuff
770
771
				for(int i = 0 ; i < grantees.getLength() ; ++i) {
772
					String id = xmlParser.getValue(grantees.item(i), "ID");
773
					if(id.length() > 0) {
774
						String canonicalUserName = xmlParser.getValue(grantees.item(i), "DisplayName");
775
						Grant grant = new Grant();
776
						Grantee grantee = new Grantee();
777
						grantee.setCanonicalUser(new CanonicalUserType(id, canonicalUserName));
778
						grant.setGrantee(grantee);
779
						grant.setPermission(permissions.get(i));
780
						grants.add(grant);
781
					} else {
782
						String groupUri = xmlParser.getValue(grantees.item(i), "URI");
783
						if(groupUri.length() == 0)
784
							throw new BindingException("malformed access control list");
785
						Grant grant = new Grant();
786
						Grantee grantee = new Grantee();
787
						grantee.setGroup(new Group(groupUri));
788
						grant.setGrantee(grantee);
789
						grant.setPermission(permissions.get(i));
790
						grants.add(grant);
791
					}
792
				}
793
			}
794
		} catch(Exception ex) {
795
			LOG.warn(ex);
356 by Neil
fixes for extended get.
796
			throw new BindingException("Unable to parse access control policy " + ex.getMessage());
286.1.2 by decker
adding more stuff
797
		}
444.52.1 by Neil
fixes LP: #517769
798
		accessControlList.setGrants(grants);
799
		accessControlPolicy.setAccessControlList(accessControlList);
286.1.2 by decker
adding more stuff
800
		return accessControlPolicy;
801
	}
802
803
804
	private AccessControlListType getAccessControlList(MappingHttpRequest httpRequest) throws BindingException {
805
		AccessControlListType accessControlList = new AccessControlListType();
444.53.5 by Neil
updated logging to handle copy object.
806
		ArrayList<Grant> grants = new ArrayList<Grant>();
286.1.2 by decker
adding more stuff
807
		try {
808
			String aclString = getMessageString(httpRequest);
809
			if(aclString.length() > 0) {
810
				XMLParser xmlParser = new XMLParser(aclString);
811
				String ownerId = xmlParser.getValue("//Owner/ID");
812
				String displayName = xmlParser.getValue("//Owner/DisplayName");
813
814
				List<String> permissions = xmlParser.getValues("/AccessControlList/Grant/Permission");
444.18.1 by Neil
fixed a number of warnings.
815
				if(permissions == null)
816
					throw new BindingException("malformed access control list");
286.1.2 by decker
adding more stuff
817
818
				DTMNodeList grantees = xmlParser.getNodes("/AccessControlList/Grant/Grantee");
444.18.1 by Neil
fixed a number of warnings.
819
				if(grantees == null)
820
					throw new BindingException("malformed access control list");
286.1.2 by decker
adding more stuff
821
822
823
				for(int i = 0 ; i < grantees.getLength() ; ++i) {
824
					String canonicalUserName = xmlParser.getValue(grantees.item(i), "DisplayName");
825
					if(canonicalUserName.length() > 0) {
826
						String id = xmlParser.getValue(grantees.item(i), "ID");
827
						Grant grant = new Grant();
828
						Grantee grantee = new Grantee();
829
						grantee.setCanonicalUser(new CanonicalUserType(id, canonicalUserName));
830
						grant.setGrantee(grantee);
831
						grant.setPermission(permissions.get(i));
832
						grants.add(grant);
833
					} else {
834
						String groupUri = xmlParser.getValue(grantees.item(i), "URI");
835
						if(groupUri.length() == 0)
836
							throw new BindingException("malformed access control list");
837
						Grant grant = new Grant();
838
						Grantee grantee = new Grantee();
839
						grantee.setGroup(new Group(groupUri));
840
						grant.setGrantee(grantee);
841
						grant.setPermission(permissions.get(i));
842
						grants.add(grant);
843
					}
844
				}
845
			}
846
		} catch(Exception ex) {
847
			LOG.warn(ex);
356 by Neil
fixes for extended get.
848
			throw new BindingException("Unable to parse access control list " + ex.getMessage());
286.1.2 by decker
adding more stuff
849
		}
444.53.5 by Neil
updated logging to handle copy object.
850
		accessControlList.setGrants(grants);
286.1.2 by decker
adding more stuff
851
		return accessControlList;
852
	}
853
854
	private String getOperationPath(MappingHttpRequest httpRequest) {
855
		return httpRequest.getServicePath().replaceAll(WalrusProperties.walrusServicePath, "");
856
	}
857
858
	private String getLocationConstraint(MappingHttpRequest httpRequest) throws BindingException {
859
		String locationConstraint = null;
860
		try {
861
			String bucketConfiguration = getMessageString(httpRequest);
862
			if(bucketConfiguration.length() > 0) {
863
				XMLParser xmlParser = new XMLParser(bucketConfiguration);
864
				locationConstraint = xmlParser.getValue("/CreateBucketConfiguration/LocationConstraint");
865
			}
866
		} catch(Exception ex) {
867
			LOG.warn(ex);
868
			throw new BindingException(ex.getMessage());
869
		}
870
		return locationConstraint;
871
	}
872
873
	private List<String> populateObject( final GroovyObject obj, final Map<String, String> paramFieldMap, final Map<String, String> params ) {
874
		List<String> failedMappings = new ArrayList<String>( );
875
		for ( Map.Entry<String, String> e : paramFieldMap.entrySet( ) ) {
876
			try {
877
				if ( obj.getClass( ).getDeclaredField( e.getValue( ) ).getType( ).equals( ArrayList.class ) ) failedMappings.addAll( populateObjectList( obj, e, params, params.size( ) ) );
878
			} catch ( NoSuchFieldException e1 ) {
879
				failedMappings.add( e.getKey( ) );
880
			}
881
		}
882
		for ( Map.Entry<String, String> e : paramFieldMap.entrySet( ) ) {
883
			if ( params.containsKey( e.getKey( ) ) && !populateObjectField( obj, e, params ) ) failedMappings.add( e.getKey( ) );
884
		}
885
		return failedMappings;
886
	}
887
888
	private void populateObjectFromBindingMap( final GroovyObject obj, final Map<String, String> paramFieldMap, final MappingHttpRequest httpRequest, final Map bindingMap)
889
	{
890
		//process headers
891
		String aclString = httpRequest.getAndRemoveHeader(WalrusProperties.AMZ_ACL);
892
		if (aclString != null) {
893
			addAccessControlList(obj, paramFieldMap, bindingMap, aclString);
894
		}
895
896
		//add meta data
897
		String metaDataString = paramFieldMap.remove("MetaData");
898
		if(metaDataString != null) {
899
			Set<String> headerNames = httpRequest.getHeaderNames();
900
			ArrayList<MetaDataEntry> metaData = new ArrayList<MetaDataEntry>();
901
			for(String key : headerNames) {
444.1.38 by Neil
minor fix to save metadata.
902
				if(key.toLowerCase().startsWith(WalrusProperties.AMZ_META_HEADER_PREFIX)) {
286.1.2 by decker
adding more stuff
903
					MetaDataEntry metaDataEntry = new MetaDataEntry();
904
					metaDataEntry.setName(key.substring(WalrusProperties.AMZ_META_HEADER_PREFIX.length()));
444.1.39 by Neil
another minor fix for saving metadata.
905
					metaDataEntry.setValue(httpRequest.getHeader(key));
286.1.2 by decker
adding more stuff
906
					metaData.add(metaDataEntry);
907
				}
908
			}
909
			obj.setProperty(metaDataString, metaData);
910
		}
911
912
		//populate from binding map (required params)
913
		Iterator bindingMapIterator = bindingMap.keySet().iterator();
914
		while(bindingMapIterator.hasNext()) {
915
			String key = (String) bindingMapIterator.next();
916
			obj.setProperty(key.substring(0, 1).toLowerCase().concat(key.substring(1)), bindingMap.get(key));
917
		}
918
	}
919
920
	private boolean populateObjectField( final GroovyObject obj, final Map.Entry<String, String> paramFieldPair, final Map<String, String> params ) {
921
		try {
922
			Class declaredType = obj.getClass( ).getDeclaredField( paramFieldPair.getValue( ) ).getType( );
923
			if ( declaredType.equals( String.class ) ) obj.setProperty( paramFieldPair.getValue( ), params.remove( paramFieldPair.getKey( ) ) );
924
			else if ( declaredType.getName( ).equals( "int" ) ) obj.setProperty( paramFieldPair.getValue( ), Integer.parseInt( params.remove( paramFieldPair.getKey( ) ) ) );
925
			else if ( declaredType.equals( Integer.class ) ) obj.setProperty( paramFieldPair.getValue( ), new Integer( params.remove( paramFieldPair.getKey( ) ) ) );
926
			else if ( declaredType.getName( ).equals( "boolean" ) ) obj.setProperty( paramFieldPair.getValue( ), Boolean.parseBoolean( params.remove( paramFieldPair.getKey( ) ) ) );
927
			else if ( declaredType.equals( Boolean.class ) ) obj.setProperty( paramFieldPair.getValue( ), new Boolean( params.remove( paramFieldPair.getKey( ) ) ) );
928
			else if ( declaredType.getName( ).equals( "long" ) ) obj.setProperty( paramFieldPair.getValue( ), Long.parseLong( params.remove( paramFieldPair.getKey( ) ) ) );
929
			else if ( declaredType.equals( Long.class ) ) obj.setProperty( paramFieldPair.getValue( ), new Long( params.remove( paramFieldPair.getKey( ) ) ) );
930
			else return false;
931
			return true;
932
		} catch ( Exception e1 ) {
933
			return false;
934
		}
935
	}
936
937
	private List<String> populateObjectList( final GroovyObject obj, final Map.Entry<String, String> paramFieldPair, final Map<String, String> params, final int paramSize ) {
938
		List<String> failedMappings = new ArrayList<String>( );
939
		try {
940
			Field declaredField = obj.getClass( ).getDeclaredField( paramFieldPair.getValue( ) );
941
			ArrayList theList = ( ArrayList ) obj.getProperty( paramFieldPair.getValue( ) );
942
			Class genericType = ( Class ) ( ( ParameterizedType ) declaredField.getGenericType( ) ).getActualTypeArguments( )[0];
943
			// :: simple case: FieldName.# :://
944
			if ( String.class.equals( genericType ) ) {
945
				if ( params.containsKey( paramFieldPair.getKey( ) ) ) {
946
					theList.add( params.remove( paramFieldPair.getKey( ) ) );
947
				} else {
948
					List<String> keys = Lists.newArrayList( params.keySet( ) );
949
					for ( String k : keys ) {
950
						if ( k.matches( paramFieldPair.getKey( ) + "\\.\\d*" ) ) {
951
							theList.add( params.remove( k ) );
952
						}
953
					}
954
				}
955
			} else if ( declaredField.isAnnotationPresent( HttpEmbedded.class ) ) {
956
				HttpEmbedded annoteEmbedded = ( HttpEmbedded ) declaredField.getAnnotation( HttpEmbedded.class );
957
				// :: build the parameter map and call populate object recursively :://
958
				if ( annoteEmbedded.multiple( ) ) {
959
					String prefix = paramFieldPair.getKey( );
960
					List<String> embeddedListFieldNames = new ArrayList<String>( );
961
					for ( String actualParameterName : params.keySet( ) )
962
						if ( actualParameterName.matches( prefix + ".1.*" ) ) embeddedListFieldNames.add( actualParameterName.replaceAll( prefix + ".1.", "" ) );
963
					for ( int i = 0; i < paramSize + 1; i++ ) {
964
						boolean foundAll = true;
965
						Map<String, String> embeddedParams = new HashMap<String, String>( );
966
						for ( String fieldName : embeddedListFieldNames ) {
967
							String paramName = prefix + "." + i + "." + fieldName;
968
							if ( !params.containsKey( paramName ) ) {
969
								failedMappings.add( "Mismatched embedded field: " + paramName );
970
								foundAll = false;
971
							} else embeddedParams.put( fieldName, params.get( paramName ) );
972
						}
973
						if ( foundAll ) failedMappings.addAll( populateEmbedded( genericType, embeddedParams, theList ) );
974
						else break;
975
					}
976
				} else failedMappings.addAll( populateEmbedded( genericType, params, theList ) );
977
			}
978
		} catch ( Exception e1 ) {
979
			failedMappings.add( paramFieldPair.getKey( ) );
980
		}
981
		return failedMappings;
982
	}
983
984
	private List<String> populateEmbedded( final Class genericType, final Map<String, String> params, final ArrayList theList ) throws InstantiationException, IllegalAccessException {
985
		GroovyObject embedded = ( GroovyObject ) genericType.newInstance( );
986
		Map<String, String> embeddedFields = buildFieldMap( genericType );
987
		int startSize = params.size( );
988
		List<String> embeddedFailures = populateObject( embedded, embeddedFields, params );
989
		if ( embeddedFailures.isEmpty( ) && !( params.size( ) - startSize == 0 ) ) theList.add( embedded );
990
		return embeddedFailures;
991
	}
992
993
	private Map<String, String> buildFieldMap( final Class targetType ) {
994
		Map<String, String> fieldMap = new HashMap<String, String>( );
995
		Field[] fields = targetType.getDeclaredFields( );
996
		for ( Field f : fields )
997
			if ( Modifier.isStatic( f.getModifiers( ) ) ) continue;
998
			else if ( f.isAnnotationPresent( HttpParameterMapping.class ) ) {
999
				fieldMap.put( f.getAnnotation( HttpParameterMapping.class ).parameter( ), f.getName( ) );
1000
				fieldMap.put( f.getName( ).substring( 0, 1 ).toUpperCase( ).concat( f.getName( ).substring( 1 ) ), f.getName( ) );
1001
			} else fieldMap.put( f.getName( ).substring( 0, 1 ).toUpperCase( ).concat( f.getName( ).substring( 1 ) ), f.getName( ) );
1002
		return fieldMap;
1003
	}
1004
1005
	private static void addAccessControlList (final GroovyObject obj, final Map<String, String> paramFieldMap, Map bindingMap, String cannedACLString) {
1006
1007
		AccessControlListType accessControlList;
1008
		ArrayList<Grant> grants;
1009
1010
		if(bindingMap.containsKey("AccessControlPolicy")) {
1011
			AccessControlPolicyType accessControlPolicy = (AccessControlPolicyType) bindingMap.get("AccessControlPolicy");
1012
			accessControlList = accessControlPolicy.getAccessControlList();
1013
			grants = accessControlList.getGrants();
1014
		} else {
1015
			accessControlList = new AccessControlListType();
1016
			grants = new ArrayList<Grant>();
1017
		}
1018
1019
		CanonicalUserType aws = new CanonicalUserType();
1020
		aws.setDisplayName("");
1021
		Grant grant = new Grant(new Grantee(aws), cannedACLString);
1022
		grants.add(grant);
1023
1024
		accessControlList.setGrants(grants);
1025
		//set obj property
1026
		String acl = paramFieldMap.remove("AccessControlList");
1027
		if(acl != null) {
1028
			obj.setProperty(acl, accessControlList );
1029
		}
1030
	}
1031
1032
	private String toUpperFirst(String string) {
1033
		return string.substring(0, 1).toUpperCase().concat(string.substring(1));
1034
	}
1035
1036
	private boolean exactMatch(JSONObject jsonObject, Map formFields, List<String> policyItemNames) {
1037
		Iterator<String> iterator = jsonObject.keys();
1038
		boolean returnValue = false;
1039
		while(iterator.hasNext()) {
1040
			String key = iterator.next();
1041
			key = key.replaceAll("\\$", "");
444.1.34 by Neil
fixed form policy verification.
1042
			policyItemNames.add(key);
286.1.2 by decker
adding more stuff
1043
			try {
1044
				if(jsonObject.get(key).equals(formFields.get(key)))
1045
					returnValue = true;
1046
				else
1047
					returnValue = false;
1048
			} catch(Exception ex) {
1049
				ex.printStackTrace();
1050
				return false;
1051
			}
1052
		}
1053
		return returnValue;
1054
	}
1055
1056
	private boolean partialMatch(JSONArray jsonArray, Map<String, String> formFields, List<String> policyItemNames) {
1057
		boolean returnValue = false;
1058
		if(jsonArray.size() != 3)
1059
			return false;
1060
		try {
1061
			String condition = (String) jsonArray.get(0);
1062
			String key = (String) jsonArray.get(1);
1063
			key = key.replaceAll("\\$", "");
444.1.34 by Neil
fixed form policy verification.
1064
			policyItemNames.add(key);
286.1.2 by decker
adding more stuff
1065
			String value = (String) jsonArray.get(2);
1066
			if(condition.contains("eq")) {
1067
				if(value.equals(formFields.get(key)))
1068
					returnValue = true;
1069
			} else if(condition.contains("starts-with")) {
444.1.34 by Neil
fixed form policy verification.
1070
				if(!formFields.containsKey(key))
286.1.2 by decker
adding more stuff
1071
					return false;
444.1.34 by Neil
fixed form policy verification.
1072
				if(formFields.get(key).startsWith(value))
286.1.2 by decker
adding more stuff
1073
					returnValue = true;
1074
			}
1075
		} catch(Exception ex) {
1076
			ex.printStackTrace();
1077
			return false;
1078
		}
1079
		return returnValue;
1080
	}
1081
1082
	private String getMessageString(MappingHttpRequest httpRequest) {
1083
		ChannelBuffer buffer = httpRequest.getContent( );
1084
		buffer.markReaderIndex( );
1085
		byte[] read = new byte[buffer.readableBytes( )];
1086
		buffer.readBytes( read );
1087
		return new String( read );
1088
	}
1089
302 by Neil
saving changes.
1090
	private void handleHttpChunk(HttpChunk httpChunk) throws Exception {
304 by Neil
saving changes.
1091
		ChannelBuffer buffer = httpChunk.getContent();
1092
		try {
1093
			buffer.markReaderIndex( );
1094
			byte[] read = new byte[buffer.readableBytes( )];
1095
			buffer.readBytes( read );
1096
			putQueue.put(WalrusDataMessage.DataMessage(read));
1097
			if(httpChunk.isLast())
1098
				putQueue.put(WalrusDataMessage.EOF());
1099
1100
		} catch (Exception ex) {
1101
			LOG.error(ex, ex);
1102
		}
1103
1104
	}
1105
1106
	private void handleFirstChunk(MappingHttpRequest httpRequest, long dataLength) {
1107
		ChannelBuffer buffer = httpRequest.getContent();
1108
		try {
1109
			putQueue.put(WalrusDataMessage.StartOfData(dataLength));
1110
			buffer.markReaderIndex( );
1111
			byte[] read = new byte[buffer.readableBytes( )];
1112
			buffer.readBytes( read );
1113
			putQueue.put(WalrusDataMessage.DataMessage(read));
321 by Neil
fixed get.
1114
			if(!httpRequest.isChunked())
1115
				putQueue.put(WalrusDataMessage.EOF());
304 by Neil
saving changes.
1116
		} catch (Exception ex) {
1117
			LOG.error(ex, ex);
1118
		}
1119
1120
	}
1121
444.1.110 by Neil
fixed POST uploads (chunked and unchunked).
1122
	private void handleFirstChunk(MappingHttpRequest httpRequest, ChannelBuffer firstChunk, long dataLength) {
444.1.37 by Neil
fixed (unchunked) form post.
1123
		try {
1124
			putQueue.put(WalrusDataMessage.StartOfData(dataLength));
444.1.110 by Neil
fixed POST uploads (chunked and unchunked).
1125
			byte[] read = new byte[firstChunk.readableBytes( )];
1126
			firstChunk.readBytes( read );
1127
			putQueue.put(WalrusDataMessage.DataMessage(read));
444.1.37 by Neil
fixed (unchunked) form post.
1128
			if(!httpRequest.isChunked())
1129
				putQueue.put(WalrusDataMessage.EOF());
1130
		} catch (Exception ex) {
1131
			LOG.error(ex, ex);
1132
		}
1133
1134
	}
444.1.329 by Neil
added import script for 1.5.0, changed lock.
1135
444.1.37 by Neil
fixed (unchunked) form post.
1136
302 by Neil
saving changes.
1137
	public static synchronized WalrusDataMessenger getWriteMessenger() {
1138
		if (putMessenger == null) {
1139
			putMessenger = new WalrusDataMessenger();
286.1.2 by decker
adding more stuff
1140
		}
302 by Neil
saving changes.
1141
		return putMessenger;
1142
	}	
304 by Neil
saving changes.
1143
1144
	class Writer extends Thread {
1145
1146
		private ChannelBuffer firstBuffer;
1147
		private long dataLength;
1148
		public Writer(ChannelBuffer firstBuffer, long dataLength) {
1149
			this.firstBuffer = firstBuffer;
1150
			this.dataLength = dataLength;
1151
		}
1152
1153
		public void run() {
1154
			byte[] bytes = new byte[DATA_MESSAGE_SIZE];
1155
1156
			try {
1157
				LOG.info("Starting upload");                
1158
				putQueue.put(WalrusDataMessage.StartOfData(dataLength));
1159
1160
				firstBuffer.markReaderIndex( );
1161
				byte[] read = new byte[firstBuffer.readableBytes( )];
1162
				firstBuffer.readBytes( read );
1163
				putQueue.put(WalrusDataMessage.DataMessage(read));
1164
				//putQueue.put(WalrusDataMessage.EOF());
1165
1166
			} catch (Exception ex) {
1167
				LOG.error(ex, ex);
1168
			}
1169
		}
1170
1171
	}
286.1.2 by decker
adding more stuff
1172
1173
}