127
132
import edu.ucsb.eucalyptus.msgs.GetBucketAccessControlPolicyType;
128
133
import edu.ucsb.eucalyptus.msgs.GetBucketLocationResponseType;
129
134
import edu.ucsb.eucalyptus.msgs.GetBucketLocationType;
135
import edu.ucsb.eucalyptus.msgs.GetBucketLoggingStatusType;
136
import edu.ucsb.eucalyptus.msgs.GetBucketLoggingStatusResponseType;
130
137
import edu.ucsb.eucalyptus.msgs.GetObjectAccessControlPolicyResponseType;
131
138
import edu.ucsb.eucalyptus.msgs.GetObjectAccessControlPolicyType;
132
139
import edu.ucsb.eucalyptus.msgs.GetObjectExtendedResponseType;
152
160
import edu.ucsb.eucalyptus.msgs.RemoveARecordType;
153
161
import edu.ucsb.eucalyptus.msgs.SetBucketAccessControlPolicyResponseType;
154
162
import edu.ucsb.eucalyptus.msgs.SetBucketAccessControlPolicyType;
163
import edu.ucsb.eucalyptus.msgs.SetBucketLoggingStatusResponseType;
164
import edu.ucsb.eucalyptus.msgs.SetBucketLoggingStatusType;
155
165
import edu.ucsb.eucalyptus.msgs.SetObjectAccessControlPolicyResponseType;
156
166
import edu.ucsb.eucalyptus.msgs.SetObjectAccessControlPolicyType;
157
167
import edu.ucsb.eucalyptus.msgs.SetRESTBucketAccessControlPolicyResponseType;
166
177
import edu.ucsb.eucalyptus.util.WalrusDataMessage;
167
178
import edu.ucsb.eucalyptus.util.WalrusDataMessenger;
168
179
import edu.ucsb.eucalyptus.util.WalrusMonitor;
180
import edu.ucsb.eucalyptus.cloud.BucketLogData;
170
182
public class WalrusManager {
171
183
private static Logger LOG = Logger.getLogger( WalrusManager.class );
173
185
private StorageManager storageManager;
174
186
private WalrusImageManager walrusImageManager;
175
private static WalrusStatistics walrusStatistics = new WalrusStatistics();
187
private static WalrusStatistics walrusStatistics = null;
188
public static void deferedInitializer() {
189
walrusStatistics = new WalrusStatistics();
177
192
public WalrusManager(StorageManager storageManager, WalrusImageManager walrusImageManager) {
178
193
this.storageManager = storageManager;
252
private void makeBucket(String userId, String bucketName, String locationConstraint, AccessControlListType accessControlList, boolean isAdministrator) throws EucalyptusCloudException {
266
public CreateBucketResponseType createBucket(CreateBucketType request) throws EucalyptusCloudException {
267
CreateBucketResponseType reply = (CreateBucketResponseType) request.getReply();
268
String userId = request.getUserId();
270
String bucketName = request.getBucket();
271
String locationConstraint = request.getLocationConstraint();
253
273
if(userId == null) {
254
274
throw new AccessDeniedException("Bucket", bucketName);
277
AccessControlListType accessControlList = request.getAccessControlList();
257
278
if (accessControlList == null) {
258
279
accessControlList = new AccessControlListType();
261
282
EntityWrapper<BucketInfo> db = WalrusControl.getEntityWrapper();
263
if(WalrusProperties.shouldEnforceUsageLimits && !isAdministrator) {
284
if(WalrusProperties.shouldEnforceUsageLimits && !request.isAdministrator()) {
264
285
BucketInfo searchBucket = new BucketInfo();
265
286
searchBucket.setOwnerId(userId);
266
287
List<BucketInfo> bucketList = db.query(searchBucket);
289
310
bucket.addGrants(userId, grantInfos, accessControlList);
290
311
bucket.setGrants(grantInfos);
291
312
bucket.setBucketSize(0L);
313
bucket.setLoggingEnabled(false);
292
314
bucket.setHidden(false);
293
315
if(locationConstraint != null)
294
316
bucket.setLocation(locationConstraint);
296
318
bucket.setLocation("US");
320
//call the storage manager to save the bucket to disk
322
storageManager.createBucket(bucketName);
323
if(WalrusProperties.trackUsageStatistics)
324
walrusStatistics.incrementBucketCount();
325
} catch (IOException ex) {
328
throw new EucalyptusCloudException("Unable to create bucket: " + bucketName);
302
public CreateBucketResponseType createBucket(CreateBucketType request) throws EucalyptusCloudException {
303
CreateBucketResponseType reply = (CreateBucketResponseType) request.getReply();
304
String userId = request.getUserId();
306
String bucketName = request.getBucket();
307
String locationConstraint = request.getLocationConstraint();
310
throw new AccessDeniedException("Bucket", bucketName);
313
AccessControlListType accessControlList = request.getAccessControlList();
314
if (accessControlList == null) {
315
accessControlList = new AccessControlListType();
318
makeBucket(userId, bucketName, locationConstraint, accessControlList, request.isAdministrator());
320
//call the storage manager to save the bucket to disk
322
storageManager.createBucket(bucketName);
323
if(WalrusProperties.trackUsageStatistics)
324
walrusStatistics.incrementBucketCount();
325
} catch (IOException ex) {
327
throw new EucalyptusCloudException(bucketName);
330
333
if(WalrusProperties.enableVirtualHosting) {
331
334
UpdateARecordType updateARecord = new UpdateARecordType();
332
335
updateARecord.setUserId(userId);
333
String address = WalrusProperties.WALRUS_IP;
337
String address = null;
339
walrusUri = new URI(EucalyptusProperties.getWalrusUrl());
340
address = walrusUri.getHost();
341
} catch (URISyntaxException e) {
342
throw new EucalyptusCloudException("Could not get Walrus URL");
334
344
String zone = WalrusProperties.WALRUS_SUBDOMAIN + ".";
335
345
updateARecord.setAddress(address);
336
346
updateARecord.setName(bucketName + "." + zone);
405
415
status.setCode(204);
406
416
status.setDescription("No Content");
407
417
reply.setStatus(status);
418
if(logData != null) {
419
updateLogData(bucketFound, logData);
420
reply.setLogData(logData);
410
throw new BucketNotEmptyException(bucketName);
424
throw new BucketNotEmptyException(bucketName, logData);
414
throw new AccessDeniedException("Bucket", bucketName);
428
throw new AccessDeniedException("Bucket", bucketName, logData);
434
448
List<BucketInfo> bucketList = db.query(bucketInfo);
436
450
AccessControlListType accessControlList = new AccessControlListType();
451
BucketLogData logData;
438
453
if (bucketList.size() > 0) {
439
454
//construct access control policy from grant infos
440
455
BucketInfo bucket = bucketList.get(0);
456
logData = bucket.getLoggingEnabled() ? request.getLogData() : null;
441
457
List<GrantInfo> grantInfos = bucket.getGrants();
442
458
if (bucket.canReadACP(userId)) {
459
if(logData != null) {
460
updateLogData(bucket, logData);
461
reply.setLogData(logData);
443
463
ownerId = bucket.getOwnerId();
444
464
ArrayList<Grant> grants = new ArrayList<Grant>();
465
bucket.readPermissions(grants);
445
466
for (GrantInfo grantInfo: grantInfos) {
446
467
String uId = grantInfo.getUserId();
448
User grantUserInfo = CredentialProvider.getUser( uId );
449
bucket.readPermissions(grants);
450
addPermission(grants, grantUserInfo, grantInfo);
470
User grantUserInfo = CredentialProvider.getUser( uId );
471
addPermission(grants, grantUserInfo, grantInfo);
473
addPermission(grants, grantInfo);
451
475
} catch ( NoSuchUserException e ) {
453
throw new AccessDeniedException("Bucket", bucketName);
477
throw new AccessDeniedException("Bucket", bucketName, logData);
456
480
accessControlList.setGrants(grants);
527
private static void addPermission(ArrayList<Grant>grants, GrantInfo grantInfo) {
528
if(grantInfo.getGrantGroup() != null) {
529
Group group = new Group(grantInfo.getGrantGroup());
531
if (grantInfo.canRead() && grantInfo.canWrite() && grantInfo.canReadACP() && grantInfo.isWriteACP()) {
532
grants.add(new Grant(new Grantee(group), "FULL_CONTROL"));
536
if (grantInfo.canRead()) {
537
grants.add(new Grant(new Grantee(group), "READ"));
540
if (grantInfo.canWrite()) {
541
grants.add(new Grant(new Grantee(group), "WRITE"));
544
if (grantInfo.canReadACP()) {
545
grants.add(new Grant(new Grantee(group), "READ_ACP"));
548
if (grantInfo.isWriteACP()) {
549
grants.add(new Grant(new Grantee(group), "WRITE_ACP"));
503
554
public PutObjectResponseType putObject(PutObjectType request) throws EucalyptusCloudException {
504
555
PutObjectResponseType reply = (PutObjectResponseType) request.getReply();
505
556
String userId = request.getUserId();
644
698
walrusStatistics.updateBytesIn(size);
645
699
walrusStatistics.updateSpaceUsed(size);
701
if(logData != null) {
702
logData.setObjectSize(size);
703
updateLogData(bucket, logData);
647
705
dbObject.commit();
706
if(logData != null) {
707
logData.setTurnAroundTime(Long.parseLong(new String(dataMessage.getPayload())));
649
710
dbObject.rollback();
650
throw new NoSuchEntityException("Could not find object: " + bucketName + "/" + objectKey);
711
throw new NoSuchEntityException("Could not find object: " + bucketName + "/" + objectKey, logData);
652
713
//restart all interrupted puts
653
714
WalrusMonitor monitor = messenger.getMonitor(key);
697
758
} catch (InterruptedException ex) {
698
759
LOG.error(ex, ex);
699
760
messenger.removeQueue(key, randomKey);
700
throw new EucalyptusCloudException();
761
throw new EucalyptusCloudException("Transfer interrupted: " + key + "." + randomKey);
704
765
messenger.removeQueue(key, randomKey);
705
throw new AccessDeniedException("Bucket", bucketName);
766
throw new AccessDeniedException("Bucket", bucketName, logData);
855
918
md5 = Hashes.getHexString(Hashes.Digest.MD5.get().digest(base64Data));
856
919
foundObject.setEtag(md5);
857
Long size = Long.parseLong(request.getContentLength());
920
Long size = (long)base64Data.length;
858
921
foundObject.setSize(size);
859
922
if(WalrusProperties.shouldEnforceUsageLimits && !request.isAdministrator()) {
860
923
Long bucketSize = bucket.getBucketSize();
861
924
long newSize = bucketSize + oldBucketSize + size;
862
925
if(newSize > WalrusProperties.MAX_BUCKET_SIZE) {
864
throw new EntityTooLargeException("Key", objectKey);
927
throw new EntityTooLargeException("Key", objectKey, logData);
866
929
bucket.setBucketSize(newSize);
877
940
foundObject.setStorageClass("STANDARD");
878
941
lastModified = new Date();
879
942
foundObject.setLastModified(lastModified);
880
} catch (/*TODO: NEIL, check if it is IOException*/Exception ex) {
943
if(logData != null) {
944
updateLogData(bucket, logData);
945
logData.setObjectSize(size);
946
reply.setLogData(logData);
948
} catch (Exception ex) {
883
951
throw new EucalyptusCloudException(bucketName);
887
throw new AccessDeniedException("Bucket", bucketName);
955
throw new AccessDeniedException("Bucket", bucketName, logData);
892
959
throw new NoSuchBucketException(bucketName);
902
public void addObject(String userId, String bucketName, String key) throws EucalyptusCloudException {
904
AccessControlListType accessControlList = new AccessControlListType();
969
public AddObjectResponseType addObject(AddObjectType request) throws EucalyptusCloudException {
971
AddObjectResponseType reply = (AddObjectResponseType) request.getReply();
972
String bucketName = request.getBucket();
973
String key = request.getKey();
974
String userId = request.getUserId();
975
String objectName = request.getObjectName();
977
AccessControlListType accessControlList = request.getAccessControlList();
905
978
if (accessControlList == null) {
906
979
accessControlList = new AccessControlListType();
934
1008
objectInfo.setObjectKey(key);
935
1009
objectInfo.setOwnerId(userId);
936
objectInfo.setSize(storageManager.getSize(bucketName, key));
937
objectInfo.setEtag("");
1010
objectInfo.setSize(storageManager.getSize(bucketName, objectName));
1011
objectInfo.setEtag(request.getEtag());
938
1012
objectInfo.setLastModified(new Date());
1013
objectInfo.setStorageClass("STANDARD");
941
1016
throw new AccessDeniedException("Bucket", bucketName);
980
1057
objectDeleter.start();
981
1058
reply.setCode("200");
982
1059
reply.setDescription("OK");
1060
if(logData != null) {
1061
updateLogData(bucketInfo, logData);
1062
reply.setLogData(logData);
985
throw new AccessDeniedException("Key", objectKey);
1066
throw new AccessDeniedException("Key", objectKey, logData);
989
throw new NoSuchEntityException(objectKey);
1070
throw new NoSuchEntityException(objectKey, logData);
1045
1126
if(bucketList.size() > 0) {
1046
1127
BucketInfo bucket = bucketList.get(0);
1128
BucketLogData logData = bucket.getLoggingEnabled() ? request.getLogData() : null;
1047
1129
if(bucket.canRead(userId)) {
1130
if(logData != null) {
1131
updateLogData(bucket, logData);
1132
reply.setLogData(logData);
1048
1134
if(request.isAdministrator()) {
1049
1135
EntityWrapper<WalrusSnapshotInfo> dbSnap = db.recast(WalrusSnapshotInfo.class);
1050
1136
WalrusSnapshotInfo walrusSnapInfo = new WalrusSnapshotInfo();
1162
1248
EntityWrapper<BucketInfo> db = WalrusControl.getEntityWrapper();
1163
1249
BucketInfo bucketInfo = new BucketInfo(bucketName);
1164
1250
List<BucketInfo> bucketList = db.query(bucketInfo);
1251
BucketLogData logData;
1166
1253
AccessControlListType accessControlList = new AccessControlListType();
1167
1254
if (bucketList.size() > 0) {
1168
1255
//construct access control policy from grant infos
1256
logData = bucketList.get(0).getLoggingEnabled() ? request.getLogData() : null;
1169
1257
EntityWrapper<ObjectInfo> dbObject = db.recast(ObjectInfo.class);
1170
1258
ObjectInfo searchObjectInfo = new ObjectInfo(bucketName, objectKey);
1171
1259
List<ObjectInfo> objectInfos = dbObject.query(searchObjectInfo);
1172
1260
if(objectInfos.size() > 0) {
1173
1261
ObjectInfo objectInfo = objectInfos.get(0);
1174
1262
if(objectInfo.canReadACP(userId)) {
1263
BucketInfo bucket = bucketList.get(0);
1264
if(logData != null) {
1265
updateLogData(bucket, logData);
1266
logData.setObjectSize(objectInfo.getSize());
1267
reply.setLogData(logData);
1175
1270
ownerId = objectInfo.getOwnerId();
1176
1271
ArrayList<Grant> grants = new ArrayList<Grant>();
1177
1272
List<GrantInfo> grantInfos = objectInfo.getGrants();
1182
1277
objectInfo.readPermissions(grants);
1183
1278
addPermission(grants, userInfo, grantInfo);
1184
1279
} catch ( NoSuchUserException e ) {
1185
throw new AccessDeniedException("Key", objectKey);
1280
throw new AccessDeniedException("Key", objectKey, logData);
1188
1283
accessControlList.setGrants(grants);
1191
throw new AccessDeniedException("Key", objectKey);
1286
throw new AccessDeniedException("Key", objectKey, logData);
1195
throw new NoSuchEntityException(objectKey);
1290
throw new NoSuchEntityException(objectKey, logData);
1455
1576
} catch (EucalyptusCloudException ex) {
1456
1577
String torrentFile = objectName + ".torrent";
1457
1578
String torrentFilePath = storageManager.getObjectPath(bucketName, torrentFile);
1458
TorrentCreator torrentCreator = new TorrentCreator(absoluteObjectPath, objectKey, objectName, torrentFilePath, WalrusProperties.TRACKER_URL);
1579
TorrentCreator torrentCreator = new TorrentCreator(absoluteObjectPath, objectKey, objectName, torrentFilePath, WalrusProperties.getTrackerUrl());
1460
1581
torrentCreator.create();
1461
1582
} catch(Exception e) {
1482
1603
Date lastModified = objectInfo.getLastModified();
1484
1605
long torrentLength = torrent.length();
1606
if(logData != null) {
1607
updateLogData(bucket, logData);
1608
logData.setObjectSize(torrentLength);
1485
1610
storageManager.sendObject(request.getChannel(), httpResponse, bucketName, torrentFile, torrentLength, null,
1486
1611
DateUtils.format(lastModified.getTime(), DateUtils.ISO8601_DATETIME_PATTERN) + ".000Z",
1487
"application/x-bittorrent", "attachment; filename=" + objectKey + ".torrent;", request.getIsCompressed());
1612
"application/x-bittorrent", "attachment; filename=" + objectKey + ".torrent;", request.getIsCompressed(),
1488
1614
if(WalrusProperties.trackUsageStatistics) {
1489
1615
walrusStatistics.updateBytesOut(torrentLength);
1530
1659
storageManager.sendObject(request.getChannel(), httpResponse, bucketName, objectName, size, etag,
1531
1660
DateUtils.format(lastModified.getTime(), DateUtils.ISO8601_DATETIME_PATTERN) + ".000Z",
1532
contentType, contentDisposition, request.getIsCompressed());
1661
contentType, contentDisposition, request.getIsCompressed(), logData);
1536
1665
storageManager.sendHeaders(request.getChannel(), httpResponse, size, etag,
1537
1666
DateUtils.format(lastModified.getTime(), DateUtils.ISO8601_DATETIME_PATTERN) + ".000Z",
1538
contentType, contentDisposition);
1667
contentType, contentDisposition, logData);
1644
1775
String contentType = objectInfo.getContentType();
1645
1776
String contentDisposition = objectInfo.getContentDisposition();
1778
if(logData != null) {
1779
updateLogData(bucket, logData);
1780
logData.setObjectSize(size);
1647
1782
if(request.getGetData()) {
1648
1783
if(WalrusProperties.trackUsageStatistics) {
1649
1784
walrusStatistics.updateBytesOut(size);
1651
1786
storageManager.sendObject(request.getChannel(), httpResponse, bucketName, objectName, byteRangeStart, byteRangeEnd, size, etag,
1652
1787
DateUtils.format(lastModified.getTime(), DateUtils.ISO8601_DATETIME_PATTERN + ".000Z"),
1653
contentType, contentDisposition, request.getIsCompressed());
1788
contentType, contentDisposition, request.getIsCompressed(), logData);
1655
1791
storageManager.sendHeaders(request.getChannel(), httpResponse, size, etag,
1656
1792
DateUtils.format(lastModified.getTime(), DateUtils.ISO8601_DATETIME_PATTERN + ".000Z"),
1657
contentType, contentDisposition);
1659
reply.setEtag(etag);
1660
reply.setLastModified(DateUtils.format(lastModified.getTime(), DateUtils.ISO8601_DATETIME_PATTERN) + ".000Z");
1661
if(byteRangeEnd > -1) {
1662
if(byteRangeEnd <= size && ((byteRangeEnd - byteRangeStart) > 0))
1663
reply.setSize(byteRangeEnd - byteRangeStart);
1667
reply.setSize(size);
1669
status.setCode(200);
1670
status.setDescription("OK");
1671
reply.setContentType("binary/octet-stream");
1672
reply.setStatus(status);
1793
contentType, contentDisposition, logData);
1675
1798
throw new AccessDeniedException("Key", objectKey);
1861
1987
throw new NoSuchBucketException(sourceBucket);
1991
public SetBucketLoggingStatusResponseType setBucketLoggingStatus(SetBucketLoggingStatusType request) throws EucalyptusCloudException {
1992
SetBucketLoggingStatusResponseType reply = (SetBucketLoggingStatusResponseType) request.getReply();
1993
String bucket = request.getBucket();
1994
String targetBucket = request.getLoggingEnabled().getTargetBucket();
1995
String targetPrefix = request.getLoggingEnabled().getTargetPrefix();
1996
List<Grant> targetGrantsList = null;
1997
TargetGrants targetGrants = request.getLoggingEnabled().getTargetGrants();
1998
if(targetGrants != null)
1999
targetGrantsList = targetGrants.getGrants();
2000
if(targetPrefix == null)
2003
EntityWrapper<BucketInfo> db = WalrusControl.getEntityWrapper();
2004
BucketInfo bucketInfo, targetBucketInfo;
2006
bucketInfo = db.getUnique(new BucketInfo(bucket));
2007
} catch(EucalyptusCloudException ex) {
2009
throw new NoSuchBucketException(bucket);
2012
targetBucketInfo = db.getUnique(new BucketInfo(targetBucket));
2013
} catch(EucalyptusCloudException ex) {
2015
throw new NoSuchBucketException(bucket);
2017
if(!targetBucketInfo.hasLoggingPerms()) {
2019
throw new InvalidTargetBucketForLoggingException(targetBucket);
2021
bucketInfo.setTargetBucket(targetBucket);
2022
bucketInfo.setTargetPrefix(targetPrefix);
2023
bucketInfo.setLoggingEnabled(true);
2024
if(targetGrantsList != null) {
2025
targetBucketInfo.addGrants(targetGrantsList);
2031
public GetBucketLoggingStatusResponseType getBucketLoggingStatus(GetBucketLoggingStatusType request) throws EucalyptusCloudException {
2032
GetBucketLoggingStatusResponseType reply = (GetBucketLoggingStatusResponseType) request.getReply();
2033
String bucket = request.getBucket();
2035
EntityWrapper<BucketInfo> db = WalrusControl.getEntityWrapper();
2037
BucketInfo bucketInfo = db.getUnique(new BucketInfo(bucket));
2038
if(bucketInfo.getLoggingEnabled()) {
2039
String targetBucket = bucketInfo.getTargetBucket();
2040
ArrayList<Grant> grants = new ArrayList<Grant>();
2042
BucketInfo targetBucketInfo = db.getUnique(new BucketInfo(targetBucket));
2043
List<GrantInfo> grantInfos = targetBucketInfo.getGrants();
2044
for (GrantInfo grantInfo: grantInfos) {
2045
String uId = grantInfo.getUserId();
2048
User grantUserInfo = CredentialProvider.getUser( uId );
2049
addPermission(grants, grantUserInfo, grantInfo);
2051
addPermission(grants, grantInfo);
2053
} catch ( NoSuchUserException e ) {
2055
throw new AccessDeniedException("Bucket", targetBucket);
2058
} catch(EucalyptusCloudException ex) {
2060
throw new InvalidTargetBucketForLoggingException(targetBucket);
2062
reply.getLoggingEnabled().setTargetBucket(bucketInfo.getTargetBucket());
2063
reply.getLoggingEnabled().setTargetPrefix(bucketInfo.getTargetPrefix());
2065
TargetGrants targetGrants = new TargetGrants();
2066
targetGrants.setGrants(grants);
2067
reply.getLoggingEnabled().setTargetGrants(targetGrants);
2069
} catch(EucalyptusCloudException ex) {
2071
throw new NoSuchBucketException(bucket);
2077
private void updateLogData(BucketInfo bucket, BucketLogData logData) {
2078
logData.setOwnerId(bucket.getOwnerId());
2079
logData.setTargetBucket(bucket.getTargetBucket());
2080
logData.setTargetPrefix(bucket.getTargetPrefix());