1
<?xml version="1.0" encoding="UTF-8"?>
2
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
3
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
4
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
6
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
8
<title>Class: DatastoreS3</title>
10
<link rel="stylesheet" href="./rdoc.css" type="text/css" media="screen" />
12
<script src="./js/jquery.js" type="text/javascript"
13
charset="utf-8"></script>
14
<script src="./js/thickbox-compressed.js" type="text/javascript"
15
charset="utf-8"></script>
16
<script src="./js/quicksearch.js" type="text/javascript"
17
charset="utf-8"></script>
18
<script src="./js/darkfish.js" type="text/javascript"
19
charset="utf-8"></script>
25
<div id="home-metadata">
26
<div id="home-section" class="section">
27
<h3 class="section-header">
28
<a href="./index.html">Home</a>
29
<a href="./index.html#classes">Classes</a>
30
<a href="./index.html#methods">Methods</a>
35
<div id="file-metadata">
36
<div id="file-list-section" class="section">
37
<h3 class="section-header">In Files</h3>
38
<div class="section-body">
41
<li><a href="./lib/datastore_s3_rb.html?TB_iframe=true&height=550&width=785"
42
class="thickbox" title="lib/datastore_s3.rb">lib/datastore_s3.rb</a></li>
51
<div id="class-metadata">
55
<div id="parent-class-section" class="section">
56
<h3 class="section-header">Parent</h3>
58
<p class="link"><a href="Datastore.html">Datastore</a></p>
63
<!-- Namespace Contents -->
66
<!-- Method Quickref -->
68
<div id="method-list-section" class="section">
69
<h3 class="section-header">Methods</h3>
70
<ul class="link-list">
72
<li><a href="#method-c-new">::new</a></li>
74
<li><a href="#method-c-parse_s3_key">::parse_s3_key</a></li>
76
<li><a href="#method-i-does_file_exist-3F">#does_file_exist?</a></li>
78
<li><a href="#method-i-get_acl">#get_acl</a></li>
80
<li><a href="#method-i-get_output_and_return_contents">#get_output_and_return_contents</a></li>
82
<li><a href="#method-i-get_output_and_save_to_fs">#get_output_and_save_to_fs</a></li>
84
<li><a href="#method-i-s3_bucket_exists-3F">#s3_bucket_exists?</a></li>
86
<li><a href="#method-i-set_acl">#set_acl</a></li>
88
<li><a href="#method-i-write_remote_file_from_local_file">#write_remote_file_from_local_file</a></li>
90
<li><a href="#method-i-write_remote_file_from_string">#write_remote_file_from_string</a></li>
96
<!-- Included Modules -->
100
<div id="project-metadata">
104
<div id="classindex-section" class="section project-section">
105
<h3 class="section-header">Class/Module Index
106
<span class="search-toggle"><img src="./images/find.png"
107
height="16" width="16" alt="[+]"
108
title="show/hide quicksearch" /></span></h3>
109
<form action="#" method="get" accept-charset="utf-8" class="initially-hidden">
111
<legend>Quicksearch</legend>
112
<input type="text" name="quicksearch" value=""
113
class="quicksearch-field" />
117
<ul class="link-list">
119
<li><a href="./AppControllerClient.html">AppControllerClient</a></li>
121
<li><a href="./BadConfigurationException.html">BadConfigurationException</a></li>
123
<li><a href="./BlobServer.html">BlobServer</a></li>
125
<li><a href="./Collectd.html">Collectd</a></li>
127
<li><a href="./CronHelper.html">CronHelper</a></li>
129
<li><a href="./Datastore.html">Datastore</a></li>
131
<li><a href="./DatastoreFactory.html">DatastoreFactory</a></li>
133
<li><a href="./DatastoreRepoOnAppScale.html">DatastoreRepoOnAppScale</a></li>
135
<li><a href="./DatastoreS3.html">DatastoreS3</a></li>
137
<li><a href="./Djinn.html">Djinn</a></li>
139
<li><a href="./DjinnJobData.html">DjinnJobData</a></li>
141
<li><a href="./DjinnServer.html">DjinnServer</a></li>
143
<li><a href="./Ejabberd.html">Ejabberd</a></li>
145
<li><a href="./EngineFactory.html">EngineFactory</a></li>
147
<li><a href="./GodInterface.html">GodInterface</a></li>
149
<li><a href="./GoogleAppEnginePullQueue.html">GoogleAppEnginePullQueue</a></li>
151
<li><a href="./GoogleAppEnginePushQueue.html">GoogleAppEnginePushQueue</a></li>
153
<li><a href="./HAProxy.html">HAProxy</a></li>
155
<li><a href="./HelperFunctions.html">HelperFunctions</a></li>
157
<li><a href="./JSONClient.html">JSONClient</a></li>
159
<li><a href="./LoadBalancer.html">LoadBalancer</a></li>
161
<li><a href="./Monitoring.html">Monitoring</a></li>
163
<li><a href="./NeptuneJobData.html">NeptuneJobData</a></li>
165
<li><a href="./Nginx.html">Nginx</a></li>
167
<li><a href="./Object.html">Object</a></li>
169
<li><a href="./PbServer.html">PbServer</a></li>
171
<li><a href="./QueueFactory.html">QueueFactory</a></li>
173
<li><a href="./RabbitMQ.html">RabbitMQ</a></li>
175
<li><a href="./Repo.html">Repo</a></li>
177
<li><a href="./TaskEngine.html">TaskEngine</a></li>
179
<li><a href="./TaskEngineAppScale.html">TaskEngineAppScale</a></li>
181
<li><a href="./TaskEngineGoogleAppEngine.html">TaskEngineGoogleAppEngine</a></li>
183
<li><a href="./TaskQueue.html">TaskQueue</a></li>
185
<li><a href="./TaskQueueAzureQueue.html">TaskQueueAzureQueue</a></li>
187
<li><a href="./TaskQueueGoogleAppEngine.html">TaskQueueGoogleAppEngine</a></li>
189
<li><a href="./TaskQueueRabbitMQ.html">TaskQueueRabbitMQ</a></li>
191
<li><a href="./TaskQueueSQS.html">TaskQueueSQS</a></li>
193
<li><a href="./UserAppClient.html">UserAppClient</a></li>
195
<li><a href="./ZKInterface.html">ZKInterface</a></li>
198
<div id="no-class-search-results" style="display: none;">No matching classes.</div>
205
<div id="documentation">
206
<h1 class="class">DatastoreS3</h1>
208
<div id="description">
210
<p>A class that abstracts away access to Amazon S3, and services that are
211
API-compatible with S3. Services store and retrieve files in the usual</p>
217
<div id="constants-list" class="section">
218
<h3 class="section-header">Constants</h3>
221
<dt><a name="NAME">NAME</a></dt>
223
<dd class="description"><p>The name of this datastore, exposed so that tests and <a
224
href="DatastoreFactory.html">DatastoreFactory</a> can refer to it without
225
worrying about mispelling it.</p></dd>
234
<div id="attribute-method-details" class="method-section section">
235
<h3 class="section-header">Attributes</h3>
238
<div id="EC2_ACCESS_KEY-attribute-method" class="method-detail">
239
<a name="EC2_ACCESS_KEY"></a>
241
<a name="EC2_ACCESS_KEY="></a>
243
<div class="method-heading attribute-method-heading">
244
<span class="method-name">EC2_ACCESS_KEY</span><span
245
class="attribute-access-type">[RW]</span>
248
<div class="method-description">
250
<p>The EC2 access key that should be used to access S3.</p>
255
<div id="EC2_SECRET_KEY-attribute-method" class="method-detail">
256
<a name="EC2_SECRET_KEY"></a>
258
<a name="EC2_SECRET_KEY="></a>
260
<div class="method-heading attribute-method-heading">
261
<span class="method-name">EC2_SECRET_KEY</span><span
262
class="attribute-access-type">[RW]</span>
265
<div class="method-description">
267
<p>The EC2 secret key that should be used to access S3.</p>
272
<div id="S3_URL-attribute-method" class="method-detail">
273
<a name="S3_URL"></a>
275
<a name="S3_URL="></a>
277
<div class="method-heading attribute-method-heading">
278
<span class="method-name">S3_URL</span><span
279
class="attribute-access-type">[RW]</span>
282
<div class="method-description">
284
<p>S3’s location (changable to support services that are S3-compatible, like
285
Google Storage and Eucalyptus Walrus).</p>
295
<div id="public-class-method-details" class="method-section section">
296
<h3 class="section-header">Public Class Methods</h3>
299
<div id="new-method" class="method-detail ">
300
<a name="method-c-new"></a>
303
<div class="method-heading">
304
<span class="method-name">new</span><span
305
class="method-args">(credentials)</span>
306
<span class="method-click-advice">click to toggle source</span>
310
<div class="method-description">
312
<p>Validates the S3 credentials that we’ve been given and returns a new S3
313
interface accordingly.</p>
317
<div class="method-source-code"
320
<span class="ruby-comment"># File lib/datastore_s3.rb, line 32</span>
321
def initialize(credentials)
322
if credentials.class != <span class="ruby-constant">Hash</span>
323
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-string">"Credentials was not a Hash"</span>)
326
if credentials[:EC2_ACCESS_KEY].nil?
327
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-string">"No EC2 access key specified"</span>)
329
<span class="ruby-ivar">@EC2_ACCESS_KEY</span> = credentials[:EC2_ACCESS_KEY]
331
if credentials[:EC2_SECRET_KEY].nil?
332
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-string">"No EC2 secret key specified"</span>)
334
<span class="ruby-ivar">@EC2_SECRET_KEY</span> = credentials[:EC2_SECRET_KEY]
336
if credentials[:S3_URL].nil?
337
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-string">"No S3 URL specified"</span>)
339
<span class="ruby-ivar">@S3_URL</span> = credentials[:S3_URL]
340
old_s3_url = <span class="ruby-constant">ENV</span>[<span class="ruby-string">'S3_URL'</span>]
341
<span class="ruby-constant">ENV</span>[<span class="ruby-string">'S3_URL'</span>] = <span class="ruby-ivar">@S3_URL</span>
342
<span class="ruby-ivar">@conn</span> = <span class="ruby-constant">RightAws</span>::<span class="ruby-constant">S3Interface</span>.new(<span class="ruby-ivar">@EC2_ACCESS_KEY</span>, <span class="ruby-ivar">@EC2_SECRET_KEY</span>)
343
<span class="ruby-constant">ENV</span>[<span class="ruby-string">'S3_URL'</span>] = old_s3_url
355
<div id="parse_s3_key-method" class="method-detail ">
356
<a name="method-c-parse_s3_key"></a>
359
<div class="method-heading">
360
<span class="method-name">parse_s3_key</span><span
361
class="method-args">(key)</span>
362
<span class="method-click-advice">click to toggle source</span>
366
<div class="method-description">
368
<p>Given a full S3 path, returns the bucket and filename.</p>
372
<div class="method-source-code"
373
id="parse_s3_key-source">
375
<span class="ruby-comment"># File lib/datastore_s3.rb, line 168</span>
376
def self.parse_s3_key(key)
377
paths = key.split(<span class="ruby-string">"/"</span>)
379
file = paths[2, paths.length - 1].join(<span class="ruby-string">"/"</span>)
394
<div id="public-instance-method-details" class="method-section section">
395
<h3 class="section-header">Public Instance Methods</h3>
398
<div id="does_file_exist-3F-method" class="method-detail ">
399
<a name="method-i-does_file_exist-3F"></a>
402
<div class="method-heading">
403
<span class="method-name">does_file_exist?</span><span
404
class="method-args">(path)</span>
405
<span class="method-click-advice">click to toggle source</span>
409
<div class="method-description">
411
<p>Checks whether the given file exists in S3.</p>
415
<div class="method-source-code"
416
id="does_file_exist-3F-source">
418
<span class="ruby-comment"># File lib/datastore_s3.rb, line 149</span>
419
def does_file_exist?(path)
420
bucket, file = self.parse_s3_key(path)
422
if !s3_bucket_exists?(bucket)
427
<span class="ruby-constant">Djinn</span>.log_debug("[does file exist] getting acl for bucket [#{bucket}] and file [#{file}] ")
428
<span class="ruby-ivar">@conn</span>.get_acl(bucket, file)
430
rescue <span class="ruby-constant">RightAws</span>::<span class="ruby-constant">AwsError</span>
431
<span class="ruby-constant">Djinn</span>.log_debug("[does file exist] saw a RightAws::AwsError on path [#{path}]")
445
<div id="get_acl-method" class="method-detail ">
446
<a name="method-i-get_acl"></a>
449
<div class="method-heading">
450
<span class="method-name">get_acl</span><span
451
class="method-args">(path)</span>
452
<span class="method-click-advice">click to toggle source</span>
456
<div class="method-description">
458
<p>Returns the access policy for the specified file. TODO(cgb): Need to
459
implement this and start using it.</p>
463
<div class="method-source-code"
466
<span class="ruby-comment"># File lib/datastore_s3.rb, line 136</span>
468
raise <span class="ruby-constant">NotImplementedError</span>.new(<span class="ruby-string">"Need to implement S3 get_acl"</span>)
480
<div id="get_output_and_return_contents-method" class="method-detail ">
481
<a name="method-i-get_output_and_return_contents"></a>
484
<div class="method-heading">
485
<span class="method-name">get_output_and_return_contents</span><span
486
class="method-args">(path)</span>
487
<span class="method-click-advice">click to toggle source</span>
491
<div class="method-description">
493
<p>Contacts S3 to retrieve a file, returning the contents of the file as a
498
<div class="method-source-code"
499
id="get_output_and_return_contents-source">
501
<span class="ruby-comment"># File lib/datastore_s3.rb, line 93</span>
502
def get_output_and_return_contents(path)
503
bucket, file = self.parse_s3_key(path)
504
return <span class="ruby-ivar">@conn</span>.get(bucket, file)[:object]
516
<div id="get_output_and_save_to_fs-method" class="method-detail ">
517
<a name="method-i-get_output_and_save_to_fs"></a>
520
<div class="method-heading">
521
<span class="method-name">get_output_and_save_to_fs</span><span
522
class="method-args">(s3_path, local_path)</span>
523
<span class="method-click-advice">click to toggle source</span>
527
<div class="method-description">
529
<p>Contacts S3 to retrieve a file, and returns the location on the local
530
filesystem that we wrote it to. If the file is a directory, we recursively
531
save all files contained in that directory.</p>
535
<div class="method-source-code"
536
id="get_output_and_save_to_fs-source">
538
<span class="ruby-comment"># File lib/datastore_s3.rb, line 61</span>
539
def get_output_and_save_to_fs(s3_path, local_path)
540
<span class="ruby-comment"># If fetching a directory, fetch all files via the prefix parameter</span>
541
<span class="ruby-comment"># TODO(cgb): presumably list_bucket only lists the first 1000 and</span>
542
<span class="ruby-comment"># returns a cursor for more, so change this accordingly</span>
543
bucket, file = self.parse_s3_key(s3_path)
544
<span class="ruby-constant">Djinn</span>.log_debug("Doing a list bucket on #{bucket}, with prefix #{file}")
545
files_to_write = <span class="ruby-ivar">@conn</span>.list_bucket(bucket, {<span class="ruby-string">'prefix'</span>=> file})
546
<span class="ruby-constant">Djinn</span>.log_debug("List bucket returned [#{files_to_write.join(', ')}]")
548
files_to_write.each { |s3_file_data|
549
s3_filename = s3_file_data[:key]
550
full_local_path = local_path + <span class="ruby-constant">File</span>::<span class="ruby-constant">Separator</span> + s3_filename
551
<span class="ruby-constant">Djinn</span>.log_debug("Writing local file #{full_local_path} from " +
552
"path #{local_path} and S3 key name #{s3_filename}")
554
<span class="ruby-comment"># if the user gives us a file to fetch that's several directories</span>
555
<span class="ruby-comment"># deep, we need to make all the directories first</span>
556
<span class="ruby-constant">FileUtils</span>.mkdir_p(<span class="ruby-constant">File</span>.dirname(full_local_path))
558
f = <span class="ruby-constant">File</span>.new(full_local_path, <span class="ruby-constant">File</span>::<span class="ruby-constant">CREAT</span>|<span class="ruby-constant">File</span>::<span class="ruby-constant">RDWR</span>)
559
result = <span class="ruby-ivar">@conn</span>.get(bucket, s3_filename) { |chunk|
577
<div id="s3_bucket_exists-3F-method" class="method-detail ">
578
<a name="method-i-s3_bucket_exists-3F"></a>
581
<div class="method-heading">
582
<span class="method-name">s3_bucket_exists?</span><span
583
class="method-args">(bucket)</span>
584
<span class="method-click-advice">click to toggle source</span>
588
<div class="method-description">
590
<p>Queries Amazon S3 with the given connection to see if the user owns the
595
<div class="method-source-code"
596
id="s3_bucket_exists-3F-source">
598
<span class="ruby-comment"># File lib/datastore_s3.rb, line 178</span>
599
def s3_bucket_exists?(bucket)
600
all_buckets = <span class="ruby-ivar">@conn</span>.list_all_my_buckets()
601
bucket_names = all_buckets.map { |b| b[:name] }
602
bucket_exists = bucket_names.include?(bucket)
603
<span class="ruby-constant">Djinn</span>.log_debug("the user owns buckets [#{bucket_names.join(', ')}] - do they own [#{bucket}]? #{bucket_exists}")
616
<div id="set_acl-method" class="method-detail ">
617
<a name="method-i-set_acl"></a>
620
<div class="method-heading">
621
<span class="method-name">set_acl</span><span
622
class="method-args">(path, acl)</span>
623
<span class="method-click-advice">click to toggle source</span>
627
<div class="method-description">
629
<p>Sets the access policy for the specified file. TODO(cgb): Need to implement
630
this and start using it.</p>
634
<div class="method-source-code"
637
<span class="ruby-comment"># File lib/datastore_s3.rb, line 143</span>
638
def set_acl(path, acl)
639
raise <span class="ruby-constant">NotImplementedError</span>.new(<span class="ruby-string">"Need to implement S3 set_acl"</span>)
651
<div id="write_remote_file_from_local_file-method" class="method-detail ">
652
<a name="method-i-write_remote_file_from_local_file"></a>
655
<div class="method-heading">
656
<span class="method-name">write_remote_file_from_local_file</span><span
657
class="method-args">(s3_path, local_path)</span>
658
<span class="method-click-advice">click to toggle source</span>
662
<div class="method-description">
664
<p>Writes a file in S3 from either a locally-specified file or from a string
665
given to us. If the file is a directory, this function also recursively
666
dives into that directory to write all files and directories contained.</p>
670
<div class="method-source-code"
671
id="write_remote_file_from_local_file-source">
673
<span class="ruby-comment"># File lib/datastore_s3.rb, line 103</span>
674
def write_remote_file_from_local_file(s3_path, local_path)
675
bucket, file = self.parse_s3_key(s3_path)
677
if <span class="ruby-constant">File</span>.directory?(path)
678
`ls #{local_path}`.split.each { |file|
679
full_s3_path = s3_path + <span class="ruby-string">"/"</span> + file
680
full_local_path = local_path + <span class="ruby-string">"/"</span> + file
681
<span class="ruby-constant">Djinn</span>.log_debug("recursive dive - now saving remote [#{full_s3_path}], local [#{full_local_path}]")
682
success = write_remote_file_from_local_file(full_s3_path, full_local_path)
689
<span class="ruby-constant">Djinn</span>.log_debug("attempting to put local file #{local_path} into " +
690
"bucket #{bucket}, location #{file}")
691
return <span class="ruby-ivar">@conn</span>.put(bucket, file, <span class="ruby-constant">File</span>.open(local_path))
706
<div id="write_remote_file_from_string-method" class="method-detail ">
707
<a name="method-i-write_remote_file_from_string"></a>
710
<div class="method-heading">
711
<span class="method-name">write_remote_file_from_string</span><span
712
class="method-args">(s3_path, string)</span>
713
<span class="method-click-advice">click to toggle source</span>
717
<div class="method-description">
719
<p>Writes a string to a file in S3.</p>
723
<div class="method-source-code"
724
id="write_remote_file_from_string-source">
726
<span class="ruby-comment"># File lib/datastore_s3.rb, line 128</span>
727
def write_remote_file_from_string(s3_path, string)
728
bucket, file = self.parse_s3_key(s3_path)
729
return <span class="ruby-ivar">@conn</span>.put(bucket, file, string, headers={<span class="ruby-string">"Content-Length"</span> => val.length})
746
<div id="validator-badges">
747
<p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
748
<p><small>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish
749
Rdoc Generator</a> 2</small>.</p>
added
removed
AppController/doc/DatastoreS3.html
