« back to all changes in this revision
Viewing changes to AppController/doc/Object.html
- browse files at revision 855
- compare with another revision
- download diff
- download tarball
- view history from revision 855
- Committer: Chris Bunch
- Date: 2012-02-27 04:59:38 UTC
- Revision ID: cgb@cs.ucsb.edu-20120227045938-mc8x1ff7520zz3sp
added datastore_s3 tests and regenerated coverage/rdoc
added
removed
AppController/doc/Object.html
407
407
408
408
<li><a href="./DatastoreFactory.html">DatastoreFactory</a></li>
409
409
410
<li><a href="./DatastoreRepo.html">DatastoreRepo</a></li>
411
412
<li><a href="./DatastoreRepoOnAppEngine.html">DatastoreRepoOnAppEngine</a></li>
413
410
414
<li><a href="./DatastoreRepoOnAppScale.html">DatastoreRepoOnAppScale</a></li>
411
415
412
416
<li><a href="./DatastoreS3.html">DatastoreS3</a></li>
896
900
<div class="method-source-code"
897
901
id="add_nodes-source">
898
902
<pre>
899
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 645</span>
903
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 631</span>
900
904
def add_nodes(node_info)
901
905
keyname = <span class="ruby-ivar">@creds</span>[<span class="ruby-string">'keyname'</span>]
902
906
new_nodes = <span class="ruby-constant">Djinn</span>.convert_location_array_to_class(node_info, keyname)
946
950
<div class="method-source-code"
947
951
id="add_roles_and_wait-source">
948
952
<pre>
949
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 692</span>
953
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 678</span>
950
954
def add_roles_and_wait(nodes, roles)
951
955
return if nodes.nil?
952
956
988
992
<div class="method-source-code"
989
993
id="average-source">
990
994
<pre>
991
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 245</span>
995
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 253</span>
992
996
def average(population)
993
997
total = 0.0
994
998
n = 0
1030
1034
<div class="method-source-code"
1031
1035
id="can_run_job-source">
1032
1036
<pre>
1033
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 266</span>
1037
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 265</span>
1034
1038
def can_run_job(job_data)
1035
1039
<span class="ruby-comment"># no input / output for appscale jobs</span>
1036
1040
return :ok if job_data[<span class="ruby-string">"@type"</span>] == <span class="ruby-string">"appscale"</span>
1037
1041
1038
1042
storage = job_data[<span class="ruby-string">"@storage"</span>]
1039
creds = <span class="ruby-constant">Djinn</span>.neptune_parse_creds(storage, job_data)
1040
1043
1041
1044
if !<span class="ruby-constant">ALLOWED_STORAGE_TYPES</span>.include?(storage)
1042
1045
return "error: bad storage type - supported types are #{ALLOWED_STORAGE_TYPES.join(', ')}"
1043
1046
end
1044
1047
1045
if !<span class="ruby-constant">Repo</span>.valid_storage_creds(storage, creds)
1046
return <span class="ruby-string">"error: bad credentials for accessing storage"</span>
1047
end
1048
datastore = <span class="ruby-constant">DatastoreFactory</span>.get_datastore(storage, job_data)
1048
1049
1049
1050
input_location = job_data[<span class="ruby-string">"@input"</span>]
1050
1051
if input_location and !<span class="ruby-constant">NO_INPUT_NEEDED</span>.include?(job_data[<span class="ruby-string">'@type'</span>])
1051
input_exists = <span class="ruby-constant">Repo</span>.does_file_exist?(input_location, storage, creds)
1052
input_exists = datastore.does_file_exist?(input_location)
1052
1053
<span class="ruby-constant">Djinn</span>.log_debug("input specified - did #{input_location} exist? #{input_exists}")
1053
1054
unless input_exists
1054
1055
return <span class="ruby-string">"error: input specified but did not exist"</span>
1058
1059
end
1059
1060
1060
1061
output_location = job_data[<span class="ruby-string">"@output"</span>]
1061
output_exists = <span class="ruby-constant">Repo</span>.does_file_exist?(output_location, storage, creds)
1062
output_exists = datastore.does_file_exist?(output_location)
1062
1063
<span class="ruby-constant">Djinn</span>.log_debug("output specified - did #{output_location} exist? #{output_exists}")
1063
1064
if output_exists
1064
1065
return <span class="ruby-string">"error: output already exists"</span>
1074
1075
end
1075
1076
end
1076
1077
1077
<span class="ruby-comment">#if is_hybrid_cloud?</span>
1078
<span class="ruby-comment"># if job_data["@nodes_to_use"].class == Array # request for hybrid cloud</span>
1079
<span class="ruby-comment"># job_data["@nodes_to_use"] = Hash[*job_data["@nodes_to_use"]]</span>
1080
<span class="ruby-comment"># else</span>
1081
<span class="ruby-comment"># return "error: must specify hybrid nodes to use in hybrid cloud runs"</span>
1082
<span class="ruby-comment"># end</span>
1083
<span class="ruby-comment">#</span>
1084
<span class="ruby-comment"># # make sure that we have enough clouds available as the user has spec'd</span>
1085
<span class="ruby-comment">#end</span>
1086
1087
1078
if !(is_cloud? or is_hybrid_cloud?)
1088
1079
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"not in cloud"</span>)
1089
1080
<span class="ruby-comment"># make sure we have enough open nodes</span>
1172
1163
<div class="method-source-code"
1173
1164
id="copyFromShadow-source">
1174
1165
<pre>
1175
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 715</span>
1166
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 701</span>
1176
1167
def copyFromShadow(location_on_shadow)
1177
1168
shadow = get_shadow
1178
1169
shadow_ip = shadow.private_ip
1212
1203
<div class="method-source-code"
1213
1204
id="execute_multiple_tasks-source">
1214
1205
<pre>
1215
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 497</span>
1206
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 500</span>
1216
1207
def execute_multiple_tasks(tasks)
1217
1208
threads = []
1218
1209
tasks.each { |task|
1363
1354
<div class="method-source-code"
1364
1355
id="get_job_name-source">
1365
1356
<pre>
1366
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 680</span>
1357
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 666</span>
1367
1358
def get_job_name(job_data)
1368
1359
job_name = job_data[<span class="ruby-string">"@type"</span>]
1369
1360
1405
1396
<div class="method-source-code"
1406
1397
id="get_language-source">
1407
1398
<pre>
1408
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 192</span>
1399
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 193</span>
1409
1400
def get_language(filename)
1410
1401
return <span class="ruby-string">"ruby"</span>
1411
1402
end</pre>
1439
1430
<div class="method-source-code"
1440
1431
id="get_length_of_all_queues-source">
1441
1432
<pre>
1442
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 431</span>
1433
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 434</span>
1443
1434
def get_length_of_all_queues(queues)
1444
1435
<span class="ruby-comment"># something to consider: do leased tasks count in the size?</span>
1445
1436
length = 0
1478
1469
<div class="method-source-code"
1479
1470
id="get_lock_file_path-source">
1480
1471
<pre>
1481
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 521</span>
1472
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 507</span>
1482
1473
def get_lock_file_path(job_data)
1483
1474
"/tmp/#{job_data['@type']}-#{job_data['@job_id']}-started"
1484
1475
end</pre>
1512
1503
<div class="method-source-code"
1513
1504
id="get_n_items_of_work-source">
1514
1505
<pre>
1515
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 441</span>
1506
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 444</span>
1516
1507
def get_n_items_of_work(n, queues)
1517
1508
items = []
1518
1509
1566
1557
<div class="method-source-code"
1567
1558
id="get_node_roles-source">
1568
1559
<pre>
1569
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 408</span>
1560
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 394</span>
1570
1561
def get_node_roles(job_data)
1571
1562
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"getting node roles"</span>)
1572
1563
job_type = job_data[<span class="ruby-string">"@type"</span>]
1617
1608
<div class="method-source-code"
1618
1609
id="get_queues_from_shadow-source">
1619
1610
<pre>
1620
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 404</span>
1611
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 407</span>
1621
1612
def get_queues_from_shadow()
1622
1613
secret = <span class="ruby-constant">HelperFunctions</span>.get_secret()
1623
1614
acc = <span class="ruby-constant">AppControllerClient</span>.new(get_shadow.public_ip, secret)
1675
1666
<div class="method-source-code"
1676
1667
id="get_queues_in_use-source">
1677
1668
<pre>
1678
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 211</span>
1669
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 216</span>
1679
1670
def get_queues_in_use(secret)
1680
1671
return <span class="ruby-constant">BAD_SECRET_MSG</span> if !valid_secret?(secret)
1681
1672
<span class="ruby-constant">Djinn</span>.log_debug("@queues_to_read is #{@queues_to_read.join(', ')}, class #{@queues_to_read.class}")
1711
1702
<div class="method-source-code"
1712
1703
id="is_storage_location-3F-source">
1713
1704
<pre>
1714
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 399</span>
1705
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 402</span>
1715
1706
def is_storage_location?(file)
1716
1707
return <span class="ruby-constant">STORAGE_PARAM_REGEX</span>.match(file)
1717
1708
end</pre>
1745
1736
<div class="method-source-code"
1746
1737
id="lock_file_exists-3F-source">
1747
1738
<pre>
1748
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 502</span>
1739
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 488</span>
1749
1740
def lock_file_exists?(job_data)
1750
1741
return <span class="ruby-constant">File</span>.exists?(get_lock_file_path(job_data))
1751
1742
end</pre>
1779
1770
<div class="method-source-code"
1780
1771
id="neptune_acquire_nodes-source">
1781
1772
<pre>
1782
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 530</span>
1773
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 516</span>
1783
1774
def neptune_acquire_nodes(job_data)
1784
1775
<span class="ruby-comment"># for jobs where no nodes need to be acquired (e.g., concurrent but not</span>
1785
1776
<span class="ruby-comment"># distributed programs), run them on the shadow node</span>
1860
1851
<div class="method-source-code"
1861
1852
id="neptune_acquire_nodes_for_cloud-source">
1862
1853
<pre>
1863
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 625</span>
1854
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 611</span>
1864
1855
def neptune_acquire_nodes_for_cloud(cloud_num, new_vms_needed, job_data)
1865
1856
return if new_vms_needed < 1
1866
1857
<span class="ruby-constant">Djinn</span>.log_debug("spawning up #{new_vms_needed} vms")
1959
1950
<div class="method-source-code"
1960
1951
id="neptune_babel_run_job-source">
1961
1952
<pre>
1962
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 131</span>
1953
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 136</span>
1963
1954
def neptune_babel_run_job(nodes, job_data, secret)
1964
1955
return <span class="ruby-constant">BAD_SECRET_MSG</span> if !valid_secret?(secret)
1965
1956
2215
2206
<div class="method-source-code"
2216
2207
id="neptune_compile_code-source">
2217
2208
<pre>
2218
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 233</span>
2209
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 232</span>
2219
2210
def neptune_compile_code(job_data, secret)
2220
2211
message = validate_environment(job_data, secret)
2221
2212
return message unless message == <span class="ruby-string">"no error"</span>
2411
2402
<div class="method-source-code"
2412
2403
id="neptune_does_file_exist-source">
2413
2404
<pre>
2414
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 153</span>
2405
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 156</span>
2415
2406
def neptune_does_file_exist(file, job_data, secret)
2416
storage = job_data[<span class="ruby-string">"@storage"</span>]
2417
creds = <span class="ruby-constant">Djinn</span>.neptune_parse_creds(storage, job_data)
2418
return <span class="ruby-constant">Repo</span>.does_file_exist?(file, storage, creds)
2407
datastore = <span class="ruby-constant">DatastoreFactory</span>.get_datastore(job_data[<span class="ruby-string">'@storage'</span>], job_data)
2408
return datastore.does_file_exist?(file)
2419
2409
end</pre>
2420
2410
</div>
2421
2411
2447
2437
<div class="method-source-code"
2448
2438
id="neptune_erlang_run_job-source">
2449
2439
<pre>
2450
<span class="ruby-comment"># File ../Neptune/erlang_helper.rb, line 11</span>
2440
<span class="ruby-comment"># File ../Neptune/erlang_helper.rb, line 18</span>
2451
2441
def neptune_erlang_run_job(nodes, job_data, secret)
2452
2442
return <span class="ruby-constant">BAD_SECRET_MSG</span> unless valid_secret?(secret)
2453
2443
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"erlang - run"</span>)
2467
2457
2468
2458
remote = job_data[<span class="ruby-string">'@code'</span>]
2469
2459
storage = job_data[<span class="ruby-string">'@storage'</span>]
2470
creds = <span class="ruby-constant">Djinn</span>.neptune_parse_creds(storage, job_data)
2471
2460
2472
<span class="ruby-constant">Repo</span>.get_output(remote, storage, creds, "/tmp/#{code}")
2461
datastore = <span class="ruby-constant">DatastoreFactory</span>.get_datastore(storage, job_data)
2462
datastore.get_output_and_save_to_fs(remote, "/tmp/#{code}")
2473
2463
2474
2464
module_name = code.split(<span class="ruby-regexp">/\./</span>)[0]
2475
2465
<span class="ruby-constant">Djinn</span>.log_debug("got code #{code}, trying to run module #{module_name}")
2476
2466
<span class="ruby-constant">Djinn</span>.log_run("chmod +x #{code}")
2477
2467
<span class="ruby-constant">Djinn</span>.log_run("cd /tmp; erl -noshell -run #{module_name} main > #{ERLANG_OUTPUT}")
2478
2468
2479
<span class="ruby-constant">Repo</span>.set_output(job_data[<span class="ruby-string">"@output"</span>], <span class="ruby-constant">ERLANG_OUTPUT</span>, storage, creds, <span class="ruby-constant">IS_FILE</span>)
2480
2469
datastore.write_remote_file_from_local_file(job_data[<span class="ruby-string">'@output'</span>], <span class="ruby-constant">ERLANG_OUTPUT</span>)
2481
2470
remove_lock_file(job_data)
2482
2483
2471
<span class="ruby-constant">Djinn</span>.log_run("rm -rfv /tmp/#{code}")
2484
2472
}
2485
2473
2515
2503
<div class="method-source-code"
2516
2504
id="neptune_find_open_nodes-source">
2517
2505
<pre>
2518
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 581</span>
2506
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 567</span>
2519
2507
def neptune_find_open_nodes(cloud, nodes_needed, job_data)
2520
2508
<span class="ruby-comment"># TODO: assigning nodes -> nodes_to_use should be atomic?</span>
2521
2509
<span class="ruby-comment"># or should going through this list be atomic?</span>
2589
2577
<div class="method-source-code"
2590
2578
id="neptune_get_acl-source">
2591
2579
<pre>
2592
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 183</span>
2580
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 184</span>
2593
2581
def neptune_get_acl(job_data, secret)
2594
2582
message = validate_environment(job_data, secret)
2595
2583
return message unless message == <span class="ruby-string">"no error"</span>
2597
2585
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"requesting acl"</span>)
2598
2586
2599
2587
type = job_data[<span class="ruby-string">"@type"</span>]
2600
storage = job_data[<span class="ruby-string">"@storage"</span>]
2601
creds = <span class="ruby-constant">Djinn</span>.neptune_parse_creds(storage, job_data)
2602
2588
2603
2589
output_location = job_data[<span class="ruby-string">"@output"</span>]
2604
2590
if output_location.nil?
2605
2591
return <span class="ruby-string">"error: output not specified"</span>
2606
2592
else
2607
if <span class="ruby-constant">Repo</span>.does_file_exist?(output_location, storage, creds)
2608
return <span class="ruby-constant">Repo</span>.get_acl(output_location, storage, creds)
2593
datastore = <span class="ruby-constant">DatastoreFactory</span>.get_datastore(job_data[<span class="ruby-string">'@storage'</span>], job_data)
2594
if datastore.does_file_exist?(output_location)
2595
return datastore.get_acl(output_location)
2609
2596
else
2610
2597
return <span class="ruby-string">"error: output does not exist"</span>
2611
2598
end
2641
2628
<div class="method-source-code"
2642
2629
id="neptune_get_output-source">
2643
2630
<pre>
2644
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 159</span>
2631
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 161</span>
2645
2632
def neptune_get_output(job_data, secret)
2646
2633
message = validate_environment(job_data, secret)
2647
2634
return message unless message == <span class="ruby-string">"no error"</span>
2649
2636
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"requesting output"</span>)
2650
2637
2651
2638
type = job_data[<span class="ruby-string">"@type"</span>]
2652
storage = job_data[<span class="ruby-string">"@storage"</span>]
2653
creds = <span class="ruby-constant">Djinn</span>.neptune_parse_creds(storage, job_data)
2654
2639
2655
2640
output_location = job_data[<span class="ruby-string">"@output"</span>]
2656
2641
if output_location.nil?
2657
2642
return <span class="ruby-string">"error: output not specified"</span>
2658
2643
else
2659
if <span class="ruby-constant">Repo</span>.does_file_exist?(output_location, storage, creds)
2644
datastore = <span class="ruby-constant">DatastoreFactory</span>.get_datastore(job_data[<span class="ruby-string">'@storage'</span>], job_data)
2645
if datastore.does_file_exist?(output_location)
2660
2646
<span class="ruby-comment"># TODO: maybe write to file or have</span>
2661
2647
<span class="ruby-comment"># special flag for this?</span>
2662
return <span class="ruby-constant">Repo</span>.get_output(output_location, storage, creds)
2648
return datastore.get_output_and_return_contents(output_location)
2663
2649
else
2664
2650
return <span class="ruby-string">"error: output does not exist"</span>
2665
2651
end
2695
2681
<div class="method-source-code"
2696
2682
id="neptune_get_ssa_num_simulations-source">
2697
2683
<pre>
2698
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 221</span>
2684
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 229</span>
2699
2685
def neptune_get_ssa_num_simulations(nodes, job_data)
2700
2686
num_nodes = nodes.length
2701
2687
num_sims = job_data[<span class="ruby-string">"@trajectories"</span>] || job_data[<span class="ruby-string">"@simulations"</span>]
2749
2735
<div class="method-source-code"
2750
2736
id="neptune_get_ssa_seed_vals-source">
2751
2737
<pre>
2752
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 208</span>
2738
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 216</span>
2753
2739
def neptune_get_ssa_seed_vals(num_vals)
2754
2740
random_numbers = []
2755
2741
loop {
2794
2780
<div class="method-source-code"
2795
2781
id="neptune_get_supported_babel_engines-source">
2796
2782
<pre>
2797
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 81</span>
2783
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 86</span>
2798
2784
def neptune_get_supported_babel_engines(job_data, secret)
2799
2785
return <span class="ruby-constant">BAD_SECRET_MSG</span> if !valid_secret?(secret)
2800
2786
2873
2859
<div class="method-source-code"
2874
2860
id="neptune_go_run_job-source">
2875
2861
<pre>
2876
<span class="ruby-comment"># File ../Neptune/go_helper.rb, line 9</span>
2862
<span class="ruby-comment"># File ../Neptune/go_helper.rb, line 16</span>
2877
2863
def neptune_go_run_job(nodes, job_data, secret)
2878
2864
return <span class="ruby-constant">BAD_SECRET_MSG</span> unless valid_secret?(secret)
2879
2865
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"go - run"</span>)
2894
2880
2895
2881
remote = job_data[<span class="ruby-string">'@code'</span>]
2896
2882
storage = job_data[<span class="ruby-string">'@storage'</span>]
2897
creds = <span class="ruby-constant">Djinn</span>.neptune_parse_creds(storage, job_data)
2898
2883
2899
<span class="ruby-constant">Repo</span>.get_output(remote, storage, creds, "#{code_loc}")
2884
datastore = <span class="ruby-constant">DatastoreFactory</span>.get_datastore(storage, job_data)
2885
datastore.get_output_and_save_to_fs(remote, code_loc)
2900
2886
2901
2887
<span class="ruby-constant">Djinn</span>.log_debug("got code #{code}, saved at #{code_loc}")
2902
2888
<span class="ruby-constant">Djinn</span>.log_run("chmod +x #{code_loc}")
2903
2889
<span class="ruby-constant">Djinn</span>.log_run("cd #{code_dir}; ./#{code} > #{output_loc}")
2904
2890
2905
<span class="ruby-constant">Repo</span>.set_output(job_data[<span class="ruby-string">"@output"</span>], output_loc, storage, creds, <span class="ruby-constant">IS_FILE</span>)
2906
2891
datastore.write_remote_file_from_local_file(job_data[<span class="ruby-string">'@output'</span>], output_loc)
2907
2892
remove_lock_file(job_data)
2908
2909
<span class="ruby-comment">#Djinn.log_run("rm -rfv #{code_dir}")</span>
2910
2893
}
2911
2894
2912
2895
return <span class="ruby-string">"OK"</span>
2941
2924
<div class="method-source-code"
2942
2925
id="neptune_is_job_running-source">
2943
2926
<pre>
2944
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 91</span>
2927
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 98</span>
2945
2928
def neptune_is_job_running(job_data, secret)
2946
2929
return <span class="ruby-constant">BAD_SECRET_MSG</span> unless valid_secret?(secret)
2947
2930
return lock_file_exists?(job_data)
2976
2959
<div class="method-source-code"
2977
2960
id="neptune_mapreduce_get_output-source">
2978
2961
<pre>
2979
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 133</span>
2962
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 134</span>
2980
2963
def neptune_mapreduce_get_output(job_data)
2981
2964
output = job_data[<span class="ruby-string">"@output"</span>]
2982
2965
output_location = "/tmp/#{output}"
3026
3009
<div class="method-source-code"
3027
3010
id="neptune_mapreduce_run_job-source">
3028
3011
<pre>
3029
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 23</span>
3012
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 28</span>
3030
3013
def neptune_mapreduce_run_job(nodes, job_data, secret)
3031
3014
return <span class="ruby-constant">BAD_SECRET_MSG</span> unless valid_secret?(secret)
3032
3015
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"mapreduce - run"</span>)
3036
3019
nodes = <span class="ruby-constant">Djinn</span>.convert_location_array_to_class(nodes, keyname)
3037
3020
3038
3021
storage = job_data[<span class="ruby-string">"@storage"</span>]
3039
creds = <span class="ruby-constant">Djinn</span>.neptune_parse_creds(storage, job_data)
3022
datastore = <span class="ruby-constant">DatastoreFactory</span>.get_datastore(storage, job_data)
3040
3023
3041
3024
mapreducejar = job_data[<span class="ruby-string">"@mapreducejar"</span>]
3042
3025
main = job_data[<span class="ruby-string">"@main"</span>]
3052
3035
3053
3036
if mapreducejar
3054
3037
<span class="ruby-constant">Djinn</span>.log_debug("need to get mr jar located at #{mapreducejar}")
3055
mr_jar = <span class="ruby-constant">Repo</span>.get_output(mapreducejar, storage, creds)
3056
3038
mr_file = mapreducejar.split(<span class="ruby-string">'/'</span>)[-1]
3057
3039
my_mrjar = "/tmp/#{mr_file}"
3058
<span class="ruby-constant">HelperFunctions</span>.write_file(my_mrjar, mr_jar)
3040
datastore.get_output_and_save_to_fs(mapreducejar, my_mrjar)
3059
3041
3060
3042
nodes.each { |node|
3061
3043
<span class="ruby-constant">HelperFunctions</span>.scp_file(my_mrjar, my_mrjar, node.private_ip, node.ssh_key)
3068
3050
3069
3051
run_mr_command = "#{HADOOP} jar #{my_mrjar} #{main} #{input} #{output}"
3070
3052
else
3071
<span class="ruby-constant">Djinn</span>.log_debug("need to get map code located at #{map}")
3072
map_code = <span class="ruby-constant">Repo</span>.get_output(map, storage, creds)
3073
3074
<span class="ruby-constant">Djinn</span>.log_debug("need to get reduce code located at #{reduce}")
3075
red_code = <span class="ruby-constant">Repo</span>.get_output(reduce, storage, creds)
3053
<span class="ruby-constant">Djinn</span>.log_debug("need to get map code located at #{map}, and reduce " +
3054
"code located at #{reduce}")
3076
3055
3077
3056
map_file = map.split(<span class="ruby-string">'/'</span>)[-1]
3078
3057
red_file = reduce.split(<span class="ruby-string">'/'</span>)[-1]
3080
3059
my_map = "/tmp/#{map_file}"
3081
3060
my_red = "/tmp/#{red_file}"
3082
3061
3083
<span class="ruby-constant">HelperFunctions</span>.write_file(my_map, map_code)
3084
<span class="ruby-constant">HelperFunctions</span>.write_file(my_red, red_code)
3062
datastore.get_output_and_save_to_fs(map, my_map)
3063
datastore.get_output_and_save_to_fs(reduce, my_red)
3085
3064
3086
3065
<span class="ruby-comment"># since the db master is the initiator of the mapreduce job, it needs</span>
3087
3066
<span class="ruby-comment"># to have both the mapper and reducer files handy</span>
3164
3143
<div class="method-source-code"
3165
3144
id="neptune_mpi_get_output-source">
3166
3145
<pre>
3167
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 110</span>
3146
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 116</span>
3168
3147
def neptune_mpi_get_output(job_data)
3169
3148
return <span class="ruby-constant">MPI_OUTPUT</span>
3170
3149
end</pre>
3198
3177
<div class="method-source-code"
3199
3178
id="neptune_mpi_run_job-source">
3200
3179
<pre>
3201
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 12</span>
3180
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 18</span>
3202
3181
def neptune_mpi_run_job(nodes, job_data, secret)
3203
3182
return <span class="ruby-constant">BAD_SECRET_MSG</span> unless valid_secret?(secret)
3204
3183
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"mpi - run"</span>)
3227
3206
<span class="ruby-constant">Djinn</span>.log_debug("remote dir is [#{remote_dir}], filename_to_exec is #{filename_to_exec}")
3228
3207
3229
3208
storage = job_data[<span class="ruby-string">'@storage'</span>]
3230
creds = <span class="ruby-constant">Djinn</span>.neptune_parse_creds(storage, job_data)
3231
3209
3232
3210
unless my_node.is_shadow?
3233
3211
<span class="ruby-constant">Djinn</span>.log_run("rm -fv /tmp/#{filename_to_exec}")
3235
3213
3236
3214
<span class="ruby-comment"># TODO(cgb): should remote_dir be remote_dir/ to prevent also getting</span>
3237
3215
<span class="ruby-comment"># remote_dir2,3, etc?</span>
3238
<span class="ruby-constant">Repo</span>.get_output(remote_dir, storage, creds, <span class="ruby-string">"/mirrornfs/"</span>)
3216
datastore = <span class="ruby-constant">DatastoreFactory</span>.get_datastore(storage, job_data)
3217
datastore.get_output_and_save_to_fs(remote_dir, <span class="ruby-string">"/mirrornfs/"</span>)
3239
3218
sleep(5)
3240
3219
<span class="ruby-constant">Djinn</span>.log_run("chmod +x /mirrornfs/#{filename_to_exec}")
3241
3220
sleep(5)
3283
3262
3284
3263
stop_nfs(nodes)
3285
3264
3286
<span class="ruby-constant">Repo</span>.set_output(job_data[<span class="ruby-string">"@output"</span>], output_file, storage, creds, <span class="ruby-constant">IS_FILE</span>)
3265
datastore.write_remote_file_from_local_file(job_data[<span class="ruby-string">'@output'</span>], output_file)
3287
3266
3288
3267
<span class="ruby-comment"># clean up after ourselves - remove the user's code</span>
3289
3268
<span class="ruby-constant">Djinn</span>.log_run(<span class="ruby-string">"rm -rfv /mirrornfs/*"</span>)
3324
3303
<div class="method-source-code"
3325
3304
id="neptune_put_input-source">
3326
3305
<pre>
3327
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 96</span>
3306
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 103</span>
3328
3307
def neptune_put_input(job_data, secret)
3329
3308
message = validate_environment(job_data, secret)
3330
3309
return message unless message == <span class="ruby-string">"no error"</span>
3332
3311
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"requesting input"</span>)
3333
3312
3334
3313
type = job_data[<span class="ruby-string">"@type"</span>]
3335
storage = job_data[<span class="ruby-string">"@storage"</span>]
3336
3314
3337
3315
[<span class="ruby-string">"type"</span>, <span class="ruby-string">"storage"</span>, <span class="ruby-string">"local"</span>, <span class="ruby-string">"remote"</span>].each { |item|
3338
3316
if job_data["@#{item}"].nil?
3340
3318
end
3341
3319
}
3342
3320
3343
creds = <span class="ruby-constant">Djinn</span>.neptune_parse_creds(storage, job_data)
3344
3345
3321
input_location = job_data[<span class="ruby-string">"@remote"</span>]
3346
3322
3347
3323
local_fs_location = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">"@local"</span>])
3357
3333
3358
3334
<span class="ruby-constant">Djinn</span>.log_debug(msg)
3359
3335
3360
ret_val = <span class="ruby-constant">Repo</span>.set_output(input_location, local_fs_location, storage, creds, is_file=true)
3336
datastore = <span class="ruby-constant">DatastoreFactory</span>.get_datastore(job_data[<span class="ruby-string">'@storage'</span>], job_data)
3337
datastore.write_remote_file_from_local_file(input_location, local_fs_location)
3361
3338
3362
3339
<span class="ruby-comment"># also, if we're running on hbase or hypertable, put a copy of the data</span>
3363
3340
<span class="ruby-comment"># into HDFS for later processing via mapreduce</span>
3377
3354
run_on_db_master(cmd)
3378
3355
end
3379
3356
3380
<span class="ruby-comment">#FileUtils.rm_rf(local_fs_location)</span>
3381
3382
3357
return ret_val
3383
3358
end</pre>
3384
3359
</div>
3411
3386
<div class="method-source-code"
3412
3387
id="neptune_r_run_job-source">
3413
3388
<pre>
3414
<span class="ruby-comment"># File ../Neptune/r_helper.rb, line 9</span>
3389
<span class="ruby-comment"># File ../Neptune/r_helper.rb, line 15</span>
3415
3390
def neptune_r_run_job(nodes, job_data, secret)
3416
3391
return <span class="ruby-constant">BAD_SECRET_MSG</span> unless valid_secret?(secret)
3417
3392
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"r - run"</span>)
3420
3395
keyname = <span class="ruby-ivar">@creds</span>[<span class="ruby-string">'keyname'</span>]
3421
3396
nodes = <span class="ruby-constant">Djinn</span>.convert_location_array_to_class(nodes, keyname)
3422
3397
3423
<span class="ruby-comment">#ENV['HOME'] = "/root"</span>
3424
3398
<span class="ruby-constant">Djinn</span>.log_debug("job data is #{job_data.inspect}")
3425
3399
3426
3400
code = job_data[<span class="ruby-string">'@code'</span>].split(<span class="ruby-regexp">/\//</span>)[-1]
3432
3406
3433
3407
remote = job_data[<span class="ruby-string">'@code'</span>]
3434
3408
storage = job_data[<span class="ruby-string">'@storage'</span>]
3435
creds = <span class="ruby-constant">Djinn</span>.neptune_parse_creds(storage, job_data)
3436
3409
3437
<span class="ruby-constant">Repo</span>.get_output(remote, storage, creds, "#{code_loc}")
3410
datastore = <span class="ruby-constant">DatastoreFactory</span>.get_datastore(storage, job_data)
3411
datastore.get_output_and_save_to_fs(remote, code_loc)
3438
3412
3439
3413
<span class="ruby-constant">Djinn</span>.log_debug("got code #{code}, saved at #{code_loc}")
3440
3414
<span class="ruby-constant">Djinn</span>.log_run("chmod +x #{code_loc}")
3441
3415
<span class="ruby-constant">Djinn</span>.log_run("Rscript --vanilla #{code_loc} > #{output_loc}")
3442
3416
3443
<span class="ruby-constant">Repo</span>.set_output(job_data[<span class="ruby-string">"@output"</span>], output_loc, storage, creds, <span class="ruby-constant">IS_FILE</span>)
3444
3417
datastore.write_remote_file_from_local_file(job_data[<span class="ruby-string">'@output'</span>], output_loc)
3445
3418
remove_lock_file(job_data)
3446
3447
<span class="ruby-comment">#Djinn.log_run("rm -rfv #{code_dir}")</span>
3448
3419
}
3449
3420
3450
3421
return <span class="ruby-string">"OK"</span>
3479
3450
<div class="method-source-code"
3480
3451
id="neptune_release_nodes-source">
3481
3452
<pre>
3482
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 665</span>
3453
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 651</span>
3483
3454
def neptune_release_nodes(nodes_to_use, job_data)
3484
3455
if is_hybrid_cloud?
3485
3456
abort(<span class="ruby-string">"hybrid cloud mode is definitely not supported"</span>)
3532
3503
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"setting acl"</span>)
3533
3504
3534
3505
type = job_data[<span class="ruby-string">"@type"</span>]
3535
storage = job_data[<span class="ruby-string">"@storage"</span>]
3536
creds = <span class="ruby-constant">Djinn</span>.neptune_parse_creds(storage, job_data)
3537
3506
3538
3507
new_acl = job_data[<span class="ruby-string">"@acl"</span>]
3539
3508
3545
3514
if output_location.nil?
3546
3515
return <span class="ruby-string">"error: output not specified"</span>
3547
3516
else
3548
if <span class="ruby-constant">Repo</span>.does_file_exist?(output_location, storage, creds)
3549
return <span class="ruby-constant">Repo</span>.set_acl(output_location, new_acl, storage, creds)
3517
datastore = <span class="ruby-constant">DatastoreFactory</span>.get_datastore(job_data[<span class="ruby-string">'@storage'</span>], job_data)
3518
if datastore.does_file_exist?(output_location)
3519
return datastore.set_acl(output_location, new_acl)
3550
3520
else
3551
3521
return <span class="ruby-string">"error: output does not exist"</span>
3552
3522
end
3582
3552
<div class="method-source-code"
3583
3553
id="neptune_ssa_get_output-source">
3584
3554
<pre>
3585
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 188</span>
3555
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 196</span>
3586
3556
def neptune_ssa_get_output(job_data)
3587
3557
return <span class="ruby-constant">SSA_HOME</span>
3588
3558
end</pre>
3616
3586
<div class="method-source-code"
3617
3587
id="neptune_ssa_run_job-source">
3618
3588
<pre>
3619
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 15</span>
3589
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 24</span>
3620
3590
def neptune_ssa_run_job(nodes, job_data, secret)
3621
3591
return <span class="ruby-constant">BAD_SECRET_MSG</span> unless valid_secret?(secret)
3622
3592
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"ssa - run"</span>)
3646
3616
3647
3617
remote = job_data[<span class="ruby-string">'@tar'</span>]
3648
3618
storage = job_data[<span class="ruby-string">'@storage'</span>]
3649
creds = <span class="ruby-constant">Djinn</span>.neptune_parse_creds(storage, job_data)
3650
3651
<span class="ruby-constant">Repo</span>.get_output(remote, storage, creds, tar)
3619
3620
datastore = <span class="ruby-constant">DatastoreFactory</span>.get_datastore(storage, job_data)
3621
datastore.get_output_and_save_to_fs(remote, tar)
3622
3652
3623
neptune_uncompress_file(tar)
3653
3624
3654
3625
num_sims = job_data[<span class="ruby-string">"@trajectories"</span>] || job_data[<span class="ruby-string">"@simulations"</span>]
3655
<span class="ruby-comment">#seeds = neptune_get_ssa_seed_vals(num_sims)</span>
3656
<span class="ruby-comment">#Djinn.log_debug("seeds are #{seeds.join(', ')}")</span>
3657
3626
3658
3627
param_num = job_data[<span class="ruby-string">'@param_num'</span>]
3659
3628
3727
3696
end
3728
3697
3729
3698
remote_location = "#{job_data['@output']}/output-#{my_trajectory}"
3730
<span class="ruby-constant">Repo</span>.set_output(remote_location, output, storage, creds, <span class="ruby-constant">IS_FILE</span>)
3699
datastore.write_remote_file_from_local_file(remote_location, output)
3731
3700
end_storage = <span class="ruby-constant">Time</span>.now
3732
3701
s_time = end_storage - start_storage
3733
3702
total_storage_time += s_time
3767
3736
<span class="ruby-constant">Djinn</span>.log_debug(timing_info)
3768
3737
3769
3738
remote_location = "#{job_data['@output']}/timing_info.txt"
3770
<span class="ruby-constant">Repo</span>.set_output(remote_location, timing_info, storage, creds, <span class="ruby-constant">NOT_A_FILE</span>)
3739
datastore.write_remote_file_from_string(remote_location, timing_info)
3771
3740
3772
3741
remove_lock_file(job_data)
3773
3742
}
3804
3773
<div class="method-source-code"
3805
3774
id="neptune_start_job-source">
3806
3775
<pre>
3807
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 37</span>
3776
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 44</span>
3808
3777
def neptune_start_job(job_data, secret)
3809
3778
message = validate_environment(job_data, secret)
3810
3779
return message unless message == <span class="ruby-string">"no error"</span>
3888
3857
<div class="method-source-code"
3889
3858
id="optimal_nodes-source">
3890
3859
<pre>
3891
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 724</span>
3860
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 710</span>
3892
3861
def optimal_nodes(job_data)
3893
3862
return job_data[<span class="ruby-string">"@nodes_to_use"</span>]
3894
3863
end</pre>
3965
3934
<div class="method-source-code"
3966
3935
id="remove_lock_file-source">
3967
3936
<pre>
3968
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 512</span>
3937
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 498</span>
3969
3938
def remove_lock_file(job_data)
3970
3939
shadow = get_shadow
3971
3940
shadow_ip = shadow.private_ip
4004
3973
<div class="method-source-code"
4005
3974
id="remove_roles-source">
4006
3975
<pre>
4007
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 704</span>
3976
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 690</span>
4008
3977
def remove_roles(nodes, roles)
4009
3978
return if nodes.nil?
4010
3979
4045
4014
<div class="method-source-code"
4046
4015
id="run_job_on_master-source">
4047
4016
<pre>
4048
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 429</span>
4017
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 415</span>
4049
4018
def run_job_on_master(master_node, nodes_to_use, job_data)
4050
4019
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"run job on master"</span>)
4051
4020
converted_nodes = <span class="ruby-constant">Djinn</span>.convert_location_class_to_array(nodes_to_use)
4108
4077
<div class="method-source-code"
4109
4078
id="run_on_db_master-source">
4110
4079
<pre>
4111
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 196</span>
4080
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 197</span>
4112
4081
def run_on_db_master(command, output=<span class="ruby-constant">WANT_OUTPUT</span>)
4113
4082
db_master = get_db_master
4114
4083
ip = db_master.public_ip
4145
4114
<div class="method-source-code"
4146
4115
id="run_task-source">
4147
4116
<pre>
4148
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 271</span>
4117
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 276</span>
4149
4118
def run_task(job_data)
4150
4119
dir = <span class="ruby-constant">Djinn</span>.create_temp_dir()
4151
4120
<span class="ruby-constant">Djinn</span>.copy_code_to_dir(job_data, dir)
4281
4250
<div class="method-source-code"
4282
4251
id="run_via_engine-source">
4283
4252
<pre>
4284
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 189</span>
4253
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 194</span>
4285
4254
def run_via_engine(engine, job_data)
4286
4255
<span class="ruby-constant">Djinn</span>.log_debug("running job with data #{job_data.inspect} via a remote engine")
4287
4256
4331
4300
<div class="method-source-code"
4332
4301
id="run_via_executor-source">
4333
4302
<pre>
4334
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 165</span>
4303
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 170</span>
4335
4304
def run_via_executor(engine, job_data)
4336
4305
<span class="ruby-constant">Djinn</span>.log_debug("running job with data #{job_data.inspect} via executor")
4337
4306
q = <span class="ruby-constant">QueueFactory</span>.get_queue(engine, job_data)
4382
4351
<div class="method-source-code"
4383
4352
id="spawn_babel_slaves-source">
4384
4353
<pre>
4385
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 468</span>
4354
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 471</span>
4386
4355
def spawn_babel_slaves(num_of_waiting_tasks)
4387
4356
<span class="ruby-comment"># TODO(cgb): move this info out of this method and include other instance types</span>
4388
4357
<span class="ruby-comment">#instance_type = "c1.xlarge"</span>
4438
4407
<div class="method-source-code"
4439
4408
id="spawn_nodes_for_neptune-3F-source">
4440
4409
<pre>
4441
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 525</span>
4410
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 511</span>
4442
4411
def spawn_nodes_for_neptune?(job_data)
4443
4412
<span class="ruby-constant">Djinn</span>.log_debug("neptune_info = #{job_data}")
4444
4413
return !job_data[<span class="ruby-string">"@nodes_to_use"</span>].nil?
4473
4442
<div class="method-source-code"
4474
4443
id="standard_deviation-source">
4475
4444
<pre>
4476
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 272</span>
4445
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 280</span>
4477
4446
def standard_deviation(population)
4478
4447
<span class="ruby-constant">Math</span>.sqrt(variance(population))
4479
4448
end</pre>
4592
4561
<div class="method-source-code"
4593
4562
id="start_babel_master-source">
4594
4563
<pre>
4595
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 221</span>
4564
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 226</span>
4596
4565
def start_babel_master()
4597
4566
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is starting babel master")
4598
4567
4642
4611
<div class="method-source-code"
4643
4612
id="start_babel_slave-source">
4644
4613
<pre>
4645
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 243</span>
4614
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 248</span>
4646
4615
def start_babel_slave()
4647
4616
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is starting babel slave")
4648
4617
4893
4862
<div class="method-source-code"
4894
4863
id="start_erlang_master-source">
4895
4864
<pre>
4896
<span class="ruby-comment"># File ../Neptune/erlang_helper.rb, line 51</span>
4865
<span class="ruby-comment"># File ../Neptune/erlang_helper.rb, line 56</span>
4897
4866
def start_erlang_master()
4898
4867
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is starting erlang master")
4899
4868
end</pre>
4927
4896
<div class="method-source-code"
4928
4897
id="start_erlang_slave-source">
4929
4898
<pre>
4930
<span class="ruby-comment"># File ../Neptune/erlang_helper.rb, line 55</span>
4899
<span class="ruby-comment"># File ../Neptune/erlang_helper.rb, line 60</span>
4931
4900
def start_erlang_slave()
4932
4901
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is starting erlang slave")
4933
4902
end</pre>
4961
4930
<div class="method-source-code"
4962
4931
id="start_go_master-source">
4963
4932
<pre>
4964
<span class="ruby-comment"># File ../Neptune/go_helper.rb, line 49</span>
4933
<span class="ruby-comment"># File ../Neptune/go_helper.rb, line 53</span>
4965
4934
def start_go_master()
4966
4935
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is starting go master")
4967
4936
end</pre>
4995
4964
<div class="method-source-code"
4996
4965
id="start_go_slave-source">
4997
4966
<pre>
4998
<span class="ruby-comment"># File ../Neptune/go_helper.rb, line 53</span>
4967
<span class="ruby-comment"># File ../Neptune/go_helper.rb, line 57</span>
4999
4968
def start_go_slave()
5000
4969
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is starting go slave")
5001
4970
end</pre>
5029
4998
<div class="method-source-code"
5030
4999
id="start_job-source">
5031
5000
<pre>
5032
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 378</span>
5001
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 364</span>
5033
5002
def start_job(nodes, job_data)
5034
5003
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"job - start"</span>)
5035
5004
5089
5058
<div class="method-source-code"
5090
5059
id="start_mapreduce_master-source">
5091
5060
<pre>
5092
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 153</span>
5061
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 154</span>
5093
5062
def start_mapreduce_master()
5094
5063
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"start mapreduce master - starting up hadoop first"</span>)
5095
5064
<span class="ruby-comment">#start_db_master</span>
5125
5094
<div class="method-source-code"
5126
5095
id="start_mapreduce_slave-source">
5127
5096
<pre>
5128
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 159</span>
5097
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 160</span>
5129
5098
def start_mapreduce_slave()
5130
5099
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"start mapreduce slave - starting up hadoop first"</span>)
5131
5100
<span class="ruby-comment">#start_db_slave</span>
5161
5130
<div class="method-source-code"
5162
5131
id="start_mpd-source">
5163
5132
<pre>
5164
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 170</span>
5133
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 176</span>
5165
5134
def start_mpd(nodes)
5166
5135
mpd_hosts_contents = <span class="ruby-string">""</span>
5167
5136
nodes.each { |node|
5209
5178
<div class="method-source-code"
5210
5179
id="start_mpi_master-source">
5211
5180
<pre>
5212
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 193</span>
5181
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 199</span>
5213
5182
def start_mpi_master()
5214
5183
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is starting mpi master")
5215
5184
end</pre>
5243
5212
<div class="method-source-code"
5244
5213
id="start_mpi_slave-source">
5245
5214
<pre>
5246
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 197</span>
5215
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 203</span>
5247
5216
def start_mpi_slave()
5248
5217
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is starting mpi slave")
5249
5218
end</pre>
5277
5246
<div class="method-source-code"
5278
5247
id="start_nfs-source">
5279
5248
<pre>
5280
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 114</span>
5249
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 120</span>
5281
5250
def start_nfs(nodes)
5282
5251
<span class="ruby-constant">Djinn</span>.log_run(<span class="ruby-string">"/etc/init.d/nfs-kernel-server start"</span>)
5283
5252
sleep(10)
5325
5294
<div class="method-source-code"
5326
5295
id="start_r_master-source">
5327
5296
<pre>
5328
<span class="ruby-comment"># File ../Neptune/r_helper.rb, line 49</span>
5297
<span class="ruby-comment"># File ../Neptune/r_helper.rb, line 51</span>
5329
5298
def start_r_master()
5330
5299
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is starting r master")
5331
5300
end</pre>
5359
5328
<div class="method-source-code"
5360
5329
id="start_r_slave-source">
5361
5330
<pre>
5362
<span class="ruby-comment"># File ../Neptune/r_helper.rb, line 53</span>
5331
<span class="ruby-comment"># File ../Neptune/r_helper.rb, line 55</span>
5363
5332
def start_r_slave()
5364
5333
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is starting r slave")
5365
5334
end</pre>
5393
5362
<div class="method-source-code"
5394
5363
id="start_ssa_master-source">
5395
5364
<pre>
5396
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 192</span>
5365
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 200</span>
5397
5366
def start_ssa_master()
5398
5367
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is starting ssa master")
5399
5368
end</pre>
5427
5396
<div class="method-source-code"
5428
5397
id="start_ssa_slave-source">
5429
5398
<pre>
5430
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 196</span>
5399
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 204</span>
5431
5400
def start_ssa_slave()
5432
5401
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is starting ssa slave")
5433
5402
end</pre>
5511
5480
<div class="method-source-code"
5512
5481
id="stop_babel_master-source">
5513
5482
<pre>
5514
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 261</span>
5483
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 266</span>
5515
5484
def stop_babel_master()
5516
5485
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is stopping babel master")
5517
5486
end</pre>
5545
5514
<div class="method-source-code"
5546
5515
id="stop_babel_slave-source">
5547
5516
<pre>
5548
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 266</span>
5517
<span class="ruby-comment"># File ../Neptune/babel_helper.rb, line 271</span>
5549
5518
def stop_babel_slave()
5550
5519
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is stopping babel slave")
5551
5520
end</pre>
5784
5753
<div class="method-source-code"
5785
5754
id="stop_erlang_master-source">
5786
5755
<pre>
5787
<span class="ruby-comment"># File ../Neptune/erlang_helper.rb, line 59</span>
5756
<span class="ruby-comment"># File ../Neptune/erlang_helper.rb, line 64</span>
5788
5757
def stop_erlang_master()
5789
5758
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is stopping erlang master")
5790
5759
end</pre>
5818
5787
<div class="method-source-code"
5819
5788
id="stop_erlang_slave-source">
5820
5789
<pre>
5821
<span class="ruby-comment"># File ../Neptune/erlang_helper.rb, line 63</span>
5790
<span class="ruby-comment"># File ../Neptune/erlang_helper.rb, line 68</span>
5822
5791
def stop_erlang_slave()
5823
5792
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is stopping erlang slave")
5824
5793
end</pre>
5852
5821
<div class="method-source-code"
5853
5822
id="stop_go_master-source">
5854
5823
<pre>
5855
<span class="ruby-comment"># File ../Neptune/go_helper.rb, line 57</span>
5824
<span class="ruby-comment"># File ../Neptune/go_helper.rb, line 61</span>
5856
5825
def stop_go_master()
5857
5826
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is stopping go master")
5858
5827
end</pre>
5886
5855
<div class="method-source-code"
5887
5856
id="stop_go_slave-source">
5888
5857
<pre>
5889
<span class="ruby-comment"># File ../Neptune/go_helper.rb, line 61</span>
5858
<span class="ruby-comment"># File ../Neptune/go_helper.rb, line 65</span>
5890
5859
def stop_go_slave()
5891
5860
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is stopping go slave")
5892
5861
end</pre>
5920
5889
<div class="method-source-code"
5921
5890
id="stop_job-source">
5922
5891
<pre>
5923
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 462</span>
5892
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 448</span>
5924
5893
def stop_job(nodes, job_data)
5925
5894
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"job - stop"</span>)
5926
5895
5978
5947
<div class="method-source-code"
5979
5948
id="stop_mapreduce_master-source">
5980
5949
<pre>
5981
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 165</span>
5950
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 166</span>
5982
5951
def stop_mapreduce_master()
5983
5952
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"stop mapreduce master - stopping hadoop"</span>)
5984
5953
<span class="ruby-comment">#stop_db_master</span>
6014
5983
<div class="method-source-code"
6015
5984
id="stop_mapreduce_slave-source">
6016
5985
<pre>
6017
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 171</span>
5986
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 172</span>
6018
5987
def stop_mapreduce_slave()
6019
5988
<span class="ruby-constant">Djinn</span>.log_debug(<span class="ruby-string">"stop mapreduce slave - stopping hadoop"</span>)
6020
5989
<span class="ruby-comment">#stop_db_slave</span>
6050
6019
<div class="method-source-code"
6051
6020
id="stop_mpd-source">
6052
6021
<pre>
6053
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 188</span>
6022
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 194</span>
6054
6023
def stop_mpd()
6055
6024
<span class="ruby-constant">Djinn</span>.log_run(<span class="ruby-string">"mpdallexit"</span>)
6056
6025
`rm -fv #{MPD_HOSTS}`
6085
6054
<div class="method-source-code"
6086
6055
id="stop_mpi_master-source">
6087
6056
<pre>
6088
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 201</span>
6057
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 207</span>
6089
6058
def stop_mpi_master()
6090
6059
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is stopping mpi master")
6091
6060
end</pre>
6119
6088
<div class="method-source-code"
6120
6089
id="stop_mpi_slave-source">
6121
6090
<pre>
6122
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 205</span>
6091
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 211</span>
6123
6092
def stop_mpi_slave()
6124
6093
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is stopping mpi slave")
6125
6094
end</pre>
6153
6122
<div class="method-source-code"
6154
6123
id="stop_nfs-source">
6155
6124
<pre>
6156
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 132</span>
6125
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 138</span>
6157
6126
def stop_nfs(nodes)
6158
6127
slave_nodes = nodes - [my_node]
6159
6128
6201
6170
<div class="method-source-code"
6202
6171
id="stop_r_master-source">
6203
6172
<pre>
6204
<span class="ruby-comment"># File ../Neptune/r_helper.rb, line 57</span>
6173
<span class="ruby-comment"># File ../Neptune/r_helper.rb, line 59</span>
6205
6174
def stop_r_master()
6206
6175
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is stopping r master")
6207
6176
end</pre>
6235
6204
<div class="method-source-code"
6236
6205
id="stop_r_slave-source">
6237
6206
<pre>
6238
<span class="ruby-comment"># File ../Neptune/r_helper.rb, line 61</span>
6207
<span class="ruby-comment"># File ../Neptune/r_helper.rb, line 63</span>
6239
6208
def stop_r_slave()
6240
6209
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is stopping r slave")
6241
6210
end</pre>
6269
6238
<div class="method-source-code"
6270
6239
id="stop_ssa_master-source">
6271
6240
<pre>
6272
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 200</span>
6241
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 208</span>
6273
6242
def stop_ssa_master()
6274
6243
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is stopping ssa master")
6275
6244
end</pre>
6303
6272
<div class="method-source-code"
6304
6273
id="stop_ssa_slave-source">
6305
6274
<pre>
6306
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 204</span>
6275
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 212</span>
6307
6276
def stop_ssa_slave()
6308
6277
<span class="ruby-constant">Djinn</span>.log_debug("#{my_node.private_ip} is stopping ssa slave")
6309
6278
end</pre>
6337
6306
<div class="method-source-code"
6338
6307
id="touch_lock_file-source">
6339
6308
<pre>
6340
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 506</span>
6309
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 492</span>
6341
6310
def touch_lock_file(job_data)
6342
6311
job_data[<span class="ruby-string">"@job_id"</span>] = rand(1000000)
6343
6312
touch_lock_file = "touch #{get_lock_file_path(job_data)}"
6373
6342
<div class="method-source-code"
6374
6343
id="unmount_nfs_store-source">
6375
6344
<pre>
6376
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 150</span>
6345
<span class="ruby-comment"># File ../Neptune/mpi_helper.rb, line 156</span>
6377
6346
def unmount_nfs_store(ip, ssh_key)
6378
6347
slave_umount = <span class="ruby-string">"umount /mirrornfs"</span>
6379
6348
6423
6392
<div class="method-source-code"
6424
6393
id="validate_environment-source">
6425
6394
<pre>
6426
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 490</span>
6395
<span class="ruby-comment"># File ../Neptune/neptune.rb, line 476</span>
6427
6396
def validate_environment(job_data, secret)
6428
6397
return <span class="ruby-constant">BAD_SECRET_MSG</span> unless valid_secret?(secret)
6429
6398
<span class="ruby-comment">#return JOB_IN_PROGRESS if lock_file_exists?(job_data)</span>
6465
6434
<div class="method-source-code"
6466
6435
id="variance-source">
6467
6436
<pre>
6468
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 257</span>
6437
<span class="ruby-comment"># File ../Neptune/ssa_helper.rb, line 265</span>
6469
6438
def variance(population)
6470
6439
n = 0
6471
6440
mean = 0.0
6559
6528
<div class="method-source-code"
6560
6529
id="wait_for_hdfs_file-source">
6561
6530
<pre>
6562
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 177</span>
6531
<span class="ruby-comment"># File ../Neptune/mapreduce_helper.rb, line 178</span>
6563
6532
def wait_for_hdfs_file(location)
6564
6533
command = "#{HADOOP} fs -ls #{location}"
6565
6534
db_master = get_db_master
Loggerhead is a web-based interface for Breezy
Version: 2.0.1