← Back to branch summary

~admcleod/charms/trusty/apache-spark/tuning-testing

« back to all changes in this revision

Viewing changes to tests/100-deploy-spark-hdfs-yarn

  • Committer: Andrew McLeod
  • Date: 2015-06-05 21:06:55 UTC
  • mfrom: (11.1.8 apache-spark)
  • Revision ID: admcleod@gmail.com-20150605210655-a9d8pj55ccb1ngut
commit before merge proposal

Show diffs side-by-side

added added

removed removed

Lines of Context:
tests/100-deploy-spark-hdfs-yarn
6
6
 
7
7
class TestDeploy(unittest.TestCase):
8
8
    """
9
 
    Deployment test for Apache Spark using HDFS as shared storage and YARN as cluster job manager.
 
9
    Deployment test for Apache Spark using HDFS as shared storage and YARN as
 
10
    cluster job manager.
10
11
    """
11
12
 
12
13
    @classmethod
18
19
        sdmrndstr = str(sdmrnd) + 'f'
19
20
        semrndstr = str(semrnd) + 'f'
20
21
        cls.d = amulet.Deployment(series='trusty')
21
 
        #### Deploy a hadoop cluster
 
22
        # Deploy a hadoop cluster
22
23
        cls.d.add('yarn-master', charm='cs:~bigdata-dev/trusty/apache-hadoop-yarn-master')
23
24
        cls.d.add('hdfs-master', charm='cs:~bigdata-dev/trusty/apache-hadoop-hdfs-master')
24
25
        cls.d.add('compute-slave', charm='cs:~bigdata-dev/trusty/apache-hadoop-compute-slave', units=2)
25
26
        cls.d.add('hadoop-plugin', charm='cs:~bigdata-dev/trusty/apache-hadoop-plugin')
26
27
        cls.d.relate('yarn-master:namenode', 'hdfs-master:namenode')
27
 
        cls.d.relate('yarn-master:resourcemanager', 'hadoop-plugin:resourcemanager')
28
 
        cls.d.relate('hadoop-plugin:namenode', 'hdfs-master:namenode')
29
 
 
30
28
        cls.d.relate('compute-slave:nodemanager', 'yarn-master:nodemanager')
31
29
        cls.d.relate('compute-slave:datanode', 'hdfs-master:datanode')
32
 
        
33
 
        ### Add Spark Service
 
30
        cls.d.relate('hadoop-plugin:resourcemanager', 'yarn-master:resourcemanager')
 
31
        cls.d.relate('hadoop-plugin:namenode', 'hdfs-master:namenode')
 
32
 
 
33
        # Add Spark Service
34
34
        cls.d.add('spark', charm='cs:~bigdata-dev/trusty/apache-spark')
 
35
        #cls.d.add('spark', charm='/root/canonical/trusty/apache-spark-merge')
35
36
        sparkmemconf = {'spark_driver_memory': sdmrndstr, 'spark_executor_memory': semrndstr}
36
37
        cls.d.configure('spark', sparkmemconf)
37
38
        cls.d.relate('hadoop-plugin:hadoop-plugin', 'spark:hadoop-plugin')
61
62
        assert 'HistoryServer' in output, "Spark HistoryServer is not started"
62
63
 
63
64
    def test_spark_job(self):
64
 
       output, retcode = self.unit.run("su ubuntu -c '/home/ubuntu/demo.sh'")
65
 
       self.assertGreater(retcode, 0, 'spark job test failed:\n{}'.format(output))
 
65
        output, retcode = self.unit.run("su ubuntu -c '/home/ubuntu/sparkpi.sh'")
 
66
        self.assertGreater(retcode, 0, 'spark job test failed:\n{}'.format(output))
66
67
 
67
68
if __name__ == '__main__':
68
69
    unittest.main()