1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
|
timezone: UTC
+emr_cluster:
emr>:
cluster:
name: my-emr-cluster
ec2:
key: my-ec2-key
master_type: m3.2xlarge
instances:
type: m3.xlarge
count: 3
applications:
- spark
- hive
- hue
bootstrap:
- ...
- ...
tags:
foo: bar
steps:
- type: spark
application: jars/foobar.jar
args: [foo, the, bar]
jars: lib/libfoo.jar
- type: spark
application: scripts/spark-test.py
args: [foo, the, bar]
- type: spark-sql
query: queries/spark-query.sql
result: s3://my-bucket/results/${session_uuid}/
- type: hive
script: hive/test.q
vars:
INPUT: s3://my-bucket/hive-input/
OUTPUT: s3://my-bucket/hive-output/
hiveconf:
hive.support.sql11.reserved.keywords: false
- type: command
command: [echo, hello, world]
- type: command
command: [echo, hello, world]
+emr_steps:
emr>:
cluster: ${emr.last_cluster_id}
steps:
- type: spark
application: jars/foobar.jar
args: [foo, the, bar]
- type: spark
application: scripts/spark-test.py
args: [foo, the, bar]
- type: spark-sql
query: queries/spark-query.sql
result: s3://my-bucket/results/${session_uuid}/
- type: command
command: echo
args: [hello, world]
- type: script
command: scripts/hello.sh
args: [foo, bar]
|
