forked from HPEEzmeral/ezua-tutorials
-
Notifications
You must be signed in to change notification settings - Fork 0
/
pod.yaml
336 lines (336 loc) · 9.49 KB
/
pod.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
apiVersion: v1
kind: Pod
metadata:
annotations:
cni.projectcalico.org/containerID: 1ad9e3ad95903c8b942fc55c8ac4ad16b15a1c8382bdea3f30640709a25c890b
cni.projectcalico.org/podIP: 10.244.1.219/32
cni.projectcalico.org/podIPs: 10.244.1.219/32
creationTimestamp: "2023-05-18T13:54:49Z"
labels:
created-by: livy
hpe-ezua/app: livy
hpe-ezua/type: app-service-user
spark-app-name: livy-session-8
spark-app-selector: spark-a4ec35530d7a45698a58b6d0befd39fa
spark-app-tag: livy-session-8-p4DDjiDZ
spark-role: driver
spark-version: 3.3.1.0-eep-910
name: livy-session-8-681a4a882f244b80-driver
namespace: spark
resourceVersion: "7683112"
uid: 387aaffa-26c5-4837-b55b-014aa461cdef
spec:
containers:
- args:
- driver
- --proxy-user
- dimpo
- --properties-file
- /opt/spark/conf/spark.properties
- --class
- org.apache.livy.rsc.driver.RSCDriverBootstrapper
- spark-internal
env:
- name: SPARK_USER
value: dimpo
- name: SPARK_APPLICATION_ID
value: spark-a4ec35530d7a45698a58b6d0befd39fa
- name: SPARK_DRIVER_BIND_ADDRESS
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: HADOOP_CONF_DIR
value: /opt/hadoop/conf
- name: SPARK_LOCAL_DIRS
value: /var/data/spark-635b9918-4c68-4e0f-849e-f8841a6a1fd1
- name: MAPR_TICKETFILE_LOCATION
value: /tmp/usersecret/CONTAINER_TICKET
- name: USER_SECRET_MOUNT_POINT
value: /tmp/usersecret
- name: MAPR_METRICSFILE_LOCATION
value: /tmp/maprticket/METRICS_TICKET
- name: SPARK_CONF_DIR
value: /opt/spark/conf
envFrom:
- configMapRef:
name: cluster-cm
optional: true
image: gcr.io/mapr-252711/spark-3.3.1:202304252229AN
imagePullPolicy: IfNotPresent
name: spark-kubernetes-driver
ports:
- containerPort: 7078
name: driver-rpc-port
protocol: TCP
- containerPort: 7079
name: blockmanager
protocol: TCP
- containerPort: 4040
name: spark-ui
protocol: TCP
resources:
limits:
cpu: "1"
memory: 1384Mi
requests:
cpu: "1"
memory: 1384Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/log/sparkhs-eventlog-storage
name: sparkhs-eventlog-storage
- mountPath: /opt/hadoop/conf
name: hadoop-properties
- mountPath: /var/data/spark-635b9918-4c68-4e0f-849e-f8841a6a1fd1
name: spark-local-dir-1
- mountPath: /tmp/usersecret
name: livy-user-secret-dimpo-volume
- mountPath: /tmp/maprticket/METRICS_TICKET
name: server-volume
subPath: maprmetricsticket
- mountPath: /opt/mapr/kubernetes/ldap-cm
name: ldap-cm
- mountPath: /opt/mapr/kubernetes/sssd-secrets
name: sssd-secrets
- mountPath: /opt/mapr/kubernetes/ssh-secrets
name: ssh-secrets
- mountPath: /opt/mapr/kubernetes/client-secrets
name: client-secrets
- mountPath: /opt/spark/conf
name: spark-conf-volume-driver
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-7h59b
readOnly: true
dnsPolicy: ClusterFirst
enableServiceLinks: true
imagePullSecrets:
- name: imagepull
nodeName: m2-lr1-ezua-worker26
preemptionPolicy: PreemptLowerPriority
priority: 0
restartPolicy: Never
schedulerName: default-scheduler
securityContext: {}
serviceAccount: hpe-spark
serviceAccountName: hpe-spark
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
volumes:
- name: sparkhs-eventlog-storage
persistentVolumeClaim:
claimName: sparkhs-pvc
- configMap:
defaultMode: 420
items:
- key: capacity-scheduler.xml
path: capacity-scheduler.xml
- key: configuration.xsl
path: configuration.xsl
- key: container-executor.cfg
path: container-executor.cfg
- key: core-site.xml
path: core-site.xml
- key: fair-scheduler.xml
path: fair-scheduler.xml
- key: hadoop-env.sh
path: hadoop-env.sh
- key: hadoop-metrics2.properties
path: hadoop-metrics2.properties
- key: hadoop-policy.xml
path: hadoop-policy.xml
- key: hadoop-user-functions.sh.example
path: hadoop-user-functions.sh.example
- key: hadoop_version
path: hadoop_version
- key: hdfs-rbf-site.xml
path: hdfs-rbf-site.xml
- key: hdfs-site.xml
path: hdfs-site.xml
- key: httpfs-env.sh
path: httpfs-env.sh
- key: httpfs-log4j.properties
path: httpfs-log4j.properties
- key: httpfs-site.xml
path: httpfs-site.xml
- key: jetty-headers.xml
path: jetty-headers.xml
- key: kms-acls.xml
path: kms-acls.xml
- key: kms-env.sh
path: kms-env.sh
- key: kms-log4j.properties
path: kms-log4j.properties
- key: kms-site.xml
path: kms-site.xml
- key: log4j.properties
path: log4j.properties
- key: mapred-env.sh
path: mapred-env.sh
- key: mapred-queues.xml.template
path: mapred-queues.xml.template
- key: mapred-site.xml
path: mapred-site.xml
- key: ssl-client.xml
path: ssl-client.xml
- key: ssl-client.xml.example
path: ssl-client.xml.example
- key: ssl-server.xml
path: ssl-server.xml
- key: ssl-server.xml.example
path: ssl-server.xml.example
- key: user_ec_policies.xml.template
path: user_ec_policies.xml.template
- key: workers
path: workers
- key: yarn-env.sh
path: yarn-env.sh
- key: yarn-site.xml.template
path: yarn-site.xml.template
- key: yarnservice-log4j.properties
path: yarnservice-log4j.properties
- key: yarn-site.xml
path: yarn-site.xml
- key: yarn-site-2023-05-12.11-45.xml
path: yarn-site-2023-05-12.11-45.xml
name: livy-session-8-681a4a882f244b80-hadoop-config
name: hadoop-properties
- emptyDir: {}
name: spark-local-dir-1
- name: livy-user-secret-dimpo-volume
secret:
defaultMode: 420
secretName: livy-user-secret-dimpo
- name: server-volume
secret:
defaultMode: 420
optional: true
secretName: server
- configMap:
defaultMode: 420
name: ldapclient-cm
optional: true
name: ldap-cm
- name: sssd-secrets
secret:
defaultMode: 420
optional: true
secretName: sssd
- name: ssh-secrets
secret:
defaultMode: 420
optional: true
secretName: ssh
- name: client-secrets
secret:
defaultMode: 420
optional: true
secretName: client
- configMap:
defaultMode: 420
items:
- key: dep-blacklist.txt
mode: 420
path: dep-blacklist.txt
- key: hbase-site.xml
mode: 420
path: hbase-site.xml
- key: hive-site.xml
mode: 420
path: hive-site.xml
- key: hive-site.xml.old
mode: 420
path: hive-site.xml.old
- key: jetty.headers.xml
mode: 420
path: jetty.headers.xml
- key: log4j2.properties
mode: 420
path: log4j2.properties
- key: log4j2.properties.old
mode: 420
path: log4j2.properties.old
- key: metrics.properties
mode: 420
path: metrics.properties
- key: spark-defaults.conf.old
mode: 420
path: spark-defaults.conf.old
- key: spark-env.sh
mode: 420
path: spark-env.sh
- key: spark-env.sh.old
mode: 420
path: spark-env.sh.old
- key: spark-jars.sh
mode: 420
path: spark-jars.sh
- key: spark.properties
mode: 420
path: spark.properties
name: spark-drv-d3ea19882f244e20-conf-map
name: spark-conf-volume-driver
- name: kube-api-access-7h59b
projected:
defaultMode: 420
sources:
- serviceAccountToken:
expirationSeconds: 3607
path: token
- configMap:
items:
- key: ca.crt
path: ca.crt
name: kube-root-ca.crt
- downwardAPI:
items:
- fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
path: namespace
status:
conditions:
- lastProbeTime: null
lastTransitionTime: "2023-05-18T13:54:49Z"
status: "True"
type: Initialized
- lastProbeTime: null
lastTransitionTime: "2023-05-18T13:54:55Z"
status: "True"
type: Ready
- lastProbeTime: null
lastTransitionTime: "2023-05-18T13:54:55Z"
status: "True"
type: ContainersReady
- lastProbeTime: null
lastTransitionTime: "2023-05-18T13:54:49Z"
status: "True"
type: PodScheduled
containerStatuses:
- containerID: containerd://7d603569737c6b86351ee9ab2125a92b7b797caeab365bb51a81a91dd4b5ea20
image: gcr.io/mapr-252711/spark-3.3.1:202304252229AN
imageID: gcr.io/mapr-252711/spark-3.3.1@sha256:a6d730a95a733283a4c3894af5bbb267818e681a20d819301010d6e263318f4a
lastState: {}
name: spark-kubernetes-driver
ready: true
restartCount: 0
started: true
state:
running:
startedAt: "2023-05-18T13:54:55Z"
hostIP: 10.227.211.14
phase: Running
podIP: 10.244.1.219
podIPs:
- ip: 10.244.1.219
qosClass: Guaranteed
startTime: "2023-05-18T13:54:49Z"