1+ kind : job
2+ metadata :
3+ name : spark-submit
4+ tag : ' '
5+ hash : e0b494cfbf67f5bb3dd5728a573f9a6a57378268
6+ project : default
7+ spec :
8+ command : ' '
9+ args : []
10+ image : mlrun/mlrun
11+ env :
12+ - name : V3IO_API
13+ value : ' '
14+ - name : V3IO_USERNAME
15+ value : ' '
16+ - name : V3IO_ACCESS_KEY
17+ value : ' '
18+ default_handler : spark_submit
19+ entry_points :
20+ get_shell_pod_name :
21+ name : get_shell_pod_name
22+ doc : ' '
23+ parameters :
24+ - name : self
25+ default : ' '
26+ - name : pod_name
27+ default : shell
28+ outputs :
29+ - default : ' '
30+ lineno : 32
31+ exec_shell_cmd :
32+ name : exec_shell_cmd
33+ doc : ' '
34+ parameters :
35+ - name : self
36+ default : ' '
37+ - name : cmd
38+ default : ' '
39+ - name : shell_pod_name
40+ default : shell
41+ outputs :
42+ - default : ' '
43+ lineno : 41
44+ nb_or_py :
45+ name : nb_or_py
46+ doc : ' '
47+ parameters :
48+ - name : pyspark_command
49+ default : ' '
50+ outputs :
51+ - default : ' '
52+ lineno : 84
53+ spark_command_builder :
54+ name : spark_command_builder
55+ doc : ' '
56+ parameters :
57+ - name : name
58+ default : ' '
59+ - name : class_name
60+ default : ' '
61+ - name : jars
62+ default : ' '
63+ - name : packages
64+ default : ' '
65+ - name : spark_options
66+ default : ' '
67+ - name : pyspark_command
68+ default : ' '
69+ - name : pyspark_params
70+ default : ' '
71+ outputs :
72+ - default : ' '
73+ lineno : 96
74+ spark_submit :
75+ name : spark_submit
76+ doc : ' spark_submit function
77+ submiting spark via shell'
78+ parameters :
79+ - name : context
80+ default : ' '
81+ - name : v3io_access_key
82+ default : ' '
83+ - name : name
84+ doc : A name of your application.
85+ default : null
86+ - name : class_name
87+ doc : Your application's main class (for Java / Scala apps). * If relative
88+ will add to the {artifact_path}
89+ default : null
90+ - name : jars
91+ doc : Comma-separated list of jars to include on the driver and executor classpaths.
92+ default : null
93+ - name : packages
94+ doc : Comma-separated list of maven coordinates of jars to include on the driver
95+ and executor classpaths. Will search the local maven repo, then maven central
96+ and any additional remote repositories given by --repositories. The format
97+ for the
98+ default : null
99+ - name : spark_options
100+ doc : spark parametes that are not included as function arguments
101+ default : ' '
102+ - name : pyspark_command
103+ doc : A python script or Jupyter notebook to be executed
104+ default : ' '
105+ - name : pyspark_params
106+ doc : parameters to the python script
107+ default : ' '
108+ outputs :
109+ - default : ' '
110+ lineno : 122
111+ description : ' '
112+ service_account : mlrun-api
113+ build :
114+ functionSourceCode: IyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKZnJvbSBtbHJ1biBpbXBvcnQgZ2V0X29yX2NyZWF0ZV9jdHgKZnJvbSBrdWJlcm5ldGVzIGltcG9ydCBjb25maWcsIGNsaWVudApmcm9tIGt1YmVybmV0ZXMuc3RyZWFtIGltcG9ydCBzdHJlYW0KaW1wb3J0IHlhbWwKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmltcG9ydCBvcwoKY2xhc3MgSzhTQ2xpZW50KG9iamVjdCk6CgogICAgZGVmIF9faW5pdF9fKHNlbGYsIGxvZ2dlciwgbmFtZXNwYWNlPSdkZWZhdWx0LXRlbmFudCcsIGNvbmZpZ19maWxlPU5vbmUpOgogICAgICAgIHNlbGYubmFtZXNwYWNlID0gbmFtZXNwYWNlCiAgICAgICAgc2VsZi5sb2dnZXIgPSBsb2dnZXIKICAgICAgICBzZWxmLl9pbml0X2s4c19jb25maWcoY29uZmlnX2ZpbGUpCiAgICAgICAgc2VsZi52MWFwaSA9IGNsaWVudC5Db3JlVjFBcGkoKQoKICAgIGRlZiBfaW5pdF9rOHNfY29uZmlnKHNlbGYsIGNvbmZpZ19maWxlKToKICAgICAgICB0cnk6CiAgICAgICAgICAgIGNvbmZpZy5sb2FkX2luY2x1c3Rlcl9jb25maWcoKQogICAgICAgICAgICBzZWxmLmxvZ2dlci5pbmZvKCd1c2luZyBpbi1jbHVzdGVyIGNvbmZpZy4nKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb246CiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIGNvbmZpZy5sb2FkX2t1YmVfY29uZmlnKGNvbmZpZ19maWxlKQogICAgICAgICAgICAgICAgc2VsZi5sb2dnZXIuaW5mbygndXNpbmcgbG9jYWwga3ViZXJuZXRlcyBjb25maWcuJykKICAgICAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbjoKICAgICAgICAgICAgICAgIHJhaXNlIFJ1bnRpbWVFcnJvcigKICAgICAgICAgICAgICAgICAgICAnY2Fubm90IGZpbmQgbG9jYWwga3ViZXJuZXRlcyBjb25maWcgZmlsZSwnCiAgICAgICAgICAgICAgICAgICAgJyBwbGFjZSBpdCBpbiB+Ly5rdWJlL2NvbmZpZyBvciBzcGVjaWZ5IGl0IGluICcKICAgICAgICAgICAgICAgICAgICAnS1VCRUNPTkZJRyBlbnYgdmFyJykKCiAgICBkZWYgZ2V0X3NoZWxsX3BvZF9uYW1lKHNlbGYsIHBvZF9uYW1lPSdzaGVsbCcpOgogICAgICAgIHNoZWxsX3BvZCA9IHNlbGYudjFhcGkubGlzdF9uYW1lc3BhY2VkX3BvZChuYW1lc3BhY2U9c2VsZi5uYW1lc3BhY2UpCiAgICAgICAgZm9yIGkgaW4gc2hlbGxfcG9kLml0ZW1zOgogICAgICAgICAgICBpZiBwb2RfbmFtZSBpbiBpLm1ldGFkYXRhLm5hbWU6CiAgICAgICAgICAgICAgICBzZWxmLmxvZ2dlci5pbmZvKCIlc1x0JXNcdCVzIiAlIChpLnN0YXR1cy5wb2RfaXAsIGkubWV0YWRhdGEubmFtZXNwYWNlLCBpLm1ldGFkYXRhLm5hbWUpKQogICAgICAgICAgICAgICAgc2hlbGxfbmFtZSA9IGkubWV0YWRhdGEubmFtZQogICAgICAgICAgICAgICAgYnJlYWsKICAgICAgICByZXR1cm4gc2hlbGxfbmFtZQoKICAgIGRlZiBleGVjX3NoZWxsX2NtZChzZWxmLCBjbWQsIHNoZWxsX3BvZF9uYW1lID0gJ3NoZWxsJyk6CiAgICAgICAgc2hlbGxfbmFtZSA9IHNlbGYuZ2V0X3NoZWxsX3BvZF9uYW1lKHNoZWxsX3BvZF9uYW1lKQogICAgICAgIGV4ZWNfY29tbWFuZCA9IFsKICAgICAgICAgICAgJy9iaW4vYmFzaCcsCiAgICAgICAgICAgICctYycsCiAgICAgICAgICAgIGNtZF0KICAgICAgICByZXNwID0gc3RyZWFtKHNlbGYudjFhcGkuY29ubmVjdF9nZXRfbmFtZXNwYWNlZF9wb2RfZXhlYywKICAgICAgICAgICAgICAgICAgICAgIHNoZWxsX25hbWUsCiAgICAgICAgICAgICAgICAgICAgICBzZWxmLm5hbWVzcGFjZSwKICAgICAgICAgICAgICAgICAgICAgIGNvbW1hbmQ9ZXhlY19jb21tYW5kLAogICAgICAgICAgICAgICAgICAgICAgc3RkZXJyPVRydWUsIHN0ZGluPUZhbHNlLAogICAgICAgICAgICAgICAgICAgICAgc3Rkb3V0PVRydWUsIHR0eT1GYWxzZSxfcHJlbG9hZF9jb250ZW50PUZhbHNlKQogICAgICAgIAogICAgICAgIAogICAgICAgIHN0ZGVyciA9IFtdCiAgICAgICAgc3Rkb3V0ID0gW10KICAgICAgICB3aGlsZSByZXNwLmlzX29wZW4oKToKICAgICAgICAgICAgcmVzcC51cGRhdGUodGltZW91dD1Ob25lKQogICAgICAgICAgICBpZiByZXNwLnBlZWtfc3RkZXJyKCk6CiAgICAgICAgICAgICAgICAgc3RkZXJyLmFwcGVuZChyZXNwLnJlYWRfc3RkZXJyKCkpCiAgICAgICAgICAgIGlmIHJlc3AucGVla19zdGRvdXQoKToKICAgICAgICAgICAgICAgICBzdGRvdXQuYXBwZW5kKHJlc3AucmVhZF9zdGRvdXQoKSkKICAgICAgICAgICAKICAgICAgICBlcnIgPSByZXNwLnJlYWRfY2hhbm5lbCgzKQogICAgICAgIGVyciA9IHlhbWwuc2FmZV9sb2FkKGVycikKICAgICAgICBpZiBlcnJbJ3N0YXR1cyddID09ICdTdWNjZXNzJzoKICAgICAgICAgICAgcmMgPSAwCiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmMgPSBpbnQoZXJyWydkZXRhaWxzJ11bJ2NhdXNlcyddWzBdWydtZXNzYWdlJ10pCiAgICAgICAgCiAgICAgICAgc3Rkb3V0X2Zvcm1hdGVkPScnCiAgICAgICAgZm9yIGVhY2ggaW4gc3Rkb3V0OgogICAgICAgICAgICBzdGRvdXRfZm9ybWF0ZWQgKz0gZWFjaAogICAgICAgIHNlbGYubG9nZ2VyLmluZm8oIlNURE9VVDogJXMiJSBzdGRvdXRfZm9ybWF0ZWQpCiAgICAgICAgc2VsZi5sb2dnZXIuaW5mbygiUkM6ICVzIiUgcmMpCiAgICAgICAgCiAgICAgICAgc3RkZXJyX2Zvcm1hdGVkPScnCiAgICAgICAgZm9yIGVhY2ggaW4gc3RkZXJyOgogICAgICAgICAgICBzdGRlcnJfZm9ybWF0ZWQgKz0gZWFjaAogICAgICAgIGlmIHJjICE9IDA6CiAgICAgICAgICAgIHNlbGYubG9nZ2VyLmluZm8oIlNUREVSUjogJXMiJSBzdGRlcnJfZm9ybWF0ZWQpCiAgICAgICAgICAgIHJhaXNlIEV4Y2VwdGlvbignRXhlY3V0aW9uX2Vycm9yJywgc3RkZXJyX2Zvcm1hdGVkICsgc3Rkb3V0X2Zvcm1hdGVkKQogICAgCmRlZiBuYl9vcl9weShweXNwYXJrX2NvbW1hbmQpOgogICAgaWYgcHlzcGFya19jb21tYW5kLmVuZHN3aXRoKCcuaXB5bmInKToKICAgICAgICBmcm9tIG1scnVuIGltcG9ydCBjb2RlX3RvX2Z1bmN0aW9uCiAgICAgICAgZGlyID0gUGF0aChweXNwYXJrX2NvbW1hbmQpLnBhcmVudAogICAgICAgIGZpbGUgPSAgUGF0aChweXNwYXJrX2NvbW1hbmQpLnN0ZW0KICAgICAgICBweXRob25fZmlsZSA9IG9zLnBhdGguam9pbihkaXIsZmlsZSArJy5weScpCiAgICAgICAgY29kZV90b19mdW5jdGlvbihraW5kPSJsb2NhbCIsZmlsZW5hbWU9cHlzcGFya19jb21tYW5kLGNvZGVfb3V0cHV0PXB5dGhvbl9maWxlKQogICAgICAgIHJldHVybiBweXRob25fZmlsZQogICAgZWxzZToKICAgICAgICByZXR1cm4gcHlzcGFya19jb21tYW5kICAgICAKCgpkZWYgc3BhcmtfY29tbWFuZF9idWlsZGVyKG5hbWUsIGNsYXNzX25hbWUsIGphcnMsIHBhY2thZ2VzLCBzcGFya19vcHRpb25zLHB5c3BhcmtfY29tbWFuZD0nJyxweXNwYXJrX3BhcmFtcz0nJyk6CiAgICBjbWQgPSAnc3Bhcmstc3VibWl0JwogICAgaWYgbmFtZToKICAgICAgICBjbWQgKz0gJyAtLW5hbWUgJyArIG5hbWUKCiAgICBpZiBjbGFzc19uYW1lOgogICAgICAgIGNtZCArPSAnIC0tY2xhc3MgJyArIGNsYXNzX25hbWUKCiAgICBpZiBqYXJzOgogICAgICAgIGNtZCArPSAnIC0tamFycyAnICsgamFycwoKICAgIGlmIHBhY2thZ2VzOgogICAgICAgIGNtZCArPSAnIC0tcGFja2FnZXMgJyArIHBhY2thZ2VzCgogICAgaWYgc3Bhcmtfb3B0aW9uczoKICAgICAgICBjbWQgKz0gJyAnICsgc3Bhcmtfb3B0aW9ucwoKICAgIGlmIHB5c3BhcmtfY29tbWFuZDoKICAgICAgICBjbWQgKz0gJyAnICsgbmJfb3JfcHkocHlzcGFya19jb21tYW5kKQogICAgICAgICAgICAKICAgIGlmIHB5c3BhcmtfcGFyYW1zOgogICAgICAgIGNtZCArPSAnICcgKyBweXNwYXJrX3BhcmFtcwogICAgICAgIAogICAgcmV0dXJuIGNtZAoKCmRlZiBzcGFya19zdWJtaXQoY29udGV4dCwgdjNpb19hY2Nlc3Nfa2V5LCBuYW1lPU5vbmUsIGNsYXNzX25hbWU9Tm9uZSwgamFycz1Ob25lLCBwYWNrYWdlcz1Ob25lLCBzcGFya19vcHRpb25zPScnLHB5c3BhcmtfY29tbWFuZD0nJyxweXNwYXJrX3BhcmFtcz0nJyk6CiAgICAiIiJzcGFya19zdWJtaXQgZnVuY3Rpb24KICAgIAogICAgc3VibWl0aW5nIHNwYXJrIHZpYSBzaGVsbAogICAgCiAgICA6cGFyYW0gbmFtZTogICAgICAgIEEgbmFtZSBvZiB5b3VyIGFwcGxpY2F0aW9uLgogICAgOnBhcmFtIGNsYXNzX25hbWU6ICBZb3VyIGFwcGxpY2F0aW9uJ3MgbWFpbiBjbGFzcyAoZm9yIEphdmEgLyBTY2FsYSBhcHBzKS4KICAgICAgICAgICAgICAgICAgICAgICAgKiBJZiByZWxhdGl2ZSB3aWxsIGFkZCB0byB0aGUge2FydGlmYWN0X3BhdGh9CiAgICA6cGFyYW0gamFyczogICAgICAgIENvbW1hLXNlcGFyYXRlZCBsaXN0IG9mIGphcnMgdG8gaW5jbHVkZSBvbiB0aGUgZHJpdmVyCiAgICAgICAgICAgICAgICAgICAgICAgIGFuZCBleGVjdXRvciBjbGFzc3BhdGhzLgogICAgOnBhcmFtIHBhY2thZ2VzOiAgICBDb21tYS1zZXBhcmF0ZWQgbGlzdCBvZiBtYXZlbiBjb29yZGluYXRlcyBvZiBqYXJzIHRvIGluY2x1ZGUKICAgICAgICAgICAgICAgICAgICAgICAgb24gdGhlIGRyaXZlciBhbmQgZXhlY3V0b3IgY2xhc3NwYXRocy4gV2lsbCBzZWFyY2ggdGhlIGxvY2FsCiAgICAgICAgICAgICAgICAgICAgICAgIG1hdmVuIHJlcG8sIHRoZW4gbWF2ZW4gY2VudHJhbCBhbmQgYW55IGFkZGl0aW9uYWwgcmVtb3RlCiAgICAgICAgICAgICAgICAgICAgICAgIHJlcG9zaXRvcmllcyBnaXZlbiBieSAtLXJlcG9zaXRvcmllcy4gVGhlIGZvcm1hdCBmb3IgdGhlCiAgICA6cGFyYW0gc3Bhcmtfb3B0aW9uczogc3BhcmsgcGFyYW1ldGVzIHRoYXQgYXJlIG5vdCBpbmNsdWRlZCBhcyBmdW5jdGlvbiBhcmd1bWVudHMKICAgIDpwYXJhbSBweXNwYXJrX2NvbW1hbmQ6IEEgcHl0aG9uIHNjcmlwdCBvciBKdXB5dGVyIG5vdGVib29rIHRvIGJlIGV4ZWN1dGVkCiAgICA6cGFyYW0gcHlzcGFya19wYXJhbXM6ICBwYXJhbWV0ZXJzIHRvIHRoZSBweXRob24gc2NyaXB0CgogICAgIiIiCiAgICBjbWQgPSBzcGFya19jb21tYW5kX2J1aWxkZXIobmFtZSwgY2xhc3NfbmFtZSwgamFycywgcGFja2FnZXMsIHNwYXJrX29wdGlvbnMscHlzcGFya19jb21tYW5kLHB5c3BhcmtfcGFyYW1zKQogICAgY29udGV4dC5sb2dnZXIuaW5mbygic3VibWl0aW5nIDoiICsgY21kKQoKICAgIGNsaSA9IEs4U0NsaWVudChjb250ZXh0LmxvZ2dlcikKICAgIGNsaS5leGVjX3NoZWxsX2NtZChjbWQpCgo=
115+ commands : []
116+ code_origin : https://github.com/marcelonyc/functions#500364cb773cf4d2f23df1449ad32b6a1e370b8c:submit.ipynb
0 commit comments