Add SciDataframe

Governance/原则.md

@@ -29,4 +29,4 @@ PifFow社区遵循[社区行为准则](https://github.com/cas-bigdatalab/piflow/
 
 ### CLA
 
-所有贡献者都必须签署PifFow CLA，请具体看[这里](https://github.com/cas-bigdatalab/piflow/blob/master/Governance/image-20211118094103884.png)。
+所有贡献者都必须签署PifFow CLA，请具体看[这里](https://github.com/cas-bigdatalab/piflow/blob/master/Governance/%CF%80Flow_Open_Source_Individual_CLA.docx)。

README.md

@@ -39,11 +39,12 @@
 ![](https://github.com/cas-bigdatalab/piflow/blob/master/doc/architecture.png) 
 ## Requirements
 * JDK 1.8 
-* Scala-2.11.8
+* Scala-2.12.18
 * Apache Maven 3.1.0 or newer  
-* Spark-2.1.0、 Spark-2.2.0、 Spark-2.3.0
-* Hadoop-2.6.0
-* Apache Livy-0.7.1
+* Spark-3.4.0
+* Hadoop-3.3.0
+
+Compatible with X86 architecture and ARM architecture, Support CentOS and Kirin system deployment
 
 ## Getting Started
 
@@ -319,12 +320,20 @@
   ![](https://github.com/cas-bigdatalab/piflow/blob/master/doc/piflow-stophublist.png)
 
 ## Contact Us
-- Name:吴老师  
-- Mobile Phone：18910263390  
-- WeChat：18910263390  
-- Email: [email protected]  
-- QQ Group：1003489545  
-  ![](https://github.com/cas-bigdatalab/piflow/blob/master/doc/PiFlowUserGroup_QQ.jpeg)  
+- Name:Yang Gang, Tian Yao  
+- Mobile Phone：13253365393, 18501260806  
+- WeChat：13253365393, 18501260806  
+- Email: [email protected], [email protected]
+- Private vulnerability contact information：[email protected]
+- Wechat User Group  
+<center>
+    <img src="https://github.com/cas-bigdatalab/piflow/blob/master/doc/wechat_user.png" width="100"/>
+</center>
+
+- Wechat Official Account
+<center>
+    <img src="https://github.com/cas-bigdatalab/piflow/blob/master/doc/tencent.jpg" width="100"/>
+</center>
 
 
 

config.properties

@@ -1,13 +1,13 @@
 spark.master=yarn
 spark.deploy.mode=cluster
-
+server.ip=172.18.32.1
 #hdfs default file system
-fs.defaultFS=hdfs://10.0.82.108:9000
+fs.defaultFS=hdfs://172.18.39.41:9000
 #yarn resourcemanager hostname
-yarn.resourcemanager.hostname=10.0.82.108
+yarn.resourcemanager.hostname=172.18.39.41
 
 #if you want to use hive, set hive metastore uris
-hive.metastore.uris=thrift://10.0.82.108:9083
+#hive.metastore.uris=thrift://10.0.82.108:9083
 
 #show data in log, set 0 if you do not show the logs
 data.show=10

piflow-bin/config.properties

@@ -2,10 +2,10 @@ spark.master=yarn
 spark.deploy.mode=cluster
 
 #hdfs default file system
-fs.defaultFS=hdfs://10.0.85.83:9000
+fs.defaultFS=hdfs://172.18.39.41:9000
 
 #yarn resourcemanager hostname
-yarn.resourcemanager.hostname=10.0.85.83
+yarn.resourcemanager.hostname=172.18.39.41
 
 #if you want to use hive, set hive metastore uris
 hive.metastore.uris=thrift://10.0.85.83:9083

piflow-bin/example/flow.json

@@ -7,45 +7,15 @@
         "paths": [
             {
                 "inport": "",
-                "from": "XmlParser",
-                "to": "SelectField",
-                "outport": ""
-            },
-            {
-                "inport": "",
-                "from": "Fork",
+                "from": "CsvParser",
                 "to": "CsvSave",
-                "outport": "out1"
-            },
-            {
-                "inport": "data2",
-                "from": "SelectField",
-                "to": "Merge",
                 "outport": ""
             },
             {
                 "inport": "",
-                "from": "Merge",
-                "to": "Fork",
-                "outport": ""
-            },
-            {
-                "inport": "data1",
                 "from": "CsvParser",
-                "to": "Merge",
+                "to": "CsvSave",
                 "outport": ""
-            },
-            {
-                "inport": "",
-                "from": "Fork",
-                "to": "JsonSave",
-                "outport": "out3"
-            },
-            {
-                "inport": "",
-                "from": "Fork",
-                "to": "PutHiveMode",
-                "outport": "out2"
             }
         ],
         "executorCores": "1",
@@ -56,7 +26,7 @@
                 "bundle": "cn.piflow.bundle.csv.CsvSave",
                 "uuid": "8a80d63f720cdd2301723a4e67a52467",
                 "properties": {
-                    "csvSavePath": "hdfs://master:9000/xjzhu/phdthesis_result.csv",
+                    "csvSavePath": "hdfs://172.18.32.1:9000/user/Yomi/test1.csv",
                     "partition": "",
                     "header": "false",
                     "saveMode": "append",
@@ -66,87 +36,18 @@
 
                 }
             },
-            {
-                "name": "PutHiveMode",
-                "bundle": "cn.piflow.bundle.hive.PutHiveMode",
-                "uuid": "8a80d63f720cdd2301723a4e67a22461",
-                "properties": {
-                    "database": "sparktest",
-                    "saveMode": "append",
-                    "table": "dblp_phdthesis"
-                },
-                "customizedProperties": {
-
-                }
-            },
             {
                 "name": "CsvParser",
                 "bundle": "cn.piflow.bundle.csv.CsvParser",
                 "uuid": "8a80d63f720cdd2301723a4e67a82470",
                 "properties": {
                     "schema": "title,author,pages",
-                    "csvPath": "hdfs://master:9000/xjzhu/phdthesis.csv",
+                    "csvPath": "hdfs://172.18.32.1:9000/user/Yomi/test.csv",
                     "delimiter": ",",
                     "header": "false"
                 },
                 "customizedProperties": {
 
-                }
-            },
-            {
-                "name": "JsonSave",
-                "bundle": "cn.piflow.bundle.json.JsonSave",
-                "uuid": "8a80d63f720cdd2301723a4e67a1245f",
-                "properties": {
-                    "jsonSavePath": "hdfs://10.0.86.191:9000/xjzhu/phdthesis.json"
-                },
-                "customizedProperties": {
-
-                }
-            },
-            {
-                "name": "XmlParser",
-                "bundle": "cn.piflow.bundle.xml.XmlParser",
-                "uuid": "8a80d63f720cdd2301723a4e67a7246d",
-                "properties": {
-                    "rowTag": "phdthesis",
-                    "xmlpath": "hdfs://master:9000/xjzhu/dblp.mini.xml"
-                },
-                "customizedProperties": {
-
-                }
-            },
-            {
-                "name": "SelectField",
-                "bundle": "cn.piflow.bundle.common.SelectField",
-                "uuid": "8a80d63f720cdd2301723a4e67aa2477",
-                "properties": {
-                    "columnNames": "title,author,pages"
-                },
-                "customizedProperties": {
-
-                }
-            },
-            {
-                "name": "Merge",
-                "bundle": "cn.piflow.bundle.common.Merge",
-                "uuid": "8a80d63f720cdd2301723a4e67a92475",
-                "properties": {
-                    "inports": "data1,data2"
-                },
-                "customizedProperties": {
-
-                }
-            },
-            {
-                "name": "Fork",
-                "bundle": "cn.piflow.bundle.common.Fork",
-                "uuid": "8a80d63f720cdd2301723a4e67a42465",
-                "properties": {
-                    "outports": "out1,out3,out2"
-                },
-                "customizedProperties": {
-
                 }
             }
         ]