multi-thread(remaining bugs)

Author: whzruc

.gitignore

@@ -35,4 +35,9 @@ build/
 .vscode/
 
 ### Mac OS ###
-.DS_Store
+.DS_Store
+
+### data ###
+#./input/
+./output/
+*.parquet

.idea/.gitignore

@@ -0,0 +1,24 @@
+# ParquetWriter
+本项目目标是利用[parquet-java](https://github.com/apache/parquet-java)
+生成tpch和clickbench的parquet的数据文件(prarrow.parquet的praquetWriter依赖于prarrow-cpp,版本比较老且无法指定rowGroupSize)
+
+本项目附属于[PixelsDB](https://github.com/pixelsdb)
+
+## 类型转换
+采取parquet的Logical Type构造message 
+
+## 多线程读取
+为了使用多线程读取文件
+需要对文件进行切分
+使用linux的 spilt命令
+```bash
+# test 
+split -l 30 test1.csv ./test1/test1-csv-
+split -l 40 test2.csv ./test2/test2-csv-
+
+```
+
+## 参考
+- https://github.com/apache/parquet-format/blob/master/LogicalTypes.md
+- https://cloud.tencent.com/developer/article/2439115
+- https://trino.io/docs/current/language/types.html

.idea/misc.xml

@@ -0,0 +1,133 @@
+import json
+
+
+field_types = {
+    "bigint": "int64",
+    "smallint": "int32",
+    "varchar(65535)": "string",
+    "timestamp": "timestamp",
+    "date": "date",
+    "int": "int32"
+}
+
+
+fields = [
+    ("watchid", "bigint"),
+    ("javaenable", "smallint"),
+    ("title", "varchar(65535)"),
+    ("goodevent", "smallint"),
+    ("eventtime", "timestamp"),
+    ("eventdate", "date"),
+    ("counterid", "int"),
+    ("clientip", "int"),
+    ("regionid", "int"),
+    ("userid", "bigint"),
+    ("counterclass", "smallint"),
+    ("os", "smallint"),
+    ("useragent", "smallint"),
+    ("url", "varchar(65535)"),
+    ("referer", "varchar(65535)"),
+    ("isrefresh", "smallint"),
+    ("referercategoryid", "smallint"),
+    ("refererregionid", "int"),
+    ("urlcategoryid", "smallint"),
+    ("urlregionid", "int"),
+    ("resolutionwidth", "smallint"),
+    ("resolutionheight", "smallint"),
+    ("resolutiondepth", "smallint"),
+    ("flashmajor", "smallint"),
+    ("flashminor", "smallint"),
+    ("flashminor2", "varchar(65535)"),
+    ("netmajor", "smallint"),
+    ("netminor", "smallint"),
+    ("useragentmajor", "smallint"),
+    ("useragentminor", "varchar(65535)"),
+    ("cookieenable", "smallint"),
+    ("javascriptenable", "smallint"),
+    ("ismobile", "smallint"),
+    ("mobilephone", "smallint"),
+    ("mobilephonemodel", "varchar(65535)"),
+    ("params", "varchar(65535)"),
+    ("ipnetworkid", "int"),
+    ("traficsourceid", "smallint"),
+    ("searchengineid", "smallint"),
+    ("searchphrase", "varchar(65535)"),
+    ("advengineid", "smallint"),
+    ("isartifical", "smallint"),
+    ("windowclientwidth", "smallint"),
+    ("windowclientheight", "smallint"),
+    ("clienttimezone", "smallint"),
+    ("clienteventtime", "timestamp"),
+    ("silverlightversion1", "smallint"),
+    ("silverlightversion2", "smallint"),
+    ("silverlightversion3", "int"),
+    ("silverlightversion4", "smallint"),
+    ("pagecharset", "varchar(65535)"),
+    ("codeversion", "int"),
+    ("islink", "smallint"),
+    ("isdownload", "smallint"),
+    ("isnotbounce", "smallint"),
+    ("funiqid", "bigint"),
+    ("originalurl", "varchar(65535)"),
+    ("hid", "int"),
+    ("isoldcounter", "smallint"),
+    ("isevent", "smallint"),
+    ("isparameter", "smallint"),
+    ("dontcounthits", "smallint"),
+    ("withhash", "smallint"),
+    ("hitcolor", "varchar(65535)"),
+    ("localeventtime", "timestamp"),
+    ("age", "smallint"),
+    ("sex", "smallint"),
+    ("income", "smallint"),
+    ("interests", "smallint"),
+    ("robotness", "smallint"),
+    ("remoteip", "int"),
+    ("windowname", "int"),
+    ("openername", "int"),
+    ("historylength", "smallint"),
+    ("browserlanguage", "varchar(65535)"),
+    ("browsercountry", "varchar(65535)"),
+    ("socialnetwork", "varchar(65535)"),
+    ("socialaction", "varchar(65535)"),
+    ("httperror", "smallint"),
+    ("sendtiming", "int"),
+    ("dnstiming", "int"),
+    ("connecttiming", "int"),
+    ("responsestarttiming", "int"),
+    ("responseendtiming", "int"),
+    ("fetchtiming", "int"),
+    ("socialsourcenetworkid", "smallint"),
+    ("socialsourcepage", "varchar(65535)"),
+    ("paramprice", "bigint"),
+    ("paramorderid", "varchar(65535)"),
+    ("paramcurrency", "varchar(65535)"),
+    ("paramcurrencyid", "smallint"),
+    ("openstatservicename", "varchar(65535)"),
+    ("openstatcampaignid", "varchar(65535)"),
+    ("openstatadid", "varchar(65535)"),
+    ("openstatsourceid", "varchar(65535)"),
+    ("utmsource", "varchar(65535)"),
+    ("utmmedium", "varchar(65535)"),
+    ("utmcampaign", "varchar(65535)"),
+    ("utmcontent", "varchar(65535)"),
+    ("utmterm", "varchar(65535)"),
+    ("fromtag", "varchar(65535)"),
+    ("hasgclid", "smallint"),
+    ("refererhash", "bigint"),
+    ("urlhash", "bigint"),
+    ("clid", "int")
+]
+
+
+converted_types = [field_types.get(field[1], field[1]) for field in fields]
+
+
+output = {
+    "schema": {
+        "hits": converted_types
+    }
+}
+print(converted_types)
+
+print(json.dumps(output, indent=2))