Skip to content

Commit 55d2a4c

Browse files
committed
Adding XLSX support
1 parent d7fcab7 commit 55d2a4c

File tree

7 files changed

+174
-47
lines changed

7 files changed

+174
-47
lines changed

README.md

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,28 @@
11
# csv-sql
2-
Command-line tool to load csv files and run sql commands
2+
Command-line tool to load csv and xlsx files and run sql commands
33

44
[![Go](https://github.com/dhamith93/csv-sql/actions/workflows/go.yml/badge.svg)](https://github.com/dhamith93/csv-sql/actions/workflows/go.yml)
55

66
## Usage
77

8-
csv-sql supports loading and saving results as CSV files with data processing with SQLite compatible sql commands
8+
csv-sql supports loading and saving results as CSV and XLSX files with data processing with SQLite compatible sql commands.
99

1010
### Loading a file
1111
```
12-
LOAD /path/to/file.csv table_name
12+
LOAD /path/to/file table_name
1313
```
14+
You can set up headers if the first row is not a header.
15+
For XLSX files, when loading, this will ask to select the sheet of the file to load.
1416

1517
### Creating a new table with a select query
1618
```sql
1719
CREATE TABLE emp_user AS SELECT emp.emp_id, emp.name, user.user_name, user.role FROM emp INNER JOIN user ON emp.user_id = user.id
1820
```
1921

2022
### Saving a table as a csv
23+
24+
This only supports CSV for now
25+
2126
```
2227
SAVE table_name /path/to/save.csv
2328
```

go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@ module csv-sql
33
go 1.15
44

55
require (
6+
github.com/360EntSecGroup-Skylar/excelize/v2 v2.4.0
67
github.com/c-bata/go-prompt v0.2.6
8+
github.com/gabriel-vasile/mimetype v1.3.0
79
github.com/mattn/go-sqlite3 v1.14.7
810
github.com/olekukonko/tablewriter v0.0.5
911
)

go.sum

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
1+
github.com/360EntSecGroup-Skylar/excelize/v2 v2.4.0 h1:X+2CWGf5W1tm2+W7Y/LLrAPLFSNlHATnqDudGoIzaxY=
2+
github.com/360EntSecGroup-Skylar/excelize/v2 v2.4.0/go.mod h1:p9lGPoVX3HYEbFRfjgrPWaaKsHe/2u4EM9DB/qoctgU=
13
github.com/c-bata/go-prompt v0.2.6 h1:POP+nrHE+DfLYx370bedwNhsqmpCUynWPxuHi0C5vZI=
24
github.com/c-bata/go-prompt v0.2.6/go.mod h1:/LMAke8wD2FsNu9EXNdHxNLbd9MedkPnCdfpU9wwHfY=
5+
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
6+
github.com/gabriel-vasile/mimetype v1.3.0 h1:4YOHITFLyYwF+iqG0ybSLGArRItynpfwdlWRmJnd75E=
7+
github.com/gabriel-vasile/mimetype v1.3.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8=
38
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
49
github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw=
510
github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
@@ -14,10 +19,28 @@ github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEg
1419
github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
1520
github.com/mattn/go-tty v0.0.3 h1:5OfyWorkyO7xP52Mq7tB36ajHDG5OHrmBGIS/DtakQI=
1621
github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0=
22+
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
23+
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
1724
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
1825
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
1926
github.com/pkg/term v1.2.0-beta.2 h1:L3y/h2jkuBVFdWiJvNfYfKmzcCnILw7mJWm2JQuMppw=
2027
github.com/pkg/term v1.2.0-beta.2/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw=
28+
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
29+
github.com/richardlehane/mscfb v1.0.3 h1:rD8TBkYWkObWO0oLDFCbwMeZ4KoalxQy+QgniCj3nKI=
30+
github.com/richardlehane/mscfb v1.0.3/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
31+
github.com/richardlehane/msoleps v1.0.1 h1:RfrALnSNXzmXLbGct/P2b4xkFz4e8Gmj/0Vj9M9xC1o=
32+
github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
33+
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
34+
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
35+
github.com/xuri/efp v0.0.0-20210322160811-ab561f5b45e3 h1:EpI0bqf/eX9SdZDwlMmahKM+CDBgNbsXMhsN28XrM8o=
36+
github.com/xuri/efp v0.0.0-20210322160811-ab561f5b45e3/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
37+
golang.org/x/crypto v0.0.0-20210415154028-4f45737414dc h1:+q90ECDSAQirdykUN6sPEiBXBsp8Csjcca8Oy7bgLTA=
38+
golang.org/x/crypto v0.0.0-20210415154028-4f45737414dc/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
39+
golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
40+
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
41+
golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
42+
golang.org/x/net v0.0.0-20210505024714-0287a6fb4125 h1:Ugb8sMTWuWRC3+sz5WeN/4kejDx9BvIwnPUiJBjJE+8=
43+
golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
2144
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
2245
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
2346
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -27,5 +50,17 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w
2750
golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
2851
golang.org/x/sys v0.0.0-20200918174421-af09f7315aff h1:1CPUrky56AcgSpxz/KfgzQWzfG09u5YOL8MvPYBlrL8=
2952
golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
53+
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
54+
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
3055
golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 h1:rF3Ohx8DRyl8h2zw9qojyLHLhrJpEMgyPOImREEryf0=
3156
golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
57+
golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
58+
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
59+
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
60+
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
61+
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
62+
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
63+
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
64+
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
65+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
66+
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

helpers/data.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@ func PopulateTables(db *sql.DB, file *entity.File) {
3232

3333
for _, c := range row {
3434
builder.WriteString("\"")
35+
// this is to handle numbers in xlsx files as they represented as float in xlsx
36+
if IsIntegral(c) {
37+
c = ConvertToIntString(c)
38+
}
3539
builder.WriteString(c)
3640
builder.WriteString("\"")
3741
builder.WriteString(",")

helpers/file.go

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,19 @@ import (
55
"encoding/csv"
66
"fmt"
77
"os"
8+
9+
"github.com/360EntSecGroup-Skylar/excelize/v2"
10+
"github.com/gabriel-vasile/mimetype"
811
)
912

10-
// ReadCSVFile read from given CSV file
13+
func GetMimeType(path string) (string, error) {
14+
mtype, err := mimetype.DetectFile(path)
15+
if err != nil {
16+
return "", err
17+
}
18+
return mtype.String(), nil
19+
}
20+
1121
func ReadCSVFile(path string) ([][]string, error) {
1222
file, err := os.Open(path)
1323

@@ -29,6 +39,26 @@ func ReadCSVFile(path string) ([][]string, error) {
2939
return records, nil
3040
}
3141

42+
func GetXLSXSheetList(path string) ([]string, error) {
43+
f, err := excelize.OpenFile(path)
44+
if err != nil {
45+
return nil, err
46+
}
47+
return f.GetSheetList(), nil
48+
}
49+
50+
func ReadXLSXFile(path string, sheet string) ([][]string, error) {
51+
f, err := excelize.OpenFile(path)
52+
if err != nil {
53+
return nil, err
54+
}
55+
rows, err := f.GetRows(sheet)
56+
if err != nil {
57+
return nil, err
58+
}
59+
return rows, nil
60+
}
61+
3262
func WriteToCSV(path string, result entity.Table) {
3363
csvFile, err := os.Create(path)
3464
if err != nil {

helpers/string.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package helpers
22

33
import (
44
"math/rand"
5+
"strconv"
56
"time"
67
"unsafe"
78
)
@@ -30,3 +31,26 @@ func RandSeq(n int) string {
3031
}
3132
return *(*string)(unsafe.Pointer(&b))
3233
}
34+
35+
func IsIntegral(val string) bool {
36+
if !isNumeric(val) {
37+
return false
38+
}
39+
num, _ := strconv.ParseFloat(val, 64)
40+
return isIntegral(num)
41+
}
42+
43+
func ConvertToIntString(val string) string {
44+
num, _ := strconv.ParseFloat(val, 64)
45+
intNum := int(num)
46+
return strconv.Itoa(intNum)
47+
}
48+
49+
func isIntegral(val float64) bool {
50+
return val == float64(int(val))
51+
}
52+
53+
func isNumeric(val string) bool {
54+
_, err := strconv.ParseFloat(val, 64)
55+
return err == nil
56+
}

main.go

Lines changed: 70 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,16 @@ func loadFile(responseArr []string, files []entity.File, db *sql.DB, tableCount
7070
if len(responseArr) == 3 {
7171
path := responseArr[1]
7272
tableName := responseArr[2]
73+
mimeType, err := helpers.GetMimeType(path)
74+
if err != nil {
75+
fmt.Printf("Error reading mimetype of file %v : %v\n", path, err.Error())
76+
return files
77+
}
78+
79+
if !validFileType(mimeType) {
80+
fmt.Printf("Error reading file %v : file type not supported\n", path)
81+
return files
82+
}
7383

7484
for i := range files {
7585
if files[i].Table == tableName {
@@ -78,59 +88,72 @@ func loadFile(responseArr []string, files []entity.File, db *sql.DB, tableCount
7888
}
7989
}
8090

81-
if _, err := os.Stat(path); err == nil || os.IsExist(err) {
82-
for {
83-
fmt.Println("File has a header row (y/n)?")
84-
response := strings.ToUpper(strings.TrimSpace(prompt.Input("> ", helpers.Completer)))
85-
if response == "Y" || response == "N" {
86-
content, fileErr := helpers.ReadCSVFile(path)
87-
if fileErr != nil {
88-
fmt.Printf("Error reading file %v : %v\n", path, fileErr.Error())
89-
break
90-
}
91+
for {
92+
fmt.Println("File has a header row (y/n)?")
93+
response := strings.ToUpper(strings.TrimSpace(prompt.Input("> ", helpers.Completer)))
94+
if response == "Y" || response == "N" {
95+
var (
96+
content [][]string
97+
fileErr error
98+
headers = make([]string, 0)
99+
)
100+
101+
if mimeType == "text/csv" {
102+
content, fileErr = helpers.ReadCSVFile(path)
103+
}
91104

92-
if len(content) == 0 {
93-
fmt.Println("Empty file")
94-
break
105+
if mimeType == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" {
106+
sheets, err := helpers.GetXLSXSheetList(path)
107+
if err != nil {
108+
fmt.Printf("Error reading file %v : %v\n", path, err.Error())
109+
return files
95110
}
111+
fmt.Printf("What is the name of the sheet to be loaded? %v\n", strings.Join(sheets, ", "))
112+
sheet := strings.TrimSpace(prompt.Choose("> ", sheets))
113+
content, fileErr = helpers.ReadXLSXFile(path, sheet)
114+
}
96115

97-
headers := make([]string, 0)
98-
99-
if response == "Y" {
100-
headers = content[0]
101-
content = content[1:]
102-
} else {
103-
for {
104-
fmt.Println("Enter " + strconv.Itoa(len(content[0])) + " headers seperated by commas")
105-
headers = strings.Split(strings.TrimSpace(prompt.Input("> ", helpers.Completer)), ",")
106-
if len(content[0]) == len(headers) {
107-
break
108-
}
109-
}
110-
}
116+
if fileErr != nil {
117+
fmt.Printf("Error reading file %v : %v\n", path, fileErr.Error())
118+
return files
119+
}
111120

112-
for i := range headers {
113-
headers[i] = strings.TrimSpace(headers[i])
114-
}
121+
if len(content) == 0 {
122+
fmt.Println("Empty file")
123+
return files
124+
}
115125

116-
file := entity.File{
117-
Path: path,
118-
Headers: headers,
119-
Table: strings.TrimSpace(tableName),
126+
if response == "Y" {
127+
headers = content[0]
128+
content = content[1:]
129+
} else {
130+
for {
131+
fmt.Println("Enter " + strconv.Itoa(len(content[0])) + " headers seperated by commas")
132+
headers = strings.Split(strings.TrimSpace(prompt.Input("> ", helpers.Completer)), ",")
133+
if len(content[0]) == len(headers) {
134+
break
135+
}
120136
}
137+
}
121138

122-
files = append(files, file)
123-
file.Content = content
124-
helpers.PopulateTables(db, &file)
125-
tableCount++
126-
break
139+
for i := range headers {
140+
headers[i] = strings.TrimSpace(headers[i])
127141
}
142+
143+
file := entity.File{
144+
Path: path,
145+
Headers: headers,
146+
Table: strings.TrimSpace(tableName),
147+
}
148+
149+
files = append(files, file)
150+
file.Content = content
151+
helpers.PopulateTables(db, &file)
152+
tableCount++
153+
break
128154
}
129-
} else {
130-
fmt.Printf("File doesn't exists: %v\n", path)
131155
}
132-
} else {
133-
fmt.Println("Please use LOAD /path/to/file.csv table_name")
156+
134157
}
135158
return files
136159
}
@@ -166,6 +189,10 @@ func createDB(dbName string) *sql.DB {
166189
return db
167190
}
168191

192+
func validFileType(mimeType string) bool {
193+
return mimeType == "text/csv" || mimeType == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
194+
}
195+
169196
func handleExit() {
170197
rawModeOff := exec.Command("/bin/stty", "-raw", "echo")
171198
rawModeOff.Stdin = os.Stdin

0 commit comments

Comments
 (0)