Skip to content

Commit

Permalink
Fixed minor csv naming bug
Browse files Browse the repository at this point in the history
  • Loading branch information
ralphr123 committed Jan 18, 2024
1 parent 003c92c commit fec6a0a
Showing 1 changed file with 71 additions and 76 deletions.
147 changes: 71 additions & 76 deletions app/api/test-chat/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,91 +5,86 @@
// 5. Filter data
// 6. Query GPT

import { queryPinecone } from "../chat/_lib/queryPinecone";
import { queryPinecone } from '../chat/_lib/queryPinecone';
import prisma from '@/_lib/server/prismadb';
import { generateApiResponse } from "../_lib/generateApiResponse";
import excelToJson from "convert-excel-to-json";
import https from "https";
import { createWriteStream } from "fs";
import { parseString } from "xml2js";
import {} from "csvtojson";
import { generateApiResponse } from '../_lib/generateApiResponse';
import excelToJson from 'convert-excel-to-json';
import https from 'https';
import { createWriteStream } from 'fs';
import { parseString } from 'xml2js';
import csv from 'csvtojson';

export async function GET(){
const userQuery = "can you tell me about the city subject thesaurus?";
// const userQuery = "Tell me about active building permits"; // JSON

//1. Query Pinecone
const pineconeRes = await queryPinecone(userQuery);
// const confidenceScore = pineconeRes.matches[0].score;
const datasetID = pineconeRes.matches[0].id;
const metadata = JSON.stringify(pineconeRes.matches[0].metadata);
export async function GET() {
const userQuery = 'can you tell me about the city subject thesaurus?';
// const userQuery = "Tell me about active building permits"; // JSON

//2. Query database
const dataset = await prisma.dataset.findUnique({
where: {
id: datasetID,
},
});

if (dataset === null){
return generateApiResponse({ status: 404, error: "Dataset not found" });
}
// console.log(dataset.name);
const datasetURL = dataset.url;
// console.log(dataset)
//1. Query Pinecone
const pineconeRes = await queryPinecone(userQuery);
// const confidenceScore = pineconeRes.matches[0].score;
const datasetID = pineconeRes.matches[0].id;
const metadata = JSON.stringify(pineconeRes.matches[0].metadata);

switch(dataset.format){
case "XLSX":
case "XLS": {
await downloadFile(datasetURL, 'tmp.xlsx');
const result = excelToJson({
sourceFile: 'tmp.xlsx'
});
console.log(result.Sheet1)
break;
}
case "XML": {
const xml = await fetch(datasetURL, {cache: "no-store"})
const result = parseString(await xml.text(), (_,result) => console.log(result));
break;
}
case "CSV": {
const csv = await fetch(datasetURL, {cache: "no-store"})
csv({
noheader:true,
output: "csv"
})
.fromString(csv)
.then((result)=>{
console.log(result)
})
}
//2. Query database
const dataset = await prisma.dataset.findUnique({
where: {
id: datasetID,
},
});

// default: // JSON
if (dataset === null) {
return generateApiResponse({ status: 404, error: 'Dataset not found' });
}
// console.log(dataset.name);
const datasetURL = dataset.url;
// console.log(dataset)

switch (dataset.format) {
case 'XLSX':
case 'XLS': {
await downloadFile(datasetURL, 'tmp.xlsx');
const result = excelToJson({
sourceFile: 'tmp.xlsx',
});
console.log(result.Sheet1);
break;
}
case 'XML': {
const xmlResp = await fetch(datasetURL, { cache: 'no-store' });
parseString(await xmlResp.text(), (_, result) => console.log(result));
break;
}
case 'CSV': {
const csvRes = await fetch(datasetURL, { cache: 'no-store' });
csv({
noheader: true,
output: 'csv',
})
.fromString(await csvRes.text())
.then(result => {
console.log(result);
});
}

return generateApiResponse({ status: 200, data: {}});

};
// default: // JSON
}

return generateApiResponse({ status: 200, data: {} });
}

export const downloadFile = (url: string, filePath: string) => {
const request = https.get(url);

return new Promise<void>((resolve, reject) => {
const file = createWriteStream(filePath);
request.on("response", (response) => {
response.pipe(file);
file.on("finish", () => {
file.close();
resolve();
});
});
request.on("error", (error) => {
reject(error);
const request = https.get(url);

return new Promise<void>((resolve, reject) => {
const file = createWriteStream(filePath);
request.on('response', response => {
response.pipe(file);
file.on('finish', () => {
file.close();
resolve();
});
});
};



request.on('error', error => {
reject(error);
});
});
};

0 comments on commit fec6a0a

Please sign in to comment.