Skip to content

fix: fix csv import #71

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nhandang02 merged 1 commit into from
Dec 6, 2025
Merged

fix: fix csv import #71

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
189 changes: 149 additions & 40 deletions src/question-banks/question-banks.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,11 +221,6 @@ export class QuestionBanksService {
throw new BadRequestException('File is required');
}

const workbook = xlsx.read(file.buffer, { type: 'buffer' });
const sheetName = workbook.SheetNames[0];
const worksheet = workbook.Sheets[sheetName];
const data = xlsx.utils.sheet_to_json(worksheet);

// Lấy danh sách questionText hiện có trong course để kiểm tra trùng lặp
const existingQuestions = await this.questionRepository.find({
where: { courseId },
Expand All @@ -235,6 +230,53 @@ export class QuestionBanksService {
existingQuestions.map((q) => q.questionText.toLowerCase().trim()),
);

// Parse file (CSV hoặc XLSX)
// xlsx library tự động xử lý cả CSV và XLSX
const workbook = xlsx.read(file.buffer, {
type: 'buffer',
cellDates: false,
cellNF: false,
cellText: false,
});
const sheetName = workbook.SheetNames[0];
const worksheet = workbook.Sheets[sheetName];

// Parse với header tự động (sẽ tạo object với keys từ dòng đầu tiên)
// Không dùng header: 1 vì nó sẽ trả về array of arrays thay vì array of objects
let data = xlsx.utils.sheet_to_json(worksheet, {
defval: '', // Giá trị mặc định cho cell rỗng
raw: false, // Convert tất cả về string
blankrows: false, // Bỏ qua dòng trống
});

// Normalize tên cột: xóa khoảng trắng thừa và chuẩn hóa
// Xử lý trường hợp xlsx có thể parse tên cột khác đi (do dấu ngoặc kép trong CSV)
data = data.map((row: any) => {
const normalizedRow: Record<string, any> = {};

// Map các tên cột có thể có (xlsx có thể parse khác do dấu ngoặc kép)
Object.keys(row).forEach((key) => {
const normalizedKey = key.trim().replace(/^["']|["']$/g, ''); // Xóa dấu ngoặc kép nếu có
const value = row[key];

// Chuẩn hóa tên cột
if (normalizedKey === 'Question Text' || normalizedKey === 'QuestionText' || normalizedKey === 'Question') {
normalizedRow['Question Text'] = value;
} else if (normalizedKey === 'Choices' || normalizedKey === 'Choice') {
normalizedRow['Choices'] = value;
} else if (normalizedKey === 'Correct Answer' || normalizedKey === 'CorrectAnswer' || normalizedKey === 'Correct') {
normalizedRow['Correct Answer'] = value;
} else if (normalizedKey === 'Difficulty') {
normalizedRow['Difficulty'] = value;
} else {
// Giữ nguyên nếu không match (có thể là tên cột khác)
normalizedRow[key] = value;
}
});

return normalizedRow;
});

// Mảng để lưu kết quả từng câu hỏi
const questionResults: Array<{
rowNumber: number;
Expand All @@ -252,30 +294,49 @@ export class QuestionBanksService {
for (const [index, row] of data.entries()) {
const rowNumber = index + 2; // 1-based index + header row
const rowData = row as Record<string, any>;
const questionText = rowData['Question Text']?.toString().trim() || '';

// Kiểm tra trùng lặp trong file
const questionTextLower = questionText.toLowerCase();
if (questionTextsInFile.has(questionTextLower)) {
questionResults.push({
rowNumber,
questionText,
success: false,
error: 'Question is duplicated in file',
});
continue;

// Debug: Log row data để kiểm tra
// console.log(`Row ${rowNumber}:`, JSON.stringify(rowData));

// Lấy questionText, xử lý các trường hợp có thể có
let questionText = '';
if (rowData['Question Text']) {
questionText = String(rowData['Question Text']).trim();
} else if (rowData['QuestionText']) {
// Fallback nếu không có khoảng trắng
questionText = String(rowData['QuestionText']).trim();
} else if (rowData['Question']) {
// Fallback nếu chỉ có "Question"
questionText = String(rowData['Question']).trim();
}
questionTextsInFile.add(questionTextLower);

// Kiểm tra trùng lặp với database
if (existingQuestionTexts.has(questionTextLower)) {
questionResults.push({
rowNumber,
questionText,
success: false,
error: 'Question already exists in question bank',
});
continue;
// Nếu questionText rỗng, bỏ qua kiểm tra trùng lặp và để mapRowToQuestionDto báo lỗi
let questionTextLower = '';
if (questionText) {
questionTextLower = questionText.toLowerCase();

// Kiểm tra trùng lặp trong file
if (questionTextsInFile.has(questionTextLower)) {
questionResults.push({
rowNumber,
questionText,
success: false,
error: 'Question is duplicated in file',
});
continue;
}
questionTextsInFile.add(questionTextLower);

// Kiểm tra trùng lặp với database
if (existingQuestionTexts.has(questionTextLower)) {
questionResults.push({
rowNumber,
questionText,
success: false,
error: 'Question already exists in question bank',
});
continue;
}
}

// Validate và map dữ liệu
Expand Down Expand Up @@ -327,41 +388,89 @@ export class QuestionBanksService {
}

private mapRowToQuestionDto(row: any, courseId: string, createdById: string): CreateQuestionDto {
if (!row['Question Text'] || !row['Choices'] || !row['Correct Answer'] || !row['Difficulty']) {
throw new Error('Missing required fields');
// Kiểm tra và convert các trường về string, xử lý null/undefined
const questionText = row['Question Text']
? String(row['Question Text']).trim()
: null;
const choicesValue = row['Choices']
? String(row['Choices']).trim()
: null;
const correctAnswerValue = row['Correct Answer'] !== undefined && row['Correct Answer'] !== null
? String(row['Correct Answer']).trim()
: null;
const difficultyValue = row['Difficulty']
? String(row['Difficulty']).trim()
: null;

// Kiểm tra từng trường và báo lỗi cụ thể
const missingFields: string[] = [];
if (!questionText) missingFields.push('Question Text');
if (!choicesValue) missingFields.push('Choices');
if (!correctAnswerValue) missingFields.push('Correct Answer');
if (!difficultyValue) missingFields.push('Difficulty');

if (missingFields.length > 0) {
throw new Error(`Missing required fields: ${missingFields.join(', ')}`);
}

// Parse Choices
if (!choicesValue) {
throw new Error('Choices is required');
}
let choices: string[];
try {
// Try parsing as JSON array first
choices = JSON.parse(row['Choices']);
choices = JSON.parse(choicesValue);
if (!Array.isArray(choices)) {
throw new Error('Choices must be a JSON array');
}
} catch {
// Fallback: split by comma or pipe if it's a string
if (typeof row['Choices'] === 'string') {
choices = row['Choices'].split('|').map((c) => c.trim());
// Fallback: split by pipe if it's a string
if (typeof choicesValue === 'string') {
choices = choicesValue.split('|').map((c) => c.trim()).filter((c) => c.length > 0);
} else {
throw new Error('Invalid choices format');
throw new Error('Invalid choices format: must be a string separated by "|" or a JSON array');
}
}

// Validate số lượng choices
if (!Array.isArray(choices) || choices.length < 2) {
throw new Error('At least 2 choices are required');
throw new Error(`At least 2 choices are required, but found ${choices.length} choice(s)`);
}

// Kiểm tra choices rỗng
const emptyChoices = choices.filter((c) => !c || c.trim().length === 0);
if (emptyChoices.length > 0) {
throw new Error(`Found ${emptyChoices.length} empty choice(s). All choices must have content.`);
}

const correctAnswer = Number(row['Correct Answer']);
// Validate Correct Answer
const correctAnswer = Number(correctAnswerValue);
if (isNaN(correctAnswer)) {
throw new Error(`Correct Answer must be a number, but got: ${correctAnswerValue}`);
}

if (correctAnswer < 0 || correctAnswer >= choices.length) {
throw new Error(`correctAnswer must be between 0 and ${choices.length - 1}`);
throw new Error(`Correct Answer must be between 0 and ${choices.length - 1}, but got: ${correctAnswer}`);
}

const difficulty = row['Difficulty'].toLowerCase();
// Validate Difficulty
if (!difficultyValue) {
throw new Error('Difficulty is required');
}
const difficulty = difficultyValue.toLowerCase();
if (!Object.values(QuestionDifficulty).includes(difficulty as QuestionDifficulty)) {
throw new Error(`Invalid difficulty: ${difficulty}`);
throw new Error(`Invalid difficulty: "${difficultyValue}". Must be one of: ${Object.values(QuestionDifficulty).join(', ')}`);
}

if (!questionText) {
throw new Error('Question Text is required');
}

return {
courseId,
createdById,
questionText: row['Question Text'],
questionText,
choices,
correctAnswer,
difficulty: difficulty as QuestionDifficulty,
Expand Down
Loading