Parsing Error

[KulAndy]

Break on parse file with non-ASCII chars e.g. french, cyrillic
Break on parse with illegal tags e.g. tournament name with " in name
No error info - only crash

[Disservin]

Can you provide examples for this ? Then I can faster debug it.

[KulAndy]

zipped file on google drive
File is generated by scid (scid correct imports file)
" is exported with backslash "

[KulAndy]

parser code

#include <fstream>
#include <memory>
#include <iostream>
#include <optional>
#include <iomanip>

#include "chess-library/include/chess.hpp"

using namespace chess;
using namespace std;

const bool validateDate(string_view date){
    if (date.size() < 4) {
        return false;
    }

    for (size_t i = 0; i < 4; ++i) {
        if (!std::isdigit(date[i])) {
            return false;
        }
    }

    return true;
}


class MyVisitor : public pgn::Visitor {
public:
public:
    MyVisitor(std::ofstream& outFile, const std::string& tableName)
    : outFile(outFile), tableName(tableName) {
        cout << endl;
    }

    virtual ~MyVisitor() {}

    void startPgn() {
        board.setFen(constants::STARTPOS);
        halfmove=2;
        movesStream.str("");
        movesStream.clear();
    }

    void header(std::string_view key, std::string_view value) {
        if(key=="Event"){
            event=value;
        }else if(key=="Site"){
            site=value;
        }else if(key=="Round"){
            round=value;
        }else if(key=="White"){
            white=value;
        }else if(key=="Black"){
            black=value;
        }else if(key=="WhiteElo"){
            whiteElo=stoi(string(value));
        }else if(key=="BlackElo"){
            blackElo=stoi(string(value));
        }else if(key=="Result"){
            result=value;
        }else if(key=="Date"){
            parseDateValue(value);
        }else if(key=="EventDate"){
            if(validateDate(value)|| year.value()==1899){
                parseDateValue(value);
            }
        }else if(key=="UTCDate"){
            if(validateDate(value)|| year.value()==1899){
                parseDateValue(value);
            }
        }
    }

    void startMoves() {}

    void move(string_view move, string_view comment) {
        try {
            const Move m = uci::parseSan(board, move);
            const string moveUci = uci::moveToUci(m);

            int src = m.from().index();
            int dest = m.to().index();

            if (m.typeOf() == Move::CASTLING) {
                switch (dest) {
                    case Square(Square::underlying::SQ_A1).index():
                    case Square(Square::underlying::SQ_A8).index():
                        dest += 2;
                        break;
                    case Square(Square::underlying::SQ_H1).index():
                    case Square(Square::underlying::SQ_H8).index():
                        dest--;
                        break;
                }
            }

            PieceType piece = PieceType::NONE;

            if (move.find("=") != string::npos) {
                piece = m.promotionType();
            }

            uint16_t packed = (static_cast<uint16_t>(src) << 10) |
            (static_cast<uint16_t>(dest) << 4) |
            (static_cast<uint16_t>(piece) & 0x07);

            movesStream << static_cast<char>((packed >> 8) & 0xFF) << static_cast<char>(packed & 0xFF);
            board.makeMove(m);
        } catch (exception &e) {
            cerr << "Error in move processing: " << e.what() << endl;
        }
    }


    void endPgn() {
        outFile
            << "insert ignore into "
            << tableName
            << " (moves_blob,Event,Site,Year,Month,Day,Round,White,Black,Result,WhiteElo,BlackElo) values("
            << "0x" ;
            for (unsigned char c : movesStream.str()) {
                outFile << std::hex << std::setw(2) << std::setfill('0')
                << static_cast<int>(c);
            }
            outFile << std::dec;

            outFile << ","
            << "\"" << event << "\"" << ","
            << "\"" << site << "\"" << ",";
        if(year.has_value()){
            outFile<< year.value();
        }else{
            outFile<< "null";
        }

        outFile << ",";

        if(month.has_value()){
            outFile<< month.value();
        }else{
            outFile<< "null";
        }

        outFile << ",";

        if(day.has_value()){
            outFile<< day.value();
        }else{
            outFile<< "null";
        }

        outFile << ","
            << "\"" << round << "\"" << ","
            << "\"" << white << "\"" << ","
            << "\"" << black << "\"" << ","
            << "\"" << result << "\"" << ",";

        if (whiteElo.has_value()) {
            outFile << whiteElo.value();
        } else {
            outFile << "NULL";
        }

        outFile << ",";

        if (blackElo.has_value()) {
            outFile << blackElo.value();
        } else {
            outFile << "NULL";
        }

        outFile
            << ");\n";

        cout << "\rPrzetworzono " << ++counter << " partii";

    }

protected:
    std::ofstream& outFile;
    std::string tableName;
    Board board;
    int halfmove = 0;
    string event = "?";
    string site = "?";
    optional<int> year;
    optional<int> month;
    optional<int> day;
    string round = "?";
    string white = "N, N";
    string black = "N, N";
    optional<int> whiteElo;
    optional<int> blackElo;
    string result = "*";
    ostringstream movesStream;
    int counter = 0;

    void parseDateValue(string_view value){
        size_t first_pos = value.find('.');
        size_t second_pos = std::string_view::npos;
        size_t third_pos = std::string_view::npos;

        if (first_pos != std::string_view::npos) {
            try{
                year = stoi(string(value.substr(0, first_pos)));
            } catch (const std::invalid_argument& e) {
                return;
            }

            second_pos = value.find('.', first_pos + 1);
            if (second_pos != std::string_view::npos) {
                try{
                    month = stoi(string(value.substr(first_pos + 1, second_pos - first_pos - 1)));
                } catch (const std::invalid_argument& e) {
                    return;
                }

                third_pos = value.find('.', second_pos + 1);
                if (third_pos != std::string_view::npos) {
                    try{
                        day = stoi(string(value.substr(second_pos + 1, third_pos - second_pos - 1)));
                    } catch (const std::invalid_argument& e) {
                        return;
                    }
                }
            }
        }
    }
};


int main(int argc, char **argv) {
    string input_file, target_table;

    if(argc>1){
        input_file = argv[1];
    }else{
        cout << "Podaj nazwę pliku PGN: ";
        cin >> input_file;
        cout << endl;
    }

    if(argc>2){
        target_table = argv[2];
    }else{
        cout << "Podaj nazwę tabeli: ";
        cin >> target_table;
        cout << endl;
    }

    ofstream outFile("insert.sql");
    auto vis = make_unique<MyVisitor>(outFile, target_table);

    ifstream file_stream(input_file);
    pgn::StreamParser parser(file_stream);
    parser.readGames(*vis);

    return 0;
}

[Disservin]

please paste the raw content of a pgn file here..
non ascii pgn and quote in header don't crash in my tests, see the tests here
https://github.com/Disservin/chess-library/pull/209/files
so please provide your test pgns (only the relevant ones..)

[KulAndy]

I can’t paste the whole file here because it’s too large, that’s why I provided drive link.

I also can’t provide a single specific game, because I don’t know exactly where the error occurs.

• For the full file (421,121 games), only 86,220 are processed.
• For the first 86,220 games / 1,527,833 lines, only 86,209 games are processed.
• For the first 86,209 games / 1,527,639 lines, only 86,198 games are processed.
  And so on.

I was cutting the games using head -n n.

After applying Python’s unidecode to transliterate everything to ASCII and removing " with sed, the problem disappeared.

[Disservin]

Then keep track of how many pgns you parsed in the the file and when it crashes you'll just have to extract the next one from the pgn.. or like split the file in half and test if it crashes and if it does half the file again and then continue like this until you found the problematic one