QWenTokenizerTests.cs

using Cnblogs.DashScope.Core;
using FluentAssertions;

namespace Cnblogs.DashScope.Sdk.UnitTests;

public class QWenTokenizerTests
{
    private const string Text =
        "很不错的一款产品，送人的额，非常喜欢，很不错，好评哦，以后还会多多支持的\n已经吃了，感觉不错。第二次购买，这个价格很实惠，卖家还赠送了美容器和尺子，很愉悦的一次购物。\n衣服的质量杠杠的，很漂亮，不枉我等了这么久啊，喜欢，以后还来这里买，我一次买了四件不到200块钱，真好\n用了一段时间了，感觉比传统的风扇好用，广度大，档数多，静音效果也不错，关键是还很漂亮！\n这个热量低，比意面还低，关键是口感还好，挺好吃，多煮一会一级棒，我的减脂早餐就靠它了\n整体评价：好 使用效果：好 包装与外观：好 三好商品\n电视机很好，我给老爸买的，老人很喜欢，图像清晰，音质很好，老人喜欢我满意。\n宝贝很轻巧，打的时候网弹力很足，儿子非常喜欢，五分好评，还送了羽毛球噢！\n终于装上，风很大，热的还可以，排风声音不小，安装的师傅说不能直吹，安装的时候非常满意，最后还帮忙把棚顶给擦干净了，非常感谢。\n颜色很美。就是别人的盒子都是平整的，唯独我的盒子有很严重的压痕。本来想直接用这个盒子装胶带的..现在这个样子强迫症看着实在很糟心。压成这样要么包装有问题，要么本来产品就有问题。而卖家一直在甩锅快递。由于包裹是被家里人拆开的，所以我也不知道是什么样子的，不好跟卖家掰扯。但我看别的人都是纸盒装的，想来我这个也是纸盒装的。那么排除包装的原因的话，很可能就是寄过来的就是个瑕疵品。比起同样今天到货的另一家，有个小瑕疵立马给我补寄了一个，这家的处理方式真的..换货..或者直接道歉说疏忽我都接受的。甩锅真的不能忍。本来还给好几个人推了这家店...就挺失望的 以后应该不会来了。这坚定的甩锅态度我也不点退换了..请不要给我打电话改评，谢谢。";

    private static readonly int[] Tokens =
    [
        109517,
        110659,
        82700,
        3837,
        36605,
        103947,
        61191,
        3837,
        115171,
        3837,
        109517,
        3837,
        102959,
        104170,
        3837,
        103934,
        104342,
        103235,
        100143,
        9370,
        198,
        99461,
        105705,
        3837,
        100681,
        100832,
        1773,
        106309,
        103946,
        3837,
        99487,
        97480,
        99165,
        109896,
        3837,
        108602,
        97706,
        109349,
        34187,
        105157,
        31548,
        33108,
        101489,
        44729,
        3837,
        99165,
        111985,
        99774,
        32571,
        102297,
        8997,
        102214,
        108042,
        103178,
        103178,
        9370,
        3837,
        117817,
        3837,
        16530,
        119117,
        35946,
        49567,
        34187,
        113260,
        103924,
        3837,
        99729,
        3837,
        103934,
        97706,
        115217,
        99565,
        3837,
        35946,
        99796,
        105275,
        63703,
        14224,
        99828,
        17,
        15,
        15,
        106734,
        3837,
        88051,
        52801,
        198,
        11622,
        99593,
        101430,
        34187,
        3837,
        100681,
        56006,
        105062,
        117814,
        52801,
        11622,
        3837,
        80942,
        26381,
        26288,
        3837,
        100104,
        8863,
        42140,
        3837,
        99541,
        78685,
        101062,
        116006,
        3837,
        111623,
        97706,
        117817,
        6313,
        198,
        99487,
        108461,
        99285,
        3837,
        56006,
        36589,
        27091,
        97706,
        99285,
        3837,
        111623,
        107816,
        108432,
        3837,
        101174,
        106678,
        3837,
        42140,
        104783,
        102947,
        105072,
        102321,
        3837,
        97611,
        99536,
        100553,
        102589,
        80158,
        99904,
        99652,
        34187,
        198,
        101932,
        103964,
        5122,
        52801,
        85658,
        101062,
        5122,
        52801,
        94305,
        227,
        98641,
        57218,
        105502,
        5122,
        52801,
        220,
        44991,
        52801,
        45943,
        198,
        115281,
        101243,
        3837,
        35946,
        89012,
        114198,
        108850,
        3837,
        102032,
        108295,
        3837,
        107553,
        104542,
        3837,
        78685,
        99178,
        101243,
        3837,
        102032,
        99729,
        35946,
        100545,
        8997,
        105882,
        99165,
        99578,
        100084,
        3837,
        75437,
        103920,
        31139,
        100080,
        47534,
        99165,
        99336,
        3837,
        102067,
        115171,
        3837,
        75108,
        17177,
        102959,
        3837,
        97706,
        36605,
        34187,
        116140,
        118401,
        6313,
        198,
        104020,
        98641,
        17447,
        3837,
        99208,
        101235,
        3837,
        99259,
        9370,
        104468,
        3837,
        59956,
        99208,
        102274,
        112592,
        3837,
        103999,
        9370,
        105476,
        36587,
        53153,
        73145,
        102203,
        3837,
        103999,
        103920,
        99491,
        100545,
        3837,
        100161,
        97706,
        106128,
        99360,
        103031,
        99743,
        89012,
        101432,
        102466,
        34187,
        3837,
        99491,
        104305,
        8997,
        102284,
        99165,
        57566,
        1773,
        99486,
        107693,
        110792,
        100132,
        112857,
        9370,
        3837,
        100473,
        99510,
        97611,
        110792,
        18830,
        99165,
        105806,
        99451,
        101629,
        1773,
        102347,
        99172,
        101041,
        11622,
        99487,
        110792,
        98641,
        100773,
        99278,
        9370,
        496,
        99601,
        99487,
        102481,
        111057,
        99769,
        101952,
        100538,
        99165,
        103336,
        63109,
        1773,
        99451,
        12857,
        99654,
        106016,
        104184,
        110686,
        3837,
        106016,
        102347,
        82700,
        104435,
        86119,
        1773,
        68536,
        108602,
        105078,
        107001,
        101240,
        104655,
        1773,
        101887,
        108232,
        20412,
        99250,
        102078,
        17340,
        101348,
        29767,
        9370,
        3837,
        107020,
        107066,
        102021,
        102481,
        9370,
        3837,
        101132,
        99557,
        108602,
        119059,
        105948,
        1773,
        105984,
        50930,
        62922,
        100623,
        100132,
        100050,
        102307,
        98641,
        9370,
        3837,
        99172,
        36407,
        35946,
        99487,
        100000,
        100050,
        102307,
        98641,
        9370,
        1773,
        100624,
        102945,
        104184,
        104249,
        100363,
        3837,
        107093,
        99486,
        101400,
        101180,
        105729,
        18947,
        117219,
        24442,
        1773,
        107957,
        101165,
        100644,
        26939,
        81668,
        9370,
        100266,
        45629,
        3837,
        104627,
        30709,
        117219,
        108628,
        104169,
        99622,
        101400,
        104059,
        3837,
        101610,
        9370,
        54542,
        75768,
        100672,
        496,
        71134,
        81668,
        496,
        100631,
        101041,
        107961,
        36587,
        100900,
        100310,
        107108,
        100669,
        9370,
        1773,
        107001,
        101240,
        100672,
        53153,
        100292,
        1773,
        102347,
        97706,
        89012,
        52801,
        112285,
        83751,
        34187,
        101610,
        71416,
        1112,
        80158,
        101174,
        106586,
        9370,
        220,
        103934,
        99730,
        99670,
        101161,
        1773,
        43288,
        102405,
        9370,
        107001,
        101240,
        102316,
        35946,
        99744,
        27442,
        55806,
        110735,
        496,
        14880,
        100148,
        104169,
        106202,
        22418,
        63379,
        3837,
        102570,
        1773
    ];

    [Fact]
    public void QWenTokenizer_Encode_SuccessAsync()
    {
        // Act
        var embeddings = QWenTokenizer.Encode(Text);

        // Assert
        embeddings.Should().BeEquivalentTo(Tokens);
    }

    [Fact]
    public void QWenTokenizer_Decode_SuccessAsync()
    {
        // Act
        var text = QWenTokenizer.Decode(Tokens);

        // Assert
        text.Should().BeEquivalentTo(Text);
    }

    [Fact]
    public void QWenTokenizer_EncodeToStrings_SuccessAsync()
    {
        // Act
        var tokens = QWenTokenizer.Tokenizer.EncodeToTokens(Text, out _);

        // Assert
        tokens.Select(t => t.Id).Should().BeEquivalentTo(Tokens);
    }

    [Fact]
    public void QWenTokenizer_CountIndex_SuccessAsync()
    {
        // Act
        var maxIndex = QWenTokenizer.GetIndexByTokenCount(Text, 100, out _, out var tokenCount);

        // Assert
        maxIndex.Should().Be(155);
        tokenCount.Should().BeLessThanOrEqualTo(100);
    }
}