diff --git a/.gitignore b/.gitignore index ed8ebf58..786c4a5e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,1050 @@ -__pycache__ \ No newline at end of file +__pycache__ + +##### Windows +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +##### Linux +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +##### MacOS +# General +.DS_Store +.AppleDouble +.LSOverride + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +##### Android +# Built application files +*.apk +*.ap_ +*.aab + +# Files for the ART/Dalvik VM +*.dex + +# Java class files +*.class + +# Generated files +bin/ +gen/ +out/ +# Uncomment the following line in case you need and you don't have the release build type files in your app +# release/ + +# Gradle files +.gradle/ +build/ + +# Local configuration file (sdk path, etc) +local.properties + +# Proguard folder generated by Eclipse +proguard/ + +# Log Files +*.log + +# Android Studio Navigation editor temp files +.navigation/ + +# Android Studio captures folder +captures/ + +# IntelliJ +*.iml +.idea/workspace.xml +.idea/tasks.xml +.idea/gradle.xml +.idea/assetWizardSettings.xml +.idea/dictionaries +.idea/libraries +# Android Studio 3 in .gitignore file. +.idea/caches +.idea/modules.xml +# Comment next line if keeping position of elements in Navigation Editor is relevant for you +.idea/navEditor.xml + +# Keystore files +# Uncomment the following lines if you do not want to check your keystore files in. +#*.jks +#*.keystore + +# External native build folder generated in Android Studio 2.2 and later +.externalNativeBuild + +# Google Services (e.g. APIs or Firebase) +# google-services.json + +# Freeline +freeline.py +freeline/ +freeline_project_description.json + +# fastlane +fastlane/report.xml +fastlane/Preview.html +fastlane/screenshots +fastlane/test_output +fastlane/readme.md + +# Version control +vcs.xml + +# lint +lint/intermediates/ +lint/generated/ +lint/outputs/ +lint/tmp/ +# lint/reports/ + +##### Backup +*.bak +*.gho +*.ori +*.orig +*.tmp + +##### GPG +secring.* + +##### Dropbox +# Dropbox settings and caches +.dropbox +.dropbox.attr +.dropbox.cache + +##### SynopsysVCS +# Waveform formats +*.vcd +*.vpd +*.evcd +*.fsdb + +# Default name of the simulation executable. A different name can be +# specified with this switch (the associated daidir database name is +# also taken from here): -o / +simv + +# Generated for Verilog and VHDL top configs +simv.daidir/ +simv.db.dir/ + +# Infrastructure necessary to co-simulate SystemC models with +# Verilog/VHDL models. An alternate directory may be specified with this +# switch: -Mdir= +csrc/ + +# Log file - the following switch allows to specify the file that will be +# used to write all messages from simulation: -l +*.log + +# Coverage results (generated with urg) and database location. The +# following switch can also be used: urg -dir .vdb +simv.vdb/ +urgReport/ + +# DVE and UCLI related files. +DVEfiles/ +ucli.key + +# When the design is elaborated for DirectC, the following file is created +# with declarations for C/C++ functions. +vc_hdrs.h + +##### SVN +.svn/ + +##### Mercurial +.hg/ +.hgignore +.hgsigs +.hgsub +.hgsubstate +.hgtags + +##### Bazaar +.bzr/ +.bzrignore + +##### CVS +/CVS/* +**/CVS/* +.cvsignore +*/.cvsignore + +##### TortoiseGit +# Project-level settings +/.tgitconfig + +##### PuTTY +# Private key +*.ppk + +##### Vim +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +##### Emacs +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data + +##### SublimeText +# Cache files for Sublime Text +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache + +# Workspace files are user-specific +*.sublime-workspace + +# Project files should be checked into the repository, unless a significant +# proportion of contributors will probably not be using Sublime Text +# *.sublime-project + +# SFTP configuration file +sftp-config.json +sftp-config-alt*.json + +# Package control specific files +Package Control.last-run +Package Control.ca-list +Package Control.ca-bundle +Package Control.system-ca-bundle +Package Control.cache/ +Package Control.ca-certs/ +Package Control.merged-ca-bundle +Package Control.user-ca-bundle +oscrypto-ca-bundle.crt +bh_unicode_properties.cache + +# Sublime-github package stores a github token in this file +# https://packagecontrol.io/packages/sublime-github +GitHub.sublime-settings + +##### Notepad++ +# Notepad++ backups # +*.bak + +##### TextMate +*.tmproj +*.tmproject +tmtags + +##### VisualStudioCode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +##### NetBeans +**/nbproject/private/ +**/nbproject/Makefile-*.mk +**/nbproject/Package-*.bash +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ + +##### JetBrains +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +##### Eclipse +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ +.apt_generated_test/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +# Uncomment this line if you wish to ignore the project description file. +# Typically, this file would be tracked if it contains build/dependency configurations: +#.project + +##### Qt +# C++ objects and libs +*.slo +*.lo +*.o +*.a +*.la +*.lai +*.so +*.so.* +*.dll +*.dylib + +# Qt-es +object_script.*.Release +object_script.*.Debug +*_plugin_import.cpp +/.qmake.cache +/.qmake.stash +*.pro.user +*.pro.user.* +*.qbs.user +*.qbs.user.* +*.moc +moc_*.cpp +moc_*.h +qrc_*.cpp +ui_*.h +*.qmlc +*.jsc +Makefile* +*build-* +*.qm +*.prl + +# Qt unit tests +target_wrapper.* + +# QtCreator +*.autosave + +# QtCreator Qml +*.qmlproject.user +*.qmlproject.user.* + +# QtCreator CMake +CMakeLists.txt.user* + +# QtCreator 4.8< compilation database +compile_commands.json + +# QtCreator local machine specific files for imported projects +*creator.user* + +##### VisualStudio +##### VisualStudio +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*[.json, .xml, .info] + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +##### Gradle +.gradle +**/build/ +!src/**/build/ + +# Ignore Gradle GUI config +gradle-app.setting + +# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +!gradle-wrapper.jar + +# Cache of project +.gradletasknamecache + +# # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# gradle/wrapper/gradle-wrapper.properties + +##### CMake +CMakeLists.txt.user +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +_deps + +##### C++ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# C/C++ binary extension file +*.bin + +##### C +# Prerequisites +*.d + +# Object files +*.o +*.ko +*.obj +*.elf + +# Linker output +*.ilk +*.map +*.exp + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +# Debug files +*.dSYM/ +*.su +*.idb +*.pdb + +# Kernel Module Compile Results +*.mod* +*.cmd +.tmp_versions/ +modules.order +Module.symvers +Mkfile.old +dkms.conf + +# Raspberry Pi Pico Object file +*.uf2 +# Raspberry Pi Pico disassembler file +*.dis + +build +quake.egg-info diff --git a/CMakeLists.txt b/CMakeLists.txt index a4d2c003..32121138 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,18 +12,6 @@ set(Torch_USE_CUDA OFF CACHE BOOL "Force disable CUDA in Torch") set(Torch_NO_CUDA ON CACHE BOOL "Force disable CUDA in Torch") set(USE_CUDA OFF CACHE BOOL "Force disable CUDA globally") -# QUAKE_ENABLE_GPU: Enable GPU support for Faiss -# Default: OFF -if(QUAKE_ENABLE_GPU) - set(FAISS_ENABLE_GPU ON) -else() - set(FAISS_ENABLE_GPU OFF) -endif() - -if(QUAKE_ENABLE_GPU) - add_compile_definitions(FAISS_ENABLE_GPU) -endif() - if(QUAKE_USE_NUMA) add_compile_definitions(QUAKE_USE_NUMA) endif() @@ -33,7 +21,7 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) # Compiler flags -set(CMAKE_CXX_FLAGS_DEBUG "-g") +set(CMAKE_CXX_FLAGS_DEBUG "-O3 -g") set(CMAKE_CXX_FLAGS_RELEASE "-O3") if(NOT DEFINED QUAKE_SET_ABI_MODE) set(QUAKE_SET_ABI_MODE ON) @@ -64,6 +52,25 @@ set(project_BINDINGS_DIR ${CPP_SOURCE}/bindings) set(project_THIRD_PARTY_DIR ${CPP_SOURCE}/third_party) set(project_TEST_DIR test/cpp) +if(QUAKE_ENABLE_GPU) + find_package(CuVS REQUIRED) + include(${project_THIRD_PARTY_DIR}/cmake/fetch_rapids.cmake) + include(rapids-cmake) + include(rapids-cpm) + include(rapids-cuda) + include(rapids-export) + include(rapids-find) + + rapids_cuda_init_architectures(quake_c) + + rapids_cpm_init() + set(BUILD_CUVS_C_LIBRARY OFF) + include(${project_THIRD_PARTY_DIR}/cmake/get_cuvs.cmake) + + add_compile_definitions(QUAKE_ENABLE_GPU) +endif() + +set(FAISS_ENABLE_GPU OFF) # --------------------------------------------------------------- # Print out Compiler and Path Information # --------------------------------------------------------------- @@ -81,7 +88,6 @@ message(STATUS "QUAKE_USE_NUMA: ${QUAKE_USE_NUMA}") # Apple-specific adjustments if(APPLE) include_directories("/opt/homebrew/opt/openblas/include") - set(FAISS_ENABLE_GPU OFF) endif() # Compiler options and definitions @@ -89,7 +95,7 @@ add_compile_options(-march=native) # Switch ABI mode if(QUAKE_SET_ABI_MODE) - add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=0) + add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=1) endif() @@ -149,6 +155,10 @@ elseif(UNIX) if(QUAKE_USE_NUMA) list(APPEND LINK_LIBS -lnuma) endif() + + if (QUAKE_ENABLE_GPU) + list(APPEND LINK_LIBS cuvs::cuvs) + endif() else() # unsupported platform message(FATAL_ERROR "Unsupported platform") @@ -156,17 +166,23 @@ endif() target_link_libraries(${PROJECT_NAME} PUBLIC ${LINK_LIBS}) -IF(CMAKE_BUILD_TYPE MATCHES Debug AND QUAKE_USE_TSAN) +IF(QUAKE_USE_TSAN) message("Using thread sanitizer") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread") set(CMAKE_MODULE_LINKER_FLAGS "{$CMAKE_MODULE_LINKER_FLAGS} -fsanitize=thread") -ENDIF(CMAKE_BUILD_TYPE MATCHES Debug AND QUAKE_USE_TSAN) +ENDIF(QUAKE_USE_TSAN) -IF(CMAKE_BUILD_TYPE MATCHES Debug AND QUAKE_USE_ASAN) +IF(QUAKE_USE_ASAN) message("Using address sanitizer") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fsanitize-recover=address") set(CMAKE_MODULE_LINKER_FLAGS "{$CMAKE_MODULE_LINKER_FLAGS} -fsanitize=address") -ENDIF(CMAKE_BUILD_TYPE MATCHES Debug AND QUAKE_USE_ASAN) +ENDIF(QUAKE_USE_ASAN) + +IF(QUAKE_USE_LSAN) + message("Using leak sanitizer") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=leak") + set(CMAKE_MODULE_LINKER_FLAGS "{$CMAKE_MODULE_LINKER_FLAGS} -fsanitize=leak") +ENDIF(QUAKE_USE_LSAN) # --------------------------------------------------------------- # Python Bindings @@ -182,7 +198,7 @@ add_dependencies(bindings _bindings) # --------------------------------------------------------------- # Benchmarks # --------------------------------------------------------------- -file(GLOB BENCHMARK_SOURCES "src/cpp/test/benchmark/*.cpp") +file(GLOB BENCHMARK_SOURCES "scripts/cpp/benchmark/*.cpp") foreach(SOURCE ${BENCHMARK_SOURCES}) get_filename_component(EXECUTABLE_NAME ${SOURCE} NAME_WE) @@ -211,7 +227,7 @@ endif() # --------------------------------------------------------------- message(STATUS "--------- Final Configuration Summary ---------") message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") -message(STATUS "GPU Enabled: ${FAISS_ENABLE_GPU}") +message(STATUS "GPU Enabled: ${QUAKE_ENABLE_GPU}") message(STATUS "NUMA Enabled: ${QUAKE_USE_NUMA}") message(STATUS "Python used: ${Python3_EXECUTABLE}") message(STATUS "Torch Path: ${TorchPath}") @@ -221,4 +237,4 @@ message(STATUS "MKL_LINK: ${MKL_LINK}") message(STATUS "MKL_INTERFACE_FULL: ${MKL_INTERFACE_FULL}") message(STATUS "MKL_THREADING: ${MKL_THREADING}") message(STATUS "MKL_MPI: ${MKL_MPI}") -message(STATUS "------------------------------------------------") \ No newline at end of file +message(STATUS "------------------------------------------------") diff --git a/README.md b/README.md index 8bab54fd..e9c75c70 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,14 @@ Quake’s Python API provides a set of core operations. The following sections d --- + +### Devesh Notes +Running Big ANN debug script: +``` +cmake -DCMAKE_PREFIX_PATH=$CONDA_PREFIX -DQUAKE_USE_NUMA=OFF -DQUAKE_USE_ASAN=OFF -DQUAKE_USE_AVX512=ON -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=OFF -DQUAKE_SET_ABI_MODE=ON .. +make big_ann_debug -j$(nproc) && ./big_ann_debug &> batch_query_debug.out +``` + ## Further Documentation For detailed instructions and advanced usage, please see the documentation: http://marius-project.org/quake/ diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 00000000..46e0f217 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,328 @@ +# Quake Architecture Overview + +Quake is a high-performance library for **dynamic approximate nearest neighbor (ANN) search**. It uses a partitioned index structure (IVF-style) with support for real-time updates, adaptive search, and automatic maintenance. + +--- + +## High-Level Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Python API Layer │ +│ (quake module, QuakeWrapper, IndexWrapper) │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ PyBind11 Bindings │ +│ (wrap.cpp - exposes C++ classes to Python via PyTorch tensors) │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ QuakeIndex │ +│ (Central orchestrator: build, search, add, remove, maintenance) │ +├──────────────────┬───────────────────┬──────────────────────────────────────┤ +│ PartitionManager │ QueryCoordinator │ MaintenancePolicy │ +└──────────────────┴───────────────────┴──────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Storage & Computation Layer │ +│ DynamicInvertedLists, IndexPartition, Clustering, List Scanning │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Core Components + +### 1. QuakeIndex (`quake_index.h`) + +The **central class** that orchestrates all index operations. It provides the main API for: + +| Operation | Description | +|-----------|-------------| +| `build()` | Create an index from vectors using k-means clustering | +| `search()` | Execute k-NN queries with configurable nprobe or recall target | +| `add()` | Insert new vectors into appropriate partitions | +| `remove()` | Delete vectors by ID | +| `maintenance()` | Trigger partition splits/merges for optimal performance | +| `save()`/`load()` | Persist and restore the index | + +**Key Members:** +- `parent_`: Optional hierarchical parent index over centroids (multi-level indexing) +- `partition_manager_`: Manages the partitioned data structure +- `query_coordinator_`: Handles search query execution +- `maintenance_policy_`: Decides when and how to restructure partitions + +--- + +### 2. PartitionManager (`partition_manager.h`) + +Manages the **partitioned (IVF) structure** of the index. Responsibilities include: + +- **Partition initialization**: Create partitions from k-means clustering +- **Vector assignment**: Assign vectors to their nearest centroid/partition +- **Dynamic operations**: Add/remove vectors, split/merge/refine partitions +- **NUMA distribution**: Distribute partitions across NUMA nodes for locality + +**Key Methods:** +```cpp +void init_partitions(parent, partitions); // Initialize from clustering +ModifyTimingInfo add(vectors, ids, assignments); // Add vectors to partitions +ModifyTimingInfo remove(ids); // Remove vectors by ID +Clustering split_partitions(partition_ids); // Split overloaded partitions +void delete_partitions(partition_ids); // Merge/delete underutilized partitions +void refine_partitions(partition_ids); // Re-cluster for better quality +``` + +--- + +### 3. QueryCoordinator (`query_coordinator.h`) + +The **query execution engine** that orchestrates search operations. Features: + +- **Multi-threaded scanning**: Parallel partition scanning with worker threads +- **NUMA-aware execution**: Pin workers to cores and use local memory +- **Batched scanning**: Efficient processing of multiple queries together +- **Adaptive search (APS)**: Dynamically determine partitions to scan based on recall target + +**Scan Modes:** +| Mode | Description | +|------|-------------| +| `serial_scan` | Sequential single-threaded scanning | +| `batched_serial_scan` | Batch queries by partition for efficiency | +| `worker_scan` | Parallel scanning with dedicated worker threads | + +**Key Structures:** +- `ScanJob`: Encapsulates a scan task (partition, query, k) +- `CoreResources`: Per-worker buffers and state +- `NUMAResources`: NUMA-local memory and job queues + +--- + +### 4. MaintenancePolicy (`maintenance_policies.h`) + +Implements **cost-based maintenance** to keep the index optimized over time: + +- **Hit tracking**: Monitor which partitions are accessed during queries +- **Split detection**: Identify overloaded partitions that slow down search +- **Merge detection**: Find underutilized partitions that waste resources +- **Local refinement**: Re-cluster nearby partitions for better boundaries + +**Key Components:** +- `HitCountTracker`: Sliding window tracker for partition access patterns +- `MaintenanceCostEstimator`: Estimates scan latency and maintenance costs + +--- + +### 5. DynamicInvertedLists (`dynamic_inverted_list.h`) + +A **dynamic, NUMA-aware inverted list** implementation that extends Faiss's InvertedLists: + +- Stores vectors and IDs in a map of `IndexPartition` objects +- Supports efficient insertion, update, and deletion +- Provides NUMA-aware memory allocation +- Maintains ID-to-location mapping for fast lookups + +--- + +### 6. IndexPartition (`index_partition.h`) + +Represents a **single partition** of the index: + +- Manages contiguous memory blocks for vectors (codes) and IDs +- Supports NUMA-local allocation +- Tracks delta changes for maintenance decisions +- Auto-resizes based on configurable thresholds + +--- + +### 7. Clustering (`clustering.h`) + +Provides **k-means clustering** implementations: + +- `kmeans_cpu()`: CPU-based k-means using Faiss +- `kmeans_cuvs_sample_and_predict()`: GPU-accelerated k-means (optional) +- `kmeans_refine_partitions()`: Iterative refinement of existing partitions + +--- + +### 8. List Scanning (`list_scanning.h`) + +Low-level **distance computation and top-k selection**: + +- SIMD-optimized distance calculations (AVX512 support) +- `TopkBuffer`: Efficient top-k result aggregation +- BLAS-accelerated batch distance computation +- Support for L2 (Euclidean) and IP (inner product) metrics + +--- + +## Python API Layer + +### Module Structure (`src/python/`) + +``` +src/python/ +├── __init__.py # Imports C++ bindings +├── utils.py # Utility functions +├── workload_generator.py # Query workload generation +├── datasets/ # Dataset loaders +└── index_wrappers/ + ├── wrapper.py # Abstract IndexWrapper base class + ├── quake.py # QuakeWrapper implementation + ├── faiss_ivf.py # Faiss IVF wrapper (comparison) + ├── faiss_hnsw.py # Faiss HNSW wrapper (comparison) + └── diskann.py # DiskANN wrapper (comparison) +``` + +### Key Python Classes + +**`QuakeWrapper`** (quake.py): High-level Python interface +```python +wrapper = QuakeWrapper() +wrapper.build(vectors, nc=1024, metric="l2") +ids, dists = wrapper.search(queries, k=10, nprobe=32) +wrapper.add(new_vectors) +wrapper.remove(ids_to_remove) +wrapper.maintenance() +``` + +--- + +## Configuration Parameters + +### IndexBuildParams +| Parameter | Default | Description | +|-----------|---------|-------------| +| `nlist` | 0 | Number of partitions (0 = flat index) | +| `niter` | 5 | K-means iterations | +| `metric` | "l2" | Distance metric ("l2" or "ip") | +| `num_workers` | 0 | Worker threads for search | +| `use_numa` | false | Enable NUMA-aware allocation | +| `use_gpu` | false | Use GPU for clustering | + +### SearchParams +| Parameter | Default | Description | +|-----------|---------|-------------| +| `k` | 1 | Number of neighbors to return | +| `nprobe` | 1 | Partitions to probe | +| `recall_target` | -1 | Adaptive search recall target (0-1) | +| `batched_scan` | false | Enable batched scanning | + +### MaintenancePolicyParams +| Parameter | Default | Description | +|-----------|---------|-------------| +| `window_size` | 1000 | Query window for hit tracking | +| `min_partition_size` | 32 | Minimum partition size | +| `alpha` | 0.9 | Maintenance policy weight | +| `refinement_iterations` | 3 | Local refinement iterations | + +--- + +## Data Flow + +### Build Flow +``` +Vectors + IDs + │ + ▼ +┌─────────────┐ +│ K-Means │ ── Cluster vectors into nlist partitions +└─────────────┘ + │ + ▼ +┌─────────────────────┐ +│ PartitionManager │ ── Initialize DynamicInvertedLists +└─────────────────────┘ + │ + ▼ +┌─────────────────────┐ +│ QueryCoordinator │ ── Initialize workers (if parallel) +└─────────────────────┘ +``` + +### Search Flow +``` +Query Vectors + │ + ▼ +┌─────────────────────┐ +│ Parent Index │ ── Find top-nprobe nearest centroids +│ (if hierarchical) │ +└─────────────────────┘ + │ + ▼ +┌─────────────────────┐ +│ QueryCoordinator │ ── Dispatch scan jobs to workers +└─────────────────────┘ + │ + ├──► Worker 1 ──► Scan Partition A + ├──► Worker 2 ──► Scan Partition B + └──► Worker N ──► Scan Partition C + │ + ▼ +┌─────────────────────┐ +│ Result Aggregation │ ── Merge top-k from all scanned partitions +└─────────────────────┘ + │ + ▼ +SearchResult (ids, distances) +``` + +--- + +## Key Design Decisions + +1. **PyTorch Integration**: Uses torch::Tensor for seamless ML workflow integration +2. **Faiss Foundation**: Builds on Faiss infrastructure (distance functions, heap operations) +3. **Dynamic Updates**: Full support for real-time add/remove without rebuild +4. **Adaptive Search**: Automatically adjusts nprobe to meet recall targets +5. **Cost-Based Maintenance**: Data-driven decisions for partition restructuring +6. **NUMA Awareness**: Optimized for multi-socket systems with local memory access +7. **Multi-Level Indexing**: Optional hierarchical index for very large datasets + +--- + +## File Organization + +``` +src/ +├── cpp/ +│ ├── include/ # Header files +│ │ ├── quake_index.h # Main index class +│ │ ├── partition_manager.h +│ │ ├── query_coordinator.h +│ │ ├── maintenance_policies.h +│ │ ├── dynamic_inverted_list.h +│ │ ├── index_partition.h +│ │ ├── clustering.h +│ │ ├── list_scanning.h +│ │ ├── common.h # Types, constants, utilities +│ │ └── ... +│ ├── src/ # Implementation files +│ │ ├── quake_index.cpp +│ │ ├── partition_manager.cpp +│ │ ├── query_coordinator.cpp +│ │ └── ... +│ ├── bindings/ +│ │ └── wrap.cpp # PyBind11 bindings +│ └── third_party/ # External dependencies +└── python/ + ├── __init__.py + ├── index_wrappers/ + └── ... +``` + +--- + +## Dependencies + +- **PyTorch**: Tensor operations and Python bindings +- **Faiss**: Distance computation, heap operations, IVF infrastructure +- **OpenBLAS**: BLAS operations for batch distance computation +- **pybind11**: C++/Python bindings +- **Optional: CuVS/CUDA**: GPU-accelerated clustering +- **Optional: libnuma**: NUMA-aware memory allocation diff --git a/environments/ubuntu-cuda/Dockerfile b/environments/ubuntu-cuda118/Dockerfile similarity index 100% rename from environments/ubuntu-cuda/Dockerfile rename to environments/ubuntu-cuda118/Dockerfile diff --git a/environments/ubuntu-cuda/conda.yaml b/environments/ubuntu-cuda118/conda.yaml similarity index 82% rename from environments/ubuntu-cuda/conda.yaml rename to environments/ubuntu-cuda118/conda.yaml index 715744f2..99265b48 100644 --- a/environments/ubuntu-cuda/conda.yaml +++ b/environments/ubuntu-cuda118/conda.yaml @@ -3,6 +3,8 @@ channels: - pytorch - defaults - conda-forge + - nvidia + - libcuvs dependencies: - python=3.11 - numpy @@ -10,6 +12,8 @@ dependencies: - faiss-gpu - matplotlib - pytest + - libcuvs + - cuda-version=12.8 - pip - pip: - sphinx diff --git a/environments/ubuntu-cuda124/Dockerfile b/environments/ubuntu-cuda124/Dockerfile new file mode 100644 index 00000000..0462c5c9 --- /dev/null +++ b/environments/ubuntu-cuda124/Dockerfile @@ -0,0 +1,64 @@ +# Use a CUDA-enabled Ubuntu base image +FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04 + +# ----------------------------- +# Set up environment variables +# ----------------------------- +ENV CONDA_DIR=/opt/miniconda +ENV PATH="${CONDA_DIR}/bin:${PATH}" +ENV DEBIAN_FRONTEND=noninteractive + +# ----------------------------- +# Install system dependencies +# ----------------------------- +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + wget \ + curl \ + build-essential \ + ca-certificates \ + swig \ + git \ + libomp5 \ + libomp-dev \ + graphviz \ + libnuma-dev \ + && rm -rf /var/lib/apt/lists/* + + +# Install CMake 3.24.2 +RUN wget -qO /tmp/cmake.sh https://github.com/Kitware/CMake/releases/download/v3.30.4/cmake-3.30.4-linux-x86_64.sh && \ + chmod +x /tmp/cmake.sh && \ + /tmp/cmake.sh --skip-license --prefix=/usr/local && \ + rm /tmp/cmake.sh + + +# ----------------------------- +# Install Miniconda +# ----------------------------- +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/miniconda.sh && \ + bash /tmp/miniconda.sh -b -p $CONDA_DIR && \ + rm /tmp/miniconda.sh + +# ----------------------------- +# Copy in your conda environment YAML +# ----------------------------- +COPY environments/ubuntu-cuda/conda.yaml /tmp/conda.yaml + +# Create quake-env +RUN conda env create -f /tmp/conda.yaml && conda clean -afy + +# ----------------------------- +# Install GPU-enabled PyTorch +# ----------------------------- +RUN conda run -n quake-env pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cu124 + +# ----------------------------- +# Debug: show conda information +# ----------------------------- +RUN echo "===== DEBUG: which conda =====" && which conda +RUN echo "===== DEBUG: conda info =====" && conda info +RUN echo "===== DEBUG: conda env list =====" && conda env list +RUN echo "===== DEBUG: quake-env check =====" && conda run -n quake-env python -c "import sys; print('OK in quake-env; python:', sys.executable)" + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/environments/ubuntu-cuda124/conda.yaml b/environments/ubuntu-cuda124/conda.yaml new file mode 100644 index 00000000..f895341b --- /dev/null +++ b/environments/ubuntu-cuda124/conda.yaml @@ -0,0 +1,24 @@ +name: quake-env +channels: + - rapidsai + - pytorch + - defaults + - conda-forge + - nvidia +dependencies: + - python=3.11 + - numpy + - pandas + - faiss-gpu + - matplotlib + - pytest + - libcuvs + - cuda-nvrtc-dev=12.4 + - cuda-version=12.4 + - pip + - pip: + - sphinx + - sphinx_rtd_theme + - sphinxcontrib-mermaid + - graphviz + - pyyaml \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..275bd883 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +--extra-index-url https://download.pytorch.org/whl/cpu +ansicolors==1.1.8 +docker==7.1.0 +h5py==3.10.0 +matplotlib==3.3.4 +numpy==1.24.2 +pyyaml==6.0 +psutil==5.9.4 +scipy==1.10.1 +scikit-learn +jinja2==3.1.2 +pandas==2.0.0 +faiss-cpu +torch diff --git a/scripts/big_ann_perf_numbers/crash_debug_scan_0.12_no_aps_refinment_wma_delete.csv b/scripts/big_ann_perf_numbers/crash_debug_scan_0.12_no_aps_refinment_wma_delete.csv new file mode 100644 index 00000000..bd346572 --- /dev/null +++ b/scripts/big_ann_perf_numbers/crash_debug_scan_0.12_no_aps_refinment_wma_delete.csv @@ -0,0 +1,11 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,2720.28,-1.0,-1.0,-1.0,-1.0,8.02,0.0184881,1000,38806 +3,insert,23.314,-1.0,-1.0,-1.0,-1.0,10.395,0.0226489,997,49334 +4,search,2056.05,-1.0,-1.0,-1.0,-1.0,9.641,0.0226571,995,49334 +5,insert,15.686,-1.0,-1.0,-1.0,-1.0,163.701,0.033261,985,73557 +6,search,1980.68,-1.0,-1.0,-1.0,-1.0,109.376,0.0333809,988,73557 +7,insert,22.491,-1.0,-1.0,-1.0,-1.0,105.84,0.043066,988,95766 +8,search,1991.55,-1.0,-1.0,-1.0,-1.0,97.167,0.0432598,992,95766 +9,insert,5.541,-1.0,-1.0,-1.0,-1.0,76.324,0.0449698,992,100025 +10,search,2061.73,-1.0,-1.0,-1.0,-1.0,81.606,0.0449918,994,100025 +11,insert,9.081,-1.0,-1.0,-1.0,-1.0,15.419,0.0501926,996,111417 diff --git a/scripts/big_ann_perf_numbers/mem_check_scan_0.1_no_aps_refinment_wma_delete.csv b/scripts/big_ann_perf_numbers/mem_check_scan_0.1_no_aps_refinment_wma_delete.csv new file mode 100644 index 00000000..a3c90368 --- /dev/null +++ b/scripts/big_ann_perf_numbers/mem_check_scan_0.1_no_aps_refinment_wma_delete.csv @@ -0,0 +1,1280 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,2336.83,-1.0,-1.0,-1.0,-1.0,4.389,0.0184873,1024,38806 +3,insert,9.285,-1.0,-1.0,-1.0,-1.0,10.822,0.0226399,1020,49334 +4,search,1637,-1.0,-1.0,-1.0,-1.0,10.937,0.0226509,1017,49334 +5,insert,14.965,-1.0,-1.0,-1.0,-1.0,157.02,0.0331879,1008,73557 +6,search,1700.63,-1.0,-1.0,-1.0,-1.0,131.037,0.0332577,1010,73557 +7,insert,32.652,-1.0,-1.0,-1.0,-1.0,128.443,0.0428645,1008,95766 +8,search,1722.13,-1.0,-1.0,-1.0,-1.0,96.171,0.0429987,1011,95766 +9,insert,3.494,-1.0,-1.0,-1.0,-1.0,89.102,0.0447152,1011,100025 +10,search,1713.11,-1.0,-1.0,-1.0,-1.0,76.216,0.044778,1013,100025 +11,insert,8.56,-1.0,-1.0,-1.0,-1.0,78.126,0.0498058,1016,111417 +12,search,1740.21,-1.0,-1.0,-1.0,-1.0,15.362,0.050202,1018,111417 +13,insert,18.508,-1.0,-1.0,-1.0,-1.0,8.029,0.0613016,1020,137139 +14,search,1737.66,-1.0,-1.0,-1.0,-1.0,13.383,0.0613061,1022,137139 +15,insert,14.621,-1.0,-1.0,-1.0,-1.0,6.594,0.0696305,1023,156710 +16,search,1765.62,-1.0,-1.0,-1.0,-1.0,15.674,0.0696334,1027,156710 +17,insert,3.996,-1.0,-1.0,-1.0,-1.0,9.588,0.0718712,1030,161773 +18,search,1764.61,-1.0,-1.0,-1.0,-1.0,10.064,0.0719439,1033,161773 +19,insert,31.771,-1.0,-1.0,-1.0,-1.0,12.815,0.0789843,1038,177161 +20,search,1655.41,-1.0,-1.0,-1.0,-1.0,3.087,0.0789843,1038,177161 +21,insert,177.536,-1.0,-1.0,-1.0,-1.0,92.183,0.200641,1084,448898 +22,search,1419.76,-1.0,-1.0,-1.0,-1.0,24.424,0.200676,1095,448898 +23,insert,188.153,-1.0,-1.0,-1.0,-1.0,140.515,0.327594,1146,733493 +24,search,1599.31,-1.0,-1.0,-1.0,-1.0,46.969,0.328483,1164,733493 +25,insert,24.87,-1.0,-1.0,-1.0,-1.0,10.945,0.345922,1168,777443 +26,search,1625.68,-1.0,-1.0,-1.0,-1.0,5.67,0.345925,1169,777443 +27,insert,14.107,-1.0,-1.0,-1.0,-1.0,7.847,0.356689,1170,805394 +28,search,1646.91,-1.0,-1.0,-1.0,-1.0,7.375,0.356776,1171,805394 +29,insert,30.787,-1.0,-1.0,-1.0,-1.0,32.308,0.383981,1186,866391 +30,search,1664.1,-1.0,-1.0,-1.0,-1.0,6.252,0.383981,1187,866391 +31,insert,254.78,-1.0,-1.0,-1.0,-1.0,203.687,0.553265,1248,1239816 +32,search,1857.4,-1.0,-1.0,-1.0,-1.0,40.651,0.553599,1257,1239816 +33,insert,24.712,-1.0,-1.0,-1.0,-1.0,15.181,0.573925,1263,1291211 +34,search,1884.75,-1.0,-1.0,-1.0,-1.0,27.534,0.5741,1268,1291211 +35,insert,16.861,-1.0,-1.0,-1.0,-1.0,15.085,0.587674,1274,1324978 +36,search,1902.28,-1.0,-1.0,-1.0,-1.0,4.367,0.587674,1274,1324978 +37,insert,21.659,-1.0,-1.0,-1.0,-1.0,14.925,0.606952,1278,1371885 +38,search,1926.21,-1.0,-1.0,-1.0,-1.0,7.017,0.606952,1279,1371885 +39,insert,4.268,-1.0,-1.0,-1.0,-1.0,3.926,0.610335,1279,1379335 +40,search,1925.94,-1.0,-1.0,-1.0,-1.0,6.523,0.610336,1280,1379335 +41,insert,17.099,-1.0,-1.0,-1.0,-1.0,8.173,0.62354,1281,1413312 +42,search,1948.95,-1.0,-1.0,-1.0,-1.0,12.054,0.623643,1282,1413312 +43,insert,6.736,-1.0,-1.0,-1.0,-1.0,8.3,0.626657,1283,1423411 +44,search,1960.35,-1.0,-1.0,-1.0,-1.0,7.253,0.626711,1284,1423411 +45,insert,116.659,-1.0,-1.0,-1.0,-1.0,21.016,0.643099,1293,1457559 +46,search,1964.04,-1.0,-1.0,-1.0,-1.0,4.698,0.643099,1293,1457559 +47,insert,159.067,-1.0,-1.0,-1.0,-1.0,151.751,0.748997,1328,1688593 +48,search,2079.41,-1.0,-1.0,-1.0,-1.0,28.83,0.748996,1332,1688593 +49,insert,18.701,-1.0,-1.0,-1.0,-1.0,16.118,0.762754,1338,1723505 +50,search,2097.02,-1.0,-1.0,-1.0,-1.0,16.35,0.762773,1340,1723505 +51,insert,197.078,-1.0,-1.0,-1.0,-1.0,160.951,0.902422,1386,2035040 +52,search,2266.09,-1.0,-1.0,-1.0,-1.0,18.447,0.902436,1389,2035040 +53,insert,28.659,-1.0,-1.0,-1.0,-1.0,31.398,0.921724,1397,2078341 +54,search,2289.3,-1.0,-1.0,-1.0,-1.0,4.352,0.921724,1397,2078341 +55,insert,206.196,-1.0,-1.0,-1.0,-1.0,280.749,1.07006,1454,2399905 +56,search,2466.19,-1.0,-1.0,-1.0,-1.0,47.658,1.0704,1467,2399905 +57,insert,11.309,-1.0,-1.0,-1.0,-1.0,8.377,1.0775,1468,2419836 +58,search,2443.28,-1.0,-1.0,-1.0,-1.0,31.209,1.07752,1473,2419836 +59,insert,127.64,-1.0,-1.0,-1.0,-1.0,78.809,1.17641,1485,2643083 +60,search,2499.14,-1.0,-1.0,-1.0,-1.0,15.664,1.17652,1488,2643083 +61,insert,5.642,-1.0,-1.0,-1.0,-1.0,7.878,1.18039,1489,2653064 +62,search,2497.54,-1.0,-1.0,-1.0,-1.0,8.4,1.18059,1490,2653064 +63,insert,2.265,-1.0,-1.0,-1.0,-1.0,4.725,1.18237,1490,2656118 +64,search,2508.82,-1.0,-1.0,-1.0,-1.0,4.816,1.18237,1490,2656118 +65,insert,39.859,-1.0,-1.0,-1.0,-1.0,55.271,1.21281,1509,2723424 +66,search,2551.77,-1.0,-1.0,-1.0,-1.0,4.928,1.21281,1509,2723424 +67,insert,22.055,-1.0,-1.0,-1.0,-1.0,43.598,1.233,1525,2769021 +68,search,2590.34,-1.0,-1.0,-1.0,-1.0,4.987,1.233,1525,2769021 +69,insert,26.783,-1.0,-1.0,-1.0,-1.0,41.015,1.25648,1537,2821777 +70,search,2602.64,-1.0,-1.0,-1.0,-1.0,7.279,1.25648,1538,2821777 +71,insert,8.187,-1.0,-1.0,-1.0,-1.0,10.274,1.26265,1540,2834921 +72,search,2624.82,-1.0,-1.0,-1.0,-1.0,5.235,1.26265,1540,2834921 +73,insert,4.355,-1.0,-1.0,-1.0,-1.0,12.37,1.26619,1542,2842044 +74,search,2611.77,-1.0,-1.0,-1.0,-1.0,5.005,1.26619,1542,2842044 +75,insert,8.118,-1.0,-1.0,-1.0,-1.0,14.921,1.27087,1546,2858890 +76,search,2635.21,-1.0,-1.0,-1.0,-1.0,6.397,1.27087,1546,2858890 +77,insert,23.39,-1.0,-1.0,-1.0,-1.0,10.204,1.27943,1547,2882818 +78,search,2624.73,-1.0,-1.0,-1.0,-1.0,5.326,1.27943,1547,2882818 +79,insert,284.164,-1.0,-1.0,-1.0,-1.0,11.058,1.30757,1549,2947806 +80,search,2653.05,-1.0,-1.0,-1.0,-1.0,5.61,1.30757,1549,2947806 +81,insert,20.671,-1.0,-1.0,-1.0,-1.0,19.613,1.3184,1552,2976895 +82,search,2666.54,-1.0,-1.0,-1.0,-1.0,10.052,1.31871,1553,2976895 +83,insert,7.349,-1.0,-1.0,-1.0,-1.0,13.113,1.32463,1555,2990682 +84,search,2674.31,-1.0,-1.0,-1.0,-1.0,5.581,1.32463,1555,2990682 +85,insert,23.783,-1.0,-1.0,-1.0,-1.0,18.624,1.33857,1559,3025613 +86,search,2681.47,-1.0,-1.0,-1.0,-1.0,5.118,1.33857,1559,3025613 +87,insert,307.676,-1.0,-1.0,-1.0,-1.0,371.562,1.54092,1640,3468632 +88,search,2909.62,-1.0,-1.0,-1.0,-1.0,64.707,1.54109,1653,3468632 +89,insert,233.126,-1.0,-1.0,-1.0,-1.0,345.566,1.70727,1738,3831692 +90,search,3129.83,-1.0,-1.0,-1.0,-1.0,37.655,1.70737,1744,3831692 +91,insert,19.274,-1.0,-1.0,-1.0,-1.0,24.634,1.71781,1749,3864067 +92,search,3132.94,-1.0,-1.0,-1.0,-1.0,7.794,1.71781,1749,3864067 +93,insert,126.897,-1.0,-1.0,-1.0,-1.0,145.668,1.82186,1786,4103827 +94,search,3242.27,-1.0,-1.0,-1.0,-1.0,22.863,1.82233,1789,4103827 +95,insert,8.147,-1.0,-1.0,-1.0,-1.0,25.319,1.8285,1795,4119898 +96,search,3251.4,-1.0,-1.0,-1.0,-1.0,8.884,1.8285,1795,4119898 +97,insert,5.289,-1.0,-1.0,-1.0,-1.0,8.189,1.83174,1795,4129367 +98,search,3255.35,-1.0,-1.0,-1.0,-1.0,8.831,1.83174,1795,4129367 +99,insert,3.001,-1.0,-1.0,-1.0,-1.0,12.28,1.83296,1796,4134502 +100,search,3254.18,-1.0,-1.0,-1.0,-1.0,8.691,1.83296,1796,4134502 +101,insert,17.69,-1.0,-1.0,-1.0,-1.0,35.67,1.84434,1805,4164482 +102,search,3292.14,-1.0,-1.0,-1.0,-1.0,9.312,1.84434,1805,4164482 +103,insert,8.039,-1.0,-1.0,-1.0,-1.0,16.545,1.84877,1807,4178601 +104,search,3286.57,-1.0,-1.0,-1.0,-1.0,9.06,1.84877,1807,4178601 +105,insert,5.41,-1.0,-1.0,-1.0,-1.0,11.712,1.85231,1808,4187774 +106,search,3286,-1.0,-1.0,-1.0,-1.0,9.157,1.85231,1808,4187774 +107,insert,9.627,-1.0,-1.0,-1.0,-1.0,8.674,1.85729,1808,4200109 +108,search,3284.27,-1.0,-1.0,-1.0,-1.0,9.146,1.85729,1808,4200109 +109,insert,231.728,-1.0,-1.0,-1.0,-1.0,299.747,2.01518,1871,4557867 +110,search,3465.45,-1.0,-1.0,-1.0,-1.0,33.959,2.0158,1878,4557867 +111,insert,16.308,-1.0,-1.0,-1.0,-1.0,38.987,2.02997,1887,4587252 +112,search,3471.71,-1.0,-1.0,-1.0,-1.0,11.009,2.02997,1887,4587252 +113,insert,6.412,-1.0,-1.0,-1.0,-1.0,10.56,2.03391,1887,4600056 +114,search,3481.72,-1.0,-1.0,-1.0,-1.0,14.974,2.03393,1888,4600056 +115,insert,17.79,-1.0,-1.0,-1.0,-1.0,28.766,2.04481,1893,4627422 +116,search,3501.25,-1.0,-1.0,-1.0,-1.0,17.313,2.04494,1895,4627422 +117,insert,7.063,-1.0,-1.0,-1.0,-1.0,19.103,2.04946,1897,4639292 +118,search,3509.01,-1.0,-1.0,-1.0,-1.0,11.485,2.04946,1897,4639292 +119,insert,2.2,-1.0,-1.0,-1.0,-1.0,10.771,2.05061,1897,4643423 +120,search,3511.29,-1.0,-1.0,-1.0,-1.0,11.462,2.05061,1897,4643423 +121,insert,9.354,-1.0,-1.0,-1.0,-1.0,23.315,2.05856,1900,4657252 +122,search,3524.37,-1.0,-1.0,-1.0,-1.0,14.789,2.05866,1901,4657252 +123,insert,211.7,-1.0,-1.0,-1.0,-1.0,260.876,2.18616,1948,4938477 +124,search,3639.87,-1.0,-1.0,-1.0,-1.0,89.885,2.18651,1959,4938477 +125,insert,29.311,-1.0,-1.0,-1.0,-1.0,37.908,2.205,1965,4980492 +126,search,3672.94,-1.0,-1.0,-1.0,-1.0,37.078,2.20509,1969,4980492 +127,insert,29.517,-1.0,-1.0,-1.0,-1.0,72.982,2.22834,1987,5030515 +128,search,3711.16,-1.0,-1.0,-1.0,-1.0,17.716,2.22838,1988,5030515 +129,delete,10.997,-1.0,-1.0,-1.0,-1.0,14.344,2.22779,1983,5009391 +130,search,3658.64,-1.0,-1.0,-1.0,-1.0,13.195,2.22779,1983,5009391 +131,delete,2.832,-1.0,-1.0,-1.0,-1.0,12.428,2.22779,1983,5004093 +132,search,3690.44,-1.0,-1.0,-1.0,-1.0,14.309,2.22779,1983,5004093 +133,delete,12.406,-1.0,-1.0,-1.0,-1.0,12.843,2.22776,1983,4991787 +134,search,3688.53,-1.0,-1.0,-1.0,-1.0,13.477,2.22776,1983,4991787 +135,delete,14.162,-1.0,-1.0,-1.0,-1.0,12.55,2.22676,1983,4974857 +136,search,3668.84,-1.0,-1.0,-1.0,-1.0,13.185,2.22676,1983,4974857 +137,delete,2.514,-1.0,-1.0,-1.0,-1.0,12.444,2.22676,1983,4971386 +138,search,3701.45,-1.0,-1.0,-1.0,-1.0,13.545,2.22676,1983,4971386 +139,delete,6.679,-1.0,-1.0,-1.0,-1.0,12.602,2.22676,1983,4963487 +140,search,3682.42,-1.0,-1.0,-1.0,-1.0,13.199,2.22676,1983,4963487 +141,delete,14.726,-1.0,-1.0,-1.0,-1.0,12.61,2.22518,1983,4941830 +142,search,3671.46,-1.0,-1.0,-1.0,-1.0,13.658,2.22518,1983,4941830 +143,delete,12.79,-1.0,-1.0,-1.0,-1.0,12.536,2.22321,1983,4925657 +144,search,3671.03,-1.0,-1.0,-1.0,-1.0,13.155,2.22321,1983,4925657 +145,delete,1.994,-1.0,-1.0,-1.0,-1.0,12.43,2.22304,1983,4922686 +146,search,3661.02,-1.0,-1.0,-1.0,-1.0,13.537,2.22304,1983,4922686 +147,delete,7.806,-1.0,-1.0,-1.0,-1.0,12.451,2.22304,1983,4913636 +148,search,3651.65,-1.0,-1.0,-1.0,-1.0,13.253,2.22304,1983,4913636 +149,delete,119.703,-1.0,-1.0,-1.0,-1.0,58.942,2.19397,1973,4717250 +150,search,3561.59,-1.0,-1.0,-1.0,-1.0,13.811,2.19397,1973,4717250 +151,delete,108.13,-1.0,-1.0,-1.0,-1.0,74.478,2.15347,1961,4560269 +152,search,3483.17,-1.0,-1.0,-1.0,-1.0,13.397,2.15347,1961,4560269 +153,delete,20.16,-1.0,-1.0,-1.0,-1.0,30.956,2.14159,1956,4521936 +154,search,3449.4,-1.0,-1.0,-1.0,-1.0,13.849,2.14159,1956,4521936 +155,delete,14.384,-1.0,-1.0,-1.0,-1.0,12.002,2.13912,1956,4502222 +156,search,3471.17,-1.0,-1.0,-1.0,-1.0,13.369,2.13912,1956,4502222 +157,delete,24.212,-1.0,-1.0,-1.0,-1.0,13.332,2.13268,1956,4456039 +158,search,3432.98,-1.0,-1.0,-1.0,-1.0,13.819,2.13268,1956,4456039 +159,delete,205.195,-1.0,-1.0,-1.0,-1.0,253.224,1.99612,1900,4143922 +160,search,3247.23,-1.0,-1.0,-1.0,-1.0,18.595,1.99632,1901,4143922 +161,delete,26.869,-1.0,-1.0,-1.0,-1.0,13.28,1.9913,1901,4101445 +162,search,3264.23,-1.0,-1.0,-1.0,-1.0,13.408,1.9913,1901,4101445 +163,delete,13.031,-1.0,-1.0,-1.0,-1.0,12.135,1.98712,1901,4080175 +164,search,3260.52,-1.0,-1.0,-1.0,-1.0,13.028,1.98712,1901,4080175 +165,delete,21.187,-1.0,-1.0,-1.0,-1.0,12.607,1.9794,1901,4047313 +166,search,3237.34,-1.0,-1.0,-1.0,-1.0,13.487,1.9794,1901,4047313 +167,delete,6.217,-1.0,-1.0,-1.0,-1.0,11.891,1.97732,1901,4040738 +168,search,3221.98,-1.0,-1.0,-1.0,-1.0,13.288,1.97732,1901,4040738 +169,delete,14.115,-1.0,-1.0,-1.0,-1.0,20.042,1.97017,1900,4022674 +170,search,3225.94,-1.0,-1.0,-1.0,-1.0,13.511,1.97017,1900,4022674 +171,delete,5.851,-1.0,-1.0,-1.0,-1.0,11.892,1.96758,1900,4015835 +172,search,3225.49,-1.0,-1.0,-1.0,-1.0,13.099,1.96758,1900,4015835 +173,delete,22.926,-1.0,-1.0,-1.0,-1.0,12.652,1.95975,1900,3986424 +174,search,3212.01,-1.0,-1.0,-1.0,-1.0,13.387,1.95975,1900,3986424 +175,delete,121.219,-1.0,-1.0,-1.0,-1.0,219.591,1.87426,1854,3804693 +176,search,3115.83,-1.0,-1.0,-1.0,-1.0,19.467,1.87445,1855,3804693 +177,delete,11.841,-1.0,-1.0,-1.0,-1.0,12.197,1.86932,1855,3786352 +178,search,3071.46,-1.0,-1.0,-1.0,-1.0,12.963,1.86932,1855,3786352 +179,delete,153.885,-1.0,-1.0,-1.0,-1.0,165.157,1.756,1799,3556994 +180,search,2960.84,-1.0,-1.0,-1.0,-1.0,12.413,1.756,1799,3556994 +181,delete,19.135,-1.0,-1.0,-1.0,-1.0,12.099,1.74619,1798,3526824 +182,search,2949.73,-1.0,-1.0,-1.0,-1.0,12.3,1.74619,1798,3526824 +183,delete,109.908,-1.0,-1.0,-1.0,-1.0,238.963,1.65983,1743,3350245 +184,search,2850.24,-1.0,-1.0,-1.0,-1.0,16.976,1.66003,1744,3350245 +185,delete,6.328,-1.0,-1.0,-1.0,-1.0,12.297,1.65808,1742,3340234 +186,search,2837.1,-1.0,-1.0,-1.0,-1.0,11.986,1.65808,1742,3340234 +187,delete,115.256,-1.0,-1.0,-1.0,-1.0,152.169,1.55932,1709,3150857 +188,search,2732.78,-1.0,-1.0,-1.0,-1.0,18.017,1.5594,1710,3150857 +189,delete,5.078,-1.0,-1.0,-1.0,-1.0,10.656,1.55758,1710,3143609 +190,search,2771.77,-1.0,-1.0,-1.0,-1.0,11.491,1.55758,1710,3143609 +191,delete,1.927,-1.0,-1.0,-1.0,-1.0,10.434,1.55701,1710,3141038 +192,search,2753.56,-1.0,-1.0,-1.0,-1.0,11.622,1.55701,1710,3141038 +193,delete,46.636,-1.0,-1.0,-1.0,-1.0,11.848,1.53278,1710,3081734 +194,search,2713.76,-1.0,-1.0,-1.0,-1.0,11.54,1.53278,1710,3081734 +195,delete,24.76,-1.0,-1.0,-1.0,-1.0,20.799,1.51748,1708,3043966 +196,search,2703.21,-1.0,-1.0,-1.0,-1.0,11.547,1.51748,1708,3043966 +197,delete,24.537,-1.0,-1.0,-1.0,-1.0,27.174,1.50337,1707,3008145 +198,search,2700.51,-1.0,-1.0,-1.0,-1.0,11.546,1.50337,1707,3008145 +199,delete,7.136,-1.0,-1.0,-1.0,-1.0,10.512,1.50019,1707,2999384 +200,search,2696.2,-1.0,-1.0,-1.0,-1.0,11.56,1.50019,1707,2999384 +201,delete,4.344,-1.0,-1.0,-1.0,-1.0,10.422,1.49727,1707,2993528 +202,search,2694.09,-1.0,-1.0,-1.0,-1.0,12.596,1.49727,1707,2993528 +203,delete,16.918,-1.0,-1.0,-1.0,-1.0,10.622,1.49322,1707,2980973 +204,search,2686.89,-1.0,-1.0,-1.0,-1.0,11.379,1.49322,1707,2980973 +205,delete,20.176,-1.0,-1.0,-1.0,-1.0,10.86,1.48257,1707,2960291 +206,search,2678.78,-1.0,-1.0,-1.0,-1.0,11.593,1.48257,1707,2960291 +207,delete,31.28,-1.0,-1.0,-1.0,-1.0,20.921,1.46165,1699,2920620 +208,search,2667.72,-1.0,-1.0,-1.0,-1.0,12.457,1.46165,1699,2920620 +209,delete,21.298,-1.0,-1.0,-1.0,-1.0,17.96,1.45286,1698,2900501 +210,search,2659.79,-1.0,-1.0,-1.0,-1.0,11.38,1.45286,1698,2900501 +211,delete,6.549,-1.0,-1.0,-1.0,-1.0,10.32,1.45001,1698,2891660 +212,search,2664.06,-1.0,-1.0,-1.0,-1.0,11.807,1.45001,1698,2891660 +213,delete,15.984,-1.0,-1.0,-1.0,-1.0,12.477,1.44168,1697,2873257 +214,search,2645.64,-1.0,-1.0,-1.0,-1.0,11.397,1.44168,1697,2873257 +215,delete,234.928,-1.0,-1.0,-1.0,-1.0,253.923,1.2605,1633,2537571 +216,search,2469.56,-1.0,-1.0,-1.0,-1.0,17.888,1.26073,1634,2537571 +217,delete,163.867,-1.0,-1.0,-1.0,-1.0,170.108,1.15096,1591,2308194 +218,search,2354.55,-1.0,-1.0,-1.0,-1.0,15.327,1.15103,1592,2308194 +219,delete,14.836,-1.0,-1.0,-1.0,-1.0,12.583,1.1441,1591,2285820 +220,search,2314.7,-1.0,-1.0,-1.0,-1.0,10.651,1.1441,1591,2285820 +221,delete,137.505,-1.0,-1.0,-1.0,-1.0,153.435,1.03423,1524,2082858 +222,search,2216.86,-1.0,-1.0,-1.0,-1.0,10.276,1.03423,1524,2082858 +223,delete,8.352,-1.0,-1.0,-1.0,-1.0,10.323,1.03088,1523,2071207 +224,search,2198.68,-1.0,-1.0,-1.0,-1.0,9.846,1.03088,1523,2071207 +225,delete,4.547,-1.0,-1.0,-1.0,-1.0,8.801,1.02931,1523,2065400 +226,search,2197.41,-1.0,-1.0,-1.0,-1.0,10.317,1.02931,1523,2065400 +227,delete,3.716,-1.0,-1.0,-1.0,-1.0,8.83,1.02778,1523,2062105 +228,search,2204.91,-1.0,-1.0,-1.0,-1.0,10.003,1.02778,1523,2062105 +229,delete,16.042,-1.0,-1.0,-1.0,-1.0,13.738,1.02037,1521,2039610 +230,search,2188.07,-1.0,-1.0,-1.0,-1.0,10.094,1.02037,1521,2039610 +231,delete,9.052,-1.0,-1.0,-1.0,-1.0,13.067,1.01385,1520,2027795 +232,search,2175.88,-1.0,-1.0,-1.0,-1.0,10.181,1.01385,1520,2027795 +233,delete,5.711,-1.0,-1.0,-1.0,-1.0,8.804,1.01131,1520,2022859 +234,search,2202.35,-1.0,-1.0,-1.0,-1.0,9.933,1.01131,1520,2022859 +235,delete,6.29,-1.0,-1.0,-1.0,-1.0,10.351,1.00832,1517,2016264 +236,search,2176.95,-1.0,-1.0,-1.0,-1.0,10.041,1.00832,1517,2016264 +237,delete,169.665,-1.0,-1.0,-1.0,-1.0,175.039,0.885904,1464,1782866 +238,search,2063.08,-1.0,-1.0,-1.0,-1.0,9.655,0.885904,1464,1782866 +239,delete,15.913,-1.0,-1.0,-1.0,-1.0,19.137,0.875419,1460,1758931 +240,search,2068.19,-1.0,-1.0,-1.0,-1.0,9.27,0.875419,1460,1758931 +241,delete,5.178,-1.0,-1.0,-1.0,-1.0,13.209,0.872546,1459,1751275 +242,search,2042.53,-1.0,-1.0,-1.0,-1.0,9.336,0.872546,1459,1751275 +243,delete,11.615,-1.0,-1.0,-1.0,-1.0,10.146,0.867792,1458,1733264 +244,search,2040.7,-1.0,-1.0,-1.0,-1.0,9.356,0.867792,1458,1733264 +245,delete,7.609,-1.0,-1.0,-1.0,-1.0,8.068,0.864776,1458,1722658 +246,search,2033.25,-1.0,-1.0,-1.0,-1.0,9.262,0.864776,1458,1722658 +247,delete,2.571,-1.0,-1.0,-1.0,-1.0,8.421,0.864104,1458,1719601 +248,search,2025.81,-1.0,-1.0,-1.0,-1.0,9.3,0.864104,1458,1719601 +249,delete,7.747,-1.0,-1.0,-1.0,-1.0,8.285,0.858977,1458,1711648 +250,search,2030.27,-1.0,-1.0,-1.0,-1.0,9.239,0.858977,1458,1711648 +251,delete,175.394,-1.0,-1.0,-1.0,-1.0,208.529,0.722563,1362,1470624 +252,search,1882.06,-1.0,-1.0,-1.0,-1.0,8.381,0.722563,1362,1470624 +253,delete,19.23,-1.0,-1.0,-1.0,-1.0,13.577,0.711876,1359,1442070 +254,search,1875.11,-1.0,-1.0,-1.0,-1.0,8.451,0.711876,1359,1442070 +255,delete,25.534,-1.0,-1.0,-1.0,-1.0,19.08,0.696489,1352,1405549 +256,search,1851.51,-1.0,-1.0,-1.0,-1.0,8.404,0.696489,1352,1405549 +257,insert,6.172,-1.0,-1.0,-1.0,-1.0,7.262,0.698638,1352,1416288 +258,search,1873.97,-1.0,-1.0,-1.0,-1.0,8.181,0.698638,1352,1416288 +259,insert,16.71,-1.0,-1.0,-1.0,-1.0,8.103,0.705291,1352,1443010 +260,search,1861.02,-1.0,-1.0,-1.0,-1.0,8.381,0.705291,1352,1443010 +261,insert,1.529,-1.0,-1.0,-1.0,-1.0,7.15,0.705618,1352,1447529 +262,search,1869.42,-1.0,-1.0,-1.0,-1.0,8.398,0.705618,1352,1447529 +263,insert,77.642,-1.0,-1.0,-1.0,-1.0,55.048,0.745655,1355,1569110 +264,search,1956.71,-1.0,-1.0,-1.0,-1.0,46.841,0.745655,1358,1569110 +265,insert,37.059,-1.0,-1.0,-1.0,-1.0,8.516,0.765288,1358,1636119 +266,search,1931.49,-1.0,-1.0,-1.0,-1.0,40.358,0.765296,1362,1636119 +267,insert,16.67,-1.0,-1.0,-1.0,-1.0,15.429,0.775432,1363,1665152 +268,search,1949.46,-1.0,-1.0,-1.0,-1.0,20.124,0.775433,1365,1665152 +269,insert,162.82,-1.0,-1.0,-1.0,-1.0,80.77,0.874826,1373,1916952 +270,search,2026.95,-1.0,-1.0,-1.0,-1.0,46.976,0.874825,1378,1916952 +271,insert,2.366,-1.0,-1.0,-1.0,-1.0,8.743,0.875937,1378,1920898 +272,search,2025.99,-1.0,-1.0,-1.0,-1.0,16.352,0.875938,1379,1920898 +273,insert,13.034,-1.0,-1.0,-1.0,-1.0,12.203,0.883856,1380,1942757 +274,search,2024.19,-1.0,-1.0,-1.0,-1.0,8.515,0.883856,1380,1942757 +275,insert,262.114,-1.0,-1.0,-1.0,-1.0,135.725,1.04525,1402,2328372 +276,search,2197,-1.0,-1.0,-1.0,-1.0,46.015,1.04533,1408,2328372 +277,insert,21.233,-1.0,-1.0,-1.0,-1.0,8.94,1.05748,1408,2363257 +278,search,2174.51,-1.0,-1.0,-1.0,-1.0,24.969,1.05748,1411,2363257 +279,insert,16.877,-1.0,-1.0,-1.0,-1.0,9.01,1.06826,1411,2388933 +280,search,2194.97,-1.0,-1.0,-1.0,-1.0,8.709,1.06826,1411,2388933 +281,insert,162.215,-1.0,-1.0,-1.0,-1.0,114.861,1.19873,1428,2684315 +282,search,2282.39,-1.0,-1.0,-1.0,-1.0,33.411,1.19879,1432,2684315 +283,insert,278.25,-1.0,-1.0,-1.0,-1.0,202.744,1.3857,1465,3109157 +284,search,2451.51,-1.0,-1.0,-1.0,-1.0,77.717,1.3864,1479,3109157 +285,insert,8.085,-1.0,-1.0,-1.0,-1.0,18.406,1.39147,1481,3122751 +286,search,2472.19,-1.0,-1.0,-1.0,-1.0,20.049,1.39147,1483,3122751 +287,insert,22.28,-1.0,-1.0,-1.0,-1.0,14.979,1.40676,1484,3163679 +288,search,2487.14,-1.0,-1.0,-1.0,-1.0,10.569,1.40676,1484,3163679 +289,insert,214.902,-1.0,-1.0,-1.0,-1.0,149.443,1.54322,1510,3472031 +290,search,2606.7,-1.0,-1.0,-1.0,-1.0,39.771,1.54321,1516,3472031 +291,insert,16.773,-1.0,-1.0,-1.0,-1.0,10.505,1.55445,1516,3495577 +292,search,2613.64,-1.0,-1.0,-1.0,-1.0,17.208,1.55445,1517,3495577 +293,insert,11.123,-1.0,-1.0,-1.0,-1.0,20.629,1.56351,1519,3515193 +294,search,2621.7,-1.0,-1.0,-1.0,-1.0,9.976,1.56351,1519,3515193 +295,insert,129.494,-1.0,-1.0,-1.0,-1.0,104.302,1.65673,1538,3723063 +296,search,2713.71,-1.0,-1.0,-1.0,-1.0,23.993,1.65695,1540,3723063 +297,insert,2.703,-1.0,-1.0,-1.0,-1.0,9.898,1.6586,1540,3727399 +298,search,2719.7,-1.0,-1.0,-1.0,-1.0,20.666,1.65885,1542,3727399 +299,insert,42.077,-1.0,-1.0,-1.0,-1.0,27.84,1.68336,1548,3788002 +300,search,2762.34,-1.0,-1.0,-1.0,-1.0,10.569,1.68336,1548,3788002 +301,insert,358.042,-1.0,-1.0,-1.0,-1.0,408.549,1.94938,1643,4370866 +302,search,3073.8,-1.0,-1.0,-1.0,-1.0,51.579,1.94972,1655,4370866 +303,insert,25.306,-1.0,-1.0,-1.0,-1.0,39.246,1.9714,1660,4420306 +304,search,3119.12,-1.0,-1.0,-1.0,-1.0,10.985,1.9714,1660,4420306 +305,insert,192.035,-1.0,-1.0,-1.0,-1.0,284.497,2.13875,1697,4789669 +306,search,3278.57,-1.0,-1.0,-1.0,-1.0,125.783,2.1392,1713,4789669 +307,insert,15.816,-1.0,-1.0,-1.0,-1.0,22.209,2.14817,1717,4815550 +308,search,3236.61,-1.0,-1.0,-1.0,-1.0,65.099,2.14818,1725,4815550 +309,insert,5.631,-1.0,-1.0,-1.0,-1.0,10.94,2.15176,1725,4823345 +310,search,3263.86,-1.0,-1.0,-1.0,-1.0,39.658,2.15183,1730,4823345 +311,insert,9.452,-1.0,-1.0,-1.0,-1.0,11.267,2.15845,1730,4843493 +312,search,3280.3,-1.0,-1.0,-1.0,-1.0,10.912,2.15845,1730,4843493 +313,insert,21.594,-1.0,-1.0,-1.0,-1.0,22.72,2.17574,1732,4885489 +314,search,3295.76,-1.0,-1.0,-1.0,-1.0,17.156,2.17574,1733,4885489 +315,insert,5.199,-1.0,-1.0,-1.0,-1.0,10.123,2.17911,1733,4892335 +316,search,3273.06,-1.0,-1.0,-1.0,-1.0,10.889,2.17911,1733,4892335 +317,insert,24.097,-1.0,-1.0,-1.0,-1.0,36.325,2.1957,1739,4932996 +318,search,3271.71,-1.0,-1.0,-1.0,-1.0,10.953,2.1957,1739,4932996 +319,insert,337.106,-1.0,-1.0,-1.0,-1.0,407.189,2.41284,1810,5430684 +320,search,3552.35,-1.0,-1.0,-1.0,-1.0,101.45,2.41481,1831,5430684 +321,insert,24.305,-1.0,-1.0,-1.0,-1.0,34.286,2.43029,1839,5471239 +322,search,3557.69,-1.0,-1.0,-1.0,-1.0,19.356,2.43032,1841,5471239 +323,insert,62.825,-1.0,-1.0,-1.0,-1.0,83.056,2.47794,1861,5575560 +324,search,3623.09,-1.0,-1.0,-1.0,-1.0,23.498,2.47831,1864,5575560 +325,insert,863.272,-1.0,-1.0,-1.0,-1.0,584.655,2.70857,1935,6069499 +326,search,3942.49,-1.0,-1.0,-1.0,-1.0,177.786,2.70878,1956,6069499 +327,insert,39.318,-1.0,-1.0,-1.0,-1.0,55.366,2.73899,1966,6139647 +328,search,3855.25,-1.0,-1.0,-1.0,-1.0,81.395,2.73962,1974,6139647 +329,insert,5.02,-1.0,-1.0,-1.0,-1.0,12.066,2.74247,1974,6146884 +330,search,3846.98,-1.0,-1.0,-1.0,-1.0,66.56,2.7428,1982,6146884 +331,insert,26.454,-1.0,-1.0,-1.0,-1.0,38.502,2.7624,1988,6188816 +332,search,3863.78,-1.0,-1.0,-1.0,-1.0,32.147,2.7624,1992,6188816 +333,insert,1.815,-1.0,-1.0,-1.0,-1.0,11.841,2.76338,1992,6190810 +334,search,3880.84,-1.0,-1.0,-1.0,-1.0,16.077,2.76346,1993,6190810 +335,insert,28.969,-1.0,-1.0,-1.0,-1.0,43.751,2.7801,2002,6230108 +336,search,3904.89,-1.0,-1.0,-1.0,-1.0,12.216,2.7801,2002,6230108 +337,insert,11.022,-1.0,-1.0,-1.0,-1.0,21.258,2.78768,2004,6249699 +338,search,3903.25,-1.0,-1.0,-1.0,-1.0,11.864,2.78768,2004,6249699 +339,insert,24.388,-1.0,-1.0,-1.0,-1.0,48.714,2.80027,2011,6290226 +340,search,3921.14,-1.0,-1.0,-1.0,-1.0,12.084,2.80027,2011,6290226 +341,insert,3.959,-1.0,-1.0,-1.0,-1.0,17.22,2.80268,2012,6297556 +342,search,3927.9,-1.0,-1.0,-1.0,-1.0,12.116,2.80268,2012,6297556 +343,insert,20.984,-1.0,-1.0,-1.0,-1.0,60.681,2.81805,2023,6330843 +344,search,3972.88,-1.0,-1.0,-1.0,-1.0,31.544,2.81868,2027,6330843 +345,insert,23.245,-1.0,-1.0,-1.0,-1.0,53.132,2.83556,2037,6370265 +346,search,4009.85,-1.0,-1.0,-1.0,-1.0,12.356,2.83556,2037,6370265 +347,insert,333.099,-1.0,-1.0,-1.0,-1.0,556.656,3.04293,2152,6831086 +348,search,4251.84,-1.0,-1.0,-1.0,-1.0,110.347,3.04561,2175,6831086 +349,insert,23.966,-1.0,-1.0,-1.0,-1.0,46.772,3.06228,2182,6876154 +350,search,4266.98,-1.0,-1.0,-1.0,-1.0,25.325,3.06242,2184,6876154 +351,insert,268.274,-1.0,-1.0,-1.0,-1.0,398.521,3.24288,2250,7288266 +352,search,4488.62,-1.0,-1.0,-1.0,-1.0,133.437,3.24398,2269,7288266 +353,insert,31.691,-1.0,-1.0,-1.0,-1.0,41.792,3.26303,2276,7333132 +354,search,4507.2,-1.0,-1.0,-1.0,-1.0,55.665,3.26312,2281,7333132 +355,insert,235.231,-1.0,-1.0,-1.0,-1.0,355.972,3.42551,2346,7696838 +356,search,4702.29,-1.0,-1.0,-1.0,-1.0,81.558,3.42605,2358,7696838 +357,insert,12.996,-1.0,-1.0,-1.0,-1.0,31.546,3.4332,2362,7714498 +358,search,4663.49,-1.0,-1.0,-1.0,-1.0,25.381,3.43343,2365,7714498 +359,insert,6.751,-1.0,-1.0,-1.0,-1.0,13.313,3.4372,2366,7725755 +360,search,4637.73,-1.0,-1.0,-1.0,-1.0,10.396,3.4372,2366,7725755 +361,insert,35.238,-1.0,-1.0,-1.0,-1.0,66.819,3.4615,2378,7789312 +362,search,4681.97,-1.0,-1.0,-1.0,-1.0,16.436,3.4615,2379,7789312 +363,insert,12.06,-1.0,-1.0,-1.0,-1.0,8.879,3.46494,2379,7803163 +364,search,4707.18,-1.0,-1.0,-1.0,-1.0,9.766,3.46494,2379,7803163 +365,insert,33.994,-1.0,-1.0,-1.0,-1.0,79.459,3.49103,2394,7863722 +366,search,4728.76,-1.0,-1.0,-1.0,-1.0,19.351,3.4913,2396,7863722 +367,insert,17.769,-1.0,-1.0,-1.0,-1.0,67.582,3.50443,2411,7895271 +368,search,4740.43,-1.0,-1.0,-1.0,-1.0,17.604,3.50485,2412,7895271 +369,insert,40.613,-1.0,-1.0,-1.0,-1.0,67.433,3.53265,2425,7961194 +370,search,4744.27,-1.0,-1.0,-1.0,-1.0,9.996,3.53265,2425,7961194 +371,insert,7.388,-1.0,-1.0,-1.0,-1.0,9.422,3.53641,2425,7970239 +372,search,4817.37,-1.0,-1.0,-1.0,-1.0,10.248,3.53641,2425,7970239 +373,insert,29.671,-1.0,-1.0,-1.0,-1.0,31.441,3.55303,2431,8010868 +374,search,4820.8,-1.0,-1.0,-1.0,-1.0,20.478,3.5533,2433,8010868 +375,insert,21.623,-1.0,-1.0,-1.0,-1.0,29.017,3.566,2437,8040698 +376,search,4790.53,-1.0,-1.0,-1.0,-1.0,11.017,3.566,2437,8040698 +377,insert,39.879,-1.0,-1.0,-1.0,-1.0,106.348,3.59464,2456,8104748 +378,search,4852.22,-1.0,-1.0,-1.0,-1.0,41.786,3.5953,2461,8104748 +379,insert,3.242,-1.0,-1.0,-1.0,-1.0,20.97,3.59731,2463,8109755 +380,search,4889.9,-1.0,-1.0,-1.0,-1.0,10.75,3.59731,2463,8109755 +381,insert,12.83,-1.0,-1.0,-1.0,-1.0,36.874,3.60633,2468,8130382 +382,search,4859.69,-1.0,-1.0,-1.0,-1.0,10.817,3.60633,2468,8130382 +383,insert,16.081,-1.0,-1.0,-1.0,-1.0,22.915,3.6142,2470,8153290 +384,search,4879.55,-1.0,-1.0,-1.0,-1.0,21.497,3.61436,2472,8153290 +385,delete,8.615,-1.0,-1.0,-1.0,-1.0,13.015,3.61436,2473,8138873 +386,search,4842.39,-1.0,-1.0,-1.0,-1.0,10.845,3.61436,2473,8138873 +387,delete,20.594,-1.0,-1.0,-1.0,-1.0,10.339,3.61371,2473,8115069 +388,search,4915.66,-1.0,-1.0,-1.0,-1.0,11.395,3.61371,2473,8115069 +389,delete,14.269,-1.0,-1.0,-1.0,-1.0,10.284,3.61371,2473,8101157 +390,search,4881.09,-1.0,-1.0,-1.0,-1.0,10.715,3.61371,2473,8101157 +391,delete,73.161,-1.0,-1.0,-1.0,-1.0,117.808,3.58733,2457,8014644 +392,search,4809.25,-1.0,-1.0,-1.0,-1.0,23.385,3.58762,2459,8014644 +393,delete,23.522,-1.0,-1.0,-1.0,-1.0,10.636,3.58717,2459,7978358 +394,search,4784.1,-1.0,-1.0,-1.0,-1.0,10.814,3.58717,2459,7978358 +395,delete,15.869,-1.0,-1.0,-1.0,-1.0,31.178,3.58119,2457,7957094 +396,search,4805.94,-1.0,-1.0,-1.0,-1.0,12.259,3.58119,2457,7957094 +397,delete,85.718,-1.0,-1.0,-1.0,-1.0,120.153,3.54694,2446,7823916 +398,search,4754.46,-1.0,-1.0,-1.0,-1.0,11.222,3.54694,2446,7823916 +399,delete,4.454,-1.0,-1.0,-1.0,-1.0,9.923,3.54666,2446,7818430 +400,search,4774,-1.0,-1.0,-1.0,-1.0,10.704,3.54666,2446,7818430 +401,delete,12.603,-1.0,-1.0,-1.0,-1.0,17.473,3.54315,2444,7800702 +402,search,4739.62,-1.0,-1.0,-1.0,-1.0,11.235,3.54315,2444,7800702 +403,delete,202.975,-1.0,-1.0,-1.0,-1.0,297.078,3.46014,2407,7517858 +404,search,4647.78,-1.0,-1.0,-1.0,-1.0,19.241,3.4602,2408,7517858 +405,delete,36.138,-1.0,-1.0,-1.0,-1.0,11.585,3.45835,2408,7451637 +406,search,4632.25,-1.0,-1.0,-1.0,-1.0,11.027,3.45835,2408,7451637 +407,delete,61.564,-1.0,-1.0,-1.0,-1.0,49.144,3.43488,2403,7360417 +408,search,4561.29,-1.0,-1.0,-1.0,-1.0,26.815,3.43538,2405,7360417 +409,delete,138.674,-1.0,-1.0,-1.0,-1.0,280.706,3.33823,2369,7131620 +410,search,4479.13,-1.0,-1.0,-1.0,-1.0,27.731,3.33917,2372,7131620 +411,delete,166.955,-1.0,-1.0,-1.0,-1.0,198.014,3.26451,2342,6867617 +412,search,4349.54,-1.0,-1.0,-1.0,-1.0,24.475,3.26486,2343,6867617 +413,delete,10.793,-1.0,-1.0,-1.0,-1.0,15.434,3.26308,2343,6850098 +414,search,4378.94,-1.0,-1.0,-1.0,-1.0,16.926,3.26308,2343,6850098 +415,delete,46.417,-1.0,-1.0,-1.0,-1.0,36.262,3.24515,2341,6782921 +416,search,4366.91,-1.0,-1.0,-1.0,-1.0,16.932,3.24515,2341,6782921 +417,delete,151.123,-1.0,-1.0,-1.0,-1.0,157.588,3.18425,2323,6562039 +418,search,4282.43,-1.0,-1.0,-1.0,-1.0,16.47,3.18425,2323,6562039 +419,delete,17.337,-1.0,-1.0,-1.0,-1.0,15.648,3.17971,2323,6536285 +420,search,4244.03,-1.0,-1.0,-1.0,-1.0,16.58,3.17971,2323,6536285 +421,delete,13.583,-1.0,-1.0,-1.0,-1.0,45.279,3.1705,2320,6510820 +422,search,4255.37,-1.0,-1.0,-1.0,-1.0,16.622,3.1705,2320,6510820 +423,delete,117.737,-1.0,-1.0,-1.0,-1.0,181.173,3.09797,2285,6334168 +424,search,4132.42,-1.0,-1.0,-1.0,-1.0,28.523,3.09827,2286,6334168 +425,delete,18.587,-1.0,-1.0,-1.0,-1.0,15.511,3.09497,2286,6316443 +426,search,4142.85,-1.0,-1.0,-1.0,-1.0,16.654,3.09497,2286,6316443 +427,delete,48.493,-1.0,-1.0,-1.0,-1.0,16.142,3.08022,2286,6266345 +428,search,4123.38,-1.0,-1.0,-1.0,-1.0,16.181,3.08022,2286,6266345 +429,delete,353.111,-1.0,-1.0,-1.0,-1.0,233.919,2.89947,2244,5883847 +430,search,3904.11,-1.0,-1.0,-1.0,-1.0,37.192,2.89983,2246,5883847 +431,delete,60.154,-1.0,-1.0,-1.0,-1.0,90.911,2.86483,2232,5801380 +432,search,3871.97,-1.0,-1.0,-1.0,-1.0,16.317,2.86483,2232,5801380 +433,delete,148.107,-1.0,-1.0,-1.0,-1.0,205.889,2.77348,2196,5607567 +434,search,3796.18,-1.0,-1.0,-1.0,-1.0,17.022,2.77348,2196,5607567 +435,delete,90.436,-1.0,-1.0,-1.0,-1.0,38.378,2.73324,2193,5512176 +436,search,3759.81,-1.0,-1.0,-1.0,-1.0,29.314,2.73323,2194,5512176 +437,delete,9.304,-1.0,-1.0,-1.0,-1.0,27.705,2.72599,2193,5501141 +438,search,3748.94,-1.0,-1.0,-1.0,-1.0,28.71,2.72599,2194,5501141 +439,delete,76.101,-1.0,-1.0,-1.0,-1.0,190.164,2.66289,2172,5369822 +440,search,3715.37,-1.0,-1.0,-1.0,-1.0,29.97,2.66339,2174,5369822 +441,delete,23.145,-1.0,-1.0,-1.0,-1.0,15.105,2.65407,2174,5338985 +442,search,3709.12,-1.0,-1.0,-1.0,-1.0,15.691,2.65407,2174,5338985 +443,delete,25.038,-1.0,-1.0,-1.0,-1.0,63.236,2.63804,2166,5307279 +444,search,3672.1,-1.0,-1.0,-1.0,-1.0,15.58,2.63804,2166,5307279 +445,delete,33.27,-1.0,-1.0,-1.0,-1.0,23.192,2.62103,2165,5268382 +446,search,3676.29,-1.0,-1.0,-1.0,-1.0,15.443,2.62103,2165,5268382 +447,delete,286.366,-1.0,-1.0,-1.0,-1.0,232.178,2.4498,2102,4922787 +448,search,3512.22,-1.0,-1.0,-1.0,-1.0,14.807,2.4498,2102,4922787 +449,delete,26.749,-1.0,-1.0,-1.0,-1.0,14.369,2.43787,2102,4887227 +450,search,3484.79,-1.0,-1.0,-1.0,-1.0,14.885,2.43787,2102,4887227 +451,delete,74.404,-1.0,-1.0,-1.0,-1.0,83.009,2.39077,2089,4795082 +452,search,3478.98,-1.0,-1.0,-1.0,-1.0,14.594,2.39077,2089,4795082 +453,delete,229.473,-1.0,-1.0,-1.0,-1.0,475.018,2.2334,2027,4500175 +454,search,3325.89,-1.0,-1.0,-1.0,-1.0,31.699,2.23395,2030,4500175 +455,delete,25.816,-1.0,-1.0,-1.0,-1.0,14.072,2.22376,2030,4462648 +456,search,3325.29,-1.0,-1.0,-1.0,-1.0,22.603,2.22415,2031,4462648 +457,delete,5.793,-1.0,-1.0,-1.0,-1.0,13.44,2.22143,2031,4456761 +458,search,3336.67,-1.0,-1.0,-1.0,-1.0,13.95,2.22143,2031,4456761 +459,delete,33.797,-1.0,-1.0,-1.0,-1.0,36.376,2.20087,2026,4419992 +460,search,3313.29,-1.0,-1.0,-1.0,-1.0,14.127,2.20087,2026,4419992 +461,delete,4.072,-1.0,-1.0,-1.0,-1.0,12.741,2.20011,2026,4415725 +462,search,3295.29,-1.0,-1.0,-1.0,-1.0,13.964,2.20011,2026,4415725 +463,delete,37.747,-1.0,-1.0,-1.0,-1.0,24.093,2.17708,2019,4370609 +464,search,3242.51,-1.0,-1.0,-1.0,-1.0,14.453,2.17708,2019,4370609 +465,delete,19.261,-1.0,-1.0,-1.0,-1.0,28.051,2.16346,2015,4345244 +466,search,3254.2,-1.0,-1.0,-1.0,-1.0,14.162,2.16346,2015,4345244 +467,delete,20.349,-1.0,-1.0,-1.0,-1.0,13.669,2.15541,2014,4319734 +468,search,3264.4,-1.0,-1.0,-1.0,-1.0,14.705,2.15541,2014,4319734 +469,delete,22.349,-1.0,-1.0,-1.0,-1.0,30.576,2.14539,2010,4298361 +470,search,3254.43,-1.0,-1.0,-1.0,-1.0,14.892,2.14539,2010,4298361 +471,delete,113.121,-1.0,-1.0,-1.0,-1.0,100.45,2.08636,1991,4188315 +472,search,3193.12,-1.0,-1.0,-1.0,-1.0,13.603,2.08636,1991,4188315 +473,delete,91.253,-1.0,-1.0,-1.0,-1.0,58.189,2.04011,1984,4074567 +474,search,3163.58,-1.0,-1.0,-1.0,-1.0,23.449,2.0404,1985,4074567 +475,delete,301.445,-1.0,-1.0,-1.0,-1.0,365.835,1.82354,1907,3691698 +476,search,2957.64,-1.0,-1.0,-1.0,-1.0,55.545,1.82393,1911,3691698 +477,delete,36.693,-1.0,-1.0,-1.0,-1.0,54.209,1.80494,1906,3642619 +478,search,2903.54,-1.0,-1.0,-1.0,-1.0,14.088,1.80494,1906,3642619 +479,delete,163.541,-1.0,-1.0,-1.0,-1.0,140.235,1.70777,1871,3421368 +480,search,2858.19,-1.0,-1.0,-1.0,-1.0,13.171,1.70777,1871,3421368 +481,delete,35.791,-1.0,-1.0,-1.0,-1.0,40.887,1.6875,1865,3378034 +482,search,2847.87,-1.0,-1.0,-1.0,-1.0,13.245,1.6875,1865,3378034 +483,delete,203.607,-1.0,-1.0,-1.0,-1.0,286.588,1.53897,1801,3115682 +484,search,2699.2,-1.0,-1.0,-1.0,-1.0,12.537,1.53897,1801,3115682 +485,delete,18.61,-1.0,-1.0,-1.0,-1.0,11.105,1.53471,1801,3094629 +486,search,2641.95,-1.0,-1.0,-1.0,-1.0,12.709,1.53471,1801,3094629 +487,delete,8.447,-1.0,-1.0,-1.0,-1.0,11.069,1.53283,1801,3086025 +488,search,2686.18,-1.0,-1.0,-1.0,-1.0,12.288,1.53283,1801,3086025 +489,delete,27.289,-1.0,-1.0,-1.0,-1.0,11.495,1.51833,1801,3052105 +490,search,2687.28,-1.0,-1.0,-1.0,-1.0,12.589,1.51833,1801,3052105 +491,delete,14.025,-1.0,-1.0,-1.0,-1.0,13.204,1.51221,1800,3038974 +492,search,2682.11,-1.0,-1.0,-1.0,-1.0,12.757,1.51221,1800,3038974 +493,delete,109.296,-1.0,-1.0,-1.0,-1.0,93.872,1.44793,1780,2911063 +494,search,2642.79,-1.0,-1.0,-1.0,-1.0,12.144,1.44793,1780,2911063 +495,delete,18.082,-1.0,-1.0,-1.0,-1.0,21.448,1.43784,1777,2888785 +496,search,2609.1,-1.0,-1.0,-1.0,-1.0,12.456,1.43784,1777,2888785 +497,delete,45.603,-1.0,-1.0,-1.0,-1.0,18.31,1.41342,1775,2829718 +498,search,2593.17,-1.0,-1.0,-1.0,-1.0,12.441,1.41342,1775,2829718 +499,delete,13.472,-1.0,-1.0,-1.0,-1.0,15.639,1.40662,1774,2816082 +500,search,2585.09,-1.0,-1.0,-1.0,-1.0,12.047,1.40662,1774,2816082 +501,delete,20.504,-1.0,-1.0,-1.0,-1.0,18.452,1.39537,1772,2790880 +502,search,2569.6,-1.0,-1.0,-1.0,-1.0,12.304,1.39537,1772,2790880 +503,delete,13.608,-1.0,-1.0,-1.0,-1.0,10.89,1.38973,1772,2774459 +504,search,2565.5,-1.0,-1.0,-1.0,-1.0,12.233,1.38973,1772,2774459 +505,delete,44.24,-1.0,-1.0,-1.0,-1.0,57.993,1.354,1759,2714683 +506,search,2547.5,-1.0,-1.0,-1.0,-1.0,11.88,1.354,1759,2714683 +507,delete,30.107,-1.0,-1.0,-1.0,-1.0,19.507,1.34295,1757,2680986 +508,search,2536.35,-1.0,-1.0,-1.0,-1.0,12.188,1.34295,1757,2680986 +509,delete,15.944,-1.0,-1.0,-1.0,-1.0,10.794,1.33628,1757,2662966 +510,search,2518.01,-1.0,-1.0,-1.0,-1.0,11.968,1.33628,1757,2662966 +511,delete,18.2,-1.0,-1.0,-1.0,-1.0,17.894,1.32637,1753,2642855 +512,search,2519.3,-1.0,-1.0,-1.0,-1.0,12.239,1.32637,1753,2642855 +513,insert,192.021,-1.0,-1.0,-1.0,-1.0,46.234,1.40705,1757,2906838 +514,search,2576.74,-1.0,-1.0,-1.0,-1.0,11.906,1.40705,1757,2906838 +515,insert,3.363,-1.0,-1.0,-1.0,-1.0,10.571,1.40763,1757,2913746 +516,search,2593.95,-1.0,-1.0,-1.0,-1.0,12.099,1.40763,1757,2913746 +517,insert,11.362,-1.0,-1.0,-1.0,-1.0,11.011,1.41094,1757,2936862 +518,search,2553.57,-1.0,-1.0,-1.0,-1.0,12.013,1.41094,1757,2936862 +519,insert,61.195,-1.0,-1.0,-1.0,-1.0,52.449,1.43576,1762,3009732 +520,search,2648.83,-1.0,-1.0,-1.0,-1.0,32.462,1.43576,1765,3009732 +521,insert,8.125,-1.0,-1.0,-1.0,-1.0,11.922,1.43872,1765,3025906 +522,search,2618.57,-1.0,-1.0,-1.0,-1.0,12.179,1.43872,1765,3025906 +523,insert,23.948,-1.0,-1.0,-1.0,-1.0,28.266,1.45084,1767,3061899 +524,search,2618.31,-1.0,-1.0,-1.0,-1.0,12.11,1.45084,1767,3061899 +525,insert,24.664,-1.0,-1.0,-1.0,-1.0,11.67,1.46028,1767,3101209 +526,search,2641.95,-1.0,-1.0,-1.0,-1.0,12.116,1.46028,1767,3101209 +527,insert,308.555,-1.0,-1.0,-1.0,-1.0,132.659,1.61253,1786,3522334 +528,search,2805.99,-1.0,-1.0,-1.0,-1.0,51.368,1.61253,1789,3522334 +529,insert,204.583,-1.0,-1.0,-1.0,-1.0,154.253,1.71662,1795,3797294 +530,search,3081,-1.0,-1.0,-1.0,-1.0,99.615,1.71662,1800,3797294 +531,insert,51.314,-1.0,-1.0,-1.0,-1.0,12.694,1.7354,1800,3853781 +532,search,3019.97,-1.0,-1.0,-1.0,-1.0,69.104,1.7354,1803,3853781 +533,insert,15.262,-1.0,-1.0,-1.0,-1.0,12.941,1.742,1803,3873352 +534,search,2982.57,-1.0,-1.0,-1.0,-1.0,68.548,1.742,1807,3873352 +535,insert,17.955,-1.0,-1.0,-1.0,-1.0,13.103,1.74945,1807,3893308 +536,search,2981.21,-1.0,-1.0,-1.0,-1.0,62.268,1.74945,1814,3893308 +537,insert,7.379,-1.0,-1.0,-1.0,-1.0,22.383,1.75355,1815,3901271 +538,search,2971.89,-1.0,-1.0,-1.0,-1.0,54.606,1.75355,1823,3901271 +539,insert,19.328,-1.0,-1.0,-1.0,-1.0,19.936,1.76154,1824,3925164 +540,search,2950.68,-1.0,-1.0,-1.0,-1.0,20.101,1.76169,1825,3925164 +541,insert,318.042,-1.0,-1.0,-1.0,-1.0,188.915,1.93219,1857,4320915 +542,search,3135.45,-1.0,-1.0,-1.0,-1.0,36.054,1.93219,1861,4320915 +543,insert,31.104,-1.0,-1.0,-1.0,-1.0,13.462,1.94706,1861,4360814 +544,search,3121.53,-1.0,-1.0,-1.0,-1.0,13.913,1.94706,1861,4360814 +545,insert,7.757,-1.0,-1.0,-1.0,-1.0,12.207,1.95022,1861,4370220 +546,search,3170.62,-1.0,-1.0,-1.0,-1.0,13.801,1.95022,1861,4370220 +547,insert,109.543,-1.0,-1.0,-1.0,-1.0,41.517,2.00816,1868,4510675 +548,search,3204.47,-1.0,-1.0,-1.0,-1.0,17.817,2.00816,1869,4510675 +549,insert,156.394,-1.0,-1.0,-1.0,-1.0,144.727,2.1117,1879,4744845 +550,search,3352.62,-1.0,-1.0,-1.0,-1.0,48.312,2.11205,1884,4744845 +551,insert,20.12,-1.0,-1.0,-1.0,-1.0,28.381,2.12418,1886,4771433 +552,search,3340.07,-1.0,-1.0,-1.0,-1.0,25.449,2.12418,1887,4771433 +553,insert,9.022,-1.0,-1.0,-1.0,-1.0,13.255,2.12916,1887,4782909 +554,search,3335.54,-1.0,-1.0,-1.0,-1.0,20.021,2.12921,1888,4782909 +555,insert,306.184,-1.0,-1.0,-1.0,-1.0,114.476,2.30188,1911,5180322 +556,search,3455.91,-1.0,-1.0,-1.0,-1.0,23.898,2.30228,1912,5180322 +557,insert,19.689,-1.0,-1.0,-1.0,-1.0,22.855,2.31295,1914,5203220 +558,search,3469.77,-1.0,-1.0,-1.0,-1.0,14.607,2.31295,1914,5203220 +559,insert,4.39,-1.0,-1.0,-1.0,-1.0,13.014,2.31511,1914,5210154 +560,search,3462.99,-1.0,-1.0,-1.0,-1.0,14.464,2.31511,1914,5210154 +561,insert,3.739,-1.0,-1.0,-1.0,-1.0,12.757,2.31665,1914,5215426 +562,search,3519.24,-1.0,-1.0,-1.0,-1.0,14.498,2.31665,1914,5215426 +563,insert,30.133,-1.0,-1.0,-1.0,-1.0,18.639,2.33201,1915,5252726 +564,search,3537.68,-1.0,-1.0,-1.0,-1.0,14.581,2.33201,1915,5252726 +565,insert,152.604,-1.0,-1.0,-1.0,-1.0,101.67,2.41641,1927,5447274 +566,search,3618.86,-1.0,-1.0,-1.0,-1.0,30.046,2.41672,1929,5447274 +567,insert,11.51,-1.0,-1.0,-1.0,-1.0,20.602,2.42334,1930,5463135 +568,search,3631.17,-1.0,-1.0,-1.0,-1.0,20.73,2.42359,1931,5463135 +569,insert,9.968,-1.0,-1.0,-1.0,-1.0,13.68,2.42949,1931,5478168 +570,search,3615.02,-1.0,-1.0,-1.0,-1.0,14.629,2.42949,1931,5478168 +571,insert,10.399,-1.0,-1.0,-1.0,-1.0,13.416,2.43617,1931,5492309 +572,search,3620.69,-1.0,-1.0,-1.0,-1.0,22.528,2.43634,1932,5492309 +573,insert,46.904,-1.0,-1.0,-1.0,-1.0,39.579,2.4625,1936,5554599 +574,search,3639.9,-1.0,-1.0,-1.0,-1.0,26.739,2.46278,1937,5554599 +575,insert,44.975,-1.0,-1.0,-1.0,-1.0,42.448,2.48538,1941,5604500 +576,search,3673.52,-1.0,-1.0,-1.0,-1.0,24.3,2.48567,1942,5604500 +577,insert,24.899,-1.0,-1.0,-1.0,-1.0,30.949,2.49933,1945,5638834 +578,search,3687.6,-1.0,-1.0,-1.0,-1.0,15.364,2.49933,1945,5638834 +579,insert,475.853,-1.0,-1.0,-1.0,-1.0,624.515,2.77208,2055,6229778 +580,search,3973.82,-1.0,-1.0,-1.0,-1.0,63.034,2.77253,2063,6229778 +581,insert,18.942,-1.0,-1.0,-1.0,-1.0,32.582,2.78478,2066,6261240 +582,search,3968.2,-1.0,-1.0,-1.0,-1.0,16.116,2.78478,2066,6261240 +583,insert,14.362,-1.0,-1.0,-1.0,-1.0,20.428,2.79299,2067,6283315 +584,search,3973.68,-1.0,-1.0,-1.0,-1.0,24.661,2.79311,2068,6283315 +585,insert,171.819,-1.0,-1.0,-1.0,-1.0,174.16,2.91558,2093,6556101 +586,search,4142.35,-1.0,-1.0,-1.0,-1.0,57.693,2.91607,2101,6556101 +587,insert,25.142,-1.0,-1.0,-1.0,-1.0,28.275,2.92975,2104,6587187 +588,search,4147.31,-1.0,-1.0,-1.0,-1.0,48.321,2.93016,2109,6587187 +589,insert,33.373,-1.0,-1.0,-1.0,-1.0,36.488,2.9495,2113,6631526 +590,search,4182.73,-1.0,-1.0,-1.0,-1.0,22.222,2.9495,2114,6631526 +591,insert,337.227,-1.0,-1.0,-1.0,-1.0,505.443,3.16397,2211,7108942 +592,search,4430.96,-1.0,-1.0,-1.0,-1.0,74.228,3.16497,2223,7108942 +593,insert,111.86,-1.0,-1.0,-1.0,-1.0,156.95,3.24707,2249,7303976 +594,search,4418.46,-1.0,-1.0,-1.0,-1.0,54.053,3.24758,2255,7303976 +595,insert,3.394,-1.0,-1.0,-1.0,-1.0,21.75,3.24926,2256,7309244 +596,search,4444.01,-1.0,-1.0,-1.0,-1.0,25.234,3.24926,2257,7309244 +597,insert,145.575,-1.0,-1.0,-1.0,-1.0,219.326,3.35653,2291,7542477 +598,search,4639.03,-1.0,-1.0,-1.0,-1.0,95.331,3.35689,2301,7542477 +599,insert,69.544,-1.0,-1.0,-1.0,-1.0,156.882,3.40088,2331,7654438 +600,search,4683.89,-1.0,-1.0,-1.0,-1.0,65.353,3.40089,2336,7654438 +601,insert,41.103,-1.0,-1.0,-1.0,-1.0,74.729,3.4279,2349,7717936 +602,search,4693.15,-1.0,-1.0,-1.0,-1.0,38.355,3.4279,2352,7717936 +603,insert,4.027,-1.0,-1.0,-1.0,-1.0,25.485,3.43054,2353,7723521 +604,search,4678.03,-1.0,-1.0,-1.0,-1.0,24.634,3.43059,2354,7723521 +605,insert,29.035,-1.0,-1.0,-1.0,-1.0,52.985,3.44933,2360,7767404 +606,search,4764.25,-1.0,-1.0,-1.0,-1.0,27.647,3.44936,2362,7767404 +607,insert,29.37,-1.0,-1.0,-1.0,-1.0,24.287,3.46899,2364,7826115 +608,search,4788.8,-1.0,-1.0,-1.0,-1.0,18.527,3.469,2365,7826115 +609,insert,127.296,-1.0,-1.0,-1.0,-1.0,255.843,3.55597,2402,8036351 +610,search,4737.42,-1.0,-1.0,-1.0,-1.0,71.877,3.55725,2412,8036351 +611,insert,14.374,-1.0,-1.0,-1.0,-1.0,31.26,3.56626,2416,8060364 +612,search,4821.49,-1.0,-1.0,-1.0,-1.0,17.646,3.56628,2417,8060364 +613,insert,131.028,-1.0,-1.0,-1.0,-1.0,221.105,3.6513,2454,8256083 +614,search,4866.67,-1.0,-1.0,-1.0,-1.0,30.642,3.65173,2457,8256083 +615,insert,242.518,-1.0,-1.0,-1.0,-1.0,412.785,3.82288,2521,8625707 +616,search,5102.47,-1.0,-1.0,-1.0,-1.0,120.483,3.82514,2543,8625707 +617,insert,200.025,-1.0,-1.0,-1.0,-1.0,388.789,3.97299,2603,8949227 +618,search,5259.7,-1.0,-1.0,-1.0,-1.0,85.16,3.97362,2615,8949227 +619,insert,140.263,-1.0,-1.0,-1.0,-1.0,310.316,4.07347,2665,9195551 +620,search,5392.28,-1.0,-1.0,-1.0,-1.0,73.69,4.07449,2678,9195551 +621,insert,22.414,-1.0,-1.0,-1.0,-1.0,68.272,4.08833,2690,9232372 +622,search,5450.47,-1.0,-1.0,-1.0,-1.0,26.665,4.08848,2692,9232372 +623,insert,12.246,-1.0,-1.0,-1.0,-1.0,26.53,4.09659,2695,9253561 +624,search,5385.22,-1.0,-1.0,-1.0,-1.0,24.483,4.09676,2696,9253561 +625,insert,43.258,-1.0,-1.0,-1.0,-1.0,102.886,4.13202,2712,9331230 +626,search,5416.87,-1.0,-1.0,-1.0,-1.0,50.789,4.13271,2718,9331230 +627,insert,30.537,-1.0,-1.0,-1.0,-1.0,50.458,4.14804,2726,9374063 +628,search,5518.11,-1.0,-1.0,-1.0,-1.0,19.811,4.14805,2727,9374063 +629,insert,11.523,-1.0,-1.0,-1.0,-1.0,30.75,4.15613,2730,9387598 +630,search,5555.18,-1.0,-1.0,-1.0,-1.0,26.831,4.15628,2732,9387598 +631,insert,26.035,-1.0,-1.0,-1.0,-1.0,34.204,4.16989,2736,9421724 +632,search,5515.15,-1.0,-1.0,-1.0,-1.0,20.486,4.17011,2737,9421724 +633,insert,33.185,-1.0,-1.0,-1.0,-1.0,51.936,4.18795,2745,9461430 +634,search,5537.33,-1.0,-1.0,-1.0,-1.0,20.319,4.18807,2746,9461430 +635,insert,15.265,-1.0,-1.0,-1.0,-1.0,33.147,4.19555,2750,9479469 +636,search,5603.49,-1.0,-1.0,-1.0,-1.0,28.843,4.1961,2752,9479469 +637,insert,338.029,-1.0,-1.0,-1.0,-1.0,651.019,4.38545,2870,9885997 +638,search,5791.68,-1.0,-1.0,-1.0,-1.0,138.752,4.38873,2894,9885997 +639,insert,256.788,-1.0,-1.0,-1.0,-1.0,551.537,4.56815,3004,10292043 +640,search,5887.28,-1.0,-1.0,-1.0,-1.0,53.134,4.56962,3012,10292043 +641,delete,124.336,-1.0,-1.0,-1.0,-1.0,155.155,4.52977,2997,10096584 +642,search,6017.26,-1.0,-1.0,-1.0,-1.0,28.82,4.53015,2998,10096584 +643,delete,11.123,-1.0,-1.0,-1.0,-1.0,15.301,4.53015,2998,10084840 +644,search,5948.27,-1.0,-1.0,-1.0,-1.0,17.502,4.53015,2998,10084840 +645,delete,17.095,-1.0,-1.0,-1.0,-1.0,15.241,4.53015,2998,10069802 +646,search,6023.01,-1.0,-1.0,-1.0,-1.0,18.046,4.53015,2998,10069802 +647,delete,75.238,-1.0,-1.0,-1.0,-1.0,103.74,4.50943,2988,9990953 +648,search,5905.68,-1.0,-1.0,-1.0,-1.0,17.262,4.50943,2988,9990953 +649,delete,29.937,-1.0,-1.0,-1.0,-1.0,13.632,4.50861,2988,9951625 +650,search,5950.29,-1.0,-1.0,-1.0,-1.0,16.659,4.50861,2988,9951625 +651,delete,19.485,-1.0,-1.0,-1.0,-1.0,15.423,4.50588,2988,9927065 +652,search,5870.95,-1.0,-1.0,-1.0,-1.0,16.941,4.50588,2988,9927065 +653,delete,78.213,-1.0,-1.0,-1.0,-1.0,116.975,4.47679,2974,9831098 +654,search,5886.23,-1.0,-1.0,-1.0,-1.0,16.579,4.47679,2974,9831098 +655,delete,280.008,-1.0,-1.0,-1.0,-1.0,334.756,4.37933,2930,9462172 +656,search,5720.72,-1.0,-1.0,-1.0,-1.0,31.41,4.37983,2933,9462172 +657,delete,169.432,-1.0,-1.0,-1.0,-1.0,184.844,4.31058,2880,9218902 +658,search,5564.1,-1.0,-1.0,-1.0,-1.0,40.484,4.31137,2884,9218902 +659,delete,82.703,-1.0,-1.0,-1.0,-1.0,124.093,4.28052,2868,9106546 +660,search,5515.9,-1.0,-1.0,-1.0,-1.0,20.093,4.2807,2869,9106546 +661,delete,30.855,-1.0,-1.0,-1.0,-1.0,25.794,4.27629,2868,9059776 +662,search,5521.08,-1.0,-1.0,-1.0,-1.0,16.078,4.27629,2868,9059776 +663,delete,55.375,-1.0,-1.0,-1.0,-1.0,62.594,4.25791,2861,8990616 +664,search,5413.12,-1.0,-1.0,-1.0,-1.0,24.444,4.2582,2862,8990616 +665,delete,42.305,-1.0,-1.0,-1.0,-1.0,33.92,4.24657,2860,8937437 +666,search,5507.95,-1.0,-1.0,-1.0,-1.0,16.179,4.24657,2860,8937437 +667,delete,137.406,-1.0,-1.0,-1.0,-1.0,199.19,4.17505,2838,8766390 +668,search,5344.07,-1.0,-1.0,-1.0,-1.0,15.292,4.17505,2838,8766390 +669,delete,156.216,-1.0,-1.0,-1.0,-1.0,264.501,4.0941,2808,8532008 +670,search,5310.38,-1.0,-1.0,-1.0,-1.0,17.26,4.0941,2808,8532008 +671,delete,52.111,-1.0,-1.0,-1.0,-1.0,31.708,4.08399,2805,8489178 +672,search,5297.66,-1.0,-1.0,-1.0,-1.0,16.358,4.08399,2805,8489178 +673,delete,56.902,-1.0,-1.0,-1.0,-1.0,75.295,4.06457,2796,8421866 +674,search,5224.77,-1.0,-1.0,-1.0,-1.0,16.462,4.06457,2796,8421866 +675,delete,112.926,-1.0,-1.0,-1.0,-1.0,47.464,4.03155,2791,8288319 +676,search,5213.4,-1.0,-1.0,-1.0,-1.0,16.407,4.03155,2791,8288319 +677,delete,152.846,-1.0,-1.0,-1.0,-1.0,164.269,3.93949,2774,8072197 +678,search,5109.77,-1.0,-1.0,-1.0,-1.0,22.645,3.93975,2775,8072197 +679,delete,24.852,-1.0,-1.0,-1.0,-1.0,29.601,3.92963,2774,8036947 +680,search,5093.55,-1.0,-1.0,-1.0,-1.0,14.051,3.92963,2774,8036947 +681,delete,7.565,-1.0,-1.0,-1.0,-1.0,13.284,3.92894,2774,8026197 +682,search,5142.02,-1.0,-1.0,-1.0,-1.0,16.459,3.92894,2774,8026197 +683,delete,247.228,-1.0,-1.0,-1.0,-1.0,86.977,3.831,2760,7744229 +684,search,4920.87,-1.0,-1.0,-1.0,-1.0,22.426,3.83113,2761,7744229 +685,delete,193.702,-1.0,-1.0,-1.0,-1.0,64.088,3.73934,2750,7541066 +686,search,4811.5,-1.0,-1.0,-1.0,-1.0,28.592,3.73948,2752,7541066 +687,delete,14.164,-1.0,-1.0,-1.0,-1.0,21.974,3.73437,2751,7522190 +688,search,4891.99,-1.0,-1.0,-1.0,-1.0,15.931,3.73437,2751,7522190 +689,delete,93.917,-1.0,-1.0,-1.0,-1.0,219.319,3.6725,2721,7409400 +690,search,4839.91,-1.0,-1.0,-1.0,-1.0,26.319,3.67289,2723,7409400 +691,delete,32.47,-1.0,-1.0,-1.0,-1.0,14.701,3.65877,2723,7375648 +692,search,4785.93,-1.0,-1.0,-1.0,-1.0,13.623,3.65877,2723,7375648 +693,delete,139.497,-1.0,-1.0,-1.0,-1.0,95.128,3.59016,2710,7222068 +694,search,4734.67,-1.0,-1.0,-1.0,-1.0,14.987,3.59016,2710,7222068 +695,delete,21.026,-1.0,-1.0,-1.0,-1.0,14.041,3.58009,2710,7196331 +696,search,4729.1,-1.0,-1.0,-1.0,-1.0,15.005,3.58009,2710,7196331 +697,delete,21.701,-1.0,-1.0,-1.0,-1.0,20.461,3.56968,2709,7170716 +698,search,4740.25,-1.0,-1.0,-1.0,-1.0,15.166,3.56968,2709,7170716 +699,delete,10.633,-1.0,-1.0,-1.0,-1.0,52.392,3.56057,2707,7155255 +700,search,4734.48,-1.0,-1.0,-1.0,-1.0,14.735,3.56057,2707,7155255 +701,delete,49.63,-1.0,-1.0,-1.0,-1.0,14.368,3.5378,2707,7097733 +702,search,4689.98,-1.0,-1.0,-1.0,-1.0,15.028,3.5378,2707,7097733 +703,delete,162.219,-1.0,-1.0,-1.0,-1.0,182.023,3.44611,2676,6924172 +704,search,4610.96,-1.0,-1.0,-1.0,-1.0,20.555,3.44628,2677,6924172 +705,delete,22.631,-1.0,-1.0,-1.0,-1.0,13.475,3.43737,2677,6896276 +706,search,4630.86,-1.0,-1.0,-1.0,-1.0,15.228,3.43737,2677,6896276 +707,delete,471.594,-1.0,-1.0,-1.0,-1.0,649.479,3.13857,2564,6385507 +708,search,4342.21,-1.0,-1.0,-1.0,-1.0,27.696,3.13877,2565,6385507 +709,delete,153.995,-1.0,-1.0,-1.0,-1.0,385.493,3.04308,2518,6207658 +710,search,4206.71,-1.0,-1.0,-1.0,-1.0,29.422,3.04352,2520,6207658 +711,delete,40.372,-1.0,-1.0,-1.0,-1.0,43.777,3.02675,2517,6162720 +712,search,4203,-1.0,-1.0,-1.0,-1.0,13.82,3.02675,2517,6162720 +713,delete,195.931,-1.0,-1.0,-1.0,-1.0,313.428,2.91293,2482,5927803 +714,search,4124.5,-1.0,-1.0,-1.0,-1.0,30.42,2.91332,2483,5927803 +715,delete,30.532,-1.0,-1.0,-1.0,-1.0,33.403,2.89727,2480,5895666 +716,search,4113.93,-1.0,-1.0,-1.0,-1.0,13.045,2.89727,2480,5895666 +717,delete,23.766,-1.0,-1.0,-1.0,-1.0,12.373,2.89091,2480,5868774 +718,search,4145.62,-1.0,-1.0,-1.0,-1.0,13.835,2.89091,2480,5868774 +719,delete,296.21,-1.0,-1.0,-1.0,-1.0,402.535,2.72206,2399,5533540 +720,search,3934.73,-1.0,-1.0,-1.0,-1.0,19.585,2.72217,2400,5533540 +721,delete,85.88,-1.0,-1.0,-1.0,-1.0,124.647,2.67524,2384,5418387 +722,search,3896.94,-1.0,-1.0,-1.0,-1.0,22.6,2.67527,2385,5418387 +723,delete,18.757,-1.0,-1.0,-1.0,-1.0,16.278,2.66796,2384,5398291 +724,search,3882.48,-1.0,-1.0,-1.0,-1.0,12.614,2.66796,2384,5398291 +725,delete,95.432,-1.0,-1.0,-1.0,-1.0,167.591,2.60731,2366,5273546 +726,search,3851.36,-1.0,-1.0,-1.0,-1.0,19.985,2.60737,2367,5273546 +727,delete,80.054,-1.0,-1.0,-1.0,-1.0,50.767,2.57023,2361,5180937 +728,search,3804.59,-1.0,-1.0,-1.0,-1.0,13.524,2.57023,2361,5180937 +729,delete,89.219,-1.0,-1.0,-1.0,-1.0,122.427,2.52013,2340,5079524 +730,search,3737.01,-1.0,-1.0,-1.0,-1.0,25.448,2.52039,2341,5079524 +731,delete,54.166,-1.0,-1.0,-1.0,-1.0,17.535,2.50078,2341,5019265 +732,search,3724.15,-1.0,-1.0,-1.0,-1.0,28.511,2.50081,2342,5019265 +733,delete,58.591,-1.0,-1.0,-1.0,-1.0,56.351,2.46966,2336,4955565 +734,search,3708.68,-1.0,-1.0,-1.0,-1.0,18.158,2.46966,2336,4955565 +735,delete,133.541,-1.0,-1.0,-1.0,-1.0,360.948,2.383,2296,4776948 +736,search,3631.86,-1.0,-1.0,-1.0,-1.0,26.472,2.38323,2297,4776948 +737,delete,98.905,-1.0,-1.0,-1.0,-1.0,67.209,2.33179,2290,4654026 +738,search,3600.01,-1.0,-1.0,-1.0,-1.0,23.732,2.33179,2291,4654026 +739,delete,54.006,-1.0,-1.0,-1.0,-1.0,78.119,2.29549,2283,4589888 +740,search,3552.5,-1.0,-1.0,-1.0,-1.0,17.236,2.29549,2283,4589888 +741,delete,131.367,-1.0,-1.0,-1.0,-1.0,117.654,2.21861,2267,4430862 +742,search,3514.94,-1.0,-1.0,-1.0,-1.0,17.403,2.21861,2267,4430862 +743,delete,153.739,-1.0,-1.0,-1.0,-1.0,284.712,2.11671,2232,4237049 +744,search,3428.02,-1.0,-1.0,-1.0,-1.0,17.544,2.11671,2232,4237049 +745,delete,202.907,-1.0,-1.0,-1.0,-1.0,361.333,1.98613,2180,4000195 +746,search,3336.28,-1.0,-1.0,-1.0,-1.0,16.378,1.98613,2180,4000195 +747,delete,146.532,-1.0,-1.0,-1.0,-1.0,169.309,1.89652,2149,3803954 +748,search,3251.84,-1.0,-1.0,-1.0,-1.0,31.236,1.89656,2150,3803954 +749,delete,78.3,-1.0,-1.0,-1.0,-1.0,52.723,1.86385,2142,3733214 +750,search,3228.36,-1.0,-1.0,-1.0,-1.0,17.277,1.86385,2142,3733214 +751,delete,25.57,-1.0,-1.0,-1.0,-1.0,28.846,1.85341,2139,3709324 +752,search,3229.38,-1.0,-1.0,-1.0,-1.0,17.233,1.85341,2139,3709324 +753,delete,59.03,-1.0,-1.0,-1.0,-1.0,96.732,1.82235,2126,3649123 +754,search,3219.01,-1.0,-1.0,-1.0,-1.0,17.066,1.82235,2126,3649123 +755,delete,29.871,-1.0,-1.0,-1.0,-1.0,14.633,1.81281,2126,3621475 +756,search,3193.91,-1.0,-1.0,-1.0,-1.0,16.573,1.81281,2126,3621475 +757,delete,32.248,-1.0,-1.0,-1.0,-1.0,34.529,1.79675,2122,3594479 +758,search,3203.44,-1.0,-1.0,-1.0,-1.0,16.78,1.79675,2122,3594479 +759,delete,25.312,-1.0,-1.0,-1.0,-1.0,25.451,1.78742,2120,3569055 +760,search,3186.49,-1.0,-1.0,-1.0,-1.0,16.778,1.78742,2120,3569055 +761,delete,40.318,-1.0,-1.0,-1.0,-1.0,20.451,1.77189,2119,3527896 +762,search,3126.14,-1.0,-1.0,-1.0,-1.0,16.188,1.77189,2119,3527896 +763,delete,22.453,-1.0,-1.0,-1.0,-1.0,26.563,1.76292,2117,3509970 +764,search,3160.73,-1.0,-1.0,-1.0,-1.0,16.614,1.76292,2117,3509970 +765,delete,309.778,-1.0,-1.0,-1.0,-1.0,317.165,1.57872,2024,3187549 +766,search,2977.5,-1.0,-1.0,-1.0,-1.0,15.355,1.57872,2024,3187549 +767,delete,330.403,-1.0,-1.0,-1.0,-1.0,456.703,1.37337,1903,2822906 +768,search,2767.85,-1.0,-1.0,-1.0,-1.0,14.371,1.37337,1903,2822906 +769,insert,7.341,-1.0,-1.0,-1.0,-1.0,12.716,1.37476,1903,2832899 +770,search,2759.64,-1.0,-1.0,-1.0,-1.0,14.186,1.37476,1903,2832899 +771,insert,27.787,-1.0,-1.0,-1.0,-1.0,19.589,1.38408,1904,2863523 +772,search,2758,-1.0,-1.0,-1.0,-1.0,14.329,1.38408,1904,2863523 +773,insert,40.632,-1.0,-1.0,-1.0,-1.0,13.924,1.39786,1904,2925169 +774,search,2772.25,-1.0,-1.0,-1.0,-1.0,14.167,1.39786,1904,2925169 +775,insert,32.929,-1.0,-1.0,-1.0,-1.0,24.025,1.40925,1905,2961356 +776,search,2809.88,-1.0,-1.0,-1.0,-1.0,14.437,1.40925,1905,2961356 +777,insert,24.67,-1.0,-1.0,-1.0,-1.0,13.102,1.41657,1905,2998678 +778,search,2802.93,-1.0,-1.0,-1.0,-1.0,14.459,1.41657,1905,2998678 +779,insert,11.176,-1.0,-1.0,-1.0,-1.0,12.565,1.41959,1905,3011024 +780,search,2804.51,-1.0,-1.0,-1.0,-1.0,14.36,1.41959,1905,3011024 +781,insert,4.955,-1.0,-1.0,-1.0,-1.0,12.462,1.42049,1905,3019515 +782,search,2797.03,-1.0,-1.0,-1.0,-1.0,14.197,1.42049,1905,3019515 +783,insert,37.875,-1.0,-1.0,-1.0,-1.0,13.467,1.43154,1905,3082298 +784,search,2832.87,-1.0,-1.0,-1.0,-1.0,14.064,1.43154,1905,3082298 +785,insert,33.313,-1.0,-1.0,-1.0,-1.0,40.725,1.44583,1908,3122090 +786,search,2818.19,-1.0,-1.0,-1.0,-1.0,36.331,1.44583,1911,3122090 +787,insert,17.105,-1.0,-1.0,-1.0,-1.0,13.783,1.45103,1911,3140294 +788,search,2840.52,-1.0,-1.0,-1.0,-1.0,14.405,1.45103,1911,3140294 +789,insert,5.169,-1.0,-1.0,-1.0,-1.0,12.701,1.4523,1911,3147819 +790,search,2837.12,-1.0,-1.0,-1.0,-1.0,14.409,1.4523,1911,3147819 +791,insert,22.431,-1.0,-1.0,-1.0,-1.0,13.133,1.46005,1911,3180668 +792,search,2860.78,-1.0,-1.0,-1.0,-1.0,14.389,1.46005,1911,3180668 +793,insert,4.059,-1.0,-1.0,-1.0,-1.0,12.404,1.46122,1911,3183235 +794,search,2879.36,-1.0,-1.0,-1.0,-1.0,14.388,1.46122,1911,3183235 +795,insert,45.856,-1.0,-1.0,-1.0,-1.0,23.218,1.47908,1912,3242089 +796,search,2873.67,-1.0,-1.0,-1.0,-1.0,14.567,1.47908,1912,3242089 +797,insert,3.95,-1.0,-1.0,-1.0,-1.0,12.846,1.47989,1912,3247192 +798,search,2900.19,-1.0,-1.0,-1.0,-1.0,14.4,1.47989,1912,3247192 +799,insert,5.295,-1.0,-1.0,-1.0,-1.0,12.784,1.48124,1912,3254626 +800,search,2904.52,-1.0,-1.0,-1.0,-1.0,14.47,1.48124,1912,3254626 +801,insert,14.833,-1.0,-1.0,-1.0,-1.0,12.748,1.48586,1912,3274953 +802,search,2904.12,-1.0,-1.0,-1.0,-1.0,15.741,1.48586,1912,3274953 +803,insert,736.635,-1.0,-1.0,-1.0,-1.0,500.742,1.81052,1978,4032688 +804,search,3244.83,-1.0,-1.0,-1.0,-1.0,152.811,1.81064,2002,4032688 +805,insert,14.074,-1.0,-1.0,-1.0,-1.0,14.496,1.81603,2002,4050220 +806,search,3259.15,-1.0,-1.0,-1.0,-1.0,27.389,1.81603,2004,4050220 +807,insert,14.245,-1.0,-1.0,-1.0,-1.0,14.506,1.82168,2004,4067940 +808,search,3265.32,-1.0,-1.0,-1.0,-1.0,15.927,1.82168,2004,4067940 +809,insert,28.751,-1.0,-1.0,-1.0,-1.0,23.068,1.83277,2005,4099369 +810,search,3278.7,-1.0,-1.0,-1.0,-1.0,15.485,1.83277,2005,4099369 +811,insert,29.022,-1.0,-1.0,-1.0,-1.0,13.877,1.84396,2005,4129509 +812,search,3292.86,-1.0,-1.0,-1.0,-1.0,15.559,1.84396,2005,4129509 +813,insert,47.156,-1.0,-1.0,-1.0,-1.0,14.932,1.86404,2005,4178669 +814,search,3304.33,-1.0,-1.0,-1.0,-1.0,15.488,1.86404,2005,4178669 +815,insert,2.313,-1.0,-1.0,-1.0,-1.0,13.755,1.86482,2005,4180960 +816,search,3297.96,-1.0,-1.0,-1.0,-1.0,15.482,1.86482,2005,4180960 +817,insert,71.695,-1.0,-1.0,-1.0,-1.0,73.243,1.89415,2013,4255359 +818,search,3313.97,-1.0,-1.0,-1.0,-1.0,15.454,1.89415,2013,4255359 +819,insert,19.295,-1.0,-1.0,-1.0,-1.0,13.929,1.90035,2013,4272706 +820,search,3300.26,-1.0,-1.0,-1.0,-1.0,15.694,1.90035,2013,4272706 +821,insert,14.281,-1.0,-1.0,-1.0,-1.0,23.035,1.90726,2014,4287851 +822,search,3324.34,-1.0,-1.0,-1.0,-1.0,22.076,1.90729,2015,4287851 +823,insert,29.444,-1.0,-1.0,-1.0,-1.0,15.065,1.92137,2015,4322788 +824,search,3334.66,-1.0,-1.0,-1.0,-1.0,15.393,1.92137,2015,4322788 +825,insert,70.094,-1.0,-1.0,-1.0,-1.0,21.356,1.95169,2016,4396408 +826,search,3341.14,-1.0,-1.0,-1.0,-1.0,15.47,1.95169,2016,4396408 +827,insert,17.47,-1.0,-1.0,-1.0,-1.0,14.3,1.95955,2016,4415721 +828,search,3343.34,-1.0,-1.0,-1.0,-1.0,15.474,1.95955,2016,4415721 +829,insert,385.982,-1.0,-1.0,-1.0,-1.0,143.465,2.13855,2036,4832079 +830,search,3495.49,-1.0,-1.0,-1.0,-1.0,15.805,2.13855,2036,4832079 +831,insert,31.484,-1.0,-1.0,-1.0,-1.0,22.39,2.1528,2037,4864525 +832,search,3538.89,-1.0,-1.0,-1.0,-1.0,15.488,2.1528,2037,4864525 +833,insert,11.253,-1.0,-1.0,-1.0,-1.0,24.174,2.15663,2039,4874443 +834,search,3543.56,-1.0,-1.0,-1.0,-1.0,15.627,2.15663,2039,4874443 +835,insert,52.528,-1.0,-1.0,-1.0,-1.0,15.479,2.18212,2039,4938267 +836,search,3570.9,-1.0,-1.0,-1.0,-1.0,15.759,2.18212,2039,4938267 +837,insert,24.011,-1.0,-1.0,-1.0,-1.0,22.086,2.1923,2040,4961147 +838,search,3571.79,-1.0,-1.0,-1.0,-1.0,21.506,2.19239,2041,4961147 +839,insert,62.471,-1.0,-1.0,-1.0,-1.0,45.599,2.22327,2046,5033740 +840,search,3596.39,-1.0,-1.0,-1.0,-1.0,20.467,2.22335,2047,5033740 +841,insert,35.289,-1.0,-1.0,-1.0,-1.0,28.626,2.2417,2049,5078736 +842,search,3600.53,-1.0,-1.0,-1.0,-1.0,23.578,2.24178,2050,5078736 +843,insert,361.049,-1.0,-1.0,-1.0,-1.0,191.729,2.41172,2075,5467376 +844,search,3766.97,-1.0,-1.0,-1.0,-1.0,56.587,2.41195,2082,5467376 +845,insert,278.896,-1.0,-1.0,-1.0,-1.0,225.736,2.56845,2119,5815547 +846,search,3918.29,-1.0,-1.0,-1.0,-1.0,59.497,2.56884,2126,5815547 +847,insert,5.395,-1.0,-1.0,-1.0,-1.0,16.385,2.57148,2126,5821782 +848,search,3921.54,-1.0,-1.0,-1.0,-1.0,38.873,2.57158,2130,5821782 +849,insert,2.493,-1.0,-1.0,-1.0,-1.0,16.209,2.57233,2130,5825594 +850,search,3890.32,-1.0,-1.0,-1.0,-1.0,17.94,2.57233,2130,5825594 +851,insert,8.501,-1.0,-1.0,-1.0,-1.0,22.053,2.57607,2131,5834880 +852,search,3925.23,-1.0,-1.0,-1.0,-1.0,26.228,2.57614,2132,5834880 +853,insert,24.906,-1.0,-1.0,-1.0,-1.0,38.625,2.58742,2135,5860274 +854,search,3947.35,-1.0,-1.0,-1.0,-1.0,17.147,2.58742,2135,5860274 +855,insert,6.576,-1.0,-1.0,-1.0,-1.0,15.572,2.5901,2135,5866064 +856,search,3943.31,-1.0,-1.0,-1.0,-1.0,17.461,2.5901,2135,5866064 +857,insert,13.692,-1.0,-1.0,-1.0,-1.0,33.249,2.59613,2138,5884456 +858,search,3962.27,-1.0,-1.0,-1.0,-1.0,17.364,2.59613,2138,5884456 +859,insert,26.927,-1.0,-1.0,-1.0,-1.0,36.18,2.60838,2140,5916786 +860,search,3979.7,-1.0,-1.0,-1.0,-1.0,23.534,2.6086,2141,5916786 +861,insert,9.33,-1.0,-1.0,-1.0,-1.0,16.291,2.61104,2141,5923467 +862,search,3978.34,-1.0,-1.0,-1.0,-1.0,17.71,2.61104,2141,5923467 +863,insert,61.168,-1.0,-1.0,-1.0,-1.0,92.296,2.64545,2155,5997844 +864,search,3982.26,-1.0,-1.0,-1.0,-1.0,17.764,2.64545,2155,5997844 +865,insert,10.42,-1.0,-1.0,-1.0,-1.0,23.241,2.64949,2156,6009561 +866,search,3999.47,-1.0,-1.0,-1.0,-1.0,17.638,2.64949,2156,6009561 +867,insert,45.412,-1.0,-1.0,-1.0,-1.0,46.745,2.67117,2161,6052876 +868,search,4017.22,-1.0,-1.0,-1.0,-1.0,23.838,2.67117,2162,6052876 +869,insert,8.101,-1.0,-1.0,-1.0,-1.0,16.1,2.67471,2162,6059234 +870,search,4017.53,-1.0,-1.0,-1.0,-1.0,17.471,2.67471,2162,6059234 +871,insert,34.635,-1.0,-1.0,-1.0,-1.0,53.859,2.69263,2166,6098225 +872,search,4063.92,-1.0,-1.0,-1.0,-1.0,24.848,2.69275,2167,6098225 +873,insert,34.627,-1.0,-1.0,-1.0,-1.0,40.176,2.71042,2170,6137308 +874,search,4063.08,-1.0,-1.0,-1.0,-1.0,30.513,2.71081,2172,6137308 +875,insert,15.802,-1.0,-1.0,-1.0,-1.0,16.466,2.71695,2172,6154089 +876,search,4061.53,-1.0,-1.0,-1.0,-1.0,17.756,2.71695,2172,6154089 +877,insert,2.655,-1.0,-1.0,-1.0,-1.0,15.987,2.71772,2172,6156061 +878,search,4057.06,-1.0,-1.0,-1.0,-1.0,17.724,2.71772,2172,6156061 +879,insert,251.467,-1.0,-1.0,-1.0,-1.0,260.633,2.84859,2210,6446004 +880,search,4168.79,-1.0,-1.0,-1.0,-1.0,56.45,2.84927,2216,6446004 +881,insert,414.907,-1.0,-1.0,-1.0,-1.0,301.357,3.08195,2267,6969764 +882,search,4352.63,-1.0,-1.0,-1.0,-1.0,97.703,3.08313,2278,6969764 +883,insert,329.906,-1.0,-1.0,-1.0,-1.0,317.612,3.28167,2335,7422501 +884,search,4585.01,-1.0,-1.0,-1.0,-1.0,94.645,3.28267,2348,7422501 +885,insert,7.337,-1.0,-1.0,-1.0,-1.0,29.984,3.28745,2350,7432053 +886,search,4578.56,-1.0,-1.0,-1.0,-1.0,43.655,3.288,2354,7432053 +887,insert,217.588,-1.0,-1.0,-1.0,-1.0,281.088,3.4449,2401,7789886 +888,search,4729.92,-1.0,-1.0,-1.0,-1.0,71.877,3.44557,2410,7789886 +889,insert,288.136,-1.0,-1.0,-1.0,-1.0,498.68,3.63516,2490,8210883 +890,search,4955.14,-1.0,-1.0,-1.0,-1.0,96.843,3.63687,2505,8210883 +891,insert,25.681,-1.0,-1.0,-1.0,-1.0,24.458,3.64619,2507,8240504 +892,search,4960.08,-1.0,-1.0,-1.0,-1.0,14.239,3.64619,2507,8240504 +893,insert,40.868,-1.0,-1.0,-1.0,-1.0,72.547,3.671,2517,8299693 +894,search,4974.43,-1.0,-1.0,-1.0,-1.0,29.236,3.67115,2519,8299693 +895,insert,40.153,-1.0,-1.0,-1.0,-1.0,56.928,3.68918,2527,8349126 +896,search,5023.05,-1.0,-1.0,-1.0,-1.0,16.714,3.68918,2527,8349126 +897,delete,54.34,-1.0,-1.0,-1.0,-1.0,85.592,3.66987,2513,8268842 +898,search,4989.18,-1.0,-1.0,-1.0,-1.0,13.827,3.66987,2513,8268842 +899,delete,12.25,-1.0,-1.0,-1.0,-1.0,11.878,3.66947,2513,8250527 +900,search,4981.15,-1.0,-1.0,-1.0,-1.0,13.801,3.66947,2513,8250527 +901,delete,43.273,-1.0,-1.0,-1.0,-1.0,50.14,3.66015,2507,8185565 +902,search,4995.77,-1.0,-1.0,-1.0,-1.0,13.464,3.66015,2507,8185565 +903,delete,38.985,-1.0,-1.0,-1.0,-1.0,82.11,3.63926,2494,8126915 +904,search,4926.5,-1.0,-1.0,-1.0,-1.0,25.543,3.63944,2496,8126915 +905,delete,17.916,-1.0,-1.0,-1.0,-1.0,12.118,3.63862,2496,8098533 +906,search,4919.55,-1.0,-1.0,-1.0,-1.0,13.523,3.63862,2496,8098533 +907,delete,18.018,-1.0,-1.0,-1.0,-1.0,25.461,3.63339,2495,8074833 +908,search,4908.04,-1.0,-1.0,-1.0,-1.0,13.756,3.63339,2495,8074833 +909,delete,25.992,-1.0,-1.0,-1.0,-1.0,12.167,3.63226,2495,8035274 +910,search,4891.24,-1.0,-1.0,-1.0,-1.0,12.855,3.63226,2495,8035274 +911,delete,51.186,-1.0,-1.0,-1.0,-1.0,22.604,3.62708,2494,7957223 +912,search,4865.19,-1.0,-1.0,-1.0,-1.0,13.215,3.62708,2494,7957223 +913,delete,27.29,-1.0,-1.0,-1.0,-1.0,39.004,3.61696,2489,7917554 +914,search,4851.38,-1.0,-1.0,-1.0,-1.0,13.089,3.61696,2489,7917554 +915,delete,46.602,-1.0,-1.0,-1.0,-1.0,64.025,3.60208,2481,7861503 +916,search,4809.36,-1.0,-1.0,-1.0,-1.0,13.291,3.60208,2481,7861503 +917,delete,11.003,-1.0,-1.0,-1.0,-1.0,11.69,3.60108,2481,7844487 +918,search,4823.35,-1.0,-1.0,-1.0,-1.0,12.952,3.60108,2481,7844487 +919,delete,17.727,-1.0,-1.0,-1.0,-1.0,11.733,3.60098,2481,7814993 +920,search,4783.52,-1.0,-1.0,-1.0,-1.0,14.089,3.60098,2481,7814993 +921,delete,30.15,-1.0,-1.0,-1.0,-1.0,25.403,3.59704,2480,7793855 +922,search,4793.3,-1.0,-1.0,-1.0,-1.0,14.398,3.59704,2480,7793855 +923,delete,52.886,-1.0,-1.0,-1.0,-1.0,29.664,3.58739,2478,7736054 +924,search,4768.24,-1.0,-1.0,-1.0,-1.0,14.525,3.58739,2478,7736054 +925,delete,125.92,-1.0,-1.0,-1.0,-1.0,83.846,3.54062,2458,7587942 +926,search,4683.34,-1.0,-1.0,-1.0,-1.0,13.875,3.54062,2458,7587942 +927,delete,16.695,-1.0,-1.0,-1.0,-1.0,19.134,3.53829,2457,7565670 +928,search,4703.62,-1.0,-1.0,-1.0,-1.0,13.772,3.53829,2457,7565670 +929,delete,32.47,-1.0,-1.0,-1.0,-1.0,12.091,3.53503,2457,7529948 +930,search,4716.79,-1.0,-1.0,-1.0,-1.0,13.378,3.53503,2457,7529948 +931,delete,372.588,-1.0,-1.0,-1.0,-1.0,567.389,3.36339,2375,7016700 +932,search,4410.93,-1.0,-1.0,-1.0,-1.0,30.877,3.36376,2378,7016700 +933,delete,24.322,-1.0,-1.0,-1.0,-1.0,25.577,3.35517,2376,6980979 +934,search,4417.15,-1.0,-1.0,-1.0,-1.0,19.296,3.35538,2377,6980979 +935,delete,16.531,-1.0,-1.0,-1.0,-1.0,57.367,3.3443,2372,6954250 +936,search,4411.37,-1.0,-1.0,-1.0,-1.0,13.03,3.3443,2372,6954250 +937,delete,24.572,-1.0,-1.0,-1.0,-1.0,17.911,3.33598,2371,6924596 +938,search,4419.23,-1.0,-1.0,-1.0,-1.0,12.798,3.33598,2371,6924596 +939,delete,113.835,-1.0,-1.0,-1.0,-1.0,19.43,3.30926,2370,6784831 +940,search,4333.55,-1.0,-1.0,-1.0,-1.0,71.374,3.30987,2371,6784831 +941,delete,30.136,-1.0,-1.0,-1.0,-1.0,12.717,3.30247,2371,6745819 +942,search,4330.67,-1.0,-1.0,-1.0,-1.0,12.732,3.30247,2371,6745819 +943,delete,3.926,-1.0,-1.0,-1.0,-1.0,11.058,3.30147,2371,6740944 +944,search,4366.41,-1.0,-1.0,-1.0,-1.0,12.535,3.30147,2371,6740944 +945,delete,93.736,-1.0,-1.0,-1.0,-1.0,199.539,3.24394,2346,6627202 +946,search,4319.36,-1.0,-1.0,-1.0,-1.0,33.006,3.24433,2348,6627202 +947,delete,18.854,-1.0,-1.0,-1.0,-1.0,19.916,3.23765,2348,6607194 +948,search,4288.22,-1.0,-1.0,-1.0,-1.0,20.621,3.23765,2348,6607194 +949,delete,42.681,-1.0,-1.0,-1.0,-1.0,48.196,3.21805,2343,6551065 +950,search,4250.98,-1.0,-1.0,-1.0,-1.0,20.464,3.21805,2343,6551065 +951,delete,26.793,-1.0,-1.0,-1.0,-1.0,69.606,3.19618,2333,6506798 +952,search,4246.45,-1.0,-1.0,-1.0,-1.0,26.338,3.19622,2334,6506798 +953,delete,32.995,-1.0,-1.0,-1.0,-1.0,19.77,3.18555,2334,6459301 +954,search,4232.84,-1.0,-1.0,-1.0,-1.0,19.86,3.18555,2334,6459301 +955,delete,16.576,-1.0,-1.0,-1.0,-1.0,33.979,3.17548,2331,6440069 +956,search,4235.73,-1.0,-1.0,-1.0,-1.0,20.421,3.17548,2331,6440069 +957,delete,268.669,-1.0,-1.0,-1.0,-1.0,221.029,3.03147,2292,6132470 +958,search,4128.47,-1.0,-1.0,-1.0,-1.0,21.843,3.03147,2292,6132470 +959,delete,56.414,-1.0,-1.0,-1.0,-1.0,34.343,3.01371,2289,6086241 +960,search,4112.55,-1.0,-1.0,-1.0,-1.0,20.916,3.01371,2289,6086241 +961,delete,26.894,-1.0,-1.0,-1.0,-1.0,18.674,3.00781,2289,6064734 +962,search,4082.74,-1.0,-1.0,-1.0,-1.0,20.617,3.00781,2289,6064734 +963,delete,113.33,-1.0,-1.0,-1.0,-1.0,48.603,2.96432,2285,5967543 +964,search,4058.9,-1.0,-1.0,-1.0,-1.0,21.554,2.96432,2285,5967543 +965,delete,67.179,-1.0,-1.0,-1.0,-1.0,55.613,2.93602,2276,5906381 +966,search,4009.15,-1.0,-1.0,-1.0,-1.0,20.161,2.93602,2276,5906381 +967,delete,47,-1.0,-1.0,-1.0,-1.0,40.206,2.91287,2274,5851849 +968,search,3965.34,-1.0,-1.0,-1.0,-1.0,20.018,2.91287,2274,5851849 +969,delete,56.338,-1.0,-1.0,-1.0,-1.0,47.274,2.88004,2268,5783023 +970,search,3975.07,-1.0,-1.0,-1.0,-1.0,20.559,2.88004,2268,5783023 +971,delete,259.243,-1.0,-1.0,-1.0,-1.0,217.078,2.74538,2234,5496517 +972,search,3856.34,-1.0,-1.0,-1.0,-1.0,19.467,2.74538,2234,5496517 +973,delete,226.259,-1.0,-1.0,-1.0,-1.0,181.513,2.61317,2200,5237981 +974,search,3754.19,-1.0,-1.0,-1.0,-1.0,30.489,2.61359,2201,5237981 +975,delete,132.689,-1.0,-1.0,-1.0,-1.0,199.342,2.53579,2150,5099794 +976,search,3657.44,-1.0,-1.0,-1.0,-1.0,18.13,2.53579,2150,5099794 +977,delete,52.254,-1.0,-1.0,-1.0,-1.0,78.576,2.50271,2140,5032210 +978,search,3597.77,-1.0,-1.0,-1.0,-1.0,18.257,2.50271,2140,5032210 +979,delete,11.77,-1.0,-1.0,-1.0,-1.0,16.311,2.4986,2140,5020211 +980,search,3597.63,-1.0,-1.0,-1.0,-1.0,18.157,2.4986,2140,5020211 +981,delete,101.316,-1.0,-1.0,-1.0,-1.0,121.87,2.44055,2120,4902291 +982,search,3574.43,-1.0,-1.0,-1.0,-1.0,17.619,2.44055,2120,4902291 +983,delete,31.714,-1.0,-1.0,-1.0,-1.0,16.322,2.42956,2120,4872577 +984,search,3520.12,-1.0,-1.0,-1.0,-1.0,17.476,2.42956,2120,4872577 +985,delete,34.577,-1.0,-1.0,-1.0,-1.0,16.548,2.41712,2120,4836730 +986,search,3530.39,-1.0,-1.0,-1.0,-1.0,17.445,2.41712,2120,4836730 +987,delete,39.239,-1.0,-1.0,-1.0,-1.0,16.788,2.40173,2120,4797164 +988,search,3523.37,-1.0,-1.0,-1.0,-1.0,17.216,2.40173,2120,4797164 +989,delete,12.688,-1.0,-1.0,-1.0,-1.0,15.714,2.39672,2120,4784789 +990,search,3512.61,-1.0,-1.0,-1.0,-1.0,17.464,2.39672,2120,4784789 +991,delete,61.333,-1.0,-1.0,-1.0,-1.0,137.212,2.36068,2113,4705420 +992,search,3479.59,-1.0,-1.0,-1.0,-1.0,38.694,2.36111,2116,4705420 +993,delete,72.683,-1.0,-1.0,-1.0,-1.0,75.269,2.32074,2107,4621807 +994,search,3426.31,-1.0,-1.0,-1.0,-1.0,16.89,2.32074,2107,4621807 +995,delete,60.223,-1.0,-1.0,-1.0,-1.0,57.913,2.28682,2100,4555849 +996,search,3383.61,-1.0,-1.0,-1.0,-1.0,17.137,2.28682,2100,4555849 +997,delete,24.865,-1.0,-1.0,-1.0,-1.0,27.743,2.27883,2099,4526975 +998,search,3369.92,-1.0,-1.0,-1.0,-1.0,17.163,2.27883,2099,4526975 +999,delete,97.553,-1.0,-1.0,-1.0,-1.0,119.046,2.21803,2079,4408448 +1000,search,3337.43,-1.0,-1.0,-1.0,-1.0,24.001,2.21824,2080,4408448 +1001,delete,97.947,-1.0,-1.0,-1.0,-1.0,100.412,2.16241,2071,4303257 +1002,search,3308.61,-1.0,-1.0,-1.0,-1.0,16.534,2.16241,2071,4303257 +1003,delete,35.279,-1.0,-1.0,-1.0,-1.0,37.221,2.14267,2066,4259551 +1004,search,3302.25,-1.0,-1.0,-1.0,-1.0,18.01,2.14267,2066,4259551 +1005,delete,28.544,-1.0,-1.0,-1.0,-1.0,15.353,2.13484,2066,4241434 +1006,search,3311.45,-1.0,-1.0,-1.0,-1.0,17.801,2.13484,2066,4241434 +1007,delete,182.116,-1.0,-1.0,-1.0,-1.0,242.601,2.02889,2030,4052722 +1008,search,3194.32,-1.0,-1.0,-1.0,-1.0,16.656,2.02889,2030,4052722 +1009,delete,277.105,-1.0,-1.0,-1.0,-1.0,256.008,1.86127,1977,3738432 +1010,search,3108.32,-1.0,-1.0,-1.0,-1.0,16.132,1.86127,1977,3738432 +1011,delete,333.503,-1.0,-1.0,-1.0,-1.0,337.499,1.67368,1909,3372539 +1012,search,2908.62,-1.0,-1.0,-1.0,-1.0,30.483,1.6741,1911,3372539 +1013,delete,10.365,-1.0,-1.0,-1.0,-1.0,22.676,1.67003,1908,3361432 +1014,search,2890.78,-1.0,-1.0,-1.0,-1.0,15.703,1.67003,1908,3361432 +1015,delete,215.704,-1.0,-1.0,-1.0,-1.0,274.977,1.53089,1849,3091692 +1016,search,2787.27,-1.0,-1.0,-1.0,-1.0,14.74,1.53089,1849,3091692 +1017,delete,263.402,-1.0,-1.0,-1.0,-1.0,316.146,1.36839,1786,2775418 +1018,search,2667.98,-1.0,-1.0,-1.0,-1.0,13.975,1.36839,1786,2775418 +1019,delete,18.369,-1.0,-1.0,-1.0,-1.0,23.121,1.36161,1784,2754073 +1020,search,2657.13,-1.0,-1.0,-1.0,-1.0,13.978,1.36161,1784,2754073 +1021,delete,99.942,-1.0,-1.0,-1.0,-1.0,63.944,1.32029,1772,2638917 +1022,search,2588.93,-1.0,-1.0,-1.0,-1.0,14.016,1.32029,1772,2638917 +1023,delete,77.689,-1.0,-1.0,-1.0,-1.0,63.857,1.27759,1757,2556494 +1024,search,2544.89,-1.0,-1.0,-1.0,-1.0,13.691,1.27759,1757,2556494 +1025,insert,24.256,-1.0,-1.0,-1.0,-1.0,13.079,1.28391,1757,2594934 +1026,search,2559.97,-1.0,-1.0,-1.0,-1.0,13.665,1.28391,1757,2594934 +1027,insert,164.284,-1.0,-1.0,-1.0,-1.0,73.742,1.34641,1763,2802105 +1028,search,2667,-1.0,-1.0,-1.0,-1.0,46.404,1.34641,1767,2802105 +1029,insert,316.617,-1.0,-1.0,-1.0,-1.0,135.059,1.48286,1782,3203975 +1030,search,2801.09,-1.0,-1.0,-1.0,-1.0,80.573,1.48314,1792,3203975 +1031,insert,488.418,-1.0,-1.0,-1.0,-1.0,295.779,1.73736,1816,3830566 +1032,search,3054.76,-1.0,-1.0,-1.0,-1.0,219.237,1.7374,1832,3830566 +1033,insert,361.841,-1.0,-1.0,-1.0,-1.0,192.924,1.91549,1862,4252669 +1034,search,3168.26,-1.0,-1.0,-1.0,-1.0,158.765,1.91581,1879,4252669 +1035,insert,200.085,-1.0,-1.0,-1.0,-1.0,219.497,2.02898,1910,4512107 +1036,search,3249.86,-1.0,-1.0,-1.0,-1.0,111.507,2.02911,1921,4512107 +1037,insert,10.06,-1.0,-1.0,-1.0,-1.0,15.174,2.03309,1921,4525815 +1038,search,3276.04,-1.0,-1.0,-1.0,-1.0,63.792,2.03309,1927,4525815 +1039,insert,61.095,-1.0,-1.0,-1.0,-1.0,20.598,2.06378,1928,4599215 +1040,search,3269.32,-1.0,-1.0,-1.0,-1.0,24.904,2.06378,1929,4599215 +1041,insert,8.694,-1.0,-1.0,-1.0,-1.0,24.941,2.0689,1931,4609851 +1042,search,3317.33,-1.0,-1.0,-1.0,-1.0,30.998,2.06891,1933,4609851 +1043,insert,12.876,-1.0,-1.0,-1.0,-1.0,14.542,2.07427,1933,4627411 +1044,search,3308.72,-1.0,-1.0,-1.0,-1.0,15.658,2.07427,1933,4627411 +1045,insert,30.762,-1.0,-1.0,-1.0,-1.0,14.782,2.0881,1933,4670607 +1046,search,3333.84,-1.0,-1.0,-1.0,-1.0,15.787,2.0881,1933,4670607 +1047,insert,4.199,-1.0,-1.0,-1.0,-1.0,14.125,2.08948,1933,4675386 +1048,search,3337.16,-1.0,-1.0,-1.0,-1.0,15.687,2.08948,1933,4675386 +1049,insert,12.827,-1.0,-1.0,-1.0,-1.0,21.172,2.09667,1934,4689998 +1050,search,3344.12,-1.0,-1.0,-1.0,-1.0,15.514,2.09667,1934,4689998 +1051,insert,84.884,-1.0,-1.0,-1.0,-1.0,30.143,2.13925,1936,4783971 +1052,search,3378.78,-1.0,-1.0,-1.0,-1.0,15.918,2.13925,1936,4783971 +1053,insert,37.651,-1.0,-1.0,-1.0,-1.0,51.936,2.1552,1941,4826219 +1054,search,3395.83,-1.0,-1.0,-1.0,-1.0,15.92,2.1552,1941,4826219 +1055,insert,32.255,-1.0,-1.0,-1.0,-1.0,23.004,2.16716,1942,4862355 +1056,search,3406.04,-1.0,-1.0,-1.0,-1.0,15.751,2.16716,1942,4862355 +1057,insert,6.316,-1.0,-1.0,-1.0,-1.0,14.279,2.1693,1942,4869564 +1058,search,3417.97,-1.0,-1.0,-1.0,-1.0,16.269,2.1693,1942,4869564 +1059,insert,22.409,-1.0,-1.0,-1.0,-1.0,14.633,2.178,1942,4898570 +1060,search,3426.67,-1.0,-1.0,-1.0,-1.0,16.246,2.178,1942,4898570 +1061,insert,9.061,-1.0,-1.0,-1.0,-1.0,14.369,2.18172,1942,4907501 +1062,search,3448.35,-1.0,-1.0,-1.0,-1.0,16.3,2.18172,1942,4907501 +1063,insert,6.513,-1.0,-1.0,-1.0,-1.0,23.057,2.18486,1943,4916386 +1064,search,3436.82,-1.0,-1.0,-1.0,-1.0,16.309,2.18486,1943,4916386 +1065,insert,242.251,-1.0,-1.0,-1.0,-1.0,246.518,2.3081,1975,5194839 +1066,search,3541.35,-1.0,-1.0,-1.0,-1.0,51.719,2.30848,1980,5194839 +1067,insert,5.253,-1.0,-1.0,-1.0,-1.0,15.723,2.31046,1980,5200182 +1068,search,3545.49,-1.0,-1.0,-1.0,-1.0,16.871,2.31046,1980,5200182 +1069,insert,31.158,-1.0,-1.0,-1.0,-1.0,18.136,2.32362,1981,5230883 +1070,search,3548.81,-1.0,-1.0,-1.0,-1.0,18.361,2.32362,1981,5230883 +1071,insert,22.407,-1.0,-1.0,-1.0,-1.0,29.521,2.33429,1983,5255659 +1072,search,3561.56,-1.0,-1.0,-1.0,-1.0,15.87,2.33429,1983,5255659 +1073,insert,28.592,-1.0,-1.0,-1.0,-1.0,35.794,2.3465,1986,5281693 +1074,search,3553.37,-1.0,-1.0,-1.0,-1.0,15.897,2.3465,1986,5281693 +1075,insert,9.203,-1.0,-1.0,-1.0,-1.0,14.308,2.34925,1986,5290081 +1076,search,3577.94,-1.0,-1.0,-1.0,-1.0,16.034,2.34925,1986,5290081 +1077,insert,25.41,-1.0,-1.0,-1.0,-1.0,14.624,2.3593,1986,5316166 +1078,search,3563.77,-1.0,-1.0,-1.0,-1.0,16.123,2.3593,1986,5316166 +1079,insert,20.262,-1.0,-1.0,-1.0,-1.0,23.153,2.36712,1988,5335531 +1080,search,3607.09,-1.0,-1.0,-1.0,-1.0,16.149,2.36712,1988,5335531 +1081,insert,309.569,-1.0,-1.0,-1.0,-1.0,154.791,2.51222,2007,5678781 +1082,search,3719.57,-1.0,-1.0,-1.0,-1.0,65.543,2.51282,2013,5678781 +1083,insert,0.752,-1.0,-1.0,-1.0,-1.0,15.909,2.51296,2013,5679848 +1084,search,3713.86,-1.0,-1.0,-1.0,-1.0,30.341,2.51296,2015,5679848 +1085,insert,22.225,-1.0,-1.0,-1.0,-1.0,34.215,2.52357,2019,5702205 +1086,search,3723.43,-1.0,-1.0,-1.0,-1.0,16.423,2.52357,2019,5702205 +1087,insert,13.901,-1.0,-1.0,-1.0,-1.0,26.36,2.5294,2021,5715514 +1088,search,3751.79,-1.0,-1.0,-1.0,-1.0,16.456,2.5294,2021,5715514 +1089,insert,342.18,-1.0,-1.0,-1.0,-1.0,279.305,2.70256,2075,6103499 +1090,search,3925.18,-1.0,-1.0,-1.0,-1.0,39.957,2.70316,2079,6103499 +1091,insert,38.646,-1.0,-1.0,-1.0,-1.0,43.807,2.72632,2086,6152505 +1092,search,3922.3,-1.0,-1.0,-1.0,-1.0,21.741,2.72641,2087,6152505 +1093,insert,13.249,-1.0,-1.0,-1.0,-1.0,26.781,2.73407,2089,6171104 +1094,search,3938.37,-1.0,-1.0,-1.0,-1.0,17.327,2.73407,2089,6171104 +1095,insert,273.956,-1.0,-1.0,-1.0,-1.0,348.398,2.90646,2144,6557850 +1096,search,4157.27,-1.0,-1.0,-1.0,-1.0,91.524,2.9065,2159,6557850 +1097,insert,15.198,-1.0,-1.0,-1.0,-1.0,18.135,2.91552,2159,6579289 +1098,search,4156.07,-1.0,-1.0,-1.0,-1.0,20.662,2.91552,2159,6579289 +1099,insert,10.807,-1.0,-1.0,-1.0,-1.0,17.967,2.9208,2159,6591442 +1100,search,4147.98,-1.0,-1.0,-1.0,-1.0,34.27,2.92117,2161,6591442 +1101,insert,30.376,-1.0,-1.0,-1.0,-1.0,46.122,2.93832,2166,6633739 +1102,search,4172.95,-1.0,-1.0,-1.0,-1.0,47.11,2.93886,2169,6633739 +1103,insert,73.701,-1.0,-1.0,-1.0,-1.0,101.663,2.98088,2187,6724357 +1104,search,4222.64,-1.0,-1.0,-1.0,-1.0,33.104,2.98095,2190,6724357 +1105,insert,1.885,-1.0,-1.0,-1.0,-1.0,25.257,2.98188,2191,6727059 +1106,search,4237.6,-1.0,-1.0,-1.0,-1.0,19.547,2.98188,2191,6727059 +1107,insert,168.696,-1.0,-1.0,-1.0,-1.0,181.969,3.0767,2214,6950869 +1108,search,4348.06,-1.0,-1.0,-1.0,-1.0,48.508,3.07793,2218,6950869 +1109,insert,5.019,-1.0,-1.0,-1.0,-1.0,25.102,3.08113,2219,6958173 +1110,search,4354.77,-1.0,-1.0,-1.0,-1.0,19.775,3.08113,2219,6958173 +1111,insert,40.06,-1.0,-1.0,-1.0,-1.0,41.543,3.09709,2224,6995273 +1112,search,4359.35,-1.0,-1.0,-1.0,-1.0,18.348,3.09709,2224,6995273 +1113,insert,40.607,-1.0,-1.0,-1.0,-1.0,81.859,3.1203,2236,7047862 +1114,search,4367.77,-1.0,-1.0,-1.0,-1.0,19.52,3.1203,2236,7047862 +1115,insert,52.365,-1.0,-1.0,-1.0,-1.0,64.424,3.15047,2245,7110727 +1116,search,4403.92,-1.0,-1.0,-1.0,-1.0,34.932,3.15087,2247,7110727 +1117,insert,13.609,-1.0,-1.0,-1.0,-1.0,25.364,3.15789,2248,7125591 +1118,search,4417.1,-1.0,-1.0,-1.0,-1.0,19.62,3.15789,2248,7125591 +1119,insert,6.412,-1.0,-1.0,-1.0,-1.0,23.022,3.1606,2249,7134442 +1120,search,4418.74,-1.0,-1.0,-1.0,-1.0,19.07,3.1606,2249,7134442 +1121,insert,25.944,-1.0,-1.0,-1.0,-1.0,39.688,3.17245,2252,7162109 +1122,search,4435.58,-1.0,-1.0,-1.0,-1.0,19.138,3.17245,2252,7162109 +1123,insert,15.439,-1.0,-1.0,-1.0,-1.0,46.499,3.17919,2257,7179509 +1124,search,4448.59,-1.0,-1.0,-1.0,-1.0,19.41,3.17919,2257,7179509 +1125,insert,6.475,-1.0,-1.0,-1.0,-1.0,35.354,3.18278,2259,7186372 +1126,search,4418.67,-1.0,-1.0,-1.0,-1.0,19.508,3.18278,2259,7186372 +1127,insert,31.978,-1.0,-1.0,-1.0,-1.0,47.127,3.20096,2264,7231450 +1128,search,4436.28,-1.0,-1.0,-1.0,-1.0,24.006,3.20106,2265,7231450 +1129,insert,8.947,-1.0,-1.0,-1.0,-1.0,22.598,3.20514,2266,7243215 +1130,search,4418.65,-1.0,-1.0,-1.0,-1.0,19.51,3.20514,2266,7243215 +1131,insert,30.431,-1.0,-1.0,-1.0,-1.0,50.219,3.2193,2270,7276107 +1132,search,4509.72,-1.0,-1.0,-1.0,-1.0,33.661,3.21953,2272,7276107 +1133,insert,11.423,-1.0,-1.0,-1.0,-1.0,36.534,3.22445,2276,7292577 +1134,search,4505.35,-1.0,-1.0,-1.0,-1.0,21.572,3.22445,2276,7292577 +1135,insert,7.718,-1.0,-1.0,-1.0,-1.0,26.536,3.22757,2277,7299867 +1136,search,4487.89,-1.0,-1.0,-1.0,-1.0,20.785,3.22757,2277,7299867 +1137,insert,25.986,-1.0,-1.0,-1.0,-1.0,26.812,3.23948,2278,7323342 +1138,search,4495.65,-1.0,-1.0,-1.0,-1.0,21.289,3.23948,2278,7323342 +1139,insert,3.293,-1.0,-1.0,-1.0,-1.0,18.499,3.24115,2278,7325844 +1140,search,4477.02,-1.0,-1.0,-1.0,-1.0,20.832,3.24115,2278,7325844 +1141,insert,203.12,-1.0,-1.0,-1.0,-1.0,238.708,3.33584,2309,7540761 +1142,search,4625.04,-1.0,-1.0,-1.0,-1.0,106.429,3.33736,2325,7540761 +1143,insert,16.621,-1.0,-1.0,-1.0,-1.0,33.945,3.34502,2328,7561926 +1144,search,4624.92,-1.0,-1.0,-1.0,-1.0,33.659,3.34516,2330,7561926 +1145,insert,9.998,-1.0,-1.0,-1.0,-1.0,34.084,3.3514,2332,7571861 +1146,search,4618.59,-1.0,-1.0,-1.0,-1.0,20.459,3.3514,2332,7571861 +1147,insert,18.524,-1.0,-1.0,-1.0,-1.0,37.566,3.35984,2335,7590881 +1148,search,4628.68,-1.0,-1.0,-1.0,-1.0,20.614,3.35984,2335,7590881 +1149,insert,6.876,-1.0,-1.0,-1.0,-1.0,36.354,3.36316,2338,7595507 +1150,search,4626.14,-1.0,-1.0,-1.0,-1.0,19.783,3.36316,2338,7595507 +1151,insert,5.569,-1.0,-1.0,-1.0,-1.0,24.519,3.36604,2339,7601664 +1152,search,4650.81,-1.0,-1.0,-1.0,-1.0,20.516,3.36604,2339,7601664 +1153,delete,21.749,-1.0,-1.0,-1.0,-1.0,19.336,3.36592,2339,7566707 +1154,search,4655.94,-1.0,-1.0,-1.0,-1.0,26.968,3.3662,2340,7566707 +1155,delete,109.228,-1.0,-1.0,-1.0,-1.0,124.225,3.33158,2328,7396192 +1156,search,4579.35,-1.0,-1.0,-1.0,-1.0,25.542,3.33161,2329,7396192 +1157,delete,220.83,-1.0,-1.0,-1.0,-1.0,269.421,3.25667,2285,7079041 +1158,search,4406.15,-1.0,-1.0,-1.0,-1.0,31.422,3.25696,2287,7079041 +1159,delete,322.134,-1.0,-1.0,-1.0,-1.0,386.976,3.08627,2207,6597115 +1160,search,4214.2,-1.0,-1.0,-1.0,-1.0,40.336,3.08714,2210,6597115 +1161,delete,193.043,-1.0,-1.0,-1.0,-1.0,168.425,2.99789,2179,6308902 +1162,search,4102.39,-1.0,-1.0,-1.0,-1.0,45.345,2.99849,2183,6308902 +1163,delete,87.126,-1.0,-1.0,-1.0,-1.0,143.403,2.9441,2164,6164514 +1164,search,4048.69,-1.0,-1.0,-1.0,-1.0,18.428,2.9441,2164,6164514 +1165,delete,30.316,-1.0,-1.0,-1.0,-1.0,34.502,2.93712,2162,6133412 +1166,search,4031.4,-1.0,-1.0,-1.0,-1.0,18.251,2.93712,2162,6133412 +1167,delete,53.048,-1.0,-1.0,-1.0,-1.0,18.145,2.92794,2162,6057218 +1168,search,4010.29,-1.0,-1.0,-1.0,-1.0,18.017,2.92794,2162,6057218 +1169,delete,17.178,-1.0,-1.0,-1.0,-1.0,63.375,2.914,2153,6028053 +1170,search,3984.79,-1.0,-1.0,-1.0,-1.0,20.121,2.914,2153,6028053 +1171,delete,30.828,-1.0,-1.0,-1.0,-1.0,17.314,2.90886,2153,6001880 +1172,search,3994.51,-1.0,-1.0,-1.0,-1.0,35.878,2.90904,2154,6001880 +1173,delete,38.68,-1.0,-1.0,-1.0,-1.0,28.453,2.89821,2152,5957180 +1174,search,3977.56,-1.0,-1.0,-1.0,-1.0,19.192,2.89821,2152,5957180 +1175,delete,18.652,-1.0,-1.0,-1.0,-1.0,16.804,2.89533,2152,5939326 +1176,search,3967.79,-1.0,-1.0,-1.0,-1.0,18.757,2.89533,2152,5939326 +1177,delete,14.312,-1.0,-1.0,-1.0,-1.0,38.737,2.8882,2150,5924145 +1178,search,3959.46,-1.0,-1.0,-1.0,-1.0,18.533,2.8882,2150,5924145 +1179,delete,57.727,-1.0,-1.0,-1.0,-1.0,69.829,2.86292,2141,5863334 +1180,search,3933.78,-1.0,-1.0,-1.0,-1.0,18.593,2.86292,2141,5863334 +1181,delete,40.985,-1.0,-1.0,-1.0,-1.0,18.509,2.8509,2140,5822703 +1182,search,3914.16,-1.0,-1.0,-1.0,-1.0,18.824,2.8509,2140,5822703 +1183,delete,37.206,-1.0,-1.0,-1.0,-1.0,31.597,2.84032,2138,5775807 +1184,search,3894.3,-1.0,-1.0,-1.0,-1.0,18.524,2.84032,2138,5775807 +1185,delete,24.606,-1.0,-1.0,-1.0,-1.0,16.228,2.83442,2138,5750627 +1186,search,3889.06,-1.0,-1.0,-1.0,-1.0,18.596,2.83442,2138,5750627 +1187,delete,136.731,-1.0,-1.0,-1.0,-1.0,120.802,2.78281,2122,5601207 +1188,search,3819.52,-1.0,-1.0,-1.0,-1.0,17.826,2.78281,2122,5601207 +1189,delete,13.763,-1.0,-1.0,-1.0,-1.0,23.318,2.78016,2121,5586425 +1190,search,3800.19,-1.0,-1.0,-1.0,-1.0,17.885,2.78016,2121,5586425 +1191,delete,11.82,-1.0,-1.0,-1.0,-1.0,16.395,2.7775,2121,5574023 +1192,search,3797.52,-1.0,-1.0,-1.0,-1.0,17.616,2.7775,2121,5574023 +1193,delete,182.738,-1.0,-1.0,-1.0,-1.0,180.278,2.67898,2099,5356157 +1194,search,3720.32,-1.0,-1.0,-1.0,-1.0,17.315,2.67898,2099,5356157 +1195,delete,15.6,-1.0,-1.0,-1.0,-1.0,15.607,2.67511,2099,5340404 +1196,search,3709.55,-1.0,-1.0,-1.0,-1.0,17.549,2.67511,2099,5340404 +1197,delete,47.874,-1.0,-1.0,-1.0,-1.0,16.469,2.65454,2099,5293259 +1198,search,3695.3,-1.0,-1.0,-1.0,-1.0,17.603,2.65454,2099,5293259 +1199,delete,10.609,-1.0,-1.0,-1.0,-1.0,15.822,2.65038,2099,5279787 +1200,search,3685.54,-1.0,-1.0,-1.0,-1.0,17.129,2.65038,2099,5279787 +1201,delete,58.635,-1.0,-1.0,-1.0,-1.0,35.241,2.62174,2096,5216567 +1202,search,3672.99,-1.0,-1.0,-1.0,-1.0,17.322,2.62174,2096,5216567 +1203,delete,19.701,-1.0,-1.0,-1.0,-1.0,15.418,2.61555,2096,5196947 +1204,search,3647.3,-1.0,-1.0,-1.0,-1.0,17.187,2.61555,2096,5196947 +1205,delete,23.253,-1.0,-1.0,-1.0,-1.0,32.809,2.60219,2094,5168447 +1206,search,3655.98,-1.0,-1.0,-1.0,-1.0,17.389,2.60219,2094,5168447 +1207,delete,18.295,-1.0,-1.0,-1.0,-1.0,37.741,2.59118,2091,5146927 +1208,search,3643.07,-1.0,-1.0,-1.0,-1.0,16.896,2.59118,2091,5146927 +1209,delete,215.407,-1.0,-1.0,-1.0,-1.0,213.528,2.45995,2057,4887612 +1210,search,3544.02,-1.0,-1.0,-1.0,-1.0,16.389,2.45995,2057,4887612 +1211,delete,7.185,-1.0,-1.0,-1.0,-1.0,14.998,2.45882,2057,4882301 +1212,search,3520.1,-1.0,-1.0,-1.0,-1.0,16.66,2.45882,2057,4882301 +1213,delete,100.927,-1.0,-1.0,-1.0,-1.0,63.868,2.41015,2050,4780915 +1214,search,3497.43,-1.0,-1.0,-1.0,-1.0,22.322,2.4103,2051,4780915 +1215,delete,12.879,-1.0,-1.0,-1.0,-1.0,15.496,2.40503,2051,4765751 +1216,search,3496.5,-1.0,-1.0,-1.0,-1.0,18.593,2.40503,2051,4765751 +1217,delete,315.641,-1.0,-1.0,-1.0,-1.0,175.292,2.24924,2009,4452306 +1218,search,3347.99,-1.0,-1.0,-1.0,-1.0,16.69,2.24924,2009,4452306 +1219,delete,91.26,-1.0,-1.0,-1.0,-1.0,109.276,2.19599,1988,4362045 +1220,search,3297.39,-1.0,-1.0,-1.0,-1.0,22.595,2.19619,1989,4362045 +1221,delete,35.976,-1.0,-1.0,-1.0,-1.0,18.362,2.18109,1987,4326917 +1222,search,3300.6,-1.0,-1.0,-1.0,-1.0,16.671,2.18109,1987,4326917 +1223,delete,195.278,-1.0,-1.0,-1.0,-1.0,181.716,2.06626,1949,4109781 +1224,search,3184.78,-1.0,-1.0,-1.0,-1.0,15.91,2.06626,1949,4109781 +1225,delete,30.747,-1.0,-1.0,-1.0,-1.0,20.406,2.05133,1947,4075879 +1226,search,3184.73,-1.0,-1.0,-1.0,-1.0,16.039,2.05133,1947,4075879 +1227,delete,67.69,-1.0,-1.0,-1.0,-1.0,61.668,2.01878,1939,4006503 +1228,search,3164.25,-1.0,-1.0,-1.0,-1.0,16.088,2.01878,1939,4006503 +1229,delete,84.219,-1.0,-1.0,-1.0,-1.0,63.244,1.9773,1932,3922625 +1230,search,3088.49,-1.0,-1.0,-1.0,-1.0,25.433,1.97747,1933,3922625 +1231,delete,74.742,-1.0,-1.0,-1.0,-1.0,117.956,1.92966,1902,3844860 +1232,search,3065.21,-1.0,-1.0,-1.0,-1.0,15.64,1.92966,1902,3844860 +1233,delete,14.49,-1.0,-1.0,-1.0,-1.0,16.067,1.92406,1901,3831414 +1234,search,3064.19,-1.0,-1.0,-1.0,-1.0,15.097,1.92406,1901,3831414 +1235,delete,129.644,-1.0,-1.0,-1.0,-1.0,95.767,1.85289,1888,3684234 +1236,search,3005.56,-1.0,-1.0,-1.0,-1.0,14.994,1.85289,1888,3684234 +1237,delete,14.282,-1.0,-1.0,-1.0,-1.0,28.324,1.84496,1886,3669074 +1238,search,2997.69,-1.0,-1.0,-1.0,-1.0,15.248,1.84496,1886,3669074 +1239,delete,49.939,-1.0,-1.0,-1.0,-1.0,14.975,1.82546,1886,3620984 +1240,search,2965.83,-1.0,-1.0,-1.0,-1.0,15.171,1.82546,1886,3620984 +1241,delete,36.557,-1.0,-1.0,-1.0,-1.0,14.228,1.81003,1886,3580113 +1242,search,2974.87,-1.0,-1.0,-1.0,-1.0,15.06,1.81003,1886,3580113 +1243,delete,44.861,-1.0,-1.0,-1.0,-1.0,21.438,1.78671,1884,3533909 +1244,search,2962.76,-1.0,-1.0,-1.0,-1.0,15.22,1.78671,1884,3533909 +1245,delete,14.323,-1.0,-1.0,-1.0,-1.0,13.523,1.78114,1884,3519590 +1246,search,2972.72,-1.0,-1.0,-1.0,-1.0,15.034,1.78114,1884,3519590 +1247,delete,51.658,-1.0,-1.0,-1.0,-1.0,73.2,1.7424,1859,3452537 +1248,search,2887.78,-1.0,-1.0,-1.0,-1.0,20.57,1.74251,1860,3452537 +1249,delete,30.881,-1.0,-1.0,-1.0,-1.0,26.128,1.72742,1858,3416711 +1250,search,2894.84,-1.0,-1.0,-1.0,-1.0,15.165,1.72742,1858,3416711 +1251,delete,28.64,-1.0,-1.0,-1.0,-1.0,25.136,1.71167,1855,3387653 +1252,search,2883.91,-1.0,-1.0,-1.0,-1.0,14.833,1.71167,1855,3387653 +1253,delete,15.4,-1.0,-1.0,-1.0,-1.0,24.615,1.70221,1852,3372157 +1254,search,2881.78,-1.0,-1.0,-1.0,-1.0,14.998,1.70221,1852,3372157 +1255,delete,88.457,-1.0,-1.0,-1.0,-1.0,94.758,1.6406,1820,3258215 +1256,search,2808.56,-1.0,-1.0,-1.0,-1.0,14.583,1.6406,1820,3258215 +1257,delete,31.328,-1.0,-1.0,-1.0,-1.0,22.694,1.62331,1816,3223657 +1258,search,2804.01,-1.0,-1.0,-1.0,-1.0,14.463,1.62331,1816,3223657 +1259,delete,41.746,-1.0,-1.0,-1.0,-1.0,36.191,1.59989,1812,3174982 +1260,search,2784.41,-1.0,-1.0,-1.0,-1.0,14.297,1.59989,1812,3174982 +1261,delete,19.12,-1.0,-1.0,-1.0,-1.0,16.33,1.59425,1811,3157038 +1262,search,2785.41,-1.0,-1.0,-1.0,-1.0,14.584,1.59425,1811,3157038 +1263,delete,88.272,-1.0,-1.0,-1.0,-1.0,86.091,1.53981,1799,3058847 +1264,search,2739.79,-1.0,-1.0,-1.0,-1.0,14.46,1.53981,1799,3058847 +1265,delete,210.687,-1.0,-1.0,-1.0,-1.0,217.952,1.40858,1715,2833531 +1266,search,2618.36,-1.0,-1.0,-1.0,-1.0,13.593,1.40858,1715,2833531 +1267,delete,56.445,-1.0,-1.0,-1.0,-1.0,42.216,1.38481,1710,2763793 +1268,search,2585.29,-1.0,-1.0,-1.0,-1.0,25.943,1.38504,1711,2763793 +1269,delete,137.857,-1.0,-1.0,-1.0,-1.0,127.213,1.31658,1686,2627384 +1270,search,2541.92,-1.0,-1.0,-1.0,-1.0,22.194,1.31668,1687,2627384 +1271,delete,72.076,-1.0,-1.0,-1.0,-1.0,61.256,1.28315,1679,2553476 +1272,search,2493.15,-1.0,-1.0,-1.0,-1.0,13.681,1.28315,1679,2553476 +1273,delete,100.724,-1.0,-1.0,-1.0,-1.0,88.624,1.22821,1657,2443067 +1274,search,2442.47,-1.0,-1.0,-1.0,-1.0,13.613,1.22821,1657,2443067 +1275,delete,24.681,-1.0,-1.0,-1.0,-1.0,22.354,1.21897,1655,2421654 +1276,search,2441.77,-1.0,-1.0,-1.0,-1.0,13.509,1.21897,1655,2421654 +1277,delete,33.139,-1.0,-1.0,-1.0,-1.0,28.164,1.20626,1650,2391396 +1278,search,2428.91,-1.0,-1.0,-1.0,-1.0,13.746,1.20626,1650,2391396 +1279,delete,28.119,-1.0,-1.0,-1.0,-1.0,11.755,1.19801,1650,2367854 +1280,search,2429.23,-1.0,-1.0,-1.0,-1.0,12.971,1.19801,1650,2367854 diff --git a/scripts/big_ann_perf_numbers/perf_debug_scan_0.13_worker_batch_clustering_score.csv b/scripts/big_ann_perf_numbers/perf_debug_scan_0.13_worker_batch_clustering_score.csv new file mode 100644 index 00000000..0bf01d50 --- /dev/null +++ b/scripts/big_ann_perf_numbers/perf_debug_scan_0.13_worker_batch_clustering_score.csv @@ -0,0 +1,21 @@ +step_num,step_type,latency_ms,worker_partition_size,worker_scan_time_ms,worker_scan_throughput,worker_result_time_ms,measured_ipc,cache_miss_rate,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors,silleheoute_mean,silleheoute_std_dev +2,search,2858.61,37.306,293.561,0.540552,482.29,-nan,-nan,0.83112,0.139674,-1.0,-1.0,7.176,0.0184881,1000,38806,0.0481552,0.0764215, +3,insert,10.949,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,30.58,0.0226893,995,49334,0.031224,0.0782707, +4,search,2353.83,65.4907,153.473,0.949213,157.785,-nan,-nan,0.84438,0.139792,-1.0,-1.0,38.521,0.0226836,988,49334,0.0306269,0.0786408, +5,insert,12.173,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,220.632,0.0330921,973,73557,0.013662,0.0728127, +6,search,1730.38,198.808,181.446,1.80123,64.426,-nan,-nan,0.91347,0.1126,-1.0,-1.0,95.583,0.0331427,974,73557,0.0136931,0.0717552, +7,insert,11.947,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,96.442,0.0428274,975,95766,0.00350572,0.0694801, +8,search,1593.52,249.649,193.196,1.62546,63.1557,-nan,-nan,0.91131,0.113758,-1.0,-1.0,20.107,0.0433047,979,95766,0.00321126,0.0697775, +9,insert,2.648,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,3.903,0.0450563,979,100025,0.00304833,0.0705768, +10,search,1736.06,253.4,176.331,1.28113,83.1989,-nan,-nan,0.91187,0.112315,-1.0,-1.0,11.029,0.0451117,983,100025,0.00274555,0.0688998, +11,insert,4.311,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,3.908,0.0499416,983,111417,-0.00256859,0.0683584, +12,search,1735.05,294.676,199.471,1.29274,73.8212,-nan,-nan,0.91046,0.11371,-1.0,-1.0,9.604,0.0500551,987,111417,-0.000659093,0.0707141, +13,insert,17.895,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,6.317,0.0610503,987,137139,-0.0030937,0.0635938, +14,search,1812.33,329.309,219.583,1.26896,77.0092,-nan,-nan,0.90656,0.116661,-1.0,-1.0,7.204,0.0610641,989,137139,-0.00323246,0.0641389, +15,insert,7.43,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,7.311,0.0695962,992,156710,-0.00404804,0.0625114, +16,search,1767.5,360.375,230.702,1.23382,71.1503,-nan,-nan,0.90344,0.11765,-1.0,-1.0,4.044,0.0695962,992,156710,-0.0021428,0.0635274, +17,insert,2.147,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,7.466,0.0718965,995,161773,-0.00578649,0.063447, +18,search,1775.58,375.119,239.23,1.14144,79.0919,-nan,-nan,0.90537,0.116405,-1.0,-1.0,6.807,0.0719006,996,161773,-0.0057422,0.0635047, +19,insert,19.104,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,5.616,0.0785021,997,177161,-0.00673169,0.0620932, +20,search,1800.4,415.047,261.647,1.20258,77.4051,-nan,-nan,0.91005,0.112435,-1.0,-1.0,5.578,0.0785021,998,177161,-0.00611008,0.0623675, +21,insert,189.577,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,103.458,0.201671,1046,448898,-0.00948841,0.0511418, diff --git a/scripts/big_ann_perf_numbers/perf_debug_scan_0.14_tuning.csv b/scripts/big_ann_perf_numbers/perf_debug_scan_0.14_tuning.csv new file mode 100644 index 00000000..95f3c854 --- /dev/null +++ b/scripts/big_ann_perf_numbers/perf_debug_scan_0.14_tuning.csv @@ -0,0 +1,651 @@ +step_num,step_type,latency_ms,worker_partition_size,worker_scan_time_ms,worker_scan_throughput,worker_result_time_ms,measured_ipc,cache_miss_rate,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,3119.98,37.4251,337.059,0.542339,590.306,-nan,-nan,0.844,0.1346,-1.0,-1.0,19.116,0.0184881,1000,38806 +3,insert,24.997,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.207,0.022655,996,49334 +4,search,2478.03,65.1954,172.243,1.02629,182.991,-nan,-nan,0.8558,0.133903,-1.0,-1.0,11.045,0.0226615,993,49334 +5,insert,16.566,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,161.104,0.033017,984,73557 +6,search,2312.4,190.054,188.271,1.6293,86.4517,-nan,-nan,0.91846,0.108847,-1.0,-1.0,140.844,0.0330521,985,73557 +7,insert,23.37,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,144.286,0.0425581,986,95766 +8,search,2398.93,238.37,199.126,1.46738,96.1916,-nan,-nan,0.9187,0.10833,-1.0,-1.0,92.957,0.0428004,989,95766 +9,insert,5.936,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,74.015,0.044627,989,100025 +10,search,2363.99,245.311,196.076,1.3594,98.7188,-nan,-nan,0.91993,0.105737,-1.0,-1.0,19.649,0.0450612,993,100025 +11,insert,8.451,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,11.19,0.049827,996,111417 +12,search,2408.5,279.141,205.534,1.18839,114.879,-nan,-nan,0.9178,0.108016,-1.0,-1.0,15.176,0.0499861,998,111417 +13,insert,19.103,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.188,0.0613151,1000,137139 +14,search,2438.46,315.833,232.396,1.16178,113.06,-nan,-nan,0.91348,0.110604,-1.0,-1.0,13.984,0.0613273,1003,137139 +15,insert,15.059,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,6.234,0.0698504,1003,156710 +16,search,2455.44,348.477,247.424,1.12924,104.21,-nan,-nan,0.91087,0.110722,-1.0,-1.0,9.026,0.06985,1004,156710 +17,insert,3.621,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,10.662,0.071841,1006,161773 +18,search,2430.87,359.776,247.507,1.13628,103.595,-nan,-nan,0.91165,0.110745,-1.0,-1.0,9.143,0.0718998,1007,161773 +19,insert,31.058,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.149,0.078733,1008,177161 +20,search,2466.25,397.871,267.363,1.1276,99.2325,-nan,-nan,0.91665,0.106402,-1.0,-1.0,9.216,0.078733,1009,177161 +21,insert,319.958,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,160.932,0.200572,1054,448898 +22,search,2680.99,804.423,477.305,1.05132,74.368,-nan,-nan,0.9029,0.113107,-1.0,-1.0,33.824,0.20057,1062,448898 +23,insert,389.466,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,252.388,0.328638,1114,733493 +24,search,2932.31,1115.83,655.345,1.02094,74.2773,-nan,-nan,0.89851,0.115125,-1.0,-1.0,71.309,0.328826,1127,733493 +25,insert,41.606,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.183,0.345294,1132,777443 +26,search,2982.07,1110.67,671.389,1.02779,79.1616,-nan,-nan,0.89765,0.115252,-1.0,-1.0,16.067,0.345306,1133,777443 +27,insert,31.667,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.746,0.356747,1134,805394 +28,search,2995.52,1139.21,686.448,0.989931,73.8757,-nan,-nan,0.89895,0.114194,-1.0,-1.0,12.743,0.35675,1135,805394 +29,insert,57.582,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,54.835,0.385014,1146,866391 +30,search,3045.4,1189.76,715.013,1.02242,68.7515,-nan,-nan,0.9013,0.113408,-1.0,-1.0,27.509,0.385013,1149,866391 +31,insert,455.223,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,345.366,0.553921,1213,1239816 +32,search,3320.02,1450.6,886.765,1.02501,75.4543,-nan,-nan,0.89498,0.117695,-1.0,-1.0,83.559,0.554228,1222,1239816 +33,insert,46.043,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,32.806,0.574066,1228,1291211 +34,search,3348.02,1457.37,891.257,1.01373,83.791,-nan,-nan,0.89479,0.117626,-1.0,-1.0,57.237,0.574272,1233,1291211 +35,insert,26.905,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,31.385,0.584538,1240,1324978 +36,search,3389.87,1474.14,903.984,0.987988,82.2582,-nan,-nan,0.89579,0.116858,-1.0,-1.0,23.963,0.584563,1243,1324978 +37,insert,42.242,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,39.834,0.604534,1249,1371885 +38,search,3452.11,1493.59,928.366,1.0299,81.4936,-nan,-nan,0.89552,0.116885,-1.0,-1.0,21.07,0.604534,1250,1371885 +39,insert,12.852,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,18.392,0.607423,1252,1379335 +40,search,3478.87,1492.43,941.284,0.95729,83.309,-nan,-nan,0.89597,0.117495,-1.0,-1.0,10.253,0.607423,1252,1379335 +41,insert,59.421,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.162,0.623012,1256,1413312 +42,search,3483.96,1517.76,959.933,0.989305,91.1776,-nan,-nan,0.89617,0.117625,-1.0,-1.0,9.987,0.623012,1256,1413312 +43,insert,14.848,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.413,0.62718,1257,1423411 +44,search,3519.87,1530.62,983.339,0.988673,75.9508,-nan,-nan,0.89649,0.117523,-1.0,-1.0,9.674,0.62718,1257,1423411 +45,insert,227.833,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.689,0.642324,1260,1457559 +46,search,3563.34,1559.9,983.746,1.03183,91.352,-nan,-nan,0.89822,0.115715,-1.0,-1.0,9.425,0.642324,1260,1457559 +47,insert,278.553,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,249.553,0.746998,1293,1688593 +48,search,3699.72,1691.59,1072.3,1.09593,86.8992,-nan,-nan,0.89996,0.115395,-1.0,-1.0,68.501,0.746997,1299,1688593 +49,insert,36.238,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.268,0.761762,1303,1723505 +50,search,3744.74,1700.35,1079.53,1.00068,89.5637,-nan,-nan,0.90036,0.11539,-1.0,-1.0,25.414,0.761763,1305,1723505 +51,insert,330.509,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,230.942,0.902099,1343,2035040 +52,search,3958.39,1892,1220.76,0.991052,92.724,-nan,-nan,0.90343,0.114527,-1.0,-1.0,39.714,0.902122,1346,2035040 +53,insert,48.276,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,63.876,0.921399,1354,2078341 +54,search,4001.99,1901.24,1234.08,0.971816,104.595,-nan,-nan,0.90307,0.114776,-1.0,-1.0,11.57,0.921399,1354,2078341 +55,insert,396.898,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,424.448,1.06795,1408,2399905 +56,search,4278.91,2074.73,1383.13,1.00095,123.162,-nan,-nan,0.9041,0.117122,-1.0,-1.0,128.021,1.06829,1430,2399905 +57,insert,26.677,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,10.075,1.0755,1430,2419836 +58,search,4336.19,2021.57,1397.92,0.982937,129.095,-nan,-nan,0.90212,0.118455,-1.0,-1.0,61.982,1.0758,1436,2419836 +59,insert,234.221,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,127.953,1.17588,1449,2643083 +60,search,4358.74,2072.32,1424.62,0.975893,128.09,-nan,-nan,0.89471,0.125998,-1.0,-1.0,40.132,1.17611,1451,2643083 +61,insert,11.949,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.128,1.18036,1453,2653064 +62,search,4359.06,2072.57,1421.73,0.984596,127.464,-nan,-nan,0.8949,0.12576,-1.0,-1.0,25.331,1.18036,1454,2653064 +63,insert,3.69,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,10.27,1.18155,1454,2656118 +64,search,4452.93,2072.72,1410.5,0.998785,123.856,-nan,-nan,0.89475,0.126188,-1.0,-1.0,10.476,1.18155,1454,2656118 +65,insert,74.112,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,96.578,1.20956,1474,2723424 +66,search,4468.76,2094.56,1439.13,0.960153,126.878,-nan,-nan,0.89558,0.125732,-1.0,-1.0,11.084,1.20956,1474,2723424 +67,insert,44.184,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,72.561,1.22986,1486,2769021 +68,search,4494.57,2113.38,1470.48,0.988697,125.167,-nan,-nan,0.89762,0.124267,-1.0,-1.0,10.621,1.22986,1486,2769021 +69,insert,53.985,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,67.337,1.25489,1496,2821777 +70,search,4531.86,2130.53,1486.5,0.993484,129.665,-nan,-nan,0.89909,0.122725,-1.0,-1.0,16.807,1.25515,1497,2821777 +71,insert,14.271,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.407,1.26031,1501,2834921 +72,search,4572.56,2132.4,1494.18,0.99627,128.363,-nan,-nan,0.89948,0.122461,-1.0,-1.0,13.111,1.26031,1501,2834921 +73,insert,9.161,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,11.476,1.26346,1501,2842044 +74,search,4575.15,2137.57,1501.21,0.994503,135.608,-nan,-nan,0.89968,0.122193,-1.0,-1.0,12.549,1.26346,1501,2842044 +75,insert,24.862,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,29.118,1.26811,1504,2858890 +76,search,4580.6,2141.92,1509.77,0.992986,135.215,-nan,-nan,0.89957,0.122393,-1.0,-1.0,12.081,1.26811,1504,2858890 +77,insert,41.526,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.098,1.27752,1506,2882818 +78,search,4596.57,2151.78,1519.25,0.932619,145.801,-nan,-nan,0.89884,0.122929,-1.0,-1.0,12.062,1.27752,1506,2882818 +79,insert,492.984,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,25.947,1.30553,1510,2947806 +80,search,4641.52,2175.43,1546.06,1.02913,139.766,-nan,-nan,0.89863,0.122712,-1.0,-1.0,11.57,1.30553,1510,2947806 +81,insert,39.362,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.687,1.3183,1512,2976895 +82,search,4644.8,2190.83,1553.99,0.983914,138.342,-nan,-nan,0.90005,0.122001,-1.0,-1.0,11.292,1.3183,1512,2976895 +83,insert,19.547,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,33.19,1.32542,1516,2990682 +84,search,4653.19,2193.12,1550.59,0.972272,139.66,-nan,-nan,0.90062,0.121522,-1.0,-1.0,11.356,1.32542,1516,2990682 +85,insert,36.636,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,18.638,1.33859,1518,3025613 +86,search,4660.56,2211,1564.52,1.01627,143.713,-nan,-nan,0.90126,0.121111,-1.0,-1.0,11.673,1.33859,1518,3025613 +87,insert,607.447,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,671.305,1.54426,1616,3468632 +88,search,5056.66,2348.14,1745.1,0.946497,155.837,-nan,-nan,0.90433,0.118725,-1.0,-1.0,73.669,1.54425,1623,3468632 +89,insert,365.088,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,465.159,1.70346,1698,3831692 +90,search,5332.99,2428.18,1881.95,0.944953,162.431,-nan,-nan,0.90861,0.113148,-1.0,-1.0,109.458,1.70374,1709,3831692 +91,insert,35.131,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,32.804,1.71572,1712,3864067 +92,search,5405.22,2426.49,1892.13,0.995646,170.177,-nan,-nan,0.90887,0.112731,-1.0,-1.0,22.945,1.71572,1713,3864067 +93,insert,277.158,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,204.951,1.81717,1741,4103827 +94,search,5545.58,2502.77,1995.22,0.960374,165.399,-nan,-nan,0.90971,0.11168,-1.0,-1.0,34.843,1.81735,1743,4103827 +95,insert,18.45,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,43.911,1.82396,1748,4119898 +96,search,5488.54,2502.47,1934.66,1.00477,164.727,-nan,-nan,0.91021,0.111124,-1.0,-1.0,16.38,1.82396,1748,4119898 +97,insert,11.233,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.243,1.82672,1749,4129367 +98,search,5575.76,2504.75,2003.35,0.983863,182.147,-nan,-nan,0.9102,0.11094,-1.0,-1.0,17.265,1.82672,1749,4129367 +99,insert,6.196,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.974,1.82794,1749,4134502 +100,search,5564.68,2509.17,1992.64,1.01704,174.827,-nan,-nan,0.91025,0.110986,-1.0,-1.0,17.057,1.82794,1749,4134502 +101,insert,41.573,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,32.98,1.83914,1752,4164482 +102,search,5607.22,2522.22,2020.24,1.01543,177.963,-nan,-nan,0.91064,0.110746,-1.0,-1.0,32.121,1.83945,1754,4164482 +103,insert,21.739,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.47,1.84604,1756,4178601 +104,search,5552.33,2523.37,1979.55,1.02499,166.706,-nan,-nan,0.91037,0.110945,-1.0,-1.0,16.78,1.84604,1756,4178601 +105,insert,11.997,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.441,1.84951,1756,4187774 +106,search,5502.13,2529,1942.3,1.00011,165.528,-nan,-nan,0.91047,0.111072,-1.0,-1.0,16.687,1.84951,1756,4187774 +107,insert,17.671,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.858,1.85395,1756,4200109 +108,search,5614.97,2532.91,2034.31,0.934197,174.541,-nan,-nan,0.91047,0.111126,-1.0,-1.0,16.569,1.85395,1756,4200109 +109,insert,496.536,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,503.732,2.01388,1821,4557867 +110,search,5859.15,2619.59,2153.42,0.98492,183.649,-nan,-nan,0.91274,0.10931,-1.0,-1.0,59.328,2.01407,1826,4557867 +111,insert,31.428,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,54.16,2.02685,1832,4587252 +112,search,5722.92,2622.96,2008.63,1.09146,162.387,-nan,-nan,0.91365,0.109019,-1.0,-1.0,20.141,2.02685,1832,4587252 +113,insert,12.457,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.949,2.03082,1832,4600056 +114,search,5920.77,2629.62,2172.37,0.975266,189.908,-nan,-nan,0.91376,0.108877,-1.0,-1.0,22.97,2.03082,1832,4600056 +115,insert,45.195,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,68.69,2.04169,1838,4627422 +116,search,6090.07,2636.04,2191.62,0.960192,188.251,-nan,-nan,0.91416,0.108522,-1.0,-1.0,20.543,2.04169,1838,4627422 +117,insert,13.667,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.382,2.04667,1838,4639292 +118,search,6053.31,2643.37,2207.96,0.941323,193.414,-nan,-nan,0.91428,0.108144,-1.0,-1.0,33.415,2.04681,1839,4639292 +119,insert,5.446,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.964,2.04888,1839,4643423 +120,search,6042.47,2644.45,2196.42,0.983484,188.806,-nan,-nan,0.91429,0.108086,-1.0,-1.0,21.309,2.04888,1839,4643423 +121,insert,20.594,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.429,2.05583,1840,4657252 +122,search,6366.19,2650.74,2216.77,0.915279,192.121,-nan,-nan,0.91444,0.108016,-1.0,-1.0,28.087,2.05583,1840,4657252 +123,insert,361.121,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,470.504,2.18573,1887,4938477 +124,search,6593.9,2705.44,2310.93,1.00702,202.617,-nan,-nan,0.91526,0.108757,-1.0,-1.0,162.907,2.18696,1901,4938477 +125,insert,62.651,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,81.409,2.20563,1909,4980492 +126,search,6330.79,2689.82,2311.44,1.0071,200.734,-nan,-nan,0.91571,0.108446,-1.0,-1.0,54.755,2.20582,1912,4980492 +127,insert,60.052,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,77.525,2.22659,1921,5030515 +128,search,6391.66,2702.06,2393.08,0.950757,206.509,-nan,-nan,0.91624,0.108335,-1.0,-1.0,28.225,2.22664,1922,5030515 +129,delete,20.672,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,45.664,2.22483,1892,5009391 +130,search,6173.87,2692.8,2300.12,0.969073,201.703,-nan,-nan,0.91493,0.109158,-1.0,-1.0,23.774,2.22483,1892,5009391 +131,delete,5.341,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.556,2.22483,1892,5004093 +132,search,6222.88,2691.3,2307.15,0.960291,195.646,-nan,-nan,0.91493,0.109168,-1.0,-1.0,23.669,2.22483,1892,5004093 +133,delete,11.638,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.681,2.22483,1892,4991787 +134,search,6319.33,2678.55,2284.4,0.944475,192.706,-nan,-nan,0.91479,0.109417,-1.0,-1.0,23.492,2.22483,1892,4991787 +135,delete,18.621,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,51.131,2.22199,1889,4974857 +136,search,6267.71,2672.07,2274.59,0.950168,198.137,-nan,-nan,0.91493,0.108993,-1.0,-1.0,23.682,2.22199,1889,4974857 +137,delete,3.551,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.043,2.22199,1889,4971386 +138,search,6135.45,2670.67,2264.2,0.976552,199.879,-nan,-nan,0.91484,0.109115,-1.0,-1.0,23.131,2.22199,1889,4971386 +139,delete,7.075,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.73,2.22199,1889,4963487 +140,search,6255.36,2665.43,2258.19,0.964832,191.974,-nan,-nan,0.91463,0.10915,-1.0,-1.0,23.403,2.22199,1889,4963487 +141,delete,24.72,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,66.937,2.21814,1884,4941830 +142,search,6103.91,2654.75,2242.44,0.945736,196.792,-nan,-nan,0.91432,0.109425,-1.0,-1.0,23.241,2.21814,1884,4941830 +143,delete,18.434,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,89.951,2.21346,1881,4925657 +144,search,6173.3,2651.79,2235.72,0.956113,192.844,-nan,-nan,0.91449,0.109325,-1.0,-1.0,23.193,2.21346,1881,4925657 +145,delete,3.096,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,36.797,2.21243,1879,4922686 +146,search,6317.42,2652.15,2256.6,0.974268,191.809,-nan,-nan,0.91429,0.109456,-1.0,-1.0,25.638,2.21243,1879,4922686 +147,delete,15.105,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.507,2.21243,1879,4913636 +148,search,6297.68,2646.02,2245.99,0.981745,195.179,-nan,-nan,0.91421,0.109628,-1.0,-1.0,24.187,2.21243,1879,4913636 +149,delete,229.755,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1097.15,2.13431,1791,4717250 +150,search,5982.28,2659.84,2148.16,0.994014,194.037,-nan,-nan,0.91348,0.110839,-1.0,-1.0,51.501,2.13472,1795,4717250 +151,delete,163.736,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1076.04,2.0627,1707,4560269 +152,search,5742.72,2690.29,2078.93,0.98504,182.555,-nan,-nan,0.91203,0.111298,-1.0,-1.0,103.877,2.06456,1716,4560269 +153,delete,30.064,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,141.871,2.05314,1702,4521936 +154,search,5764.15,2668.31,2082.45,0.989946,187.56,-nan,-nan,0.91109,0.112262,-1.0,-1.0,47.999,2.054,1704,4521936 +155,delete,19.888,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,53.721,2.05131,1702,4502222 +156,search,5702.89,2653.95,2046.29,0.985664,176.835,-nan,-nan,0.91109,0.112556,-1.0,-1.0,23.013,2.05131,1702,4502222 +157,delete,48.022,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,194.882,2.03743,1692,4456039 +158,search,5679.42,2638.63,2022.64,0.974714,187.012,-nan,-nan,0.9103,0.113402,-1.0,-1.0,29.164,2.03757,1693,4456039 +159,delete,313.229,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1745.53,1.87962,1540,4143922 +160,search,5342.63,2727.37,1903.13,0.938314,168.37,-nan,-nan,0.90936,0.114122,-1.0,-1.0,164.018,1.88237,1557,4143922 +161,delete,49.433,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,56.236,1.87788,1554,4101445 +162,search,5256.49,2654.68,1869.35,0.949412,164.865,-nan,-nan,0.90946,0.113838,-1.0,-1.0,25.957,1.87797,1555,4101445 +163,delete,18.094,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,43.874,1.87572,1553,4080175 +164,search,5241.98,2634.23,1856.13,0.961747,164.409,-nan,-nan,0.90907,0.114248,-1.0,-1.0,20.717,1.87572,1553,4080175 +165,delete,33.206,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,101.111,1.86841,1548,4047313 +166,search,5200.23,2620.48,1828.15,0.969852,161.334,-nan,-nan,0.90907,0.113915,-1.0,-1.0,20.57,1.86841,1548,4047313 +167,delete,7.663,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.066,1.86813,1548,4040738 +168,search,5154.33,2614.41,1797.74,1.00728,156.924,-nan,-nan,0.90899,0.114089,-1.0,-1.0,21.372,1.86813,1548,4040738 +169,delete,19.35,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,75.445,1.86351,1543,4022674 +170,search,5180.75,2610.95,1824.08,0.989895,162.014,-nan,-nan,0.90917,0.113788,-1.0,-1.0,21.449,1.86351,1543,4022674 +171,delete,9.413,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.08,1.86335,1543,4015835 +172,search,5137.31,2601.98,1814.88,0.95135,156.338,-nan,-nan,0.90889,0.114082,-1.0,-1.0,32.57,1.86382,1544,4015835 +173,delete,35.789,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.193,1.86126,1543,3986424 +174,search,5139.56,2573.99,1800.7,0.957831,158.223,-nan,-nan,0.90863,0.114403,-1.0,-1.0,20.414,1.86126,1543,3986424 +175,delete,174.245,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,713.485,1.76769,1463,3804693 +176,search,4872.68,2601.26,1733.65,0.952756,155.905,-nan,-nan,0.90618,0.115333,-1.0,-1.0,72.885,1.76896,1470,3804693 +177,delete,14.147,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,59.708,1.76481,1467,3786352 +178,search,4891.45,2578.82,1721.18,0.960007,154.281,-nan,-nan,0.90611,0.115112,-1.0,-1.0,21.652,1.76481,1467,3786352 +179,delete,248.68,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,973.429,1.63587,1323,3556994 +180,search,4492.95,2667.94,1608.86,0.958026,144.687,-nan,-nan,0.8976,0.119639,-1.0,-1.0,113.414,1.63765,1334,3556994 +181,delete,28.711,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,52.795,1.63217,1331,3526824 +182,search,4479.13,2611.69,1598.06,0.984283,147.314,-nan,-nan,0.89765,0.119537,-1.0,-1.0,32.716,1.63214,1333,3526824 +183,delete,174.912,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,719.155,1.5443,1234,3350245 +184,search,4181.42,2660.31,1499.62,0.929557,135.305,-nan,-nan,0.88174,0.137083,-1.0,-1.0,82.931,1.5452,1243,3350245 +185,delete,11.65,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,18.613,1.54382,1240,3340234 +186,search,4191.33,2619.78,1488.06,0.92047,136.115,-nan,-nan,0.88693,0.130466,-1.0,-1.0,53.968,1.54422,1243,3340234 +187,delete,205.665,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,291.214,1.45723,1204,3150857 +188,search,4079.54,2627.6,1457.29,0.968674,129.879,-nan,-nan,0.89034,0.127356,-1.0,-1.0,62.354,1.45767,1210,3150857 +189,delete,6.968,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.007,1.4572,1210,3143609 +190,search,4083.33,2594.37,1448.14,0.968465,130.257,-nan,-nan,0.89007,0.127593,-1.0,-1.0,15.952,1.4572,1210,3143609 +191,delete,3.783,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.554,1.45683,1210,3141038 +192,search,4084.93,2592.33,1440.65,0.967038,127.735,-nan,-nan,0.89001,0.127553,-1.0,-1.0,16.508,1.45683,1210,3141038 +193,delete,71.104,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,154.749,1.43672,1193,3081734 +194,search,4029.34,2573.01,1421.82,0.890801,133.388,-nan,-nan,0.88948,0.12832,-1.0,-1.0,34.654,1.43709,1195,3081734 +195,delete,39.719,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,139.44,1.42064,1183,3043966 +196,search,3971.16,2555.16,1397.31,0.93881,130.796,-nan,-nan,0.88878,0.128897,-1.0,-1.0,36.577,1.42102,1185,3043966 +197,delete,38.677,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,56.572,1.41188,1183,3008145 +198,search,3959.37,2528.73,1378.79,0.91426,126.8,-nan,-nan,0.88826,0.12902,-1.0,-1.0,24.28,1.41215,1184,3008145 +199,delete,11.402,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,37.694,1.40915,1181,2999384 +200,search,3959.24,2526.77,1389.88,0.926275,128.491,-nan,-nan,0.88834,0.128971,-1.0,-1.0,22.968,1.40923,1182,2999384 +201,delete,7.293,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,37.332,1.40604,1180,2993528 +202,search,3950.49,2523.18,1372.69,0.931916,126.439,-nan,-nan,0.88821,0.128993,-1.0,-1.0,15.551,1.40604,1180,2993528 +203,delete,14.402,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.209,1.40533,1180,2980973 +204,search,3962.57,2509.24,1373.83,0.92538,122.851,-nan,-nan,0.88817,0.129028,-1.0,-1.0,15.636,1.40533,1180,2980973 +205,delete,23.899,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.905,1.39941,1171,2960291 +206,search,3893.34,2498.18,1349.04,0.960659,125.283,-nan,-nan,0.88676,0.129922,-1.0,-1.0,15.591,1.39941,1171,2960291 +207,delete,52.213,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,125.092,1.37669,1129,2920620 +208,search,3708.77,2513,1269.17,0.98803,109.681,-nan,-nan,0.88416,0.132068,-1.0,-1.0,11.886,1.37669,1129,2920620 +209,delete,19.344,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,51.617,1.36786,1121,2900501 +210,search,3626.68,2505.61,1231.94,1.09098,111.394,-nan,-nan,0.88237,0.133141,-1.0,-1.0,27.015,1.36821,1122,2900501 +211,delete,10.979,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.794,1.36505,1121,2891660 +212,search,5205.09,2497.8,1275.52,0.959387,111.152,-nan,-nan,0.88187,0.133377,-1.0,-1.0,19.38,1.36505,1121,2891660 +213,delete,27.519,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,71.577,1.3592,1117,2873257 +214,search,3865.96,2486.2,1323.75,0.903887,125.706,-nan,-nan,0.88117,0.133737,-1.0,-1.0,14.285,1.3592,1117,2873257 +215,delete,395.994,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1357.35,1.16219,933,2537571 +216,search,3222.31,2657.24,1152.5,0.94589,102.934,-nan,-nan,0.87554,0.136874,-1.0,-1.0,259.399,1.16844,957,2537571 +217,delete,209.973,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,813.686,1.05035,840,2308194 +218,search,2904.63,2652.79,1044.8,0.96584,92.8842,-nan,-nan,0.86986,0.142867,-1.0,-1.0,244.69,1.05529,860,2308194 +219,delete,18.495,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.716,1.05021,855,2285820 +220,search,2906.42,2550.53,1029.63,0.966664,93.1492,-nan,-nan,0.86836,0.144028,-1.0,-1.0,105.551,1.0503,864,2285820 +221,delete,193.586,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,620.026,0.940495,728,2082858 +222,search,2530.26,2622.43,904.714,0.90041,90.0116,-nan,-nan,0.85837,0.150245,-1.0,-1.0,95.821,0.942306,737,2082858 +223,delete,10.206,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.376,0.940115,735,2071207 +224,search,2508.28,2555.35,884.903,0.990844,85.9444,-nan,-nan,0.85682,0.151192,-1.0,-1.0,31.389,0.940432,736,2071207 +225,delete,5.31,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.328,0.93901,735,2065400 +226,search,2520.54,2547.76,884.844,0.913892,89.4254,-nan,-nan,0.85646,0.151252,-1.0,-1.0,11.912,0.93901,735,2065400 +227,delete,6.361,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,10.221,0.938724,735,2062105 +228,search,2521.43,2541.49,895.966,0.926843,82.9567,-nan,-nan,0.85621,0.151572,-1.0,-1.0,10.895,0.938724,735,2062105 +229,delete,23.131,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.47,0.933072,730,2039610 +230,search,2500.99,2521.25,879.472,0.964288,85.292,-nan,-nan,0.85587,0.151941,-1.0,-1.0,10.777,0.933072,730,2039610 +231,delete,13.38,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,34.758,0.927676,726,2027795 +232,search,2485.17,2522.09,869.296,0.900644,84.4392,-nan,-nan,0.85425,0.152922,-1.0,-1.0,11.52,0.927676,726,2027795 +233,delete,7.187,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.01,0.925643,725,2022859 +234,search,2491.34,2516.05,877.788,0.962138,81.4811,-nan,-nan,0.85426,0.152802,-1.0,-1.0,11.051,0.925643,725,2022859 +235,delete,6.751,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.186,0.924847,722,2016264 +236,search,2484.66,2506.24,867.155,0.908193,84.3323,-nan,-nan,0.85411,0.152761,-1.0,-1.0,10.972,0.924847,722,2016264 +237,delete,294.884,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,708.471,0.801818,608,1782866 +238,search,2163.34,2652.13,785.124,0.963902,76.3068,-nan,-nan,0.84687,0.159541,-1.0,-1.0,97.717,0.803651,617,1782866 +239,delete,18.222,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,62.486,0.792048,606,1758931 +240,search,2122.95,2601.9,759.864,1.02348,75.4475,-nan,-nan,0.84349,0.160427,-1.0,-1.0,26.194,0.792114,607,1758931 +241,delete,6.829,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.489,0.789734,605,1751275 +242,search,2124.32,2595.67,756.378,0.987941,74.222,-nan,-nan,0.8433,0.160536,-1.0,-1.0,9.049,0.789734,605,1751275 +243,delete,18.444,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.121,0.786902,603,1733264 +244,search,2108.07,2564.42,751.187,0.934967,71.9551,-nan,-nan,0.84141,0.161307,-1.0,-1.0,8.986,0.786902,603,1733264 +245,delete,10.512,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,33.126,0.783914,601,1722658 +246,search,2108.6,2556.56,750.249,0.928179,69.6817,-nan,-nan,0.84101,0.161257,-1.0,-1.0,9.034,0.783914,601,1722658 +247,delete,2.778,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,7.997,0.783893,601,1719601 +248,search,2106.69,2546.65,749.016,0.928909,72.4485,-nan,-nan,0.84063,0.161243,-1.0,-1.0,9.126,0.783893,601,1719601 +249,delete,8.929,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.93,0.781018,599,1711648 +250,search,2080.21,2547.53,741.967,1.00086,69.5811,-nan,-nan,0.83905,0.162415,-1.0,-1.0,9.491,0.781018,599,1711648 +251,delete,296.113,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,318.081,0.661145,516,1470624 +252,search,1855.65,2601.77,669.428,0.891225,62.9496,-nan,-nan,0.83577,0.163568,-1.0,-1.0,48.934,0.662017,520,1470624 +253,delete,27.426,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,53.145,0.652972,513,1442070 +254,search,1815.55,2544.87,646.314,0.99285,60.69,-nan,-nan,0.83341,0.165021,-1.0,-1.0,10.746,0.652972,513,1442070 +255,delete,40.315,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,94.667,0.636496,494,1405549 +256,search,1775.22,2579.27,639.926,0.906175,59.7011,-nan,-nan,0.83039,0.168087,-1.0,-1.0,9.772,0.636496,494,1405549 +257,insert,14.28,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,8.949,0.639346,494,1416288 +258,search,1780.53,2589.26,639.613,0.91383,61.9908,-nan,-nan,0.83085,0.167718,-1.0,-1.0,9.6,0.639346,494,1416288 +259,insert,34.154,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,55.089,0.650013,499,1443010 +260,search,1778.92,2610.5,642.719,0.91388,61.071,-nan,-nan,0.83011,0.168087,-1.0,-1.0,9.815,0.650013,499,1443010 +261,insert,7.598,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.199,0.651968,499,1447529 +262,search,1788.85,2628.2,649.531,0.905286,62.631,-nan,-nan,0.8309,0.167727,-1.0,-1.0,9.943,0.651968,499,1447529 +263,insert,150.195,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,133.467,0.699697,507,1569110 +264,search,1854.68,2809.97,694.638,0.876424,64.2551,-nan,-nan,0.83182,0.169497,-1.0,-1.0,95.869,0.699709,513,1569110 +265,insert,93.525,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,103.594,0.727706,524,1636119 +266,search,1928.42,2850.1,727.629,0.896463,67.9419,-nan,-nan,0.83796,0.165634,-1.0,-1.0,54.602,0.727739,527,1636119 +267,insert,40.101,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,49.174,0.740083,531,1665152 +268,search,1969.56,2879.53,745.666,0.887858,70.9218,-nan,-nan,0.84103,0.165134,-1.0,-1.0,37.979,0.740084,534,1665152 +269,insert,330.362,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,250.968,0.853001,555,1916952 +270,search,2088.76,3140.66,830.922,0.979276,76.5083,-nan,-nan,0.84236,0.163815,-1.0,-1.0,106.265,0.853688,564,1916952 +271,insert,3.158,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.451,0.854762,566,1920898 +272,search,2123.62,3057.04,830.939,0.887736,79.0532,-nan,-nan,0.84263,0.163666,-1.0,-1.0,77.196,0.854853,570,1920898 +273,insert,22.143,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.456,0.861784,572,1942757 +274,search,2148.09,3067.55,843.741,0.90569,82.8889,-nan,-nan,0.84556,0.160568,-1.0,-1.0,49.184,0.861905,575,1942757 +275,insert,464.525,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,542.082,1.04008,633,2328372 +276,search,2425.56,3315.68,986.072,0.926865,93.6758,-nan,-nan,0.85166,0.158041,-1.0,-1.0,207.098,1.04144,655,2328372 +277,insert,46.243,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,99.281,1.05765,668,2363257 +278,search,2495.31,3175.4,991.974,0.942719,92.6733,-nan,-nan,0.85397,0.156245,-1.0,-1.0,65.297,1.0577,674,2363257 +279,insert,25.786,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,48.021,1.06754,680,2388933 +280,search,2535.01,3145.14,1009.77,0.936905,93.0498,-nan,-nan,0.8549,0.15567,-1.0,-1.0,50.452,1.06783,685,2388933 +281,insert,361.164,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,383.58,1.20586,714,2684315 +282,search,2682.9,3248.13,1076.02,0.925667,102.788,-nan,-nan,0.85488,0.156405,-1.0,-1.0,193.772,1.20714,724,2684315 +283,insert,472.037,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,661.599,1.40157,810,3109157 +284,search,2984.65,3342.3,1190.21,0.936701,103.014,-nan,-nan,0.86294,0.149501,-1.0,-1.0,311.221,1.40287,844,3109157 +285,insert,11.331,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.974,1.40702,845,3122751 +286,search,3126.2,3213.48,1248.06,0.971081,114.496,-nan,-nan,0.86401,0.149061,-1.0,-1.0,52.337,1.40723,848,3122751 +287,insert,39.912,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,114.106,1.42425,857,3163679 +288,search,3030.1,3210.58,1258.04,0.97031,114.301,-nan,-nan,0.86428,0.148925,-1.0,-1.0,21.199,1.42425,859,3163679 +289,insert,174.462,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,324.388,1.56134,917,3472031 +290,search,3317.68,3292.11,1350.97,0.99248,121.242,-nan,-nan,0.86804,0.147939,-1.0,-1.0,272.948,1.56309,942,3472031 +291,insert,24.653,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,42.588,1.57097,947,3495577 +292,search,3480.88,3232.97,1394.44,0.981707,125.948,-nan,-nan,0.87076,0.145814,-1.0,-1.0,124.123,1.5712,957,3495577 +293,insert,18.312,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,52.411,1.58047,961,3515193 +294,search,3524.67,3202.36,1399.05,0.984173,127.978,-nan,-nan,0.87109,0.145969,-1.0,-1.0,98.847,1.58067,970,3515193 +295,insert,250.354,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,388.107,1.67397,1021,3723063 +296,search,3719.63,3198.42,1480.91,0.9343,130.964,-nan,-nan,0.87522,0.142628,-1.0,-1.0,140.467,1.67547,1036,3723063 +297,insert,4.094,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.352,1.67726,1037,3727399 +298,search,3770.39,3144.85,1487.46,0.93947,133.954,-nan,-nan,0.87586,0.142256,-1.0,-1.0,21.496,1.67734,1038,3727399 +299,insert,75.026,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,116.729,1.70086,1056,3788002 +300,search,3815.37,3142.3,1505.16,0.991111,129.856,-nan,-nan,0.87783,0.140215,-1.0,-1.0,14.739,1.70086,1056,3788002 +301,insert,711.948,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1151.94,1.96941,1200,4370866 +302,search,4411.25,3308.93,1796.27,0.927771,158.499,-nan,-nan,0.90189,0.122331,-1.0,-1.0,236.101,1.96972,1233,4370866 +303,insert,51.316,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,100.479,1.98875,1243,4420306 +304,search,4505.1,3165.42,1780.07,1.00229,152.747,-nan,-nan,0.90126,0.122996,-1.0,-1.0,51.223,1.98899,1247,4420306 +305,insert,357.459,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,525.415,2.1503,1287,4789669 +306,search,4734.69,3307.35,1914.79,0.964804,164.309,-nan,-nan,0.90386,0.120864,-1.0,-1.0,234.461,2.15158,1304,4789669 +307,insert,26.048,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,64.288,2.15939,1311,4815550 +308,search,4787.48,3262.94,1934.08,0.949036,163.629,-nan,-nan,0.90443,0.120826,-1.0,-1.0,137.265,2.15946,1318,4815550 +309,insert,10.153,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,25.824,2.16286,1320,4823345 +310,search,4716.47,3246.66,1920.54,0.975944,166.067,-nan,-nan,0.90474,0.120459,-1.0,-1.0,58.869,2.16286,1323,4823345 +311,insert,14.234,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.297,2.1694,1324,4843493 +312,search,4810.9,3246.86,1932.07,0.961382,163.584,-nan,-nan,0.90465,0.120696,-1.0,-1.0,34.03,2.16961,1327,4843493 +313,insert,43.667,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,121.153,2.18781,1340,4885489 +314,search,4834.66,3227.94,1931.34,0.981969,167.749,-nan,-nan,0.90472,0.12067,-1.0,-1.0,12.587,2.18781,1340,4885489 +315,insert,8.99,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,11.111,2.19087,1340,4892335 +316,search,4851.97,3229.15,1934.84,0.9724,171.36,-nan,-nan,0.90455,0.120828,-1.0,-1.0,12.396,2.19087,1340,4892335 +317,insert,49.366,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,82.615,2.20676,1350,4932996 +318,search,4901.76,3228.8,1958.07,0.944005,167.448,-nan,-nan,0.90515,0.120305,-1.0,-1.0,12.862,2.20676,1350,4932996 +319,insert,633.704,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,874.654,2.4201,1459,5430684 +320,search,5169.48,3337.87,2028.49,1.08163,167.372,-nan,-nan,0.90973,0.116803,-1.0,-1.0,214.674,2.42296,1487,5430684 +321,insert,45.793,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,112.017,2.43951,1502,5471239 +322,search,5247.36,3269.28,2009.9,1.12298,163.949,-nan,-nan,0.91116,0.11561,-1.0,-1.0,20.345,2.43968,1503,5471239 +323,insert,126.017,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,182.823,2.48445,1524,5575560 +324,search,5351.22,3298.63,2056.36,1.05913,168.933,-nan,-nan,0.91218,0.114537,-1.0,-1.0,52.625,2.48462,1529,5575560 +325,insert,1491.98,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,762.285,2.70948,1591,6069499 +326,search,5872.44,3440.28,2419.75,0.953453,207.755,-nan,-nan,0.91425,0.11297,-1.0,-1.0,367.756,2.71255,1622,6069499 +327,insert,75.937,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,140.14,2.74124,1638,6139647 +328,search,6019.08,3377.16,2463.69,0.942684,219.682,-nan,-nan,0.91586,0.111093,-1.0,-1.0,254.287,2.74312,1659,6139647 +329,insert,8.953,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.338,2.74521,1659,6146884 +330,search,6073.37,3340.18,2460.74,0.929441,223,-nan,-nan,0.9161,0.110782,-1.0,-1.0,131.826,2.74589,1671,6146884 +331,insert,47.619,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,106.486,2.76332,1682,6188816 +332,search,5913.15,3317.47,2310.06,1.05937,187.04,-nan,-nan,0.91632,0.110938,-1.0,-1.0,58.834,2.76382,1688,6188816 +333,insert,3.901,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.466,2.76497,1689,6190810 +334,search,6130.51,3302.21,2474.73,0.957747,224.101,-nan,-nan,0.91641,0.110775,-1.0,-1.0,15.637,2.76497,1689,6190810 +335,insert,49.511,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,98.866,2.78109,1702,6230108 +336,search,6167.41,3302.08,2491.02,0.954316,216.113,-nan,-nan,0.91697,0.110357,-1.0,-1.0,15.684,2.78109,1702,6230108 +337,insert,24.578,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,46.249,2.78848,1705,6249699 +338,search,6068.57,3305.24,2410.19,1.02507,201.097,-nan,-nan,0.91706,0.11031,-1.0,-1.0,15.866,2.78848,1705,6249699 +339,insert,56.579,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,97.613,2.80294,1717,6290226 +340,search,6251.32,3301,2513.74,0.956294,226.386,-nan,-nan,0.91803,0.109306,-1.0,-1.0,15.875,2.80294,1717,6290226 +341,insert,11.688,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.174,2.80687,1717,6297556 +342,search,6083.77,3304.81,2406.17,1.04631,199.812,-nan,-nan,0.91818,0.109279,-1.0,-1.0,61.21,2.80721,1720,6297556 +343,insert,36.475,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,84.635,2.8196,1729,6330843 +344,search,6248.54,3302.72,2516,0.970047,221.259,-nan,-nan,0.91845,0.109099,-1.0,-1.0,16.21,2.8196,1729,6330843 +345,insert,45.812,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.435,2.83251,1733,6370265 +346,search,6281.9,3317.48,2534.94,0.946145,222.455,-nan,-nan,0.91843,0.109172,-1.0,-1.0,16.17,2.83251,1733,6370265 +347,insert,615.67,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,878.088,3.03727,1846,6831086 +348,search,6466.39,3354.92,2607.98,1.05804,223.535,-nan,-nan,0.92184,0.106616,-1.0,-1.0,144.009,3.04049,1869,6831086 +349,insert,31.235,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,69.814,3.05756,1877,6876154 +350,search,6770.07,3333.27,2718.65,0.963157,241.855,-nan,-nan,0.92242,0.106231,-1.0,-1.0,78.874,3.05856,1882,6876154 +351,insert,444.702,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,719.657,3.24799,1963,7288266 +352,search,6809.96,3355.05,2781.75,1.0092,234.913,-nan,-nan,0.92211,0.107035,-1.0,-1.0,353.395,3.25198,1996,7288266 +353,insert,57.533,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,58.604,3.27016,2005,7333132 +354,search,6737.11,3310.48,2917.64,0.953575,254.695,-nan,-nan,0.92288,0.106379,-1.0,-1.0,66.167,3.27084,2014,7333132 +355,insert,216.084,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,509.919,3.43054,2108,7696838 +356,search,7356.4,3302.74,2964.54,1.01311,246.742,-nan,-nan,0.92465,0.104911,-1.0,-1.0,188.86,3.43166,2127,7696838 +357,insert,26.111,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,58.64,3.43982,2132,7714498 +358,search,7694.46,3270.07,3072.11,0.974918,268.417,-nan,-nan,0.92501,0.104806,-1.0,-1.0,72.24,3.44013,2136,7714498 +359,insert,13.767,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,40.473,3.4436,2138,7725755 +360,search,7420.81,3268.3,2942.82,1.06859,247.021,-nan,-nan,0.92527,0.104708,-1.0,-1.0,41.055,3.44364,2140,7725755 +361,insert,72.245,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,108.521,3.46786,2150,7789312 +362,search,7797.57,3275.97,3013.26,1.00553,266.174,-nan,-nan,0.92628,0.103674,-1.0,-1.0,59.562,3.46801,2153,7789312 +363,insert,23.523,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,51.925,3.4732,2156,7803163 +364,search,7769.07,3273.58,3100.46,0.959911,266.563,-nan,-nan,0.9262,0.103727,-1.0,-1.0,37.449,3.4735,2157,7803163 +365,insert,68.959,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,120.683,3.50032,2168,7863722 +366,search,7972.26,3280.32,3132.12,0.959428,276.069,-nan,-nan,0.92701,0.10321,-1.0,-1.0,70.967,3.50117,2172,7863722 +367,insert,36.396,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,101.301,3.51448,2182,7895271 +368,search,7921.59,3275.94,3153.66,0.965028,279.418,-nan,-nan,0.92761,0.102772,-1.0,-1.0,73.37,3.51513,2187,7895271 +369,insert,78.936,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,186.831,3.54127,2205,7961194 +370,search,7964.06,3277.22,3163.65,1.03215,278.991,-nan,-nan,0.92822,0.101962,-1.0,-1.0,64.212,3.54213,2208,7961194 +371,insert,12.037,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,33.089,3.54478,2209,7970239 +372,search,8516.98,3276.67,3165.71,1.00892,274.236,-nan,-nan,0.92855,0.101739,-1.0,-1.0,27.083,3.54478,2209,7970239 +373,insert,56.363,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,88.787,3.55899,2217,8010868 +374,search,9959.97,3280.33,3162.56,0.989672,274.802,-nan,-nan,0.92889,0.10133,-1.0,-1.0,39.469,3.55906,2218,8010868 +375,insert,43.074,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,85.927,3.57032,2224,8040698 +376,search,11252,3277.98,4004.77,0.773153,328.923,-nan,-nan,0.92883,0.101317,-1.0,-1.0,51.07,3.57053,2225,8040698 +377,insert,80.514,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,167.057,3.59673,2237,8104748 +378,search,10984.5,3284.57,3703.12,0.830653,323.944,-nan,-nan,0.92944,0.100792,-1.0,-1.0,32.915,3.59673,2237,8104748 +379,insert,8.771,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,36.662,3.59897,2238,8109755 +380,search,8310.39,3284.81,3670.94,0.847375,321.899,-nan,-nan,0.92948,0.100729,-1.0,-1.0,21.938,3.59897,2238,8109755 +381,insert,22.379,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,40.728,3.60881,2241,8130382 +382,search,6799.04,3290.02,3026.57,1.07178,256.706,-nan,-nan,0.92949,0.100682,-1.0,-1.0,21.657,3.60881,2241,8130382 +383,insert,25.739,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,68.979,3.61719,2248,8153290 +384,search,6607.82,3290.08,2910.76,1.1493,245.828,-nan,-nan,0.92958,0.100719,-1.0,-1.0,19.871,3.61719,2248,8153290 +385,delete,13.399,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.371,3.61719,2248,8138873 +386,search,6322.71,3284.48,2700.25,1.25694,224.586,-nan,-nan,0.92954,0.100798,-1.0,-1.0,21.106,3.61719,2248,8138873 +387,delete,18.681,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,122,3.61094,2243,8115069 +388,search,7603.67,3282.02,2918.05,1.14305,243.597,-nan,-nan,0.92981,0.100564,-1.0,-1.0,38.316,3.61113,2245,8115069 +389,delete,12.058,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,62.955,3.60887,2242,8101157 +390,search,7122.21,3274.12,2426.34,1.36244,200.847,-nan,-nan,0.92947,0.100616,-1.0,-1.0,30.997,3.60914,2243,8101157 +391,delete,67.747,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,438.603,3.57372,2211,8014644 +392,search,6934.31,3284.93,2327.29,1.46986,192.579,-nan,-nan,0.92901,0.100344,-1.0,-1.0,126.106,3.57513,2223,8014644 +393,delete,32.137,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,70.174,3.57216,2221,7978358 +394,search,7035.48,3248.01,2316.65,1.45456,196.482,-nan,-nan,0.92825,0.101069,-1.0,-1.0,41.546,3.57223,2223,7978358 +395,delete,24.207,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,173.946,3.56098,2215,7957094 +396,search,7029.54,3246.88,2392.96,1.40385,199.989,-nan,-nan,0.92804,0.101247,-1.0,-1.0,41.018,3.56141,2217,7957094 +397,delete,88.756,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,744.735,3.50092,2180,7823916 +398,search,7258.37,3254.57,2647.19,1.20546,216.002,-nan,-nan,0.92788,0.101241,-1.0,-1.0,123.152,3.50274,2190,7823916 +399,delete,5.717,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,58.809,3.50059,2188,7818430 +400,search,7317.85,3240.49,2614.33,1.2245,219.375,-nan,-nan,0.9278,0.101411,-1.0,-1.0,44.786,3.50103,2189,7818430 +401,delete,21.766,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,60.618,3.49705,2186,7800702 +402,search,7779.17,3232.72,2608.14,1.25248,219.646,-nan,-nan,0.92694,0.102691,-1.0,-1.0,25.38,3.49705,2186,7800702 +403,delete,243.986,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1651.94,3.36351,2079,7517858 +404,search,10490,3278.3,3081.8,0.924907,271.745,-nan,-nan,0.92496,0.104087,-1.0,-1.0,128.764,3.36642,2095,7517858 +405,delete,44.86,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,237.49,3.35226,2085,7451637 +406,search,6975.19,3223.2,2533.83,1.20341,205.506,-nan,-nan,0.9252,0.104017,-1.0,-1.0,49.248,3.3526,2087,7451637 +407,delete,86.914,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,582.834,3.30731,2054,7360417 +408,search,6805.75,3227.11,2490.82,1.23267,198.103,-nan,-nan,0.92445,0.104946,-1.0,-1.0,56.566,3.30809,2060,7360417 +409,delete,131.726,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,581.422,3.20732,2025,7131620 +410,search,6712.63,3205.17,2442.78,1.19439,197.941,-nan,-nan,0.92433,0.104829,-1.0,-1.0,108.229,3.20827,2032,7131620 +411,delete,246.847,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1592.16,3.09473,1935,6867617 +412,search,6500.61,3223.7,2352.71,1.19726,191.469,-nan,-nan,0.92418,0.105013,-1.0,-1.0,199.88,3.09853,1955,6867617 +413,delete,15.077,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,74.288,3.09355,1951,6850098 +414,search,6625.14,3187.44,2435.66,1.16785,201.184,-nan,-nan,0.92431,0.104867,-1.0,-1.0,32.152,3.09356,1952,6850098 +415,delete,68.682,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,480.811,3.06045,1926,6782921 +416,search,6514.53,3193.65,2406.96,1.19461,193.813,-nan,-nan,0.92399,0.105186,-1.0,-1.0,63.964,3.06126,1930,6782921 +417,delete,212.107,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1097.53,2.97239,1866,6562039 +418,search,6414.29,3189.42,2406.72,1.08947,195.276,-nan,-nan,0.9231,0.105314,-1.0,-1.0,189.743,2.97472,1879,6562039 +419,delete,27.807,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.302,2.97239,1877,6536285 +420,search,6441.52,3158.81,2404.45,1.09911,194.264,-nan,-nan,0.92349,0.105234,-1.0,-1.0,22.79,2.97239,1877,6536285 +421,delete,25.027,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,87.081,2.96205,1872,6510820 +422,search,6684.13,3158.8,2621.75,0.96842,230.983,-nan,-nan,0.92391,0.104579,-1.0,-1.0,34.807,2.96212,1873,6510820 +423,delete,215.535,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,628.916,2.88847,1814,6334168 +424,search,6489.04,3165.54,2546.06,0.989822,235.46,-nan,-nan,0.92257,0.105145,-1.0,-1.0,118.023,2.89051,1823,6334168 +425,delete,19.474,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,40.635,2.8887,1822,6316443 +426,search,6490.65,3146.51,2546.11,0.957333,225.064,-nan,-nan,0.92288,0.105253,-1.0,-1.0,22.789,2.8887,1822,6316443 +427,delete,63.584,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,60.966,2.88323,1819,6266345 +428,search,6453.44,3115.78,2511.01,0.958659,225.548,-nan,-nan,0.92244,0.105881,-1.0,-1.0,21.797,2.88323,1819,6266345 +429,delete,557.224,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,2543.05,2.66144,1596,5883847 +430,search,5926.21,3384.79,2418.77,0.935547,219.439,-nan,-nan,0.91064,0.113696,-1.0,-1.0,499.515,2.67126,1657,5883847 +431,delete,69.745,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,308.684,2.63392,1631,5801380 +432,search,5800.48,3214.99,2251.75,1.04425,192.462,-nan,-nan,0.90907,0.115458,-1.0,-1.0,99.302,2.63486,1640,5801380 +433,delete,225.372,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1090.7,2.53741,1565,5607567 +434,search,5615.36,3236.68,2208.56,0.984912,189.901,-nan,-nan,0.90576,0.117897,-1.0,-1.0,167.465,2.54168,1579,5607567 +435,delete,87.931,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,286.953,2.50908,1554,5512176 +436,search,5607.39,3191.84,2212.8,0.962684,200.727,-nan,-nan,0.9053,0.118105,-1.0,-1.0,174.743,2.51176,1562,5512176 +437,delete,11.648,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,55.958,2.50704,1560,5501141 +438,search,5519.96,3170.06,2133.36,1.02262,188.265,-nan,-nan,0.9052,0.118537,-1.0,-1.0,103.875,2.50704,1564,5501141 +439,delete,110.255,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,454.888,2.44382,1545,5369822 +440,search,5554.66,3174.7,2184.79,0.963067,202.11,-nan,-nan,0.90782,0.115544,-1.0,-1.0,120.383,2.44633,1553,5369822 +441,delete,28.05,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,53.492,2.44106,1550,5338985 +442,search,5512.97,3147.13,2128.91,1.00086,189.06,-nan,-nan,0.9072,0.116345,-1.0,-1.0,41.334,2.44108,1552,5338985 +443,delete,29.698,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,98.771,2.42699,1544,5307279 +444,search,5523.23,3147.04,2163.42,0.968176,189.954,-nan,-nan,0.9078,0.115715,-1.0,-1.0,35.39,2.42737,1546,5307279 +445,delete,41.985,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,94.737,2.41579,1541,5268382 +446,search,5506.72,3129.04,2143.9,0.951002,193.554,-nan,-nan,0.90761,0.116014,-1.0,-1.0,17.574,2.41579,1541,5268382 +447,delete,349.785,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,983.052,2.24036,1383,4922787 +448,search,4895.33,3239.82,1981.06,0.953832,170.538,-nan,-nan,0.90091,0.120346,-1.0,-1.0,175.582,2.24344,1399,4922787 +449,delete,30.952,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,104.11,2.23526,1390,4887227 +450,search,5040.08,3194.47,1980.42,0.910561,175.049,-nan,-nan,0.90127,0.120146,-1.0,-1.0,98.822,2.23677,1393,4887227 +451,delete,122.582,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,349.999,2.19352,1361,4795082 +452,search,4923.31,3193.35,1937.36,0.919689,169.789,-nan,-nan,0.90064,0.12119,-1.0,-1.0,88.807,2.19365,1365,4795082 +453,delete,358.232,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1402.52,2.04477,1245,4500175 +454,search,4553.62,3261.13,1813.11,0.941748,161.812,-nan,-nan,0.89663,0.124143,-1.0,-1.0,317.203,2.0515,1269,4500175 +455,delete,30.412,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,72.332,2.04331,1265,4462648 +456,search,4494.7,3196.97,1730.68,1.04412,146.843,-nan,-nan,0.89717,0.123658,-1.0,-1.0,88.854,2.04407,1271,4462648 +457,delete,6.793,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.339,2.04262,1270,4456761 +458,search,4571.44,3180.45,1804.42,0.964246,158.702,-nan,-nan,0.89697,0.123567,-1.0,-1.0,14.987,2.04262,1270,4456761 +459,delete,45.441,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,201.136,2.02011,1256,4419992 +460,search,4512.91,3190.04,1760.27,0.979904,155.252,-nan,-nan,0.89718,0.122965,-1.0,-1.0,36.747,2.02033,1258,4419992 +461,delete,5.275,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.998,2.02003,1258,4415725 +462,search,4527.99,3179.59,1768.03,0.95511,156.571,-nan,-nan,0.89771,0.122821,-1.0,-1.0,14.951,2.02003,1258,4415725 +463,delete,56.769,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,251.944,1.99514,1230,4370609 +464,search,4421.98,3225.74,1776.17,0.94472,161.865,-nan,-nan,0.89622,0.123792,-1.0,-1.0,32.635,1.99554,1232,4370609 +465,delete,23.72,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,46.296,1.98445,1226,4345244 +466,search,4375.37,3211.69,1694.1,0.982545,149.862,-nan,-nan,0.89448,0.125143,-1.0,-1.0,23.191,1.98467,1227,4345244 +467,delete,30.491,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,67.468,1.97683,1223,4319734 +468,search,4305.94,3197.13,1648.38,1.02547,141.488,-nan,-nan,0.89421,0.125262,-1.0,-1.0,14.514,1.97683,1223,4319734 +469,delete,20.704,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,51.383,1.96637,1217,4298361 +470,search,4417.13,3196.68,1745.85,0.937627,153.634,-nan,-nan,0.89306,0.125789,-1.0,-1.0,14.451,1.96637,1217,4298361 +471,delete,150.649,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,397.33,1.91272,1169,4188315 +472,search,4281.12,3242.5,1706.12,0.922692,156.967,-nan,-nan,0.89226,0.125864,-1.0,-1.0,120.308,1.91438,1180,4188315 +473,delete,127.49,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,423.125,1.86271,1140,4074567 +474,search,4168.78,3218.7,1658.13,0.964827,152.138,-nan,-nan,0.89039,0.126814,-1.0,-1.0,108.959,1.86441,1149,4074567 +475,delete,418.496,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1611.42,1.65922,984,3691698 +476,search,3693.85,3394.64,1511.17,0.922887,135.588,-nan,-nan,0.88208,0.131647,-1.0,-1.0,383.8,1.66581,1023,3691698 +477,delete,40.924,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,181.378,1.64545,1010,3642619 +478,search,3724.25,3235.96,1493.02,0.97218,134.463,-nan,-nan,0.88194,0.131655,-1.0,-1.0,117.382,1.64701,1020,3642619 +479,delete,245.182,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1256.27,1.54268,923,3421368 +480,search,3383.89,3336.37,1339.81,1.04599,116.766,-nan,-nan,0.87648,0.142482,-1.0,-1.0,407.254,1.55264,944,3421368 +481,delete,42.286,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,115.917,1.5364,934,3378034 +482,search,3392.72,3226.07,1321.01,1.02928,115.508,-nan,-nan,0.87855,0.139709,-1.0,-1.0,165.119,1.53696,940,3378034 +483,delete,315.582,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,946.037,1.4003,830,3115682 +484,search,3153.05,3341.49,1278.01,0.921464,117.384,-nan,-nan,0.87625,0.139827,-1.0,-1.0,288.602,1.40513,855,3115682 +485,delete,19.031,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,38.607,1.40036,852,3094629 +486,search,3167.43,3220.19,1267.1,0.982993,120.341,-nan,-nan,0.87714,0.139433,-1.0,-1.0,87.118,1.40117,859,3094629 +487,delete,7.665,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,12.937,1.4009,859,3086025 +488,search,3122.8,3187.43,1225.02,0.962306,105.307,-nan,-nan,0.87685,0.139646,-1.0,-1.0,15.719,1.4009,859,3086025 +489,delete,50.19,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,102.351,1.38948,853,3052105 +490,search,3148.59,3177.11,1252.74,0.928128,114.3,-nan,-nan,0.87619,0.140249,-1.0,-1.0,27.086,1.38965,854,3052105 +491,delete,20.632,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.5,1.38604,852,3038974 +492,search,3154.69,3168.2,1250.23,0.904875,117.847,-nan,-nan,0.87583,0.140492,-1.0,-1.0,14.545,1.38604,852,3038974 +493,delete,187.163,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,458.372,1.3115,793,2911063 +494,search,2974.37,3267.42,1196.59,0.936926,109.166,-nan,-nan,0.87365,0.141887,-1.0,-1.0,70.731,1.31218,799,2911063 +495,delete,23.863,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,65.626,1.30448,792,2888785 +496,search,2912.16,3238.16,1155.26,0.924685,106.022,-nan,-nan,0.8717,0.142806,-1.0,-1.0,24.588,1.30452,793,2888785 +497,delete,72.247,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,143.75,1.28237,778,2829718 +498,search,2926.54,3216.66,1147.88,1.01822,102.065,-nan,-nan,0.86987,0.143957,-1.0,-1.0,20.725,1.28248,779,2829718 +499,delete,18.337,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,34.102,1.28042,778,2816082 +500,search,2882.21,3200.22,1148.47,0.937005,98.604,-nan,-nan,0.8697,0.144089,-1.0,-1.0,12.744,1.28042,778,2816082 +501,delete,31.683,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,93.026,1.2693,770,2790880 +502,search,2880.69,3205.43,1140.79,0.944859,104.155,-nan,-nan,0.86956,0.144239,-1.0,-1.0,22.434,1.26939,771,2790880 +503,delete,17.939,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.233,1.26629,770,2774459 +504,search,2868.03,3184.47,1132.81,0.923681,101.729,-nan,-nan,0.86923,0.144694,-1.0,-1.0,38.1,1.26687,772,2774459 +505,delete,76.395,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,160.501,1.23505,753,2714683 +506,search,2827.97,3187.6,1116.04,0.955544,108.453,-nan,-nan,0.86776,0.1453,-1.0,-1.0,53.484,1.23592,756,2714683 +507,delete,47.616,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,77.66,1.22446,749,2680986 +508,search,2805.87,3173.92,1107.84,0.919727,102.299,-nan,-nan,0.86699,0.145583,-1.0,-1.0,23.262,1.22464,750,2680986 +509,delete,23.655,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,75.38,1.21547,745,2662966 +510,search,2774.08,3168.7,1094.19,0.92405,99.8279,-nan,-nan,0.86697,0.145321,-1.0,-1.0,38.675,1.21607,747,2662966 +511,delete,25.996,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,56.976,1.21035,742,2642855 +512,search,2750.39,3154.82,1087.65,0.908356,101.78,-nan,-nan,0.86512,0.146382,-1.0,-1.0,22.626,1.21064,743,2642855 +513,insert,410.054,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,382.081,1.31505,770,2906838 +514,search,4197.41,3287.01,1263.18,0.928532,112.515,-nan,-nan,0.86792,0.144679,-1.0,-1.0,92.617,1.31577,777,2906838 +515,insert,9.314,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.274,1.31796,778,2913746 +516,search,3053.49,3255.62,1225.68,0.905076,112.051,-nan,-nan,0.86811,0.144618,-1.0,-1.0,11.963,1.31796,778,2913746 +517,insert,24.947,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,37.534,1.32195,781,2936862 +518,search,2629.15,3281.46,909.081,1.33386,81.5704,-nan,-nan,0.86963,0.143432,-1.0,-1.0,12.368,1.32195,781,2936862 +519,insert,126.058,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,164.45,1.35524,793,3009732 +520,search,2678.8,3312.84,935.21,1.31857,83.4887,-nan,-nan,0.87109,0.143398,-1.0,-1.0,62.243,1.35633,801,3009732 +521,insert,14.221,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.8,1.36152,803,3025906 +522,search,2689.25,3293.24,951.685,1.24261,83.0405,-nan,-nan,0.87132,0.14344,-1.0,-1.0,17.301,1.36176,804,3025906 +523,insert,32.75,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,55.68,1.37724,810,3061899 +524,search,2716.75,3304.14,963.869,1.29259,83.0145,-nan,-nan,0.87195,0.143205,-1.0,-1.0,34.202,1.37767,814,3061899 +525,insert,33.475,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,55.41,1.3936,821,3101209 +526,search,2779.22,3299.06,999.956,1.34183,85.362,-nan,-nan,0.8719,0.142935,-1.0,-1.0,10.744,1.3936,821,3101209 +527,insert,407.719,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,565.275,1.57791,906,3522334 +528,search,3133.36,3451.61,1174.63,1.21736,101.431,-nan,-nan,0.88115,0.136557,-1.0,-1.0,154.374,1.57969,926,3522334 +529,insert,216.392,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,379.045,1.69791,986,3797294 +530,search,3494.85,3463.63,1352.43,1.13953,113.996,-nan,-nan,0.88934,0.130951,-1.0,-1.0,273.325,1.7003,1005,3797294 +531,insert,94.709,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,119.643,1.72708,1015,3853781 +532,search,3634.71,3418.89,1381.1,1.10883,119.234,-nan,-nan,0.89036,0.129858,-1.0,-1.0,174.437,1.72741,1023,3853781 +533,insert,28.763,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,67.439,1.7364,1028,3873352 +534,search,3746.09,3390.87,1486.36,1.08824,126.386,-nan,-nan,0.89029,0.129856,-1.0,-1.0,93.025,1.7364,1033,3873352 +535,insert,25.806,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,79.607,1.74404,1042,3893308 +536,search,3748.28,3358.79,1464.43,1.10509,124.704,-nan,-nan,0.89095,0.129461,-1.0,-1.0,68.761,1.74406,1047,3893308 +537,insert,4.544,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.604,1.74509,1047,3901271 +538,search,3765.83,3342.63,1469.94,1.07062,124.819,-nan,-nan,0.89132,0.129347,-1.0,-1.0,27.822,1.74539,1048,3901271 +539,insert,27.274,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,72.86,1.75392,1054,3925164 +540,search,3884.22,3345.65,1570.93,0.976927,139.261,-nan,-nan,0.8918,0.129055,-1.0,-1.0,13.099,1.75392,1054,3925164 +541,insert,288.32,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,519.889,1.93769,1127,4320915 +542,search,4084.86,3461.81,1670.9,1.01651,138.417,-nan,-nan,0.89573,0.126208,-1.0,-1.0,202.101,1.94017,1156,4320915 +543,insert,28.873,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,66.671,1.95381,1168,4360814 +544,search,4202.85,3373.14,1693.28,1.05137,139.602,-nan,-nan,0.89722,0.124989,-1.0,-1.0,97.05,1.95428,1181,4360814 +545,insert,5.346,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,25.415,1.95676,1184,4370220 +546,search,4241.93,3331.92,1701.85,1.00673,138.713,-nan,-nan,0.8975,0.124906,-1.0,-1.0,83.794,1.9572,1191,4370220 +547,insert,209.241,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,364.985,2.021,1239,4510675 +548,search,4567.81,3299.72,1849.74,0.947838,162.107,-nan,-nan,0.90004,0.122891,-1.0,-1.0,73.443,2.02165,1246,4510675 +549,insert,257.674,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,312.431,2.12122,1269,4744845 +550,search,4681.89,3336.1,1904.54,0.927239,168.101,-nan,-nan,0.89577,0.12691,-1.0,-1.0,132.637,2.12177,1277,4744845 +551,insert,27.507,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,26.652,2.13299,1278,4771433 +552,search,4714.65,3323.77,1912.82,0.956521,169.388,-nan,-nan,0.89601,0.126477,-1.0,-1.0,109.352,2.13329,1285,4771433 +553,insert,16.972,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.513,2.13754,1286,4782909 +554,search,4740.64,3317.78,1903.05,0.987529,166.893,-nan,-nan,0.89763,0.12497,-1.0,-1.0,93.398,2.13779,1293,4782909 +555,insert,590.088,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,888.21,2.31344,1425,5180322 +556,search,5192.21,3284.86,2104.19,0.946767,184.134,-nan,-nan,0.90288,0.120178,-1.0,-1.0,74.753,2.31367,1434,5180322 +557,insert,35.496,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,32.682,2.32339,1437,5203220 +558,search,5088.91,3262.23,1971.66,1.11496,164.422,-nan,-nan,0.90292,0.120111,-1.0,-1.0,21.822,2.32339,1438,5203220 +559,insert,9.132,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.811,2.32615,1440,5210154 +560,search,5244.83,3254.97,2098.99,0.975502,179.698,-nan,-nan,0.90276,0.12038,-1.0,-1.0,15.566,2.32615,1440,5210154 +561,insert,6.901,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,17.711,2.32748,1440,5215426 +562,search,5357.42,3257.23,2086.95,0.988507,182.967,-nan,-nan,0.90285,0.120276,-1.0,-1.0,14.475,2.32748,1440,5215426 +563,insert,42.684,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,85.641,2.34257,1453,5252726 +564,search,5750.36,3258.23,2121.53,0.997865,176.306,-nan,-nan,0.90351,0.119574,-1.0,-1.0,35.906,2.34291,1455,5252726 +565,insert,300.44,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,324.057,2.43018,1487,5447274 +566,search,5414.88,3299.14,2186,0.984106,190.227,-nan,-nan,0.90484,0.118567,-1.0,-1.0,146.234,2.43139,1499,5447274 +567,insert,18.843,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.598,2.43713,1500,5463135 +568,search,5450.98,3270.13,2188.45,0.990443,190.185,-nan,-nan,0.9047,0.11883,-1.0,-1.0,86.554,2.43763,1505,5463135 +569,insert,16.618,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,44.543,2.44201,1508,5478168 +570,search,5488.66,3263.24,2194.1,0.949507,195.226,-nan,-nan,0.90486,0.118672,-1.0,-1.0,30.574,2.44203,1509,5478168 +571,insert,21.508,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.812,2.44837,1510,5492309 +572,search,5483.59,3261.43,2199.61,0.973606,193.734,-nan,-nan,0.90452,0.118866,-1.0,-1.0,16.91,2.44837,1510,5492309 +573,insert,73.805,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,83.905,2.46859,1518,5554599 +574,search,5501.21,3281.59,2232.38,0.968093,197.332,-nan,-nan,0.90504,0.118751,-1.0,-1.0,37.625,2.46874,1520,5554599 +575,insert,66.537,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,125.606,2.48957,1528,5604500 +576,search,5552.39,3289.59,2237.96,0.93861,194.361,-nan,-nan,0.90504,0.119129,-1.0,-1.0,24.598,2.48971,1529,5604500 +577,insert,49.331,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,100.814,2.5027,1541,5638834 +578,search,5612.9,3286.42,2263.08,0.933446,203.447,-nan,-nan,0.90557,0.11875,-1.0,-1.0,43.297,2.50312,1544,5638834 +579,insert,921.812,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1049.24,2.77858,1686,6229778 +580,search,6198.2,3367.18,2533.78,0.948054,221.55,-nan,-nan,0.91155,0.113814,-1.0,-1.0,273.502,2.78126,1714,6229778 +581,insert,32.198,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,86.985,2.79252,1721,6261240 +582,search,6247.4,3304.33,2521.12,0.950074,221.225,-nan,-nan,0.91178,0.11357,-1.0,-1.0,62.298,2.79271,1723,6261240 +583,insert,29.668,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,46.764,2.80273,1727,6283315 +584,search,6294.17,3305.15,2540.07,0.977091,229.678,-nan,-nan,0.91193,0.113599,-1.0,-1.0,73.745,2.80329,1732,6283315 +585,insert,348.634,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,471.345,2.92397,1782,6556101 +586,search,6486.97,3336.12,2638.93,0.952909,232.137,-nan,-nan,0.91432,0.112018,-1.0,-1.0,140.904,2.92509,1795,6556101 +587,insert,50.283,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,94.562,2.94017,1805,6587187 +588,search,6518.19,3303.91,2636.88,0.946984,226.113,-nan,-nan,0.91442,0.111961,-1.0,-1.0,30.543,2.94017,1806,6587187 +589,insert,62.427,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,119.223,2.95659,1819,6631526 +590,search,6579.98,3299.38,2654.39,0.992715,231.502,-nan,-nan,0.91503,0.111588,-1.0,-1.0,46.33,2.95697,1822,6631526 +591,insert,630.219,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,960.175,3.16588,1946,7108942 +592,search,6976.57,3335.4,2792.79,0.980557,233.35,-nan,-nan,0.91895,0.108614,-1.0,-1.0,203.224,3.16824,1967,7108942 +593,insert,204.785,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,220.135,3.2526,1985,7303976 +594,search,7209.96,3341.94,2911.33,0.919127,252.608,-nan,-nan,0.91992,0.108033,-1.0,-1.0,225.724,3.25401,2003,7303976 +595,insert,7.768,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.231,3.25689,2003,7309244 +596,search,7284.99,3315.55,2932.25,0.913999,255.544,-nan,-nan,0.92024,0.107825,-1.0,-1.0,84.131,3.25729,2008,7309244 +597,insert,311.703,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,422.512,3.36054,2048,7542477 +598,search,8086.42,3353.63,3209.02,0.903918,283.145,-nan,-nan,0.92204,0.10616,-1.0,-1.0,163.802,3.36183,2061,7542477 +599,insert,136.898,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,308.085,3.40494,2095,7654438 +600,search,7725.67,3333.91,3126.17,0.944382,278.702,-nan,-nan,0.92289,0.105346,-1.0,-1.0,58.802,3.40555,2099,7654438 +601,insert,80.825,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,158.596,3.43443,2117,7717936 +602,search,7726.37,3327.24,3113.9,0.950857,280.943,-nan,-nan,0.92325,0.104945,-1.0,-1.0,98.633,3.43507,2121,7717936 +603,insert,10.421,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,39.171,3.43685,2121,7723521 +604,search,9026.73,3326.14,3185.6,0.916304,278.722,-nan,-nan,0.92326,0.104931,-1.0,-1.0,58.796,3.43685,2122,7723521 +605,insert,63.938,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,123.804,3.45496,2134,7767404 +606,search,7293.93,3325.32,3644.05,0.887783,291.846,-nan,-nan,0.92371,0.104463,-1.0,-1.0,17.728,3.45496,2134,7767404 +607,insert,41.398,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,43.45,3.47631,2139,7826115 +608,search,6400.15,3328.48,2991.45,1.01256,240.241,-nan,-nan,0.92344,0.104735,-1.0,-1.0,33.77,3.47661,2141,7826115 +609,insert,139.64,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,256.395,3.56466,2187,8036351 +610,search,6543.08,3333.72,3052.62,1.04986,246.967,-nan,-nan,0.92431,0.104408,-1.0,-1.0,57.398,3.56575,2196,8036351 +611,insert,14.244,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,39.248,3.57484,2201,8060364 +612,search,6611.43,3319.32,3071.02,1.07292,247.763,-nan,-nan,0.92453,0.104122,-1.0,-1.0,23.28,3.57502,2202,8060364 +613,insert,151.595,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,194.95,3.65807,2234,8256083 +614,search,6642.16,3343.8,3090.92,1.07635,246.281,-nan,-nan,0.92525,0.103876,-1.0,-1.0,61.755,3.65927,2241,8256083 +615,insert,224.735,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,472.06,3.82817,2314,8625707 +616,search,6924.73,3359.04,3243.95,1.07001,260.333,-nan,-nan,0.92696,0.101693,-1.0,-1.0,154.013,3.83139,2339,8625707 +617,insert,193.035,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,361.739,3.97728,2394,8949227 +618,search,8308.68,3357.09,3361.04,1.03692,269.194,-nan,-nan,0.92893,0.100305,-1.0,-1.0,158.401,3.97902,2415,8949227 +619,insert,165.097,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,263.688,4.0813,2453,9195551 +620,search,8599.9,3346.84,3443.01,1.05903,280.2,-nan,-nan,0.93015,0.0988606,-1.0,-1.0,131.083,4.08361,2473,9195551 +621,insert,26.135,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,51.8,4.096,2481,9232372 +622,search,8856.68,3324.85,3595.29,0.967755,309.678,-nan,-nan,0.93056,0.0985834,-1.0,-1.0,33.617,4.09611,2482,9232372 +623,insert,22.026,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,102.632,4.10599,2491,9253561 +624,search,8989.16,3317.5,3626.47,0.979616,317.024,-nan,-nan,0.9307,0.09854,-1.0,-1.0,29.194,4.10605,2492,9253561 +625,insert,94.028,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,194.258,4.14002,2513,9331230 +626,search,9051.39,3319.93,3652.93,0.944916,320.057,-nan,-nan,0.93104,0.0982309,-1.0,-1.0,50.333,4.14109,2518,9331230 +627,insert,40.412,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,51.341,4.1601,2525,9374063 +628,search,8809.06,3319.13,3520.75,0.992993,289.97,-nan,-nan,0.93116,0.098111,-1.0,-1.0,38.621,4.16019,2527,9374063 +629,insert,21.594,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,55.369,4.16604,2531,9387598 +630,search,9027.31,3314.86,3604.02,0.962741,306.666,-nan,-nan,0.9312,0.0981281,-1.0,-1.0,27.889,4.16604,2532,9387598 +631,insert,46.738,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,70.579,4.17799,2538,9421724 +632,search,9112.2,3311.61,3655.45,0.988235,317.289,-nan,-nan,0.93133,0.0979402,-1.0,-1.0,23.896,4.17799,2538,9421724 +633,insert,81.893,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,105.468,4.19351,2546,9461430 +634,search,9005.32,3319.16,3535.16,1.03552,298.877,-nan,-nan,0.93184,0.0973423,-1.0,-1.0,57.731,4.19418,2549,9461430 +635,insert,39.126,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,66.181,4.20128,2554,9479469 +636,search,8847.88,3313.36,3408.07,1.06638,271.81,-nan,-nan,0.93188,0.0973955,-1.0,-1.0,23.088,4.20128,2554,9479469 +637,insert,608.819,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,891.752,4.38232,2649,9885997 +638,search,9602.96,3342.58,3907.55,0.953936,342.251,-nan,-nan,0.93359,0.0967035,-1.0,-1.0,269.076,4.38526,2666,9885997 +639,insert,575.348,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,892.254,4.56333,2759,10292043 +640,search,10507.1,3349.81,4241.34,0.904714,377.715,-nan,-nan,0.93604,0.0942613,-1.0,-1.0,313.619,4.56821,2792,10292043 +641,delete,182.705,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1340.25,4.49191,2744,10096584 +642,search,9898.43,3314.72,3999.5,0.954243,356.012,-nan,-nan,0.93608,0.0938486,-1.0,-1.0,207.152,4.49672,2758,10096584 +643,delete,12.621,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,89.15,4.49331,2755,10084840 +644,search,12659,3298.38,4066.13,0.922694,351.789,-nan,-nan,0.93602,0.0939665,-1.0,-1.0,66.539,4.49375,2758,10084840 +645,delete,18.887,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,33.29,4.49375,2758,10069802 +646,search,9949.69,3287.36,3879.37,1.00881,330.537,-nan,-nan,0.93603,0.0940173,-1.0,-1.0,21.62,4.49375,2758,10069802 +647,delete,76.592,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,529.306,4.45581,2729,9990953 +648,search,9162.67,3293.7,3459.92,1.13966,271.225,-nan,-nan,0.93546,0.0939989,-1.0,-1.0,105.387,4.45765,2737,9990953 +649,delete,32.802,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,94.562,4.4531,2733,9951625 +650,search,9170.99,3277.39,3453.53,1.14992,272.191,-nan,-nan,0.93565,0.0941163,-1.0,-1.0,22.056,4.4531,2733,9951625 +651,delete,27.888,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,125.592,4.44462,2728,9927065 diff --git a/scripts/big_ann_perf_numbers/perf_debug_scan_0.14_worker_batch_tuning_breakdown.csv b/scripts/big_ann_perf_numbers/perf_debug_scan_0.14_worker_batch_tuning_breakdown.csv new file mode 100644 index 00000000..38c944e9 --- /dev/null +++ b/scripts/big_ann_perf_numbers/perf_debug_scan_0.14_worker_batch_tuning_breakdown.csv @@ -0,0 +1,2 @@ +step_num,step_type,latency_ms,worker_partition_size,worker_scan_time_ms,worker_scan_throughput,worker_result_time_ms,measured_ipc,cache_miss_rate,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,3056.05,37.3917,337.096,0.511741,602.905,-nan,-nan,0.83796,0.13696,-1.0,-1.0,22.208,0.0184881,1000,38806 diff --git a/scripts/big_ann_perf_numbers/perf_debug_scan_0.14_worker_batch_tuning_lower.csv b/scripts/big_ann_perf_numbers/perf_debug_scan_0.14_worker_batch_tuning_lower.csv new file mode 100644 index 00000000..b840a2d9 --- /dev/null +++ b/scripts/big_ann_perf_numbers/perf_debug_scan_0.14_worker_batch_tuning_lower.csv @@ -0,0 +1,1280 @@ +step_num,step_type,latency_ms,worker_partition_size,worker_scan_time_ms,worker_scan_throughput,worker_result_time_ms,measured_ipc,cache_miss_rate,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,3239.02,37.4251,357.034,0.493315,641.35,-nan,-nan,0.844,0.1346,-1.0,-1.0,6.551,0.0184881,1000,38806 +3,insert,25.368,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.157,0.0226893,995,49334 +4,search,2488.24,65.147,196.543,1.11729,190.883,-nan,-nan,0.85634,0.133964,-1.0,-1.0,22.186,0.0226836,988,49334 +5,insert,19.622,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,169.101,0.032957,974,73557 +6,search,2350.69,192.202,188.762,1.59827,101.731,-nan,-nan,0.92009,0.107539,-1.0,-1.0,130.653,0.0330313,976,73557 +7,insert,24.303,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,136.573,0.0431297,975,95766 +8,search,2482.21,240.917,203.001,1.42565,101.501,-nan,-nan,0.9186,0.108923,-1.0,-1.0,85.887,0.0433072,978,95766 +9,insert,3.381,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,75.798,0.0450273,978,100025 +10,search,2381.78,249.211,206.812,1.42462,101.247,-nan,-nan,0.9196,0.106614,-1.0,-1.0,80.907,0.0450497,980,100025 +11,insert,8.515,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,17.308,0.0500298,983,111417 +12,search,2454.36,285.67,212.925,1.18862,126.949,-nan,-nan,0.91799,0.108448,-1.0,-1.0,23.612,0.0500294,985,111417 +13,insert,28.768,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,6.276,0.0610503,985,137139 +14,search,2464.69,326.778,242.915,1.11263,112.087,-nan,-nan,0.91447,0.110692,-1.0,-1.0,15.99,0.0610625,989,137139 +15,insert,14.996,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,8.747,0.0695942,990,156710 +16,search,2489.49,357.432,257.646,1.10803,103.857,-nan,-nan,0.91224,0.110953,-1.0,-1.0,6.546,0.0695942,990,156710 +17,insert,4.064,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,8.162,0.0716893,991,161773 +18,search,2429.11,375.625,256.253,1.08406,103.661,-nan,-nan,0.91322,0.110943,-1.0,-1.0,8.383,0.0717064,992,161773 +19,insert,30.481,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,8.719,0.0784282,993,177161 +20,search,2448.18,413.973,278.293,1.15302,99.1912,-nan,-nan,0.91811,0.107416,-1.0,-1.0,9.051,0.0784282,994,177161 +21,insert,324.191,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,161.699,0.201152,1037,448898 +22,search,2693.54,829.028,491.659,1.01798,77.3204,-nan,-nan,0.90318,0.111969,-1.0,-1.0,44.068,0.201151,1044,448898 +23,insert,394.341,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,293.323,0.328245,1094,733493 +24,search,3036.7,1151.67,702.744,0.995295,84.6762,-nan,-nan,0.89874,0.113726,-1.0,-1.0,90.338,0.328429,1112,733493 +25,insert,36.072,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,35.727,0.345909,1118,777443 +26,search,2990.88,1120.29,672.05,1.03339,77.8536,-nan,-nan,0.89684,0.115357,-1.0,-1.0,17.728,0.346002,1120,777443 +27,insert,28.516,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.763,0.358141,1121,805394 +28,search,3046.89,1147.7,695.02,1.04244,71.0955,-nan,-nan,0.89761,0.114545,-1.0,-1.0,9.002,0.358141,1121,805394 +29,insert,55.221,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,54.638,0.385159,1131,866391 +30,search,2988.23,1209.87,740.95,1.00112,72.9892,-nan,-nan,0.90032,0.113898,-1.0,-1.0,18.417,0.385159,1133,866391 +31,insert,448.111,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,332.948,0.554264,1191,1239816 +32,search,3353.37,1495.98,923.705,0.991814,75.0187,-nan,-nan,0.89218,0.120469,-1.0,-1.0,69.733,0.554263,1198,1239816 +33,insert,46.191,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.169,0.574814,1203,1291211 +34,search,3350.85,1498.08,917.448,1.00292,72.914,-nan,-nan,0.89213,0.120177,-1.0,-1.0,56.957,0.574941,1208,1291211 +35,insert,29.373,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,30.295,0.586937,1214,1324978 +36,search,3331.52,1521.07,966.584,0.926052,75.8825,-nan,-nan,0.89341,0.11922,-1.0,-1.0,39.033,0.586948,1217,1324978 +37,insert,57.408,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.283,0.605487,1220,1371885 +38,search,3343.68,1549.12,998.988,0.97988,78.3444,-nan,-nan,0.89441,0.11829,-1.0,-1.0,13.733,0.605487,1221,1371885 +39,insert,12.916,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,8.388,0.608722,1221,1379335 +40,search,3318.76,1550.37,989.235,0.975417,86.3315,-nan,-nan,0.89449,0.118777,-1.0,-1.0,8.823,0.608722,1221,1379335 +41,insert,45.764,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.185,0.623868,1225,1413312 +42,search,3364.19,1575.61,1000.7,0.987989,76.259,-nan,-nan,0.89563,0.117823,-1.0,-1.0,5.993,0.623868,1225,1413312 +43,insert,7.366,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,6.312,0.627645,1225,1423411 +44,search,3423.3,1588.91,995.033,0.996097,67.0249,-nan,-nan,0.8958,0.118005,-1.0,-1.0,8.893,0.627645,1225,1423411 +45,insert,172.698,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.959,0.641578,1230,1457559 +46,search,3373.28,1610.72,968.708,1.02897,76.0286,-nan,-nan,0.89681,0.117022,-1.0,-1.0,8.742,0.641578,1230,1457559 +47,insert,197.627,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,199.281,0.748462,1261,1688593 +48,search,3558.05,1744.31,1116.74,0.992678,79.6673,-nan,-nan,0.89773,0.118657,-1.0,-1.0,56.707,0.74846,1271,1688593 +49,insert,18.387,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.748,0.76251,1274,1723505 +50,search,3606.29,1746.5,1105.27,1.01215,81.4916,-nan,-nan,0.89881,0.117586,-1.0,-1.0,32.53,0.76251,1280,1723505 +51,insert,298.56,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,299.64,0.905831,1328,2035040 +52,search,3646.86,1900.84,1239.21,0.942485,92.116,-nan,-nan,0.90108,0.116045,-1.0,-1.0,24.097,0.906126,1333,2035040 +53,insert,28.237,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,17.899,0.92353,1336,2078341 +54,search,3222.39,1919.19,1179.75,1.05211,102.982,-nan,-nan,0.90223,0.115801,-1.0,-1.0,6.52,0.92353,1336,2078341 +55,insert,199.364,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,273.513,1.07194,1395,2399905 +56,search,4016.56,2079.68,1290.64,1.02876,104.923,-nan,-nan,0.90345,0.117146,-1.0,-1.0,105.482,1.07206,1411,2399905 +57,insert,23.742,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.379,1.07953,1414,2419836 +58,search,4161.97,2035.15,1304.06,1.0911,100.361,-nan,-nan,0.90193,0.118316,-1.0,-1.0,40.666,1.07955,1418,2419836 +59,insert,212.387,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,134.414,1.18011,1433,2643083 +60,search,3922.69,2091.58,1426.44,0.96094,114.17,-nan,-nan,0.8957,0.12527,-1.0,-1.0,29.111,1.18025,1437,2643083 +61,insert,7.494,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,7.07,1.18439,1437,2653064 +62,search,4174.03,2092.84,1433.79,0.999942,120.149,-nan,-nan,0.89566,0.125219,-1.0,-1.0,6.565,1.18439,1437,2653064 +63,insert,2.264,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,6.214,1.18568,1437,2656118 +64,search,3831.67,2094.01,1455.05,1.01849,119.298,-nan,-nan,0.89579,0.125109,-1.0,-1.0,6.614,1.18568,1437,2656118 +65,insert,41.662,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,52.378,1.21047,1453,2723424 +66,search,3624.52,2126.81,1415.59,1.03841,119.033,-nan,-nan,0.89669,0.12415,-1.0,-1.0,8.637,1.21061,1454,2723424 +67,insert,23.692,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,48.871,1.23206,1469,2769021 +68,search,3659.36,2131.9,1436.3,1.05355,120.01,-nan,-nan,0.89779,0.123401,-1.0,-1.0,16.706,1.23223,1473,2769021 +69,insert,24.821,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,39.243,1.25341,1484,2821777 +70,search,3695.22,2134.13,1451.23,1.05794,124.672,-nan,-nan,0.89908,0.122793,-1.0,-1.0,10.167,1.25367,1485,2821777 +71,insert,8.385,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,11.34,1.25929,1487,2834921 +72,search,4325.33,2142.27,1415.32,1.05746,117.452,-nan,-nan,0.90003,0.12229,-1.0,-1.0,16.161,1.25935,1488,2834921 +73,insert,7.535,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,17.589,1.26219,1489,2842044 +74,search,4570.26,2143.59,1543.09,0.993555,121.9,-nan,-nan,0.89981,0.122471,-1.0,-1.0,12.545,1.26219,1489,2842044 +75,insert,25.834,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.791,1.26926,1490,2858890 +76,search,4574.79,2156.74,1546.15,0.97941,131.771,-nan,-nan,0.90005,0.122329,-1.0,-1.0,11.288,1.26926,1490,2858890 +77,insert,35.561,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,17.788,1.27844,1491,2882818 +78,search,4615.18,2167.85,1564.21,0.939767,135.21,-nan,-nan,0.89984,0.122496,-1.0,-1.0,13.272,1.27844,1491,2882818 +79,insert,481.218,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.99,1.30742,1492,2947806 +80,search,4517.42,2200.72,1502.66,1.09335,124.85,-nan,-nan,0.8995,0.122604,-1.0,-1.0,11.711,1.30742,1492,2947806 +81,insert,43.179,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,39.441,1.32074,1497,2976895 +82,search,4684.37,2208.95,1605.37,0.924379,145.748,-nan,-nan,0.90072,0.121644,-1.0,-1.0,18.958,1.3208,1498,2976895 +83,insert,16.358,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.67,1.3266,1501,2990682 +84,search,4547.94,2210.08,1610.83,0.975595,141.36,-nan,-nan,0.90136,0.121149,-1.0,-1.0,13.405,1.32668,1502,2990682 +85,insert,24.411,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,18.028,1.34056,1506,3025613 +86,search,4094.95,2221.68,1509.22,1.02205,130.943,-nan,-nan,0.90185,0.120941,-1.0,-1.0,11.399,1.34056,1506,3025613 +87,insert,606.709,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,638.653,1.54251,1595,3468632 +88,search,4921.17,2374.19,1760.66,1.00098,147.64,-nan,-nan,0.90522,0.117856,-1.0,-1.0,86.125,1.54251,1605,3468632 +89,insert,364.446,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,307.623,1.70253,1677,3831692 +90,search,5332.64,2472.17,1946.52,0.967054,159.683,-nan,-nan,0.90929,0.113609,-1.0,-1.0,48.134,1.70253,1683,3831692 +91,insert,29.098,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.404,1.71237,1685,3864067 +92,search,5335.62,2477.16,1916.86,0.968151,162.061,-nan,-nan,0.90925,0.113488,-1.0,-1.0,13.903,1.71237,1685,3864067 +93,insert,210.068,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,165.551,1.8181,1715,4103827 +94,search,5600.08,2542.53,2065.02,0.927304,184.717,-nan,-nan,0.91003,0.112621,-1.0,-1.0,41.096,1.81861,1718,4103827 +95,insert,14.982,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,18.433,1.823,1719,4119898 +96,search,5477.38,2548.23,1978.07,1.05451,162.504,-nan,-nan,0.91037,0.112289,-1.0,-1.0,16.248,1.823,1719,4119898 +97,insert,9.692,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,34.187,1.82599,1722,4129367 +98,search,5442.4,2547.51,1924.89,1.03489,159.298,-nan,-nan,0.91056,0.112102,-1.0,-1.0,23.581,1.82619,1723,4129367 +99,insert,6.962,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.909,1.82783,1723,4134502 +100,search,5597.96,2549.3,2062.75,0.969111,184.331,-nan,-nan,0.91068,0.111999,-1.0,-1.0,15.915,1.82783,1723,4134502 +101,insert,36.206,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.113,1.83822,1727,4164482 +102,search,5507.75,2562.36,2070.23,0.951069,181.729,-nan,-nan,0.91064,0.11203,-1.0,-1.0,15.905,1.83822,1727,4164482 +103,insert,16.916,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,45.014,1.84428,1732,4178601 +104,search,5579.7,2559.78,2029.35,0.999137,170.795,-nan,-nan,0.91084,0.11206,-1.0,-1.0,15.866,1.84428,1732,4178601 +105,insert,9.308,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.508,1.8465,1733,4187774 +106,search,5508.6,2564.05,1975.38,0.998379,161.914,-nan,-nan,0.91085,0.112368,-1.0,-1.0,16.772,1.8465,1733,4187774 +107,insert,19.42,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.175,1.85099,1733,4200109 +108,search,4853.57,2568.7,2083.29,0.942483,179.987,-nan,-nan,0.91059,0.112565,-1.0,-1.0,10.957,1.85099,1733,4200109 +109,insert,268.984,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,353.487,2.01503,1806,4557867 +110,search,5764.89,2652.85,2176.41,0.964127,183.428,-nan,-nan,0.91289,0.110385,-1.0,-1.0,101.807,2.01605,1818,4557867 +111,insert,28.423,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,69.688,2.02776,1827,4587252 +112,search,5942.74,2631.78,2200.08,0.966084,184.989,-nan,-nan,0.91342,0.109909,-1.0,-1.0,24.935,2.02776,1828,4587252 +113,insert,13.455,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,30.299,2.03296,1830,4600056 +114,search,5886.97,2630.34,2126.88,0.991812,175.459,-nan,-nan,0.91374,0.109701,-1.0,-1.0,20.094,2.03296,1830,4600056 +115,insert,29.989,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,55.753,2.04262,1836,4627422 +116,search,5974.57,2636.82,2191.19,0.990823,183.027,-nan,-nan,0.9144,0.109338,-1.0,-1.0,31.072,2.04287,1837,4627422 +117,insert,13.504,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,46.542,2.04819,1840,4639292 +118,search,5949.29,2636.57,2165.42,1.01338,181.97,-nan,-nan,0.91432,0.109371,-1.0,-1.0,23.314,2.04819,1840,4639292 +119,insert,5.368,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.807,2.0492,1840,4643423 +120,search,6060.09,2638.79,2263.57,0.97286,199.949,-nan,-nan,0.91438,0.109429,-1.0,-1.0,21.5,2.0492,1840,4643423 +121,insert,17.716,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.694,2.0547,1840,4657252 +122,search,6060.43,2646.35,2268.34,0.978382,197.196,-nan,-nan,0.91461,0.109221,-1.0,-1.0,21.499,2.0547,1840,4657252 +123,insert,400.891,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,457.173,2.18718,1885,4938477 +124,search,6270.59,2708.85,2372.8,0.960018,206.372,-nan,-nan,0.91551,0.108998,-1.0,-1.0,169.289,2.18801,1901,4938477 +125,insert,60.857,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,77.422,2.20631,1910,4980492 +126,search,6297.01,2684.65,2363.09,0.913016,206.927,-nan,-nan,0.91578,0.109018,-1.0,-1.0,93.82,2.20686,1919,4980492 +127,insert,65.32,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,94.677,2.22905,1932,5030515 +128,search,6285.35,2670.76,2316.4,1.00421,191.923,-nan,-nan,0.91628,0.108731,-1.0,-1.0,38.75,2.22907,1934,5030515 +129,delete,19,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,52.595,2.22697,1901,5009391 +130,search,6206.9,2663.29,2303.53,0.948941,196.303,-nan,-nan,0.91474,0.1097,-1.0,-1.0,24.941,2.22697,1901,5009391 +131,delete,5.103,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.09,2.22697,1901,5004093 +132,search,6239.44,2659.88,2330.52,0.933567,200.013,-nan,-nan,0.9148,0.109624,-1.0,-1.0,24.584,2.22697,1901,5004093 +133,delete,11.991,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,31.917,2.22617,1900,4991787 +134,search,6256.17,2650.47,2333.63,0.959278,204.392,-nan,-nan,0.91468,0.109914,-1.0,-1.0,24.479,2.22617,1900,4991787 +135,delete,17.271,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,48.595,2.22358,1897,4974857 +136,search,6121.25,2643.61,2234.65,1.00228,185.537,-nan,-nan,0.91447,0.10983,-1.0,-1.0,24.585,2.22358,1897,4974857 +137,delete,3.223,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.963,2.22358,1897,4971386 +138,search,6199.54,2640.46,2296.19,0.952652,195.776,-nan,-nan,0.91441,0.109872,-1.0,-1.0,24.711,2.22358,1897,4971386 +139,delete,9.49,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.705,2.22358,1897,4963487 +140,search,6239.46,2635.72,2315.73,0.932836,199.945,-nan,-nan,0.9144,0.109995,-1.0,-1.0,27.144,2.22358,1897,4963487 +141,delete,40.931,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,104.844,2.21723,1890,4941830 +142,search,6116.81,2629.62,2230.89,0.968625,182.233,-nan,-nan,0.91429,0.110094,-1.0,-1.0,24.336,2.21723,1890,4941830 +143,delete,17.801,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,106.489,2.2119,1886,4925657 +144,search,6241.41,2626.18,2314.91,0.954353,206.691,-nan,-nan,0.91456,0.109763,-1.0,-1.0,24.75,2.2119,1886,4925657 +145,delete,3.511,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.53,2.2119,1886,4922686 +146,search,6171.69,2625.56,2280.18,0.95377,193.989,-nan,-nan,0.91446,0.109852,-1.0,-1.0,24.855,2.2119,1886,4922686 +147,delete,14.843,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,31.69,2.21112,1885,4913636 +148,search,5968.34,2618.4,2105.17,1.09628,169.414,-nan,-nan,0.91408,0.110126,-1.0,-1.0,25.277,2.21112,1885,4913636 +149,delete,223.864,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1302.35,2.12677,1783,4717250 +150,search,5884.96,2661.2,2206.16,0.937183,200.538,-nan,-nan,0.91312,0.110701,-1.0,-1.0,84.428,2.12769,1790,4717250 +151,delete,159.894,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1075.16,2.05462,1703,4560269 +152,search,5568.99,2679.2,2064.38,1.00481,172.801,-nan,-nan,0.91218,0.112074,-1.0,-1.0,127.943,2.05722,1715,4560269 +153,delete,27.506,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,128.201,2.04439,1702,4521936 +154,search,5546.05,2659.42,2030.95,0.994625,174.079,-nan,-nan,0.91132,0.112694,-1.0,-1.0,23.197,2.04439,1702,4521936 +155,delete,23.292,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,52.073,2.03931,1699,4502222 +156,search,5619.6,2650.09,2110,0.973728,191.059,-nan,-nan,0.91081,0.112923,-1.0,-1.0,32.594,2.03955,1700,4502222 +157,delete,41.423,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,183.427,2.0254,1688,4456039 +158,search,5480.81,2638.31,1995.63,1.0085,171.259,-nan,-nan,0.91089,0.113378,-1.0,-1.0,22.469,2.0254,1688,4456039 +159,delete,299.097,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1835.36,1.87414,1527,4143922 +160,search,4997.27,2749.53,1857.87,1.0208,151.177,-nan,-nan,0.90946,0.113899,-1.0,-1.0,261.886,1.88039,1562,4143922 +161,delete,41.329,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,72.183,1.8742,1558,4101445 +162,search,4951.31,2633.96,1735.78,1.10226,136.504,-nan,-nan,0.90957,0.113751,-1.0,-1.0,21.404,1.8742,1558,4101445 +163,delete,19.045,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,45.495,1.87226,1555,4080175 +164,search,5152.87,2617.81,1911.65,0.940008,171.623,-nan,-nan,0.90901,0.114164,-1.0,-1.0,21.459,1.87226,1555,4080175 +165,delete,30.365,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,105.488,1.86473,1549,4047313 +166,search,5104.63,2606.38,1884.37,0.940985,166.942,-nan,-nan,0.90922,0.113894,-1.0,-1.0,21.046,1.86473,1549,4047313 +167,delete,6.923,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.665,1.8645,1549,4040738 +168,search,5013.91,2601.87,1825.3,1.01188,149.682,-nan,-nan,0.90932,0.113879,-1.0,-1.0,21.053,1.8645,1549,4040738 +169,delete,22.424,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,48.181,1.86229,1547,4022674 +170,search,5083.86,2590.93,1876.55,0.939891,164.343,-nan,-nan,0.90937,0.113885,-1.0,-1.0,21.122,1.86229,1547,4022674 +171,delete,6.422,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.966,1.86229,1547,4015835 +172,search,5083.46,2584.95,1874.79,0.911814,162.678,-nan,-nan,0.90922,0.113929,-1.0,-1.0,21.08,1.86229,1547,4015835 +173,delete,35.471,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,32.943,1.85826,1546,3986424 +174,search,4968.43,2557.32,1780.44,1.00246,156.052,-nan,-nan,0.9087,0.114186,-1.0,-1.0,22.524,1.85826,1546,3986424 +175,delete,196.307,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,667.664,1.77137,1467,3804693 +176,search,4788.91,2585.27,1747.98,0.999854,149.163,-nan,-nan,0.90692,0.115298,-1.0,-1.0,67.559,1.77212,1472,3804693 +177,delete,19.12,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,79.175,1.76656,1467,3786352 +178,search,4822.86,2567.41,1780.5,0.960918,155.313,-nan,-nan,0.9067,0.11531,-1.0,-1.0,32.461,1.76676,1468,3786352 +179,delete,277.027,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1056.85,1.63374,1319,3556994 +180,search,4273.56,2659.04,1532.45,1.05382,128.427,-nan,-nan,0.89591,0.121702,-1.0,-1.0,90.475,1.6356,1327,3556994 +181,delete,27.897,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,67.27,1.62952,1323,3526824 +182,search,4409.88,2619.66,1642.78,0.97329,143.468,-nan,-nan,0.89651,0.121068,-1.0,-1.0,33.071,1.62965,1325,3526824 +183,delete,177.725,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,766.07,1.54039,1215,3350245 +184,search,4110.83,2703.35,1557.89,0.953996,134.8,-nan,-nan,0.8801,0.139869,-1.0,-1.0,74.203,1.54177,1224,3350245 +185,delete,10.713,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.32,1.53979,1219,3340234 +186,search,4088.08,2650.72,1525.64,0.909161,139.139,-nan,-nan,0.88494,0.131323,-1.0,-1.0,76.376,1.54156,1222,3340234 +187,delete,208.538,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,293.087,1.44974,1179,3150857 +188,search,3991.52,2669.46,1510.83,0.879377,133.039,-nan,-nan,0.88838,0.128941,-1.0,-1.0,51.964,1.45025,1183,3150857 +189,delete,5.765,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.249,1.45025,1183,3143609 +190,search,3947.97,2645.46,1480.44,0.928995,132.252,-nan,-nan,0.88771,0.129211,-1.0,-1.0,15.574,1.45025,1183,3143609 +191,delete,3.72,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.248,1.44994,1183,3141038 +192,search,3944.9,2642.42,1477.28,0.938585,134.787,-nan,-nan,0.88765,0.129279,-1.0,-1.0,15.468,1.44994,1183,3141038 +193,delete,61.533,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,195.747,1.42745,1163,3081734 +194,search,3822.43,2636.17,1408.42,0.985098,127.08,-nan,-nan,0.88646,0.130074,-1.0,-1.0,28.949,1.4277,1165,3081734 +195,delete,41.773,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,181.97,1.41055,1148,3043966 +196,search,3810.08,2631.49,1410.79,0.983833,126.422,-nan,-nan,0.88619,0.130706,-1.0,-1.0,56.306,1.41123,1152,3043966 +197,delete,46.868,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,60.084,1.39985,1148,3008145 +198,search,3791.53,2601.6,1390.59,0.993336,125.647,-nan,-nan,0.88571,0.130621,-1.0,-1.0,15.204,1.39985,1148,3008145 +199,delete,11.049,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,34.533,1.3969,1146,2999384 +200,search,3824.69,2602.5,1416.69,0.916913,128.082,-nan,-nan,0.88524,0.130848,-1.0,-1.0,15.232,1.3969,1146,2999384 +201,delete,5.915,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,40.514,1.39463,1144,2993528 +202,search,3831.01,2597.51,1417.67,0.953188,128.698,-nan,-nan,0.8851,0.130857,-1.0,-1.0,28.99,1.39499,1145,2993528 +203,delete,12.204,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.984,1.39463,1145,2980973 +204,search,3828.48,2583.25,1411.55,0.885326,128.193,-nan,-nan,0.88494,0.131202,-1.0,-1.0,24.844,1.39485,1146,2980973 +205,delete,23.455,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,40.644,1.3891,1137,2960291 +206,search,3793.82,2572.39,1401.72,0.944292,124.19,-nan,-nan,0.88421,0.131611,-1.0,-1.0,14.97,1.3891,1137,2960291 +207,delete,47.035,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,127.458,1.3685,1091,2920620 +208,search,3632.76,2585.86,1343.31,0.930398,117.831,-nan,-nan,0.88051,0.134818,-1.0,-1.0,18.274,1.3685,1091,2920620 +209,delete,23.518,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,52.039,1.36027,1084,2900501 +210,search,3241.03,2581.93,1331.42,0.92744,115.644,-nan,-nan,0.87964,0.135081,-1.0,-1.0,12.16,1.36027,1084,2900501 +211,delete,5.549,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.726,1.35739,1083,2891660 +212,search,3451.61,2575.67,1270.13,0.992371,111.12,-nan,-nan,0.87942,0.13526,-1.0,-1.0,27.299,1.35742,1084,2891660 +213,delete,20.776,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,77.62,1.35031,1078,2873257 +214,search,3454.82,2563.77,1320.7,0.944427,114.247,-nan,-nan,0.8783,0.13611,-1.0,-1.0,13.176,1.35031,1078,2873257 +215,delete,242.353,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1494.38,1.14861,879,2537571 +216,search,2819.05,2831.25,1086.51,0.997684,89.8229,-nan,-nan,0.87291,0.139252,-1.0,-1.0,283.342,1.15486,915,2537571 +217,delete,124.286,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,614.541,1.03598,777,2308194 +218,search,2242.3,2886.24,973.77,1.05279,82.8769,-nan,-nan,0.86617,0.144809,-1.0,-1.0,208.54,1.0438,816,2308194 +219,delete,10.622,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,35.81,1.03865,811,2285820 +220,search,2258.81,2664.37,937.309,1.0087,84.0616,-nan,-nan,0.8636,0.146864,-1.0,-1.0,54.036,1.0392,820,2285820 +221,delete,116.315,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,432.079,0.92963,679,2082858 +222,search,2236.92,2784.41,826.658,1.07193,73.2636,-nan,-nan,0.85355,0.153915,-1.0,-1.0,72.805,0.930751,687,2082858 +223,delete,9.198,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.679,0.927826,683,2071207 +224,search,2195.84,2728.25,859.636,1.05604,75.1523,-nan,-nan,0.8511,0.155631,-1.0,-1.0,25.913,0.928493,684,2071207 +225,delete,4.851,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,12.223,0.927506,683,2065400 +226,search,2325.03,2719.38,897.337,0.944892,84.1392,-nan,-nan,0.85107,0.155546,-1.0,-1.0,10.974,0.927506,683,2065400 +227,delete,3.13,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,10.145,0.927506,683,2062105 +228,search,2250.82,2713.96,906.232,0.928092,83.1459,-nan,-nan,0.85072,0.15576,-1.0,-1.0,10.109,0.927506,683,2062105 +229,delete,23.204,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,52.131,0.921084,677,2039610 +230,search,2362.79,2703.35,907.629,0.923592,82.8306,-nan,-nan,0.84902,0.156903,-1.0,-1.0,19.165,0.92136,678,2039610 +231,delete,12.669,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,50.528,0.916266,672,2027795 +232,search,2275.71,2712.62,858.851,0.978512,78.9798,-nan,-nan,0.84977,0.1567,-1.0,-1.0,40.02,0.916559,675,2027795 +233,delete,5.004,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.083,0.915492,674,2022859 +234,search,2361.8,2689.37,903.813,0.872645,90.1896,-nan,-nan,0.84893,0.157088,-1.0,-1.0,8.291,0.915492,674,2022859 +235,delete,4.035,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.819,0.915136,673,2016264 +236,search,2241.97,2684.54,838.206,1.04464,76.3599,-nan,-nan,0.84875,0.157118,-1.0,-1.0,11.084,0.915136,673,2016264 +237,delete,300.516,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,741.319,0.792515,551,1782866 +238,search,1970.75,2928.14,758.2,0.966397,66.7084,-nan,-nan,0.84117,0.164179,-1.0,-1.0,168.095,0.796057,568,1782866 +239,delete,19.763,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,63.324,0.786082,558,1758931 +240,search,1961.31,2822.31,744.12,1.04752,66.199,-nan,-nan,0.8384,0.164827,-1.0,-1.0,56.02,0.786385,563,1758931 +241,delete,6.942,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,34.026,0.782659,560,1751275 +242,search,1958.21,2792.76,733.607,1.01614,68.7278,-nan,-nan,0.83696,0.165016,-1.0,-1.0,28.018,0.782948,562,1751275 +243,delete,19.528,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,26.667,0.779254,559,1733264 +244,search,2014.6,2762.46,784.075,0.891631,75.9705,-nan,-nan,0.83554,0.166284,-1.0,-1.0,16.458,0.779466,560,1733264 +245,delete,9.118,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.89,0.7777,559,1722658 +246,search,1877.35,2739.45,745.89,1.00469,66.3436,-nan,-nan,0.83478,0.166521,-1.0,-1.0,8.38,0.7777,559,1722658 +247,delete,2.398,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,8.024,0.7777,559,1719601 +248,search,1999.46,2735.36,768.661,0.968369,71.9837,-nan,-nan,0.83448,0.166603,-1.0,-1.0,8.833,0.7777,559,1719601 +249,delete,9.259,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.894,0.774906,557,1711648 +250,search,1782.66,2729.12,744.329,0.989099,65.7997,-nan,-nan,0.83288,0.167399,-1.0,-1.0,6.575,0.774906,557,1711648 +251,delete,182.878,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,309.894,0.654489,462,1470624 +252,search,1712.59,2899.67,691.328,0.815954,63.6184,-nan,-nan,0.82899,0.168827,-1.0,-1.0,113.097,0.657254,474,1470624 +253,delete,17.523,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,33.714,0.648981,467,1442070 +254,search,1657.67,2772.52,650.173,0.905936,59.6014,-nan,-nan,0.82805,0.169312,-1.0,-1.0,9.59,0.648981,467,1442070 +255,delete,38.396,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,108.01,0.631023,448,1405549 +256,search,1607.2,2827.02,608.059,1.0469,54.6439,-nan,-nan,0.82206,0.17264,-1.0,-1.0,21.74,0.631206,450,1405549 +257,insert,12.883,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,7.83,0.634335,450,1416288 +258,search,1679.41,2822.47,658.116,0.848679,58.8,-nan,-nan,0.82329,0.172639,-1.0,-1.0,15.503,0.634382,451,1416288 +259,insert,31.684,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,53.947,0.645258,456,1443010 +260,search,1684.51,2829.23,670.994,0.949099,63.0048,-nan,-nan,0.82194,0.173316,-1.0,-1.0,6.224,0.645258,456,1443010 +261,insert,5.011,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.508,0.647009,458,1447529 +262,search,1474.77,2822.5,640.758,0.892868,57.7263,-nan,-nan,0.8238,0.172442,-1.0,-1.0,6.118,0.647009,458,1447529 +263,insert,83.539,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,78.918,0.697252,466,1569110 +264,search,1525.41,3022.32,687.238,0.896871,60.5443,-nan,-nan,0.82631,0.17282,-1.0,-1.0,48.836,0.697488,470,1569110 +265,insert,50.39,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,89.455,0.72656,488,1636119 +266,search,1771.95,3007.27,711.995,0.936005,67.3647,-nan,-nan,0.83096,0.1699,-1.0,-1.0,110.28,0.7266,496,1636119 +267,insert,32.684,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,65.499,0.740769,502,1665152 +268,search,1881.52,2988.14,767.584,0.861896,68.0138,-nan,-nan,0.83476,0.168296,-1.0,-1.0,65.923,0.74091,508,1665152 +269,insert,301.231,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,297.547,0.85372,536,1916952 +270,search,2050,3189.22,859.202,0.901939,76.5703,-nan,-nan,0.83636,0.166429,-1.0,-1.0,155.045,0.854696,549,1916952 +271,insert,5.159,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.066,0.855858,551,1920898 +272,search,2000.86,3101.84,862.519,0.910505,77.5885,-nan,-nan,0.83768,0.165902,-1.0,-1.0,37.579,0.856075,556,1920898 +273,insert,10.793,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,31.858,0.864167,562,1942757 +274,search,2053.78,3078.13,843.481,0.91838,74.443,-nan,-nan,0.8402,0.164089,-1.0,-1.0,29.238,0.864406,564,1942757 +275,insert,379.972,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,565.93,1.04288,639,2328372 +276,search,2336.01,3271.08,993.646,0.975758,86.1996,-nan,-nan,0.84997,0.158664,-1.0,-1.0,226.884,1.04488,665,2328372 +277,insert,37.932,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,58.403,1.05856,673,2363257 +278,search,2460.17,3131.63,1008.49,0.97955,88.1646,-nan,-nan,0.85234,0.157049,-1.0,-1.0,67.228,1.05894,677,2363257 +279,insert,31.067,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.569,1.06985,679,2388933 +280,search,2520.9,3142.43,1045.16,0.931414,97.2971,-nan,-nan,0.85343,0.156073,-1.0,-1.0,53.05,1.06985,683,2388933 +281,insert,298.475,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,389.568,1.20574,718,2684315 +282,search,2672.5,3253.24,1122.57,0.899991,102.917,-nan,-nan,0.85521,0.156094,-1.0,-1.0,151.358,1.20709,731,2684315 +283,insert,463.968,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,632.265,1.40051,821,3109157 +284,search,3023.2,3313.63,1278.21,0.935867,112.351,-nan,-nan,0.86272,0.151953,-1.0,-1.0,235.403,1.40196,850,3109157 +285,insert,13.913,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.436,1.40656,852,3122751 +286,search,3119.25,3205.68,1297.79,0.887082,115.605,-nan,-nan,0.86484,0.150778,-1.0,-1.0,71.309,1.40759,857,3122751 +287,insert,41.042,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,68.449,1.42637,863,3163679 +288,search,3173.1,3206.3,1316,0.852168,114.964,-nan,-nan,0.86478,0.1509,-1.0,-1.0,44.762,1.4271,867,3163679 +289,insert,358.136,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,481.725,1.56119,918,3472031 +290,search,3380.11,3305.24,1429.79,0.93077,127.844,-nan,-nan,0.8675,0.149583,-1.0,-1.0,244.059,1.56208,934,3472031 +291,insert,27.425,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,63.04,1.57295,943,3495577 +292,search,3475.38,3232.06,1458.91,0.880425,129.647,-nan,-nan,0.86951,0.147906,-1.0,-1.0,94.5,1.57296,951,3495577 +293,insert,19.206,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,45.569,1.58128,955,3515193 +294,search,3348.67,3209.65,1448.97,0.925965,128.635,-nan,-nan,0.86933,0.148408,-1.0,-1.0,32.365,1.58128,957,3515193 +295,insert,243.151,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,353.285,1.66991,1000,3723063 +296,search,3613.18,3267.32,1482.85,1.02531,131.682,-nan,-nan,0.87427,0.145541,-1.0,-1.0,154.276,1.67165,1021,3723063 +297,insert,3.672,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.66,1.6729,1022,3727399 +298,search,3679.88,3195.06,1516.01,0.93827,135.027,-nan,-nan,0.87476,0.145191,-1.0,-1.0,51.892,1.67363,1027,3727399 +299,insert,76.237,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,125.217,1.69647,1047,3788002 +300,search,3641.7,3171.8,1455.56,1.03589,123.293,-nan,-nan,0.87695,0.142589,-1.0,-1.0,20.079,1.69647,1048,3788002 +301,insert,726.209,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1123.27,1.9695,1193,4370866 +302,search,4327.18,3343.84,1834.65,0.945757,157.284,-nan,-nan,0.90184,0.122855,-1.0,-1.0,213.132,1.96994,1223,4370866 +303,insert,57.755,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,91.145,1.99022,1233,4420306 +304,search,4354.31,3221.57,1776.23,0.996096,146.162,-nan,-nan,0.90056,0.123259,-1.0,-1.0,22.087,1.99022,1235,4420306 +305,insert,389.532,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,487.363,2.15567,1282,4789669 +306,search,4670.03,3317.69,1969.46,0.90406,174.906,-nan,-nan,0.90337,0.121587,-1.0,-1.0,234.755,2.15679,1300,4789669 +307,insert,28.818,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,39.672,2.16441,1304,4815550 +308,search,4570.46,3281.93,1853.73,1.02517,154.763,-nan,-nan,0.90379,0.121146,-1.0,-1.0,120.304,2.16472,1314,4815550 +309,insert,9.142,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,18.184,2.1675,1315,4823345 +310,search,4599.27,3262.17,1847.45,1.06239,155.242,-nan,-nan,0.90476,0.120367,-1.0,-1.0,41.025,2.1675,1318,4823345 +311,insert,21.887,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,17.995,2.17652,1319,4843493 +312,search,4743.15,3259.62,1991.83,0.931453,175.201,-nan,-nan,0.90413,0.120811,-1.0,-1.0,12.58,2.17652,1319,4843493 +313,insert,45.049,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,62.42,2.19327,1325,4885489 +314,search,4593.04,3273.18,1828.36,1.08159,146.448,-nan,-nan,0.90448,0.120895,-1.0,-1.0,54.135,2.19378,1330,4885489 +315,insert,7.759,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,12.028,2.19653,1330,4892335 +316,search,4695.79,3265.1,1927.02,1.01447,159.444,-nan,-nan,0.90464,0.120699,-1.0,-1.0,28.56,2.19677,1332,4892335 +317,insert,46.084,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,75.319,2.2123,1340,4932996 +318,search,4691.72,3274.45,1903.16,0.998541,157.142,-nan,-nan,0.90497,0.120457,-1.0,-1.0,24.507,2.21274,1341,4932996 +319,insert,619.613,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,849.439,2.4253,1450,5430684 +320,search,5150.5,3367.53,2157.44,0.971539,181.974,-nan,-nan,0.90971,0.115656,-1.0,-1.0,348.74,2.43284,1498,5430684 +321,insert,48.216,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,78.077,2.44999,1507,5471239 +322,search,5396.26,3253.34,2251.74,0.980214,203.623,-nan,-nan,0.90969,0.115915,-1.0,-1.0,70.375,2.4501,1511,5471239 +323,insert,110.738,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,163.358,2.49315,1534,5575560 +324,search,5372.27,3262.58,2197.35,1.01611,187.663,-nan,-nan,0.91074,0.114487,-1.0,-1.0,64.482,2.49331,1540,5575560 +325,insert,1350.21,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,744.909,2.72229,1607,6069499 +326,search,5819.25,3387.17,2488.83,0.885272,222.315,-nan,-nan,0.91389,0.112886,-1.0,-1.0,356.899,2.72298,1637,6069499 +327,insert,74.622,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,135.3,2.75082,1652,6139647 +328,search,5961.96,3329.96,2517.82,0.890001,224.411,-nan,-nan,0.91588,0.111257,-1.0,-1.0,195.588,2.75146,1670,6139647 +329,insert,12.29,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.56,2.75487,1671,6146884 +330,search,5994.27,3296.34,2510.96,0.933243,224.867,-nan,-nan,0.91578,0.111413,-1.0,-1.0,79.178,2.75487,1677,6146884 +331,insert,52.508,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,65.245,2.77313,1683,6188816 +332,search,5812.49,3297.48,2411.6,0.997272,203.743,-nan,-nan,0.91598,0.111387,-1.0,-1.0,53.674,2.77336,1687,6188816 +333,insert,3.445,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.552,2.774,1688,6190810 +334,search,5887.92,3286.63,2408.84,1.01514,205.762,-nan,-nan,0.916,0.111293,-1.0,-1.0,16.24,2.774,1688,6190810 +335,insert,53.085,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,125.146,2.79201,1705,6230108 +336,search,6047.76,3275.75,2503.97,0.947008,216.864,-nan,-nan,0.91662,0.110599,-1.0,-1.0,24.521,2.79213,1706,6230108 +337,insert,23.83,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,53.62,2.80041,1710,6249699 +338,search,6015.1,3277.55,2464.29,0.969751,209.416,-nan,-nan,0.91734,0.109968,-1.0,-1.0,34.473,2.80082,1712,6249699 +339,insert,55.837,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,71.357,2.8143,1721,6290226 +340,search,5897.39,3273.64,2369.4,1.04496,199.284,-nan,-nan,0.91768,0.109371,-1.0,-1.0,24.682,2.81446,1722,6290226 +341,insert,10.201,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,32.565,2.81807,1724,6297556 +342,search,5910.98,3271.67,2356.73,1.03452,189.78,-nan,-nan,0.91811,0.109216,-1.0,-1.0,33.96,2.81816,1726,6297556 +343,insert,37.572,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,97.79,2.83089,1739,6330843 +344,search,6093.56,3267.8,2504.03,0.956056,212.847,-nan,-nan,0.91839,0.109113,-1.0,-1.0,16.161,2.83089,1739,6330843 +345,insert,47.194,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,99.111,2.84804,1751,6370265 +346,search,6133.34,3270.59,2510.62,1.06201,213.915,-nan,-nan,0.91898,0.108793,-1.0,-1.0,32.223,2.84845,1753,6370265 +347,insert,576.613,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1024.4,3.0527,1892,6831086 +348,search,6572.64,3264.31,2682.5,1.01507,230.679,-nan,-nan,0.92245,0.10564,-1.0,-1.0,158.023,3.05492,1911,6831086 +349,insert,46.141,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,94.702,3.071,1922,6876154 +350,search,6686.96,3240,2705.34,1.01986,233.088,-nan,-nan,0.92341,0.105273,-1.0,-1.0,91.691,3.07265,1931,6876154 +351,insert,511.096,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,590.774,3.25412,2004,7288266 +352,search,7007.87,3269.66,2866.06,1.02988,242.647,-nan,-nan,0.92153,0.107744,-1.0,-1.0,255.957,3.2571,2032,7288266 +353,insert,53.329,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,112.764,3.27437,2044,7333132 +354,search,7216.24,3227.19,2964.49,0.950497,261.174,-nan,-nan,0.92235,0.107172,-1.0,-1.0,92.679,3.27452,2050,7333132 +355,insert,420.443,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,572.177,3.4318,2118,7696838 +356,search,7260.17,3276.21,2901.06,1.07277,235.65,-nan,-nan,0.9243,0.105546,-1.0,-1.0,229.617,3.43481,2139,7696838 +357,insert,22.523,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.438,3.44102,2139,7714498 +358,search,7425,3251.61,2986.2,0.998383,251.154,-nan,-nan,0.92479,0.105277,-1.0,-1.0,69.872,3.44118,2143,7714498 +359,insert,14.244,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,64.194,3.44596,2148,7725755 +360,search,7417.76,3242.59,2992.08,1.0052,244.118,-nan,-nan,0.92497,0.105267,-1.0,-1.0,63.994,3.44655,2152,7725755 +361,insert,70.044,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,103.542,3.47145,2162,7789312 +362,search,7601.94,3245.29,3117.35,0.923066,263.508,-nan,-nan,0.92538,0.104971,-1.0,-1.0,57.172,3.4719,2165,7789312 +363,insert,22.973,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,63.568,3.47686,2169,7803163 +364,search,7621.73,3241.9,3132.04,0.942892,270.597,-nan,-nan,0.9255,0.104621,-1.0,-1.0,33.343,3.47698,2170,7803163 +365,insert,70.156,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,153.612,3.50109,2184,7863722 +366,search,7647.05,3246.73,3118.19,0.965928,269.034,-nan,-nan,0.92613,0.103962,-1.0,-1.0,75.502,3.50161,2189,7863722 +367,insert,37.557,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,99.471,3.51436,2199,7895271 +368,search,6683.07,3238.16,3017.39,1.09757,254.009,-nan,-nan,0.92648,0.103757,-1.0,-1.0,39.418,3.51474,2204,7895271 +369,insert,37.727,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,50.886,3.53846,2213,7961194 +370,search,7339.35,3241.68,2964.15,1.12009,235.826,-nan,-nan,0.92691,0.103002,-1.0,-1.0,25.758,3.5386,2214,7961194 +371,insert,9.048,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.53,3.54207,2215,7970239 +372,search,6519.07,3244.53,3064.67,1.02654,250.138,-nan,-nan,0.92707,0.102901,-1.0,-1.0,29.001,3.54244,2217,7970239 +373,insert,30.932,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,50.014,3.55884,2225,8010868 +374,search,7336.98,3249.95,3090.05,1.00083,252.335,-nan,-nan,0.92739,0.102535,-1.0,-1.0,34.458,3.55898,2226,8010868 +375,insert,42.219,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,93.999,3.57271,2235,8040698 +376,search,7581.04,3244.14,2997.68,1.07074,247.267,-nan,-nan,0.92744,0.102489,-1.0,-1.0,42.693,3.57301,2237,8040698 +377,insert,72.714,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,177.459,3.60131,2256,8104748 +378,search,7513.57,3237.15,3154.38,1.01857,266.033,-nan,-nan,0.92808,0.1021,-1.0,-1.0,31.998,3.60174,2258,8104748 +379,insert,3.647,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.455,3.60404,2261,8109755 +380,search,7364.77,3231.13,2987.09,1.12252,236.187,-nan,-nan,0.92815,0.102041,-1.0,-1.0,40.443,3.60412,2262,8109755 +381,insert,29.792,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.958,3.61216,2262,8130382 +382,search,7798.09,3235.2,3134.42,1.03693,262.457,-nan,-nan,0.92813,0.102121,-1.0,-1.0,48.38,3.61223,2263,8130382 +383,insert,45.598,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,75.628,3.62141,2268,8153290 +384,search,7966.5,3237.25,3245.31,0.97257,274.341,-nan,-nan,0.92847,0.101723,-1.0,-1.0,38.161,3.62163,2269,8153290 +385,delete,28.269,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,71.244,3.61938,2267,8138873 +386,search,7664.89,3231.25,3021.7,1.05249,252.126,-nan,-nan,0.92853,0.101602,-1.0,-1.0,29.353,3.61938,2267,8138873 +387,delete,33.958,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,172.401,3.61055,2260,8115069 +388,search,7726.69,3231.92,3220.18,0.963919,275.844,-nan,-nan,0.92851,0.101499,-1.0,-1.0,26.995,3.61059,2261,8115069 +389,delete,12.182,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,79.7,3.60667,2255,8101157 +390,search,7391.48,3233.11,2963.31,1.09291,236.608,-nan,-nan,0.92851,0.101745,-1.0,-1.0,47.989,3.60692,2257,8101157 +391,delete,118.449,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,507.92,3.57028,2222,8014644 +392,search,7563.68,3244.09,2995.7,1.06331,240.643,-nan,-nan,0.92826,0.101543,-1.0,-1.0,130.327,3.57251,2235,8014644 +393,delete,33.51,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,76.213,3.56852,2232,7978358 +394,search,7751.33,3206.08,3138.53,0.985914,264.828,-nan,-nan,0.92772,0.101938,-1.0,-1.0,38.174,3.56871,2233,7978358 +395,delete,22.898,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,268.689,3.55564,2221,7957094 +396,search,7704.65,3211.25,3096.29,1.04991,258.24,-nan,-nan,0.92721,0.102259,-1.0,-1.0,71.18,3.55647,2226,7957094 +397,delete,125.734,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,967.34,3.49258,2179,7823916 +398,search,7174.57,3224.18,2858.87,1.12191,228.633,-nan,-nan,0.92698,0.102485,-1.0,-1.0,82.525,3.49517,2191,7823916 +399,delete,3.406,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,37.283,3.49282,2189,7818430 +400,search,7138.88,3206.27,2907.31,1.04754,235.689,-nan,-nan,0.92682,0.102751,-1.0,-1.0,53.717,3.49359,2192,7818430 +401,delete,23.358,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,93.92,3.4873,2187,7800702 +402,search,7439.94,3200.29,2942.05,1.08446,240.888,-nan,-nan,0.92615,0.103543,-1.0,-1.0,27.273,3.4873,2187,7800702 +403,delete,318.744,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1971.26,3.35721,2066,7517858 +404,search,7318.57,3259.51,3021.14,0.966763,260.4,-nan,-nan,0.92413,0.104666,-1.0,-1.0,345.83,3.36418,2103,7517858 +405,delete,57.87,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,349.815,3.34562,2087,7451637 +406,search,7316.51,3199.82,2979.55,0.929866,259.63,-nan,-nan,0.9246,0.104458,-1.0,-1.0,48.129,3.34573,2089,7451637 +407,delete,119.526,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,727.889,3.29527,2050,7360417 +408,search,6820.57,3215.4,2999.82,0.940746,268.129,-nan,-nan,0.924,0.105083,-1.0,-1.0,61.222,3.29728,2058,7360417 +409,delete,123.392,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,548.886,3.19649,2015,7131620 +410,search,6648.72,3207.76,2740.55,1.08397,226.292,-nan,-nan,0.9239,0.10507,-1.0,-1.0,142.863,3.19919,2027,7131620 +411,delete,264.933,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1700.66,3.07728,1909,6867617 +412,search,5593.06,3259.32,2669.2,1.08803,217.401,-nan,-nan,0.9227,0.105923,-1.0,-1.0,181.132,3.08294,1942,6867617 +413,delete,8.448,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,50.516,3.07899,1940,6850098 +414,search,6411.45,3189.65,2667.6,1.04305,219.436,-nan,-nan,0.92311,0.105568,-1.0,-1.0,54.791,3.07974,1945,6850098 +415,delete,46.648,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,469.809,3.04976,1920,6782921 +416,search,5909.27,3193.23,2604.48,1.0886,212.761,-nan,-nan,0.92253,0.105951,-1.0,-1.0,24.338,3.05014,1922,6782921 +417,delete,132.022,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,803.787,2.95764,1853,6562039 +418,search,6111.72,3201.93,2558.21,1.06611,207.496,-nan,-nan,0.92219,0.106296,-1.0,-1.0,182.306,2.96093,1870,6562039 +419,delete,23.63,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,75.679,2.95761,1866,6536285 +420,search,5556.45,3168.45,2531.05,1.09674,207.653,-nan,-nan,0.92306,0.10547,-1.0,-1.0,36.236,2.95802,1869,6536285 +421,delete,11.328,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,88.129,2.94541,1860,6510820 +422,search,6001.67,3168.88,2316.04,1.17717,182.709,-nan,-nan,0.92304,0.105414,-1.0,-1.0,42.363,2.94554,1862,6510820 +423,delete,170.011,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,739.241,2.8699,1799,6334168 +424,search,6360.56,3182.36,2600.05,0.940237,221.347,-nan,-nan,0.92143,0.106491,-1.0,-1.0,97.606,2.87113,1807,6334168 +425,delete,18.416,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,34.612,2.86931,1806,6316443 +426,search,6253.94,3161.74,2506.7,1.0065,211.416,-nan,-nan,0.92151,0.106427,-1.0,-1.0,21.693,2.86931,1806,6316443 +427,delete,50.857,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,75.528,2.86257,1802,6266345 +428,search,6028.08,3132.44,2567.41,0.92593,220.994,-nan,-nan,0.92139,0.10683,-1.0,-1.0,14.319,2.86257,1802,6266345 +429,delete,295.611,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1903.53,2.64175,1548,5883847 +430,search,5328.28,3497.19,2354.47,1.003,190.465,-nan,-nan,0.90995,0.116189,-1.0,-1.0,554.999,2.65396,1623,5883847 +431,delete,63.918,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,326.193,2.61605,1597,5801380 +432,search,4857.96,3296.31,2196.26,1.1097,178.81,-nan,-nan,0.90944,0.116149,-1.0,-1.0,54.862,2.61728,1608,5801380 +433,delete,117.794,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,678.466,2.52854,1534,5607567 +434,search,4978.56,3302.96,2200.51,1.06557,179.068,-nan,-nan,0.9066,0.118421,-1.0,-1.0,286.748,2.53286,1550,5607567 +435,delete,108.783,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,382.521,2.49578,1517,5512176 +436,search,5266.66,3283.45,2195.8,1.02103,186.471,-nan,-nan,0.90601,0.118772,-1.0,-1.0,167.033,2.49699,1525,5512176 +437,delete,10.078,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,61.464,2.4925,1522,5501141 +438,search,5481.27,3262.13,2259.23,1.0075,200.761,-nan,-nan,0.90592,0.118847,-1.0,-1.0,109.563,2.49249,1525,5501141 +439,delete,111.867,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,446.088,2.42841,1505,5369822 +440,search,5320.47,3260.99,2178.06,1.02306,191.051,-nan,-nan,0.90804,0.116471,-1.0,-1.0,194.688,2.43167,1518,5369822 +441,delete,26.272,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,95.35,2.42335,1513,5338985 +442,search,5320.39,3220.76,2165.24,0.999202,186.557,-nan,-nan,0.9074,0.116871,-1.0,-1.0,42.142,2.42366,1515,5338985 +443,delete,30.05,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,72.807,2.40932,1510,5307279 +444,search,5405.07,3223,2234.82,0.931145,196.857,-nan,-nan,0.90823,0.116297,-1.0,-1.0,41.614,2.40992,1513,5307279 +445,delete,40.164,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,101.597,2.39975,1509,5268382 +446,search,4683.97,3202.52,2119.25,1.04606,180.897,-nan,-nan,0.90795,0.116547,-1.0,-1.0,18.87,2.39994,1510,5268382 +447,delete,248.542,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1031.85,2.22482,1344,4922787 +448,search,4541.22,3348.01,1974.92,0.932628,163.588,-nan,-nan,0.90048,0.121312,-1.0,-1.0,210.619,2.22825,1368,4922787 +449,delete,34.397,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,143.694,2.21642,1357,4887227 +450,search,4870.89,3276.33,2023.3,0.985415,177.908,-nan,-nan,0.90037,0.120997,-1.0,-1.0,145.681,2.2198,1366,4887227 +451,delete,99.202,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,346.317,2.17986,1329,4795082 +452,search,4721.15,3270.38,1926.78,0.99336,161.347,-nan,-nan,0.90048,0.120866,-1.0,-1.0,137.706,2.18065,1338,4795082 +453,delete,329.461,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1442.69,2.03676,1209,4500175 +454,search,4420.59,3342.82,1857.81,0.925934,168.886,-nan,-nan,0.89464,0.125924,-1.0,-1.0,301.371,2.04091,1233,4500175 +455,delete,27.789,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,113.083,2.03225,1227,4462648 +456,search,4306.75,3290.96,1734.81,1.02701,145.63,-nan,-nan,0.89481,0.12525,-1.0,-1.0,155.158,2.03378,1235,4462648 +457,delete,5.562,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.765,2.03339,1235,4456761 +458,search,4269.38,3260.06,1689.96,0.993304,140.033,-nan,-nan,0.89492,0.125163,-1.0,-1.0,70.001,2.03339,1237,4456761 +459,delete,37.797,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,192.458,2.01224,1222,4419992 +460,search,4459.62,3268.48,1858.21,0.891393,166.899,-nan,-nan,0.89516,0.124989,-1.0,-1.0,92.441,2.01284,1228,4419992 +461,delete,3.852,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.341,2.01284,1228,4415725 +462,search,4442.84,3251.87,1847.68,0.942499,165.987,-nan,-nan,0.89501,0.125028,-1.0,-1.0,14.627,2.01284,1228,4415725 +463,delete,59.055,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,308.44,1.98591,1196,4370609 +464,search,4366.06,3318.33,1836.73,0.91728,163.494,-nan,-nan,0.89476,0.124912,-1.0,-1.0,102.767,1.98709,1206,4370609 +465,delete,19.726,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,79.21,1.97564,1199,4345244 +466,search,4369.32,3293.81,1822.88,0.939983,163.672,-nan,-nan,0.89366,0.125433,-1.0,-1.0,46.321,1.97577,1202,4345244 +467,delete,43.129,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,73.013,1.96762,1198,4319734 +468,search,4330.04,3268.19,1812.18,0.879917,153.832,-nan,-nan,0.89333,0.125693,-1.0,-1.0,35.585,1.96823,1200,4319734 +469,delete,17.339,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,40.855,1.95915,1196,4298361 +470,search,4341.33,3258.25,1802.04,0.891265,157.429,-nan,-nan,0.89268,0.126084,-1.0,-1.0,32.088,1.95943,1197,4298361 +471,delete,138.787,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,351.396,1.90577,1155,4188315 +472,search,4204.73,3283.2,1752.54,0.927696,154.141,-nan,-nan,0.89105,0.127821,-1.0,-1.0,95.806,1.90754,1165,4188315 +473,delete,79.44,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,481.991,1.85001,1115,4074567 +474,search,4086.5,3300.73,1711.85,0.930999,153.052,-nan,-nan,0.88992,0.12902,-1.0,-1.0,197.874,1.85301,1135,4074567 +475,delete,425.331,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1660.32,1.65476,960,3691698 +476,search,3523.61,3483.51,1460.97,1.03829,125.974,-nan,-nan,0.88364,0.133768,-1.0,-1.0,416.558,1.66312,1005,3691698 +477,delete,42.84,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,209.953,1.64053,988,3642619 +478,search,3524.29,3319.99,1437.12,1.01488,118.959,-nan,-nan,0.88312,0.134124,-1.0,-1.0,109.978,1.6417,997,3642619 +479,delete,255.693,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1164.04,1.53338,904,3421368 +480,search,3485.2,3626.92,1543.99,0.865196,127.066,-nan,-nan,0.89538,0.123596,-1.0,-1.0,495.089,1.54338,938,3421368 +481,delete,50.333,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,141.596,1.52656,924,3378034 +482,search,3450.14,3298.65,1449.3,0.86281,125.651,-nan,-nan,0.8883,0.129703,-1.0,-1.0,81.255,1.52738,930,3378034 +483,delete,318.314,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1133.11,1.3873,803,3115682 +484,search,3066.05,3472.48,1322.41,0.895326,117.215,-nan,-nan,0.88012,0.135622,-1.0,-1.0,368.188,1.39393,842,3115682 +485,delete,18.781,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.006,1.38961,839,3094629 +486,search,3117.72,3278.94,1315.78,0.868895,116.71,-nan,-nan,0.88154,0.134697,-1.0,-1.0,78.613,1.39016,844,3094629 +487,delete,6.788,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.138,1.39016,844,3086025 +488,search,3135.63,3251.99,1316.74,0.893284,111.673,-nan,-nan,0.882,0.133896,-1.0,-1.0,14.172,1.39016,844,3086025 +489,delete,36.173,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,152.789,1.37753,833,3052105 +490,search,3062.95,3259.06,1273.43,0.868054,113.733,-nan,-nan,0.88072,0.135296,-1.0,-1.0,37.515,1.37774,835,3052105 +491,delete,14.971,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.361,1.37463,833,3038974 +492,search,2874.97,3244.8,1251.84,0.931173,103.951,-nan,-nan,0.88044,0.13549,-1.0,-1.0,11.328,1.37463,833,3038974 +493,delete,103.237,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,421.812,1.30586,780,2911063 +494,search,2920.62,3320.84,1236.47,0.948831,108.147,-nan,-nan,0.87808,0.136384,-1.0,-1.0,115.199,1.30818,791,2911063 +495,delete,21.528,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,65.277,1.2991,784,2888785 +496,search,2797.64,3267.04,1209.24,0.914553,107.277,-nan,-nan,0.87664,0.137255,-1.0,-1.0,16.476,1.29913,785,2888785 +497,delete,39.008,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,128.743,1.27627,768,2829718 +498,search,2830.1,3261.38,1194.34,0.907964,105.703,-nan,-nan,0.87503,0.138125,-1.0,-1.0,33.781,1.27684,770,2829718 +499,delete,16.216,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,11.746,1.27684,770,2816082 +500,search,2702.68,3229.99,1093.33,0.991909,94.7944,-nan,-nan,0.87498,0.138244,-1.0,-1.0,12.873,1.27684,770,2816082 +501,delete,27.868,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,73.406,1.26517,761,2790880 +502,search,2830.65,3239.62,1181.66,0.85648,107.407,-nan,-nan,0.87437,0.138568,-1.0,-1.0,12.431,1.26517,761,2790880 +503,delete,18.896,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,45.387,1.26025,757,2774459 +504,search,2776.97,3234.33,1154.66,0.886522,101.472,-nan,-nan,0.87338,0.13908,-1.0,-1.0,11.914,1.26025,757,2774459 +505,delete,71.802,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,153.495,1.2286,739,2714683 +506,search,2678.03,3238,1085.57,1.03208,91.0106,-nan,-nan,0.87194,0.139822,-1.0,-1.0,31.474,1.22915,741,2714683 +507,delete,40.973,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,70.144,1.21642,733,2680986 +508,search,2644.47,3225.37,1067.41,1.00653,93.7509,-nan,-nan,0.87144,0.139895,-1.0,-1.0,22.687,1.21655,734,2680986 +509,delete,22.05,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,100.48,1.20635,728,2662966 +510,search,2555.58,3229.94,1104.99,1.05744,94.2531,-nan,-nan,0.87039,0.14035,-1.0,-1.0,40.91,1.20711,731,2662966 +511,delete,14.743,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,56.617,1.19964,725,2642855 +512,search,2669.19,3215.05,1124.77,0.894691,97.1487,-nan,-nan,0.86934,0.141055,-1.0,-1.0,8.295,1.19964,725,2642855 +513,insert,195.502,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,254.596,1.30471,762,2906838 +514,search,2416.3,3328.46,1149.99,0.964045,97.2351,-nan,-nan,0.87251,0.139927,-1.0,-1.0,27.752,1.30485,765,2906838 +515,insert,3.077,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.523,1.30655,767,2913746 +516,search,2737.19,3317.83,1147.62,1.01443,99.5861,-nan,-nan,0.87328,0.139457,-1.0,-1.0,27.153,1.30677,769,2913746 +517,insert,25.525,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.33,1.31227,771,2936862 +518,search,2728.04,3344.85,1200.94,0.935599,105.289,-nan,-nan,0.87356,0.13903,-1.0,-1.0,11.367,1.31227,771,2936862 +519,insert,82.086,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,116.43,1.34433,780,3009732 +520,search,2836.87,3386.3,1177.05,1.05478,95.9398,-nan,-nan,0.87595,0.138859,-1.0,-1.0,71.62,1.34441,785,3009732 +521,insert,21.068,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,62.293,1.35021,789,3025906 +522,search,2980.94,3374.04,1274.3,0.951973,117.037,-nan,-nan,0.87668,0.138098,-1.0,-1.0,22.975,1.35021,790,3025906 +523,insert,51.085,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,92.046,1.36607,797,3061899 +524,search,2913.28,3385.74,1212.04,0.996898,101.61,-nan,-nan,0.87733,0.13801,-1.0,-1.0,41.261,1.36614,800,3061899 +525,insert,61.289,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,93,1.38449,808,3101209 +526,search,3059.52,3378.99,1302.23,0.926886,117.73,-nan,-nan,0.87841,0.137317,-1.0,-1.0,40.447,1.38489,811,3101209 +527,insert,632.079,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,943.075,1.57427,929,3522334 +528,search,3290.57,3368.9,1333.65,1.08197,108.472,-nan,-nan,0.88664,0.131366,-1.0,-1.0,204.792,1.57645,949,3522334 +529,insert,360.872,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,570.691,1.69686,1014,3797294 +530,search,3607.5,3364.32,1481.83,1.08783,120.726,-nan,-nan,0.89122,0.129196,-1.0,-1.0,199.66,1.69771,1030,3797294 +531,insert,97.791,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,144.098,1.72301,1044,3853781 +532,search,3857.6,3310.69,1636.18,0.896864,146.149,-nan,-nan,0.89321,0.12743,-1.0,-1.0,125.415,1.72333,1053,3853781 +533,insert,24.94,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,56.771,1.73058,1058,3873352 +534,search,3887.09,3285.75,1636.73,0.920917,148.222,-nan,-nan,0.89371,0.126638,-1.0,-1.0,103.111,1.73064,1065,3873352 +535,insert,25.506,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,70.745,1.73762,1072,3893308 +536,search,3896.47,3265,1643.05,0.911835,145.753,-nan,-nan,0.89457,0.126209,-1.0,-1.0,77.399,1.73852,1077,3893308 +537,insert,7.53,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,26.423,1.74185,1078,3901271 +538,search,3886.99,3247.83,1619.63,0.935806,141.383,-nan,-nan,0.89409,0.12659,-1.0,-1.0,15.509,1.74185,1078,3901271 +539,insert,27.201,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,42.022,1.74886,1081,3925164 +540,search,3904.94,3262.73,1624.96,0.957087,141.7,-nan,-nan,0.89496,0.125682,-1.0,-1.0,41.966,1.74897,1084,3925164 +541,insert,541.349,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,810.14,1.93325,1175,4320915 +542,search,4294.68,3330.28,1823.84,0.940486,161.564,-nan,-nan,0.89872,0.123679,-1.0,-1.0,318.031,1.93665,1208,4320915 +543,insert,58.958,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,88.189,1.95324,1218,4360814 +544,search,4383.73,3235.13,1818.1,0.921105,162.295,-nan,-nan,0.89994,0.122732,-1.0,-1.0,117.234,1.95376,1231,4360814 +545,insert,13.054,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.251,1.95761,1232,4370220 +546,search,4335.78,3192.1,1756.79,1.10201,147.775,-nan,-nan,0.90014,0.122741,-1.0,-1.0,22.882,1.9577,1233,4370220 +547,insert,200.853,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,306.213,2.02076,1275,4510675 +548,search,4555.29,3202.21,1889.69,0.970138,161.03,-nan,-nan,0.90249,0.121012,-1.0,-1.0,46.641,2.02076,1280,4510675 +549,insert,284.665,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,322.286,2.12186,1308,4744845 +550,search,4624.74,3229.6,1891.05,0.989333,159.109,-nan,-nan,0.89802,0.1265,-1.0,-1.0,100.958,2.12202,1313,4744845 +551,insert,33.162,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,49.424,2.13498,1316,4771433 +552,search,4621.49,3227.45,1869.98,0.987076,153.509,-nan,-nan,0.89837,0.12565,-1.0,-1.0,41.662,2.13498,1317,4771433 +553,insert,11.263,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,29.049,2.13784,1319,4782909 +554,search,4757.75,3220.34,1963.82,0.904206,172.122,-nan,-nan,0.89903,0.124551,-1.0,-1.0,37.499,2.13784,1320,4782909 +555,insert,549.768,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,793.104,2.31015,1440,5180322 +556,search,5117.88,3235.65,2122.06,0.927396,181.356,-nan,-nan,0.90436,0.119942,-1.0,-1.0,115.84,2.31125,1451,5180322 +557,insert,24.254,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,34.509,2.31758,1454,5203220 +558,search,5196.07,3224.55,2166.88,0.92585,192.478,-nan,-nan,0.90495,0.119581,-1.0,-1.0,41.523,2.31758,1456,5203220 +559,insert,6.812,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,35.477,2.31965,1458,5210154 +560,search,5087.95,3221.71,2066.14,1.02345,171.341,-nan,-nan,0.90543,0.1195,-1.0,-1.0,36.149,2.32,1460,5210154 +561,insert,9.354,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.334,2.32314,1460,5215426 +562,search,5185.32,3219.45,2153.78,0.986308,186.755,-nan,-nan,0.90535,0.119572,-1.0,-1.0,16.395,2.32314,1460,5215426 +563,insert,61.564,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,97.79,2.33689,1470,5252726 +564,search,5209.55,3220.33,2148.85,0.950768,187.635,-nan,-nan,0.90526,0.120055,-1.0,-1.0,29.443,2.33725,1471,5252726 +565,insert,269.718,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,290.897,2.41953,1495,5447274 +566,search,5376.48,3267.05,2264.47,0.96646,199.791,-nan,-nan,0.90562,0.119668,-1.0,-1.0,168.679,2.42028,1505,5447274 +567,insert,14.908,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.658,2.42666,1506,5463135 +568,search,5298.81,3238.46,2177.32,1.02302,182.77,-nan,-nan,0.90516,0.119967,-1.0,-1.0,70.124,2.42666,1508,5463135 +569,insert,23.048,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.035,2.43331,1509,5478168 +570,search,5401.51,3243.65,2256.06,0.905999,193.525,-nan,-nan,0.90599,0.119095,-1.0,-1.0,67.514,2.43379,1513,5478168 +571,insert,15.332,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.442,2.43966,1513,5492309 +572,search,5321.92,3236.52,2184.71,0.98884,183.577,-nan,-nan,0.90528,0.119742,-1.0,-1.0,29.056,2.43966,1514,5492309 +573,insert,85.703,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,126.855,2.463,1527,5554599 +574,search,5367.89,3246.75,2195.59,0.979563,185.358,-nan,-nan,0.90623,0.118705,-1.0,-1.0,36.847,2.46332,1529,5554599 +575,insert,70.111,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,98.898,2.4839,1538,5604500 +576,search,5561,3253.36,2316.89,0.923053,206.188,-nan,-nan,0.90647,0.11878,-1.0,-1.0,58.285,2.48492,1542,5604500 +577,insert,47.359,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,107.951,2.49914,1556,5638834 +578,search,5456.52,3237.65,2232.55,1.03513,185.643,-nan,-nan,0.90686,0.118668,-1.0,-1.0,30.767,2.49953,1557,5638834 +579,insert,880.001,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1218.2,2.77831,1710,6229778 +580,search,6020.95,3289.3,2481.6,0.980136,208.252,-nan,-nan,0.91188,0.113878,-1.0,-1.0,171.488,2.78084,1731,6229778 +581,insert,40.896,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,82.969,2.79687,1739,6261240 +582,search,6092.58,3245.73,2484.98,0.990595,206.871,-nan,-nan,0.9123,0.113578,-1.0,-1.0,26.224,2.79699,1740,6261240 +583,insert,25.523,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,44.306,2.80489,1743,6283315 +584,search,6177.28,3251.56,2538.72,0.937662,220.15,-nan,-nan,0.91255,0.113324,-1.0,-1.0,29.244,2.80508,1744,6283315 +585,insert,335.267,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,470.112,2.93011,1796,6556101 +586,search,6351.27,3285.48,2622.21,1.01137,221.724,-nan,-nan,0.91435,0.112005,-1.0,-1.0,181.241,2.93203,1816,6556101 +587,insert,43.185,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,59.765,2.9446,1821,6587187 +588,search,6479.2,3250.62,2695.91,0.964076,234.471,-nan,-nan,0.91431,0.11221,-1.0,-1.0,21.193,2.9446,1821,6587187 +589,insert,66.944,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,93.11,2.95949,1830,6631526 +590,search,6508.61,3260.34,2711.98,0.920957,234.645,-nan,-nan,0.9154,0.111066,-1.0,-1.0,33.831,2.9596,1831,6631526 +591,insert,631.616,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,766.511,3.16777,1932,7108942 +592,search,6889.86,3344.43,2864.48,0.988992,239.442,-nan,-nan,0.91832,0.10931,-1.0,-1.0,210.849,3.16954,1956,7108942 +593,insert,223.933,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,311.76,3.25236,1986,7303976 +594,search,7057.03,3325.76,2925.93,0.959346,246.421,-nan,-nan,0.91982,0.108437,-1.0,-1.0,169.082,3.25365,2001,7303976 +595,insert,6.419,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,38.514,3.25597,2003,7309244 +596,search,7055.58,3296.36,2901.17,1.00031,241.156,-nan,-nan,0.9196,0.108593,-1.0,-1.0,91.327,3.25615,2008,7309244 +597,insert,289.991,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,385.055,3.35893,2056,7542477 +598,search,7243.73,3323.32,2986.39,0.990598,249.285,-nan,-nan,0.92169,0.107232,-1.0,-1.0,171.463,3.36043,2071,7542477 +599,insert,136.018,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,244.52,3.4069,2103,7654438 +600,search,7506.33,3291.79,3143.12,0.925781,271.656,-nan,-nan,0.92225,0.10684,-1.0,-1.0,95.268,3.40767,2110,7654438 +601,insert,73.423,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,132.068,3.43467,2123,7717936 +602,search,7610.74,3294.55,3195.13,0.928727,279.136,-nan,-nan,0.92304,0.106142,-1.0,-1.0,73.587,3.43538,2128,7717936 +603,insert,7.882,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,25.409,3.4372,2128,7723521 +604,search,7200.54,3289.43,2853.97,1.08113,225.357,-nan,-nan,0.92296,0.106217,-1.0,-1.0,26.176,3.4372,2128,7723521 +605,insert,59.984,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,96.315,3.45308,2137,7767404 +606,search,7586.58,3294.9,3164.64,0.942439,267.742,-nan,-nan,0.92347,0.105808,-1.0,-1.0,45.502,3.45338,2139,7767404 +607,insert,111.187,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,253.392,3.4791,2143,7826115 +608,search,7688.8,3300.04,3209.88,0.911506,278.456,-nan,-nan,0.9232,0.106089,-1.0,-1.0,44.442,3.47936,2144,7826115 +609,insert,296.191,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,446.524,3.5665,2193,8036351 +610,search,7580.56,3320.81,3070.59,1.09276,245.937,-nan,-nan,0.92499,0.104082,-1.0,-1.0,164.089,3.56864,2207,8036351 +611,insert,26.523,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,68.695,3.57696,2212,8060364 +612,search,7769.11,3300.26,3195.35,1.07089,276.979,-nan,-nan,0.92495,0.104065,-1.0,-1.0,34.344,3.57697,2213,8060364 +613,insert,260.877,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,361.691,3.66201,2256,8256083 +614,search,7957.98,3306.31,3278.48,0.969623,274.814,-nan,-nan,0.92589,0.1035,-1.0,-1.0,98.291,3.66374,2265,8256083 +615,insert,449.977,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,650.5,3.83651,2331,8625707 +616,search,8194.06,3331.6,3381.48,1.0007,279.662,-nan,-nan,0.92772,0.102026,-1.0,-1.0,217.908,3.83863,2353,8625707 +617,insert,384.552,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,591.813,3.98468,2409,8949227 +618,search,8346.97,3350.24,3409.64,1.02301,276.429,-nan,-nan,0.9304,0.0993631,-1.0,-1.0,244.827,3.98732,2435,8949227 +619,insert,283.327,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,418.714,4.08879,2484,9195551 +620,search,8770.52,3319.01,3652.06,0.979456,308.381,-nan,-nan,0.93131,0.0988982,-1.0,-1.0,224.755,4.09265,2509,9195551 +621,insert,43.127,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,67.125,4.10566,2516,9232372 +622,search,8816.77,3289.65,3647.23,0.95471,308.845,-nan,-nan,0.93183,0.0985834,-1.0,-1.0,80.485,4.10638,2522,9232372 +623,insert,24.303,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,63.535,4.11548,2528,9253561 +624,search,8855.14,3283,3690.23,0.958383,310.701,-nan,-nan,0.93194,0.0984911,-1.0,-1.0,42.076,4.11585,2530,9253561 +625,insert,91.121,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,170.833,4.14724,2545,9331230 +626,search,8847.04,3288.19,3624.09,1.03692,295.504,-nan,-nan,0.93248,0.0980937,-1.0,-1.0,84.096,4.14776,2551,9331230 +627,insert,75.572,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,94.628,4.16479,2560,9374063 +628,search,8948.65,3284.76,3669.7,0.980996,307.108,-nan,-nan,0.93258,0.0979593,-1.0,-1.0,65.597,4.16537,2564,9374063 +629,insert,23.527,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,33.121,4.17065,2565,9387598 +630,search,9115.21,3280.88,3800.65,0.946131,331.027,-nan,-nan,0.93279,0.0978494,-1.0,-1.0,23.063,4.17065,2565,9387598 +631,insert,67.558,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,72.66,4.1864,2570,9421724 +632,search,9095.11,3283.66,3807.36,0.981128,326.061,-nan,-nan,0.93269,0.0978042,-1.0,-1.0,42.644,4.18651,2572,9421724 +633,insert,67.904,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,98.937,4.20004,2582,9461430 +634,search,9159,3282.43,3832.8,0.92092,336.461,-nan,-nan,0.93299,0.0974583,-1.0,-1.0,34.94,4.20014,2584,9461430 +635,insert,34.397,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,51.556,4.2067,2587,9479469 +636,search,9097.27,3281.65,3765.01,0.964946,318.965,-nan,-nan,0.9331,0.0973854,-1.0,-1.0,38.435,4.20685,2589,9479469 +637,insert,636.425,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,863.023,4.3964,2692,9885997 +638,search,9415.43,3311.95,3899.64,0.954167,329.644,-nan,-nan,0.9351,0.0958871,-1.0,-1.0,216.011,4.39911,2714,9885997 +639,insert,565.677,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,740.376,4.57865,2808,10292043 +640,search,9941.9,3307.56,4179.87,0.91994,359.217,-nan,-nan,0.93724,0.0939594,-1.0,-1.0,139.276,4.57969,2820,10292043 +641,delete,217.588,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1833.86,4.49654,2758,10096584 +642,search,9645.97,3306.26,4007.03,0.985974,337.27,-nan,-nan,0.93667,0.0943049,-1.0,-1.0,277.994,4.49996,2780,10096584 +643,delete,13.049,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,45.929,4.49915,2779,10084840 +644,search,9315.37,3287.58,3893.13,0.999192,318.117,-nan,-nan,0.93685,0.0942236,-1.0,-1.0,31.665,4.49944,2781,10084840 +645,delete,17.857,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,18.44,4.49944,2781,10069802 +646,search,9710.94,3277.38,4009.11,0.953299,344.175,-nan,-nan,0.93676,0.094291,-1.0,-1.0,25.873,4.49944,2781,10069802 +647,delete,108.264,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,746.711,4.45895,2743,9990953 +648,search,9745.07,3292.77,4083.91,0.926413,354.58,-nan,-nan,0.93599,0.0946826,-1.0,-1.0,122.649,4.46073,2752,9990953 +649,delete,47.061,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,153.143,4.45433,2747,9951625 +650,search,9597.89,3275.16,3955.58,0.97772,340.551,-nan,-nan,0.93601,0.0947069,-1.0,-1.0,26.18,4.45433,2747,9951625 +651,delete,32.124,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,237.751,4.44308,2737,9927065 +652,search,9674.43,3278.14,4048.95,0.952878,349.319,-nan,-nan,0.93563,0.0950678,-1.0,-1.0,42.675,4.44332,2738,9927065 +653,delete,111.208,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,767.84,4.39339,2699,9831098 +654,search,9559.78,3296.82,3997.68,0.923438,343.085,-nan,-nan,0.93507,0.0955935,-1.0,-1.0,97.221,4.39493,2706,9831098 +655,delete,393.376,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,2648,4.22352,2553,9462172 +656,search,9088.8,3337.81,3819.4,0.939661,323.689,-nan,-nan,0.93215,0.0989428,-1.0,-1.0,441.031,4.23052,2598,9462172 +657,delete,248.831,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,805.843,4.14089,2500,9218902 +658,search,8785.22,3312.64,3621.88,1.01696,306.948,-nan,-nan,0.92901,0.101543,-1.0,-1.0,121.603,4.14251,2510,9218902 +659,delete,158.515,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,686.806,4.08757,2470,9106546 +660,search,8589.87,3310.2,3516.63,1.01436,291.557,-nan,-nan,0.92807,0.102185,-1.0,-1.0,71.996,4.08802,2474,9106546 +661,delete,62.917,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,177.053,4.07751,2466,9059776 +662,search,8689.94,3292.71,3591.6,0.995117,308.226,-nan,-nan,0.92799,0.102562,-1.0,-1.0,24.414,4.07751,2466,9059776 +663,delete,111.783,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,360.318,4.05513,2447,8990616 +664,search,8712.99,3288.01,3651.25,0.900074,321.137,-nan,-nan,0.92703,0.10347,-1.0,-1.0,49.393,4.05522,2449,8990616 +665,delete,61.884,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,368.893,4.03409,2437,8937437 +666,search,8673.75,3284.6,3623.98,0.962454,319.77,-nan,-nan,0.9271,0.103282,-1.0,-1.0,56.773,4.03447,2440,8937437 +667,delete,221.628,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1216.57,3.94932,2368,8766390 +668,search,8459.26,3307.04,3552.03,0.909869,310.033,-nan,-nan,0.92629,0.104205,-1.0,-1.0,162.889,3.95188,2382,8766390 +669,delete,258.568,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,2073.57,3.83919,2268,8532008 +670,search,8140.76,3360.68,3426.4,0.946264,293.12,-nan,-nan,0.92603,0.104299,-1.0,-1.0,383.893,3.84421,2304,8532008 +671,delete,42.679,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,232.255,3.82834,2293,8489178 +672,search,8180.28,3297.18,3408.37,0.931198,297.496,-nan,-nan,0.9259,0.104167,-1.0,-1.0,103.511,3.82927,2297,8489178 +673,delete,81.055,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,319.232,3.79866,2276,8421866 +674,search,8046.67,3295.45,3332.28,1.00095,287.387,-nan,-nan,0.92554,0.104397,-1.0,-1.0,86.08,3.79943,2281,8421866 +675,delete,181.053,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,842.799,3.74487,2235,8288319 +676,search,7904.63,3299.38,3254.47,0.975021,280.612,-nan,-nan,0.92489,0.105515,-1.0,-1.0,79.637,3.74522,2239,8288319 +677,delete,246.514,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,454.167,3.66822,2213,8072197 +678,search,7796.11,3264.86,3202.44,0.925357,275.602,-nan,-nan,0.92749,0.102292,-1.0,-1.0,102.958,3.66904,2219,8072197 +679,delete,39.997,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,328.588,3.65016,2209,8036947 +680,search,7760.4,3260.04,3169.25,0.9914,270.976,-nan,-nan,0.92761,0.102059,-1.0,-1.0,56.248,3.65058,2211,8036947 +681,delete,12.87,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,31.684,3.65058,2211,8026197 +682,search,7788.1,3254.69,3192.28,0.949667,273.708,-nan,-nan,0.92772,0.10182,-1.0,-1.0,31.897,3.65058,2211,8026197 +683,delete,344.993,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1739.17,3.53022,2099,7744229 +684,search,7446.18,3308.5,3064.19,1.01988,267.961,-nan,-nan,0.92699,0.103042,-1.0,-1.0,233.063,3.53481,2123,7744229 +685,delete,255.497,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1413.79,3.43509,2027,7541066 +686,search,7075.42,3338.17,2877.98,1.04052,236.815,-nan,-nan,0.92403,0.105621,-1.0,-1.0,449.89,3.44371,2073,7541066 +687,delete,14.792,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,100.752,3.43771,2069,7522190 +688,search,7141.55,3253.29,2879.69,1.08136,239.199,-nan,-nan,0.92397,0.105999,-1.0,-1.0,74.399,3.4378,2072,7522190 +689,delete,152.383,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,550.761,3.38116,2034,7409400 +690,search,7184.57,3265.98,2951.58,0.964671,250.458,-nan,-nan,0.92259,0.106918,-1.0,-1.0,112.715,3.38232,2041,7409400 +691,delete,48.128,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,92.759,3.37629,2036,7375648 +692,search,7133.73,3246.25,2904.7,0.989166,249.998,-nan,-nan,0.92278,0.106719,-1.0,-1.0,39.792,3.37637,2037,7375648 +693,delete,227.718,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,591.912,3.30543,2007,7222068 +694,search,6878.86,3228.44,2745.95,1.0217,229.58,-nan,-nan,0.9227,0.106788,-1.0,-1.0,41.252,3.30544,2008,7222068 +695,delete,34.577,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,224.431,3.29162,2003,7196331 +696,search,6997.54,3237.2,2837.33,0.98719,241.093,-nan,-nan,0.92377,0.105733,-1.0,-1.0,86.885,3.29243,2007,7196331 +697,delete,33.48,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,170.132,3.28305,1999,7170716 +698,search,7102.67,3224.74,2928.48,0.928866,264.564,-nan,-nan,0.92339,0.106288,-1.0,-1.0,40.712,3.28317,2000,7170716 +699,delete,14.509,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,89.675,3.27644,1997,7155255 +700,search,6927.19,3227.36,2785.67,0.967335,234.12,-nan,-nan,0.92383,0.105874,-1.0,-1.0,70.937,3.27691,2001,7155255 +701,delete,74.228,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,135.817,3.26529,1994,7097733 +702,search,6835.54,3201.78,2711.68,1.08648,225.869,-nan,-nan,0.92316,0.106502,-1.0,-1.0,37.848,3.26548,1995,7097733 +703,delete,262.506,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,856.063,3.17323,1934,6924172 +704,search,6632.87,3218.5,2642.62,1.0407,216.845,-nan,-nan,0.92275,0.106903,-1.0,-1.0,153.93,3.17563,1945,6924172 +705,delete,27.772,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,127.647,3.1674,1936,6896276 +706,search,6802.06,3205.79,2759.98,0.973302,236.047,-nan,-nan,0.92327,0.106137,-1.0,-1.0,60.425,3.16774,1939,6896276 +707,delete,638.581,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,3396.94,2.8861,1661,6385507 +708,search,6175.15,3510.99,2659.52,0.915727,234.345,-nan,-nan,0.91978,0.107625,-1.0,-1.0,916.921,2.90278,1773,6385507 +709,delete,178.742,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,979.522,2.80925,1690,6207658 +710,search,5934.96,3276.84,2517.29,0.936159,221.821,-nan,-nan,0.91581,0.110963,-1.0,-1.0,324.571,2.81528,1724,6207658 +711,delete,49.498,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,241.066,2.79655,1713,6162720 +712,search,5716.21,3182.24,2292.91,1.07444,183.238,-nan,-nan,0.91483,0.1117,-1.0,-1.0,86.556,2.79725,1719,6162720 +713,delete,329.933,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1253.69,2.67509,1639,5927803 +714,search,5628.29,3209.1,2302.97,1.05357,196.002,-nan,-nan,0.91193,0.114737,-1.0,-1.0,99.351,2.67708,1651,5927803 +715,delete,23.293,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,146.326,2.66421,1639,5895666 +716,search,5657.55,3189.56,2399.94,0.941059,212.49,-nan,-nan,0.91173,0.115093,-1.0,-1.0,98.142,2.66562,1644,5895666 +717,delete,31.932,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,100.692,2.65694,1637,5868774 +718,search,5643.63,3182.37,2282.38,0.994469,190.666,-nan,-nan,0.91138,0.115746,-1.0,-1.0,25.864,2.65699,1639,5868774 +719,delete,281.875,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1706.61,2.49714,1470,5533540 +720,search,5215.64,3365.4,2151.1,0.956341,182.325,-nan,-nan,0.90702,0.118125,-1.0,-1.0,460.029,2.50549,1515,5533540 +721,delete,99.453,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,303.756,2.45168,1485,5418387 +722,search,5066.1,3255.83,2205.46,0.923712,189.657,-nan,-nan,0.90554,0.119664,-1.0,-1.0,202.548,2.45331,1499,5418387 +723,delete,18.613,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,76.981,2.44739,1494,5398291 +724,search,5277.47,3209.83,2152.41,0.952442,188.22,-nan,-nan,0.90587,0.119432,-1.0,-1.0,140.266,2.44772,1500,5398291 +725,delete,146.883,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,640.088,2.39318,1452,5273546 +726,search,4987.36,3204.66,2116.23,0.919338,181.374,-nan,-nan,0.90319,0.121324,-1.0,-1.0,206.113,2.39437,1466,5273546 +727,delete,104.103,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,559.546,2.34976,1420,5180937 +728,search,5005.79,3224.46,2056.94,1.01626,178.183,-nan,-nan,0.90313,0.120974,-1.0,-1.0,205.656,2.35276,1443,5180937 +729,delete,123.701,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,362.516,2.30332,1402,5079524 +730,search,4861.51,3173.26,2036.11,0.941638,179.697,-nan,-nan,0.90119,0.121815,-1.0,-1.0,71.166,2.30503,1412,5079524 +731,delete,42.817,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,206.033,2.28547,1396,5019265 +732,search,4792.51,3131.57,2019.15,0.948738,175.901,-nan,-nan,0.90094,0.122367,-1.0,-1.0,32.216,2.28554,1398,5019265 +733,delete,81.285,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,189.133,2.26634,1385,4955565 +734,search,4810.11,3102.42,1925.99,0.990051,165.545,-nan,-nan,0.89995,0.123379,-1.0,-1.0,49.28,2.26686,1387,4955565 +735,delete,200.653,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,593.938,2.17991,1340,4776948 +736,search,4686.78,3217.05,1973.6,0.92489,168.102,-nan,-nan,0.90812,0.115023,-1.0,-1.0,97.832,2.18202,1351,4776948 +737,delete,87.326,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,487.88,2.12778,1312,4654026 +738,search,4519.28,3131.25,1901.16,0.945156,166.783,-nan,-nan,0.90378,0.118179,-1.0,-1.0,124.719,2.12884,1323,4654026 +739,delete,65.781,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,310.02,2.09967,1304,4589888 +740,search,4520.29,3103.79,1821.12,1.02824,154.008,-nan,-nan,0.90319,0.118758,-1.0,-1.0,73.66,2.10051,1308,4589888 +741,delete,223.343,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,687.337,2.01808,1254,4430862 +742,search,4355.51,3123.65,1732.2,1.02468,144.42,-nan,-nan,0.90281,0.119315,-1.0,-1.0,144.062,2.02066,1268,4430862 +743,delete,130.975,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,882.123,1.9235,1194,4237049 +744,search,4199.5,3158.27,1740.92,0.896037,151.911,-nan,-nan,0.90151,0.120762,-1.0,-1.0,317.96,1.92923,1219,4237049 +745,delete,282.582,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1143.26,1.80541,1119,4000195 +746,search,3950,3219.77,1605.8,0.985772,137.949,-nan,-nan,0.8976,0.122308,-1.0,-1.0,255.591,1.80837,1142,4000195 +747,delete,208.048,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,957.105,1.70946,1073,3803954 +748,search,3915.35,3200.49,1621.58,0.913101,146.957,-nan,-nan,0.8953,0.123325,-1.0,-1.0,193.268,1.7128,1089,3803954 +749,delete,82.513,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,212.06,1.68856,1069,3733214 +750,search,3836.31,3154.33,1578.78,0.877801,135.642,-nan,-nan,0.89344,0.124756,-1.0,-1.0,52.054,1.68907,1071,3733214 +751,delete,29.689,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,114.009,1.67885,1062,3709324 +752,search,3770.38,3158.88,1523.88,0.964586,133.075,-nan,-nan,0.89302,0.12482,-1.0,-1.0,31.356,1.67912,1063,3709324 +753,delete,84.481,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,254.955,1.64869,1041,3649123 +754,search,3729.84,3172.58,1536.94,0.971186,133.044,-nan,-nan,0.89149,0.126626,-1.0,-1.0,46.117,1.64889,1043,3649123 +755,delete,35.269,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,105.989,1.63976,1036,3621475 +756,search,3679.84,3162.32,1481.41,1.02511,127.971,-nan,-nan,0.89225,0.125695,-1.0,-1.0,85.422,1.64084,1040,3621475 +757,delete,32.849,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,89.63,1.62761,1034,3594479 +758,search,3722.8,3149.65,1532.03,0.885333,136.826,-nan,-nan,0.89126,0.12634,-1.0,-1.0,42.611,1.62761,1035,3594479 +759,delete,30.204,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,149.995,1.61305,1024,3569055 +760,search,3711.2,3162.86,1532.54,0.920926,134.638,-nan,-nan,0.89122,0.126015,-1.0,-1.0,63.03,1.61392,1028,3569055 +761,delete,52.999,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,107.032,1.59909,1018,3527896 +762,search,3674.31,3141.54,1509.6,0.966863,129.099,-nan,-nan,0.89026,0.126915,-1.0,-1.0,18.227,1.59909,1018,3527896 +763,delete,26.175,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,75.447,1.59218,1013,3509970 +764,search,3641.66,3133.94,1495.75,0.907093,131.236,-nan,-nan,0.88937,0.127486,-1.0,-1.0,18.18,1.59218,1013,3509970 +765,delete,510.968,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1063.08,1.42964,881,3187549 +766,search,3295.33,3320.29,1393.79,0.893897,124.094,-nan,-nan,0.88662,0.128795,-1.0,-1.0,197.499,1.43297,900,3187549 +767,delete,490.407,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1365.86,1.24934,733,2822906 +768,search,2716.7,3527.14,1137.09,0.977732,97.0231,-nan,-nan,0.87531,0.138129,-1.0,-1.0,473.138,1.25893,783,2822906 +769,insert,15.111,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,10.802,1.26179,783,2832899 +770,search,2872.49,3271.73,1198.58,0.995883,101.445,-nan,-nan,0.87683,0.137203,-1.0,-1.0,56.456,1.26225,787,2832899 +771,insert,46.494,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,87.718,1.27537,794,2863523 +772,search,2948.49,3262.69,1238.96,0.881719,109.559,-nan,-nan,0.87816,0.136004,-1.0,-1.0,31.475,1.27553,796,2863523 +773,insert,93.552,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,115.638,1.29805,806,2925169 +774,search,2990.84,3303.88,1272.32,0.930932,109.366,-nan,-nan,0.87943,0.135069,-1.0,-1.0,58.015,1.29815,811,2925169 +775,insert,53.55,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,89.013,1.31457,816,2961356 +776,search,3037.12,3291.58,1285.71,0.905024,105.763,-nan,-nan,0.88021,0.135223,-1.0,-1.0,30.459,1.31474,818,2961356 +777,insert,58.072,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,85.293,1.32966,826,2998678 +778,search,3015.84,3299.7,1249.12,1.00098,109.006,-nan,-nan,0.88102,0.134273,-1.0,-1.0,25.782,1.32969,827,2998678 +779,insert,20.436,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,36.608,1.33358,829,3011024 +780,search,3014.03,3298.15,1246.7,0.976868,103.562,-nan,-nan,0.88197,0.133864,-1.0,-1.0,33.321,1.33396,831,3011024 +781,insert,13.754,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.621,1.33712,833,3019515 +782,search,3086.71,3288.45,1313.97,0.925489,110.587,-nan,-nan,0.88129,0.134572,-1.0,-1.0,14.607,1.33712,833,3019515 +783,insert,107.026,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,158.094,1.36263,847,3082298 +784,search,3149.25,3309.21,1339.96,0.937545,117.843,-nan,-nan,0.88256,0.133388,-1.0,-1.0,25.422,1.36274,848,3082298 +785,insert,57.939,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,77.089,1.37973,853,3122090 +786,search,3187.06,3331.58,1357.01,0.880487,121.091,-nan,-nan,0.88446,0.132369,-1.0,-1.0,51.038,1.3799,856,3122090 +787,insert,44.613,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,66.366,1.38862,862,3140294 +788,search,3203.84,3314.85,1363.36,0.879509,117.208,-nan,-nan,0.88468,0.132421,-1.0,-1.0,15.435,1.38862,862,3140294 +789,insert,18.052,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.853,1.39165,863,3147819 +790,search,3202.61,3319.15,1355.65,0.884131,114.052,-nan,-nan,0.88451,0.132673,-1.0,-1.0,13.812,1.39165,863,3147819 +791,insert,59.499,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,56.935,1.40444,868,3180668 +792,search,3151.96,3336.91,1303.25,1.02505,109.574,-nan,-nan,0.88512,0.132256,-1.0,-1.0,14.257,1.40444,868,3180668 +793,insert,3.601,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.091,1.40549,868,3183235 +794,search,3173.59,3338.48,1331.45,0.982895,112.068,-nan,-nan,0.8851,0.13227,-1.0,-1.0,14.158,1.40549,868,3183235 +795,insert,108.945,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,138.501,1.43114,881,3242089 +796,search,3273.06,3347.06,1396.94,0.917444,119.104,-nan,-nan,0.88497,0.132689,-1.0,-1.0,26.49,1.43122,882,3242089 +797,insert,8.403,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.156,1.43276,883,3247192 +798,search,3269.69,3343.06,1385.7,0.93564,124.171,-nan,-nan,0.88492,0.13288,-1.0,-1.0,14.171,1.43276,883,3247192 +799,insert,21.171,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.446,1.4365,884,3254626 +800,search,3239.99,3349.05,1364.26,0.928144,113.689,-nan,-nan,0.88496,0.13286,-1.0,-1.0,14.331,1.4365,884,3254626 +801,insert,41.99,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,111.227,1.44574,893,3274953 +802,search,3338.52,3334.96,1423.37,0.897877,121.546,-nan,-nan,0.88592,0.132115,-1.0,-1.0,61.6,1.44662,897,3274953 +803,insert,1319.56,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1492.12,1.80789,1045,4032688 +804,search,3955.04,3586.04,1755.96,0.919145,151.07,-nan,-nan,0.89453,0.125222,-1.0,-1.0,599.469,1.81082,1107,4032688 +805,insert,29.193,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,56.274,1.81963,1111,4050220 +806,search,4107.15,3364.8,1751.44,0.923408,156.953,-nan,-nan,0.89566,0.124153,-1.0,-1.0,162.287,1.82008,1126,4050220 +807,insert,24.215,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,70.174,1.82768,1131,4067940 +808,search,4119.01,3317.57,1737,0.935885,147.661,-nan,-nan,0.89627,0.123533,-1.0,-1.0,66.288,1.82811,1137,4067940 +809,insert,47.526,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,91.962,1.83993,1146,4099369 +810,search,4110.38,3292.69,1707.16,0.962366,141.682,-nan,-nan,0.89647,0.12303,-1.0,-1.0,32.572,1.84028,1148,4099369 +811,insert,47.788,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,66.324,1.8496,1155,4129509 +812,search,4108.83,3297.03,1695.62,1.00192,139.028,-nan,-nan,0.8966,0.122933,-1.0,-1.0,35.432,1.85006,1157,4129509 +813,insert,84.401,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,121.814,1.8675,1172,4178669 +814,search,4247.29,3287.98,1779.47,0.909623,154.293,-nan,-nan,0.8984,0.121541,-1.0,-1.0,14.212,1.8675,1172,4178669 +815,insert,4.012,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,12.159,1.86812,1172,4180960 +816,search,4166.05,3288.41,1767.86,0.958176,153.946,-nan,-nan,0.89835,0.121585,-1.0,-1.0,9.166,1.86812,1172,4180960 +817,insert,53.453,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,125.024,1.89503,1189,4255359 +818,search,4173.4,3294.31,1824.37,0.930659,164.538,-nan,-nan,0.89893,0.121313,-1.0,-1.0,43.464,1.89565,1194,4255359 +819,insert,23.605,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.929,1.90092,1196,4272706 +820,search,4215.1,3291.88,1836.33,0.971918,156.09,-nan,-nan,0.89935,0.120874,-1.0,-1.0,35.885,1.90131,1198,4272706 +821,insert,22.257,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,45.253,1.90882,1200,4287851 +822,search,4232.66,3291.23,1843.67,0.917557,160.74,-nan,-nan,0.89988,0.120557,-1.0,-1.0,42.168,1.90929,1202,4287851 +823,insert,46.029,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,73.245,1.92423,1207,4322788 +824,search,4307.89,3295.01,1777.45,1.0256,151.129,-nan,-nan,0.89929,0.1212,-1.0,-1.0,45.685,1.92456,1209,4322788 +825,insert,117.736,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,174.173,1.95508,1225,4396408 +826,search,4356.31,3294.59,1792.33,1.02452,147.836,-nan,-nan,0.90016,0.12039,-1.0,-1.0,40.736,1.95556,1228,4396408 +827,insert,24.769,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,32.727,1.96276,1230,4415721 +828,search,4290.6,3287.46,1720.53,1.01926,139.704,-nan,-nan,0.89985,0.120601,-1.0,-1.0,29.51,1.96282,1231,4415721 +829,insert,509.979,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,938.835,2.16135,1339,4832079 +830,search,4720.21,3302.54,1933.5,0.98671,160.24,-nan,-nan,0.90402,0.119484,-1.0,-1.0,226.112,2.16324,1361,4832079 +831,insert,46.498,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,74.658,2.17477,1368,4864525 +832,search,4742.23,3249.95,2011.67,0.91734,172.355,-nan,-nan,0.90462,0.119129,-1.0,-1.0,33.322,2.17512,1371,4864525 +833,insert,7.248,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,25.989,2.17674,1374,4874443 +834,search,4810.25,3241.06,2082.09,0.923987,180.818,-nan,-nan,0.90469,0.11905,-1.0,-1.0,16.226,2.17674,1374,4874443 +835,insert,101.847,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,153.998,2.20278,1391,4938267 +836,search,4816.55,3248.8,1940.92,1.01293,163.212,-nan,-nan,0.90528,0.118381,-1.0,-1.0,27.586,2.20294,1392,4938267 +837,insert,23.761,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.383,2.20761,1392,4961147 +838,search,4831.9,3260.72,1946.46,1.00984,159.069,-nan,-nan,0.90517,0.118636,-1.0,-1.0,15.68,2.20761,1392,4961147 +839,insert,77.766,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,96.515,2.24071,1409,5033740 +840,search,4866.19,3276.73,2114.95,0.959087,180.5,-nan,-nan,0.907,0.117484,-1.0,-1.0,75.82,2.24136,1416,5033740 +841,insert,71.438,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,60.774,2.25922,1421,5078736 +842,search,4951.91,3274.4,2007.42,1.00925,169.264,-nan,-nan,0.90719,0.117427,-1.0,-1.0,37.296,2.25933,1423,5078736 +843,insert,523.905,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,696.301,2.43231,1513,5467376 +844,search,5310.44,3318.26,2185.3,0.983241,181.994,-nan,-nan,0.90989,0.115561,-1.0,-1.0,261.173,2.43513,1536,5467376 +845,insert,403.395,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,671.019,2.59483,1618,5815547 +846,search,5593.13,3317.35,2266.97,1.03329,185.94,-nan,-nan,0.91429,0.113114,-1.0,-1.0,152.873,2.59569,1632,5815547 +847,insert,8.305,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.698,2.59744,1632,5821782 +848,search,5513.69,3292.98,2256.71,1.03303,180.017,-nan,-nan,0.9143,0.11294,-1.0,-1.0,50.94,2.59744,1636,5821782 +849,insert,3.287,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.437,2.59799,1636,5825594 +850,search,4999.02,3284.82,2361.3,1.03238,190.633,-nan,-nan,0.91445,0.112822,-1.0,-1.0,16.462,2.59821,1637,5825594 +851,insert,5.925,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,26.711,2.60192,1640,5834880 +852,search,5576.8,3283.64,2332.64,1.05082,188.839,-nan,-nan,0.91452,0.112841,-1.0,-1.0,42.859,2.60244,1641,5834880 +853,insert,41.654,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,45.563,2.61223,1643,5860274 +854,search,5818.24,3292.92,2494.25,0.916033,225.314,-nan,-nan,0.9148,0.112683,-1.0,-1.0,30.451,2.61227,1644,5860274 +855,insert,11.595,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,29.191,2.61362,1645,5866064 +856,search,5809.71,3293.4,2398.37,0.962088,203.074,-nan,-nan,0.91484,0.112508,-1.0,-1.0,19.149,2.61362,1645,5866064 +857,insert,26.349,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,46.176,2.61922,1648,5884456 +858,search,5758.67,3299.35,2484.32,0.951832,221.426,-nan,-nan,0.91471,0.112715,-1.0,-1.0,19.522,2.61922,1648,5884456 +859,insert,49.066,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,136.789,2.6314,1660,5916786 +860,search,5845.32,3291.79,2408.54,0.977346,204.7,-nan,-nan,0.91482,0.112543,-1.0,-1.0,28.226,2.63157,1661,5916786 +861,insert,15.435,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.76,2.63399,1661,5923467 +862,search,5738.55,3294.19,2488.64,0.917706,218.751,-nan,-nan,0.9148,0.112568,-1.0,-1.0,36.327,2.63425,1662,5923467 +863,insert,104.693,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,151.244,2.6673,1676,5997844 +864,search,5818.18,3300.26,2422.45,0.953566,201.917,-nan,-nan,0.91374,0.114248,-1.0,-1.0,44.908,2.66783,1682,5997844 +865,insert,10.728,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,32.945,2.67141,1685,6009561 +866,search,5673.55,3289.61,2450.85,1.04521,208.034,-nan,-nan,0.91357,0.114376,-1.0,-1.0,18.144,2.67141,1685,6009561 +867,insert,70.412,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,106.844,2.68983,1696,6052876 +868,search,5831.31,3286.33,2422.45,1.06762,201.136,-nan,-nan,0.91408,0.11392,-1.0,-1.0,27.101,2.69001,1698,6052876 +869,insert,5.054,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.128,2.692,1699,6059234 +870,search,5941.35,3281.35,2565.06,0.949293,224.649,-nan,-nan,0.9139,0.114053,-1.0,-1.0,18.169,2.692,1699,6059234 +871,insert,67.132,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,98.353,2.71167,1706,6098225 +872,search,5918.17,3284.25,2484.84,0.987154,211.227,-nan,-nan,0.91391,0.11407,-1.0,-1.0,37.946,2.71191,1708,6098225 +873,insert,65.028,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,65.355,2.728,1713,6137308 +874,search,6093.23,3287.1,2548.37,0.944851,218.922,-nan,-nan,0.91415,0.113946,-1.0,-1.0,41.397,2.72831,1716,6137308 +875,insert,27.522,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.383,2.73491,1718,6154089 +876,search,5938.3,3280.18,2475.92,1.01872,207.334,-nan,-nan,0.91431,0.114022,-1.0,-1.0,43.872,2.73538,1720,6154089 +877,insert,1.247,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,17.398,2.73538,1720,6156061 +878,search,5996.48,3278.76,2446.63,1.02037,201.245,-nan,-nan,0.91432,0.11407,-1.0,-1.0,21.26,2.73538,1720,6156061 +879,insert,381.713,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,562.948,2.86652,1774,6446004 +880,search,6206.05,3323.15,2539.18,1.01071,211.29,-nan,-nan,0.91595,0.113047,-1.0,-1.0,252.891,2.86869,1800,6446004 +881,insert,618.207,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,722.392,3.10055,1893,6969764 +882,search,6544.4,3364.9,2859.76,0.957884,250.627,-nan,-nan,0.919,0.110036,-1.0,-1.0,295.054,3.10513,1925,6969764 +883,insert,522.298,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,836.883,3.30137,2034,7422501 +884,search,6926.79,3348.38,2916.95,1.03984,242.016,-nan,-nan,0.92245,0.107041,-1.0,-1.0,189.052,3.30575,2069,7422501 +885,insert,7.92,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.374,3.30958,2069,7432053 +886,search,7124.19,3296.04,2998.04,0.982795,253.085,-nan,-nan,0.92316,0.10638,-1.0,-1.0,66.484,3.31014,2074,7432053 +887,insert,494.913,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,631.358,3.46521,2151,7789886 +888,search,7693.01,3306.91,3241.41,0.952449,289.456,-nan,-nan,0.92506,0.105464,-1.0,-1.0,265.571,3.4691,2179,7789886 +889,insert,563.434,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,780.283,3.65742,2275,8210883 +890,search,8017.01,3292.22,3325.65,0.983852,287.081,-nan,-nan,0.92712,0.10434,-1.0,-1.0,259.332,3.66144,2306,8210883 +891,insert,39.86,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,47.997,3.66902,2309,8240504 +892,search,7992.98,3249.05,3252.46,0.974783,271.663,-nan,-nan,0.92737,0.104403,-1.0,-1.0,83.785,3.66996,2314,8240504 +893,insert,60.129,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,75.148,3.686,2319,8299693 +894,search,8117.78,3251.6,3317.01,0.999886,280.084,-nan,-nan,0.92751,0.104143,-1.0,-1.0,42.049,3.68622,2320,8299693 +895,insert,69.983,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,136.821,3.70384,2333,8349126 +896,search,8113.74,3253.71,3302.22,1.00195,276.903,-nan,-nan,0.92801,0.103786,-1.0,-1.0,32.913,3.70384,2333,8349126 +897,delete,107.01,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,365.217,3.67093,2315,8268842 +898,search,8074.13,3253.85,3288.01,0.999854,274.309,-nan,-nan,0.92779,0.104051,-1.0,-1.0,81.474,3.67193,2321,8268842 +899,delete,22.77,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,108.085,3.6675,2317,8250527 +900,search,8088.5,3242.6,3292.28,0.962454,281.626,-nan,-nan,0.92761,0.104164,-1.0,-1.0,30.915,3.6675,2317,8250527 +901,delete,83.667,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,287.098,3.64756,2298,8185565 +902,search,8154.12,3241.14,3387.37,0.908795,300.697,-nan,-nan,0.92688,0.104648,-1.0,-1.0,61.751,3.64789,2301,8185565 +903,delete,69.543,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,218.688,3.61996,2283,8126915 +904,search,8034.43,3238.69,3300.78,0.946933,285.974,-nan,-nan,0.92564,0.10492,-1.0,-1.0,29.656,3.61996,2283,8126915 +905,delete,36.536,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,115.761,3.61443,2277,8098533 +906,search,7945.63,3233.05,3247.51,0.939565,278.312,-nan,-nan,0.92546,0.104989,-1.0,-1.0,29.345,3.61443,2277,8098533 +907,delete,29.764,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,225.379,3.6035,2270,8074833 +908,search,7925.06,3233.93,3241.39,0.978235,280.245,-nan,-nan,0.92526,0.105143,-1.0,-1.0,115.45,3.60473,2276,8074833 +909,delete,46.568,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,91.035,3.60011,2273,8035274 +910,search,7785.41,3215.71,3106.96,1.06291,260.183,-nan,-nan,0.92549,0.105064,-1.0,-1.0,59.935,3.60048,2275,8035274 +911,delete,119.353,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,241.704,3.58123,2261,7957223 +912,search,7854.02,3197.13,3179.16,0.976882,274.541,-nan,-nan,0.92478,0.105723,-1.0,-1.0,31.974,3.58123,2261,7957223 +913,delete,57.601,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,290.935,3.56155,2243,7917554 +914,search,7864.08,3207.09,3205.65,0.979685,278.804,-nan,-nan,0.92357,0.106165,-1.0,-1.0,111.485,3.56304,2249,7917554 +915,delete,81.007,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,313.246,3.53708,2227,7861503 +916,search,7803.9,3202.53,3182.93,0.976423,278.074,-nan,-nan,0.92276,0.107216,-1.0,-1.0,62.28,3.53754,2230,7861503 +917,delete,23.842,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,129.825,3.53248,2227,7844487 +918,search,7632.59,3198.26,3128.41,0.997114,268.273,-nan,-nan,0.92286,0.107006,-1.0,-1.0,29.288,3.53248,2227,7844487 +919,delete,37.034,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,85.072,3.5276,2223,7814993 +920,search,7583.42,3188.16,3091.53,1.02796,264.319,-nan,-nan,0.92292,0.107079,-1.0,-1.0,40.259,3.52784,2224,7814993 +921,delete,29.02,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,190.299,3.5179,2218,7793855 +922,search,7596.54,3189.39,3111.35,0.963566,267.746,-nan,-nan,0.9229,0.10725,-1.0,-1.0,42.209,3.51818,2219,7793855 +923,delete,69.475,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,248.205,3.50086,2206,7736054 +924,search,7521.33,3177.53,3082.05,0.961771,258.391,-nan,-nan,0.92273,0.107332,-1.0,-1.0,46.193,3.5013,2208,7736054 +925,delete,177.43,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,765.301,3.42751,2151,7587942 +926,search,7351.42,3190.16,2988.32,0.992236,251.693,-nan,-nan,0.92175,0.107907,-1.0,-1.0,149.963,3.42879,2160,7587942 +927,delete,33.092,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,60.499,3.42569,2157,7565670 +928,search,7385.68,3171.99,3020.81,0.965283,251.215,-nan,-nan,0.92136,0.10823,-1.0,-1.0,30.691,3.42569,2157,7565670 +929,delete,61.15,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,180.051,3.41487,2148,7529948 +930,search,7296.59,3168.95,2964.01,1.03978,256.231,-nan,-nan,0.92134,0.108231,-1.0,-1.0,26.125,3.41487,2148,7529948 +931,delete,478.48,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,3311.49,3.16362,1892,7016700 +932,search,6697.84,3342.03,2877.13,0.903922,254.159,-nan,-nan,0.91973,0.10893,-1.0,-1.0,568.619,3.1745,1954,7016700 +933,delete,23.057,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,119.944,3.16085,1946,6980979 +934,search,6714.43,3222.34,2858.92,0.921249,254.983,-nan,-nan,0.92039,0.107853,-1.0,-1.0,145.23,3.16216,1957,6980979 +935,delete,25.269,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,248.98,3.14955,1948,6954250 +936,search,6709.16,3208.5,2775.81,0.985029,237.485,-nan,-nan,0.92031,0.107893,-1.0,-1.0,90.886,3.15102,1952,6954250 +937,delete,32.051,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,133.949,3.14085,1944,6924596 +938,search,6663.56,3205.81,2744.8,1.01366,228.985,-nan,-nan,0.92017,0.108189,-1.0,-1.0,43.405,3.14133,1947,6924596 +939,delete,129.532,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,605.188,3.0938,1902,6784831 +940,search,6521.43,3200.31,2599.81,1.0319,213.406,-nan,-nan,0.91993,0.108758,-1.0,-1.0,62.355,3.09503,1910,6784831 +941,delete,25.125,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.633,3.09278,1908,6745819 +942,search,6057.95,3160.23,2638.82,1.00439,224.668,-nan,-nan,0.91899,0.109367,-1.0,-1.0,15.516,3.09278,1908,6745819 +943,delete,2.633,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,25.605,3.09182,1907,6740944 +944,search,5504.76,3158.82,2594.97,0.966075,208.419,-nan,-nan,0.91863,0.109577,-1.0,-1.0,15.307,3.09182,1907,6740944 +945,delete,76.567,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,681.426,3.01797,1844,6627202 +946,search,6479.5,3215.93,2687.87,0.941706,227.006,-nan,-nan,0.91736,0.110218,-1.0,-1.0,204.152,3.02129,1862,6627202 +947,delete,25.394,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,47.344,3.0179,1860,6607194 +948,search,6027.77,3167.16,2668.15,0.91248,227.992,-nan,-nan,0.91686,0.110702,-1.0,-1.0,42.133,3.01857,1863,6607194 +949,delete,40.603,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,141.827,2.99561,1851,6551065 +950,search,6404.01,3155.74,2612.58,0.940302,221.198,-nan,-nan,0.91657,0.110803,-1.0,-1.0,35.622,2.99588,1852,6551065 +951,delete,42.784,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,151.022,2.97454,1839,6506798 +952,search,6362.57,3157.42,2665.3,0.89236,233.043,-nan,-nan,0.9171,0.110359,-1.0,-1.0,52.017,2.97467,1841,6506798 +953,delete,68.291,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,209.461,2.95841,1833,6459301 +954,search,6194.95,3148.65,2540.95,0.956064,214.564,-nan,-nan,0.91648,0.110686,-1.0,-1.0,23.784,2.95841,1833,6459301 +955,delete,24.091,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,32.782,2.95034,1831,6440069 +956,search,6374.56,3147.56,2579.81,0.96323,216.39,-nan,-nan,0.91687,0.110172,-1.0,-1.0,22.646,2.95034,1831,6440069 +957,delete,370.613,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1512.16,2.79081,1703,6132470 +958,search,5997.66,3222.53,2436.98,0.979421,203.501,-nan,-nan,0.91351,0.112397,-1.0,-1.0,269.536,2.79636,1728,6132470 +959,delete,50.178,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,82.837,2.78235,1718,6086241 +960,search,5663.35,3166.32,2371.64,1.01319,195.826,-nan,-nan,0.91224,0.113599,-1.0,-1.0,53.578,2.78275,1721,6086241 +961,delete,26.244,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,77.374,2.77777,1716,6064734 +962,search,5330.47,3158.4,2401.35,1.00028,197.055,-nan,-nan,0.91262,0.113105,-1.0,-1.0,23.055,2.77777,1717,6064734 +963,delete,105.844,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,670.609,2.7184,1663,5967543 +964,search,5522.16,3215.08,2440.74,0.924841,210.347,-nan,-nan,0.91166,0.113295,-1.0,-1.0,82.222,2.72035,1677,5967543 +965,delete,40.779,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,170.586,2.69087,1655,5906381 +966,search,5595.88,3184.75,2305.39,0.980491,194.997,-nan,-nan,0.9106,0.114366,-1.0,-1.0,91.981,2.69247,1664,5906381 +967,delete,49.146,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,226.633,2.67365,1649,5851849 +968,search,5579.82,3158.03,2366.47,0.909297,199.373,-nan,-nan,0.9098,0.114781,-1.0,-1.0,42.222,2.67401,1651,5851849 +969,delete,66.303,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,170.778,2.64791,1640,5783023 +970,search,5641.13,3146.2,2249.6,1.04527,183.558,-nan,-nan,0.90893,0.115404,-1.0,-1.0,19.418,2.64791,1640,5783023 +971,delete,336.887,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1244.59,2.51385,1518,5496517 +972,search,5448.45,3230.97,2250.63,0.918068,186.2,-nan,-nan,0.90756,0.116626,-1.0,-1.0,349.087,2.51878,1552,5496517 +973,delete,285.464,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1018.7,2.39619,1449,5237981 +974,search,4935.48,3210.28,2129.86,0.943171,185.561,-nan,-nan,0.9045,0.119436,-1.0,-1.0,350.097,2.40256,1482,5237981 +975,delete,181.318,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,501.32,2.32805,1416,5099794 +976,search,4721.16,3199.4,2038.06,1.01523,167.26,-nan,-nan,0.90304,0.119631,-1.0,-1.0,101.611,2.33056,1436,5099794 +977,delete,38.095,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,177.039,2.30081,1415,5032210 +978,search,4768.35,3141.31,1957.72,0.990747,161.444,-nan,-nan,0.90137,0.120592,-1.0,-1.0,39.032,2.30135,1419,5032210 +979,delete,9.386,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,25.359,2.29791,1417,5020211 +980,search,4681.12,3126.39,1953.15,0.982788,159.179,-nan,-nan,0.90147,0.120695,-1.0,-1.0,17.881,2.29791,1417,5020211 +981,delete,151.79,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,500.331,2.23675,1377,4902291 +982,search,4755.27,3141.65,1989.99,0.982125,165.793,-nan,-nan,0.89886,0.122328,-1.0,-1.0,63.963,2.23768,1381,4902291 +983,delete,38.134,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,51.482,2.23312,1378,4872577 +984,search,4784,3117.67,1920.96,1.05989,160.374,-nan,-nan,0.89864,0.122252,-1.0,-1.0,18,2.23312,1378,4872577 +985,delete,51.766,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,148.472,2.22007,1367,4836730 +986,search,4818.29,3112.1,1957.1,0.965964,169.206,-nan,-nan,0.89839,0.122823,-1.0,-1.0,38.186,2.22033,1369,4836730 +987,delete,46.385,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,101.708,2.20976,1360,4797164 +988,search,4780.68,3093.87,1931.25,0.944867,164.328,-nan,-nan,0.89798,0.123455,-1.0,-1.0,39.799,2.20997,1362,4797164 +989,delete,16.727,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.561,2.20934,1362,4784789 +990,search,4809.28,3082.73,1964.96,0.950913,163.932,-nan,-nan,0.89771,0.123738,-1.0,-1.0,16.827,2.20934,1362,4784789 +991,delete,100.099,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,398.417,2.17137,1331,4705420 +992,search,4739.41,3144.54,1933.05,0.938592,164.542,-nan,-nan,0.90188,0.121299,-1.0,-1.0,151.635,2.17357,1342,4705420 +993,delete,103.314,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,249.255,2.13586,1321,4621807 +994,search,4666.38,3088.42,1900.98,0.923779,165.871,-nan,-nan,0.89891,0.123056,-1.0,-1.0,93.2,2.13629,1329,4621807 +995,delete,87.906,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,381.372,2.09667,1300,4555849 +996,search,4604.49,3086.99,1880.12,0.907386,158.568,-nan,-nan,0.89785,0.123605,-1.0,-1.0,76.485,2.09763,1307,4555849 +997,delete,31.707,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,144.24,2.08419,1301,4526975 +998,search,4513.21,3064.2,1794.3,0.941411,149.117,-nan,-nan,0.89774,0.123485,-1.0,-1.0,66.534,2.08516,1305,4526975 +999,delete,133.73,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,555.722,2.0225,1265,4408448 +1000,search,4454.5,3062.53,1784.84,0.927572,151.794,-nan,-nan,0.89652,0.124518,-1.0,-1.0,123.834,2.02393,1275,4408448 +1001,delete,129.242,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,568.33,1.96835,1232,4303257 +1002,search,4356.01,3071.96,1763.27,0.933816,157.947,-nan,-nan,0.89421,0.126469,-1.0,-1.0,154.688,1.9708,1245,4303257 +1003,delete,48.265,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,195.787,1.94868,1229,4259551 +1004,search,4292.47,3048.88,1717.73,0.932801,142.078,-nan,-nan,0.89382,0.126572,-1.0,-1.0,59.265,1.9491,1233,4259551 +1005,delete,23.494,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,61.493,1.94286,1229,4241434 +1006,search,4277.17,3030.66,1709.1,0.943552,142.721,-nan,-nan,0.89363,0.126881,-1.0,-1.0,14.584,1.94286,1229,4241434 +1007,delete,223.761,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,838.083,1.85308,1166,4052722 +1008,search,4124.64,3059.54,1651.82,1.02057,140.536,-nan,-nan,0.88964,0.12993,-1.0,-1.0,190.751,1.85616,1179,4052722 +1009,delete,382.663,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1100.77,1.69755,1067,3738432 +1010,search,3842.4,3130.69,1582.23,0.921132,134.401,-nan,-nan,0.88676,0.130735,-1.0,-1.0,243.024,1.70115,1084,3738432 +1011,delete,515.85,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1340.08,1.51095,932,3372539 +1012,search,3293.24,3277.58,1317.42,1.068,110.161,-nan,-nan,0.88472,0.130929,-1.0,-1.0,313.144,1.51652,962,3372539 +1013,delete,10.524,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,58.121,1.51169,956,3361432 +1014,search,3365.78,3177.3,1343.29,0.989856,114.279,-nan,-nan,0.88478,0.131664,-1.0,-1.0,45.259,1.51191,959,3361432 +1015,delete,340.819,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1030.48,1.38205,853,3091692 +1016,search,3188.14,3312.89,1350.37,0.902043,114.629,-nan,-nan,0.87991,0.134066,-1.0,-1.0,308.398,1.38728,881,3091692 +1017,delete,373.018,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1287.47,1.23363,757,2775418 +1018,search,2870.8,3416.08,1241.73,0.893312,111.917,-nan,-nan,0.87557,0.137265,-1.0,-1.0,408.173,1.24009,796,2775418 +1019,delete,18.313,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,75.69,1.23148,790,2754073 +1020,search,2929.11,3253.59,1232.59,0.933021,112.664,-nan,-nan,0.8779,0.135259,-1.0,-1.0,85.616,1.23189,796,2754073 +1021,delete,131.864,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,388.617,1.18381,754,2638917 +1022,search,2811.92,3254.98,1189.04,0.904379,104.996,-nan,-nan,0.87558,0.137214,-1.0,-1.0,106.027,1.18544,762,2638917 +1023,delete,102.543,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,268.055,1.1483,725,2556494 +1024,search,2611.47,3276.68,1060.89,1.03394,90.1421,-nan,-nan,0.87272,0.139466,-1.0,-1.0,93.949,1.14961,733,2556494 +1025,insert,56.576,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,97.452,1.16517,741,2594934 +1026,search,2771.12,3256.75,1171.59,0.892604,100.597,-nan,-nan,0.87413,0.138497,-1.0,-1.0,34.325,1.16527,743,2594934 +1027,insert,338.185,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,361.912,1.25338,771,2802105 +1028,search,2890.78,3355.08,1246.93,0.915737,105.212,-nan,-nan,0.87483,0.139294,-1.0,-1.0,159.782,1.25447,784,2802105 +1029,insert,679.525,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,860.88,1.43301,880,3203975 +1030,search,3290.37,3374.04,1417.13,0.953309,120.618,-nan,-nan,0.88207,0.134455,-1.0,-1.0,202.648,1.43399,899,3203975 +1031,insert,904.64,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1065.05,1.72035,989,3830566 +1032,search,3640.03,3520.22,1561,0.997386,126.064,-nan,-nan,0.88724,0.133039,-1.0,-1.0,517.892,1.72271,1017,3830566 +1033,insert,646.268,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,840.538,1.90804,1109,4252669 +1034,search,4096.61,3499.59,1770.55,0.975485,144.375,-nan,-nan,0.89499,0.127553,-1.0,-1.0,631.966,1.91164,1152,4252669 +1035,insert,329.382,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,430.296,2.02542,1194,4512107 +1036,search,4387.99,3422.38,1891.25,0.969788,160.763,-nan,-nan,0.89883,0.12461,-1.0,-1.0,475.673,2.02937,1234,4512107 +1037,insert,14.99,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,26.813,2.03324,1236,4525815 +1038,search,4378.77,3307.26,1794.6,0.988119,141.456,-nan,-nan,0.89988,0.123483,-1.0,-1.0,220.573,2.03326,1249,4525815 +1039,insert,104.204,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,203.865,2.06433,1275,4599215 +1040,search,4431.38,3283.4,1793.06,1.0504,140.956,-nan,-nan,0.90128,0.122274,-1.0,-1.0,139.77,2.06491,1286,4599215 +1041,insert,13.857,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,59.278,2.0701,1290,4609851 +1042,search,4664.4,3261.5,1930.29,0.916854,164.806,-nan,-nan,0.90235,0.121348,-1.0,-1.0,76.995,2.0707,1296,4609851 +1043,insert,30.261,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.82,2.07747,1298,4627411 +1044,search,4519.74,3246,1958.23,0.910254,170.97,-nan,-nan,0.90269,0.121305,-1.0,-1.0,25.951,2.07772,1299,4627411 +1045,insert,62.088,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,123.901,2.09343,1311,4670607 +1046,search,4742.83,3262.46,1993.84,0.889041,176.87,-nan,-nan,0.9032,0.121304,-1.0,-1.0,24.488,2.09345,1312,4670607 +1047,insert,8.144,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,35.252,2.09553,1314,4675386 +1048,search,4459.65,3255.48,1909.69,0.98907,157.457,-nan,-nan,0.90305,0.121356,-1.0,-1.0,9.929,2.09553,1314,4675386 +1049,insert,10.894,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.334,2.10047,1315,4689998 +1050,search,4506.8,3257.45,1873.52,1.0344,151.139,-nan,-nan,0.90369,0.120862,-1.0,-1.0,15.844,2.10047,1315,4689998 +1051,insert,167.865,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,190.758,2.14225,1342,4783971 +1052,search,4487.68,3250.86,1941.95,0.994664,158.679,-nan,-nan,0.90316,0.120919,-1.0,-1.0,34.815,2.14243,1344,4783971 +1053,insert,63.391,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,72.781,2.15831,1352,4826219 +1054,search,4542.87,3257.82,1955.95,0.976178,162.332,-nan,-nan,0.90373,0.120619,-1.0,-1.0,15.945,2.15848,1353,4826219 +1055,insert,30.149,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,69.142,2.17345,1365,4862355 +1056,search,4686.35,3257.97,1963.49,1.02686,165.31,-nan,-nan,0.90425,0.119892,-1.0,-1.0,47.518,2.17385,1368,4862355 +1057,insert,9.905,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,26.848,2.17573,1369,4869564 +1058,search,4811.14,3253.76,1956.75,1.04125,160.321,-nan,-nan,0.90433,0.119782,-1.0,-1.0,16.651,2.17573,1369,4869564 +1059,insert,53.775,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,59.305,2.18792,1374,4898570 +1060,search,4716.48,3263.84,1997.5,1.01339,168.056,-nan,-nan,0.90465,0.119572,-1.0,-1.0,21.518,2.18818,1376,4898570 +1061,insert,5.383,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.502,2.19038,1378,4907501 +1062,search,4409.23,3262.46,1993.16,0.944083,169.037,-nan,-nan,0.90439,0.11957,-1.0,-1.0,10.004,2.19038,1378,4907501 +1063,insert,6.16,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.853,2.19332,1380,4916386 +1064,search,4690.03,3263.34,2023.1,0.948156,169.685,-nan,-nan,0.90471,0.119431,-1.0,-1.0,29.833,2.19348,1381,4916386 +1065,insert,446.955,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,556.525,2.31082,1438,5194839 +1066,search,4648.48,3295.45,2152.48,0.957748,177.65,-nan,-nan,0.90691,0.118037,-1.0,-1.0,89.203,2.31207,1453,5194839 +1067,insert,5.102,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,17.72,2.31435,1455,5200182 +1068,search,4946.77,3259.49,2154.46,0.943549,186.496,-nan,-nan,0.90711,0.117932,-1.0,-1.0,42.406,2.31435,1457,5200182 +1069,insert,57.449,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,91.841,2.32858,1468,5230883 +1070,search,5148.51,3253.88,2113.89,0.999821,174.524,-nan,-nan,0.90804,0.117284,-1.0,-1.0,24.868,2.32863,1469,5230883 +1071,insert,35.271,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,86.293,2.33848,1477,5255659 +1072,search,4502.96,3244.5,2057.62,1.04338,167.686,-nan,-nan,0.90803,0.117259,-1.0,-1.0,34.878,2.339,1479,5255659 +1073,insert,25.441,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,38.025,2.35012,1485,5281693 +1074,search,5060.08,3244.52,2205.24,0.914463,192.152,-nan,-nan,0.90828,0.116864,-1.0,-1.0,33.511,2.35035,1486,5281693 +1075,insert,16.838,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,43.232,2.35376,1490,5290081 +1076,search,5037.78,3234.19,2215.5,0.962368,195.095,-nan,-nan,0.90841,0.116937,-1.0,-1.0,15.015,2.35397,1491,5290081 +1077,insert,20.829,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,25.984,2.36417,1494,5316166 +1078,search,4400.31,3241.65,2080.9,1.03782,169.265,-nan,-nan,0.90879,0.11668,-1.0,-1.0,15.938,2.36423,1495,5316166 +1079,insert,20.597,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,39.112,2.37186,1500,5335531 +1080,search,5157.45,3240.04,2151.48,0.9785,179.071,-nan,-nan,0.90901,0.116445,-1.0,-1.0,30.237,2.37205,1501,5335531 +1081,insert,595.974,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,688.596,2.53025,1582,5678781 +1082,search,5569.47,3262.67,2300.44,1.01978,187.238,-nan,-nan,0.91105,0.115299,-1.0,-1.0,97.589,2.53205,1590,5678781 +1083,insert,2.178,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.574,2.53263,1590,5679848 +1084,search,5689.33,3244.69,2375.64,0.925475,207.901,-nan,-nan,0.9111,0.115137,-1.0,-1.0,17.4,2.53263,1590,5679848 +1085,insert,45.297,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.8,2.54305,1593,5702205 +1086,search,5464.76,3255.26,2246.62,1.00452,185.484,-nan,-nan,0.91154,0.114971,-1.0,-1.0,22.107,2.5432,1595,5702205 +1087,insert,12.058,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.502,2.54677,1597,5715514 +1088,search,5680.85,3253.58,2404.37,0.902058,206.15,-nan,-nan,0.91145,0.114931,-1.0,-1.0,17.667,2.54677,1597,5715514 +1089,insert,623.688,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,612.774,2.71882,1703,6103499 +1090,search,6073.81,3261.48,2548.49,0.934832,222.431,-nan,-nan,0.91473,0.113361,-1.0,-1.0,82.197,2.71944,1712,6103499 +1091,insert,75.685,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,128.193,2.74176,1728,6152505 +1092,search,6077.19,3237.81,2508.04,0.967515,209.56,-nan,-nan,0.91513,0.113379,-1.0,-1.0,28.467,2.74193,1729,6152505 +1093,insert,25.994,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.201,2.74881,1731,6171104 +1094,search,6060.12,3239.69,2481.19,1.01685,207.841,-nan,-nan,0.91549,0.113125,-1.0,-1.0,14.463,2.74881,1731,6171104 +1095,insert,480.729,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,837.857,2.92301,1833,6557850 +1096,search,6388.65,3261.93,2604.18,0.95059,216.821,-nan,-nan,0.91765,0.111764,-1.0,-1.0,152.482,2.92361,1848,6557850 +1097,insert,22.789,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.138,2.9302,1849,6579289 +1098,search,6307.63,3234.51,2611.06,1.01043,222.265,-nan,-nan,0.91752,0.111966,-1.0,-1.0,28.286,2.93071,1851,6579289 +1099,insert,10.906,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,29.924,2.93607,1854,6591442 +1100,search,6573.15,3234.64,2747.61,0.898512,241.49,-nan,-nan,0.9178,0.111578,-1.0,-1.0,25.793,2.93607,1854,6591442 +1101,insert,61.25,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,170.561,2.95584,1871,6633739 +1102,search,6527.86,3226.18,2746.6,0.915327,240.705,-nan,-nan,0.91821,0.111188,-1.0,-1.0,32.683,2.95586,1872,6633739 +1103,insert,132.34,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,227.155,2.9965,1900,6724357 +1104,search,5952.23,3231.49,2670.39,1.016,218.682,-nan,-nan,0.92025,0.109731,-1.0,-1.0,40.499,2.99662,1904,6724357 +1105,insert,3.487,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,25.005,2.99874,1906,6727059 +1106,search,6605.25,3223.83,2729.5,0.983744,235.876,-nan,-nan,0.92011,0.109839,-1.0,-1.0,17.154,2.99874,1906,6727059 +1107,insert,263.657,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,386.665,3.09005,1951,6950869 +1108,search,6318.72,3267.2,2865.85,0.974536,244.294,-nan,-nan,0.92202,0.108309,-1.0,-1.0,63.686,3.09091,1960,6950869 +1109,insert,6.359,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.755,3.09389,1960,6958173 +1110,search,6765.47,3252.89,2864.33,0.960367,242.737,-nan,-nan,0.92208,0.108343,-1.0,-1.0,22.935,3.09389,1960,6958173 +1111,insert,76.03,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,92.934,3.10972,1970,6995273 +1112,search,6856.06,3256.74,2913.61,0.92153,249.335,-nan,-nan,0.92205,0.108126,-1.0,-1.0,32.376,3.10979,1971,6995273 +1113,insert,86.557,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,106.605,3.13243,1982,7047862 +1114,search,7071.38,3264.3,2945.77,0.928656,254.565,-nan,-nan,0.92269,0.107714,-1.0,-1.0,59.245,3.13307,1986,7047862 +1115,insert,92.851,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,104.933,3.15978,1997,7110727 +1116,search,7106.78,3272.86,2961.72,0.979007,251.075,-nan,-nan,0.92313,0.107396,-1.0,-1.0,68.3,3.16022,2001,7110727 +1117,insert,19.325,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.622,3.1648,2003,7125591 +1118,search,7115.06,3272.11,2963.47,0.980256,258.641,-nan,-nan,0.92335,0.107025,-1.0,-1.0,35.809,3.16501,2004,7125591 +1119,insert,10.044,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,48.855,3.16745,2007,7134442 +1120,search,7166.21,3269.02,2993.36,0.963582,255.281,-nan,-nan,0.92291,0.107377,-1.0,-1.0,26.659,3.16745,2007,7134442 +1121,insert,40.499,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,82.465,3.17772,2013,7162109 +1122,search,7196.69,3272.12,3004.92,0.920676,265.424,-nan,-nan,0.92307,0.107277,-1.0,-1.0,52.749,3.17786,2016,7162109 +1123,insert,24.635,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,85.43,3.18388,2023,7179509 +1124,search,7249.23,3258.97,3018.08,0.949831,264.103,-nan,-nan,0.92339,0.107001,-1.0,-1.0,73.501,3.18437,2026,7179509 +1125,insert,18.534,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,40.492,3.18868,2028,7186372 +1126,search,7203.99,3254.27,3004.16,0.967837,261.618,-nan,-nan,0.92327,0.106982,-1.0,-1.0,24.278,3.18868,2028,7186372 +1127,insert,65.978,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,145.384,3.20768,2038,7231450 +1128,search,7168.64,3258.19,2943.62,0.928387,248.795,-nan,-nan,0.92369,0.106766,-1.0,-1.0,70.547,3.2085,2043,7231450 +1129,insert,21.146,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,43.16,3.21306,2045,7243215 +1130,search,7102.41,3251.91,2859.07,1.00815,233.279,-nan,-nan,0.92377,0.106834,-1.0,-1.0,44.489,3.21348,2047,7243215 +1131,insert,69.413,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,68.444,3.22643,2051,7276107 +1132,search,6909.54,3253.76,2943,1.0144,243.564,-nan,-nan,0.92378,0.106792,-1.0,-1.0,27.183,3.22667,2053,7276107 +1133,insert,19.731,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,44.37,3.23416,2060,7292577 +1134,search,6950.39,3244.6,2862.07,0.999231,231.775,-nan,-nan,0.92364,0.106976,-1.0,-1.0,29.465,3.23433,2061,7292577 +1135,insert,9.737,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,29.656,3.23812,2063,7299867 +1136,search,6648.65,3243.02,2915.51,0.986755,239.622,-nan,-nan,0.92358,0.106998,-1.0,-1.0,15.808,3.23812,2063,7299867 +1137,insert,22.685,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,33.283,3.24807,2067,7323342 +1138,search,6963.59,3247.64,2866.57,1.01621,229.424,-nan,-nan,0.92372,0.107019,-1.0,-1.0,59.286,3.24848,2069,7323342 +1139,insert,3.063,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.056,3.24897,2069,7325844 +1140,search,7111.16,3246.78,2958.38,0.994153,243.608,-nan,-nan,0.92375,0.106985,-1.0,-1.0,38.727,3.24908,2070,7325844 +1141,insert,315.751,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,279.657,3.34656,2115,7540761 +1142,search,7118.12,3267.28,3004.05,0.984749,245.614,-nan,-nan,0.92434,0.106285,-1.0,-1.0,275.121,3.34934,2139,7540761 +1143,insert,33.736,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,59.202,3.35755,2143,7561926 +1144,search,7434.27,3225.77,2995.35,1.04848,248.808,-nan,-nan,0.92443,0.106309,-1.0,-1.0,98.442,3.35775,2146,7561926 +1145,insert,16.451,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,45.04,3.36165,2148,7571861 +1146,search,7617.27,3226.28,3151.85,0.94853,276.629,-nan,-nan,0.92445,0.106244,-1.0,-1.0,75.937,3.36165,2151,7571861 +1147,insert,34.911,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,53.829,3.3673,2154,7590881 +1148,search,7156.31,3227.89,3171.91,0.911265,276.094,-nan,-nan,0.92466,0.106063,-1.0,-1.0,41.454,3.36779,2158,7590881 +1149,insert,4.727,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.529,3.37033,2160,7595507 +1150,search,7468.62,3219.49,3094.68,0.97557,260.237,-nan,-nan,0.92476,0.106112,-1.0,-1.0,27.908,3.37033,2160,7595507 +1151,insert,11.826,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,25.111,3.37268,2160,7601664 +1152,search,7195.41,3222.43,3105.43,0.987124,258.673,-nan,-nan,0.92474,0.106103,-1.0,-1.0,17.886,3.37268,2160,7601664 +1153,delete,24.689,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,143.198,3.36327,2153,7566707 +1154,search,7183.88,3217.59,3128.36,0.948217,265,-nan,-nan,0.9246,0.106044,-1.0,-1.0,24.316,3.3635,2154,7566707 +1155,delete,106.129,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,621.12,3.29238,2104,7396192 +1156,search,6139.06,3221.54,2931.38,0.978245,234.878,-nan,-nan,0.92428,0.105736,-1.0,-1.0,69.353,3.29436,2112,7396192 +1157,delete,209.255,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1416.08,3.15616,1962,7079041 +1158,search,5755.24,3300.25,2807.52,1.01444,221.403,-nan,-nan,0.92245,0.107878,-1.0,-1.0,276.835,3.16314,2007,7079041 +1159,delete,298.805,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,914.855,2.95198,1854,6597115 +1160,search,6442.25,3253.84,2724.99,0.916236,227.551,-nan,-nan,0.91762,0.110606,-1.0,-1.0,320.283,2.95888,1879,6597115 +1161,delete,307.372,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1690.45,2.82687,1761,6308902 +1162,search,6270.04,3268.33,2589.96,0.966959,217.986,-nan,-nan,0.91569,0.112064,-1.0,-1.0,303.483,2.83197,1791,6308902 +1163,delete,130.781,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,865.752,2.76526,1745,6164514 +1164,search,6227.98,3241.27,2578.49,0.918979,222.166,-nan,-nan,0.91585,0.11117,-1.0,-1.0,204.427,2.76905,1764,6164514 +1165,delete,39.004,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,160.526,2.75753,1755,6133412 +1166,search,6165.8,3210.83,2505.95,0.956245,209.22,-nan,-nan,0.91567,0.111051,-1.0,-1.0,42.35,2.75783,1757,6133412 +1167,delete,93.827,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,231.887,2.73848,1743,6057218 +1168,search,6151.53,3186.24,2526.36,0.927492,213.602,-nan,-nan,0.91522,0.110819,-1.0,-1.0,21.899,2.73848,1743,6057218 +1169,delete,30.204,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,155.105,2.72194,1729,6028053 +1170,search,6172.31,3194.29,2542.87,0.939943,220.614,-nan,-nan,0.91234,0.11275,-1.0,-1.0,52.536,2.7225,1732,6028053 +1171,delete,31.925,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,111.514,2.71635,1727,6001880 +1172,search,6038.26,3181.57,2446.36,0.960407,204.136,-nan,-nan,0.91182,0.112911,-1.0,-1.0,33.931,2.71649,1728,6001880 +1173,delete,56.354,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,103.256,2.70904,1723,5957180 +1174,search,6055.06,3161.52,2466.21,0.971374,209.902,-nan,-nan,0.91156,0.113488,-1.0,-1.0,20.88,2.70904,1723,5957180 +1175,delete,22.667,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.906,2.70875,1723,5939326 +1176,search,5995.23,3150.14,2408.66,1.00837,196.479,-nan,-nan,0.91173,0.11335,-1.0,-1.0,20.197,2.70875,1723,5939326 +1177,delete,14.492,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,35.417,2.70529,1722,5924145 +1178,search,5922.97,3145.89,2352,1.06397,190.117,-nan,-nan,0.91183,0.113314,-1.0,-1.0,20.726,2.70529,1722,5924145 +1179,delete,77.339,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,347.724,2.67395,1699,5863334 +1180,search,5996.46,3150.76,2442.67,0.924563,206.308,-nan,-nan,0.9112,0.113941,-1.0,-1.0,32.157,2.67419,1700,5863334 +1181,delete,45.501,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,35.964,2.67101,1699,5822703 +1182,search,5961.94,3122.92,2424.82,0.955642,207.795,-nan,-nan,0.91114,0.114233,-1.0,-1.0,19.629,2.67101,1699,5822703 +1183,delete,53.213,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,150.948,2.65428,1690,5775807 +1184,search,5909.32,3111.49,2401.8,0.923841,205.977,-nan,-nan,0.91062,0.114719,-1.0,-1.0,19.47,2.65428,1690,5775807 +1185,delete,34.728,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,156.138,2.64394,1682,5750627 +1186,search,5913.77,3110.13,2404.08,0.909585,210.614,-nan,-nan,0.91058,0.114688,-1.0,-1.0,19.326,2.64394,1682,5750627 +1187,delete,188.726,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,955.969,2.57096,1618,5601207 +1188,search,5720.59,3139.71,2321.65,0.941009,199.815,-nan,-nan,0.90976,0.115495,-1.0,-1.0,154.904,2.57369,1629,5601207 +1189,delete,16.61,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,34.297,2.57213,1628,5586425 +1190,search,5616.57,3115.07,2226.75,1.01029,186.865,-nan,-nan,0.90968,0.115659,-1.0,-1.0,21.036,2.57213,1628,5586425 +1191,delete,19.747,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.819,2.56958,1626,5574023 +1192,search,5746.12,3113.22,2346.6,0.93763,207.346,-nan,-nan,0.90971,0.115562,-1.0,-1.0,19.297,2.56958,1626,5574023 +1193,delete,289.465,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1200.74,2.46405,1545,5356157 +1194,search,5434.01,3148.92,2208.42,0.961239,189.746,-nan,-nan,0.90702,0.117283,-1.0,-1.0,164.887,2.46635,1558,5356157 +1195,delete,18.397,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.134,2.46485,1557,5340404 +1196,search,5488.29,3115.33,2240.06,0.893272,192.986,-nan,-nan,0.90753,0.11643,-1.0,-1.0,18.838,2.46485,1557,5340404 +1197,delete,64.915,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,58.101,2.45771,1553,5293259 +1198,search,5326.07,3085.46,2095.62,0.994595,171.471,-nan,-nan,0.90728,0.117008,-1.0,-1.0,18.294,2.45771,1553,5293259 +1199,delete,16.95,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,61.415,2.45164,1551,5279787 +1200,search,5472.43,3083.22,2213.28,0.929211,193.071,-nan,-nan,0.90727,0.117018,-1.0,-1.0,18.251,2.45164,1551,5279787 +1201,delete,88.592,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,426.6,2.42044,1521,5216567 +1202,search,5217.55,3110.43,2060.61,1.06206,171.817,-nan,-nan,0.90614,0.117881,-1.0,-1.0,83.186,2.42172,1527,5216567 +1203,delete,28.438,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.577,2.41959,1527,5196947 +1204,search,5311.89,3080.49,2122.38,0.920915,181.444,-nan,-nan,0.90601,0.118062,-1.0,-1.0,27.649,2.4197,1528,5196947 +1205,delete,31.689,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,44.234,2.41344,1526,5168447 +1206,search,5270.82,3064.76,2092.17,0.937265,173.691,-nan,-nan,0.90571,0.118337,-1.0,-1.0,17.3,2.41344,1526,5168447 +1207,delete,26.511,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,120.599,2.40283,1518,5146927 +1208,search,5149.93,3066.41,2001.98,1.00931,162.524,-nan,-nan,0.90454,0.119549,-1.0,-1.0,56.278,2.40367,1521,5146927 +1209,delete,337.631,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,923.496,2.27413,1451,4887612 +1210,search,5109.96,3066.35,2070.94,0.928789,176.315,-nan,-nan,0.90411,0.119034,-1.0,-1.0,100.6,2.27543,1459,4887612 +1211,delete,6.629,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,29.618,2.2739,1458,4882301 +1212,search,4987.77,3051.97,1950.87,0.997806,162.322,-nan,-nan,0.90448,0.118843,-1.0,-1.0,17.349,2.2739,1458,4882301 +1213,delete,150.072,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,509.277,2.21203,1415,4780915 +1214,search,4951.69,3075.11,1963.33,0.983012,165.743,-nan,-nan,0.90263,0.120879,-1.0,-1.0,62.163,2.21251,1418,4780915 +1215,delete,25.264,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,58.672,2.20724,1415,4765751 +1216,search,4916.88,3065.92,1947.45,0.990483,160.306,-nan,-nan,0.90277,0.120751,-1.0,-1.0,34.734,2.20753,1416,4765751 +1217,delete,482.466,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1601.91,2.03999,1263,4452306 +1218,search,4426.29,3193.51,1778.91,1.00594,150.357,-nan,-nan,0.89656,0.125488,-1.0,-1.0,307.754,2.0457,1291,4452306 +1219,delete,117.064,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,385.585,1.99903,1244,4362045 +1220,search,4481.08,3180.58,1845.13,0.93269,161.292,-nan,-nan,0.89545,0.126907,-1.0,-1.0,180.052,2.0022,1259,4362045 +1221,delete,38.735,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,104.012,1.99039,1251,4326917 +1222,search,4476.12,3138.85,1831.19,0.909644,159.615,-nan,-nan,0.89542,0.126868,-1.0,-1.0,38.067,1.99054,1253,4326917 +1223,delete,281.475,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,896.497,1.88366,1174,4109781 +1224,search,4240.16,3180.96,1749,0.909114,150.806,-nan,-nan,0.8914,0.128912,-1.0,-1.0,180.441,1.8862,1189,4109781 +1225,delete,33.502,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,189.55,1.87079,1179,4075879 +1226,search,4236.22,3135.86,1730.99,0.949195,152.637,-nan,-nan,0.89087,0.129167,-1.0,-1.0,99.3,1.87257,1186,4075879 +1227,delete,90.591,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,433.009,1.83105,1156,4006503 +1228,search,4137.93,3168.59,1698.97,0.961493,146.321,-nan,-nan,0.89091,0.129308,-1.0,-1.0,152.246,1.83421,1168,4006503 +1229,delete,103.67,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,505.966,1.78748,1123,3922625 +1230,search,4051.45,3187.96,1670.47,0.951063,144.226,-nan,-nan,0.89014,0.129244,-1.0,-1.0,143.117,1.78983,1135,3922625 +1231,delete,89.355,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,366.43,1.74898,1090,3844860 +1232,search,3926.82,3223.18,1612.22,0.931313,137.361,-nan,-nan,0.88521,0.131135,-1.0,-1.0,139.671,1.75087,1103,3844860 +1233,delete,13.009,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,57.114,1.74637,1101,3831414 +1234,search,3987.85,3173.4,1633.71,0.885209,143.665,-nan,-nan,0.88513,0.131591,-1.0,-1.0,31.9,1.7466,1102,3831414 +1235,delete,179.976,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,692.848,1.67096,1044,3684234 +1236,search,3813.82,3201.5,1587.4,0.934488,140.292,-nan,-nan,0.88144,0.135014,-1.0,-1.0,133.629,1.67336,1055,3684234 +1237,delete,14.063,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.137,1.67182,1054,3669074 +1238,search,3793.04,3169.91,1560.75,0.933636,133.603,-nan,-nan,0.88167,0.134354,-1.0,-1.0,40.305,1.67211,1056,3669074 +1239,delete,60.95,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,84.417,1.66057,1047,3620984 +1240,search,3762.59,3131.5,1540.38,0.911211,135.353,-nan,-nan,0.88146,0.134489,-1.0,-1.0,27.15,1.66068,1048,3620984 +1241,delete,41.744,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,81.41,1.65354,1043,3580113 +1242,search,3757.74,3099.18,1524.12,0.926788,134.761,-nan,-nan,0.88133,0.134234,-1.0,-1.0,17.46,1.65354,1043,3580113 +1243,delete,59.004,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,250.871,1.62558,1020,3533909 +1244,search,3656.35,3122.9,1487.97,0.929771,132.286,-nan,-nan,0.87977,0.135232,-1.0,-1.0,42.885,1.62612,1022,3533909 +1245,delete,14.547,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,38.366,1.6235,1020,3519590 +1246,search,3593.66,3110.72,1442.89,0.968076,122.645,-nan,-nan,0.87964,0.135429,-1.0,-1.0,17.252,1.6235,1020,3519590 +1247,delete,75.79,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,116.151,1.5895,996,3452537 +1248,search,3577.43,3125.93,1445.25,0.894424,120.004,-nan,-nan,0.88126,0.133253,-1.0,-1.0,47.465,1.58995,999,3452537 +1249,delete,38.898,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,111.044,1.58048,991,3416711 +1250,search,3522.93,3106.46,1410.81,1.00246,123.055,-nan,-nan,0.88087,0.133839,-1.0,-1.0,16.34,1.58048,991,3416711 +1251,delete,38.856,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,82.257,1.56858,983,3387653 +1252,search,3467.03,3100.82,1411.56,1.05259,122.019,-nan,-nan,0.88061,0.134227,-1.0,-1.0,27.932,1.56871,984,3387653 +1253,delete,18.183,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,49.758,1.56074,980,3372157 +1254,search,3468.8,3097.95,1421.26,0.949843,124.194,-nan,-nan,0.88063,0.134351,-1.0,-1.0,18.312,1.56074,980,3372157 +1255,delete,167.292,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,175.685,1.50294,949,3258215 +1256,search,3412.51,3096.42,1382.45,0.921565,123.794,-nan,-nan,0.8775,0.135857,-1.0,-1.0,52.547,1.50317,952,3258215 +1257,delete,51.289,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,119.908,1.4855,941,3223657 +1258,search,3303.99,3083.01,1301.56,1.00455,111.083,-nan,-nan,0.87674,0.136303,-1.0,-1.0,57.725,1.48601,944,3223657 +1259,delete,59.788,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,193.63,1.46375,927,3174982 +1260,search,3234.68,3078.64,1280.54,1.00397,106.723,-nan,-nan,0.87499,0.137272,-1.0,-1.0,45.937,1.46432,929,3174982 +1261,delete,25.255,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.959,1.46279,929,3157038 +1262,search,3266.43,3051.53,1278.55,1.00793,110.198,-nan,-nan,0.87591,0.136551,-1.0,-1.0,15.509,1.46279,929,3157038 +1263,delete,133.502,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,314.586,1.41461,905,3058847 +1264,search,3241.26,3052.12,1314.07,0.931101,111.88,-nan,-nan,0.87369,0.137713,-1.0,-1.0,100.687,1.41579,911,3058847 +1265,delete,401.346,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,499.2,1.29095,818,2833531 +1266,search,2983.71,3124.46,1219.18,0.924809,108.777,-nan,-nan,0.86839,0.140311,-1.0,-1.0,136.649,1.29221,829,2833531 +1267,delete,74.814,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,226.107,1.26164,809,2763793 +1268,search,2825.32,3078.26,1157.63,1.01784,98.1842,-nan,-nan,0.86823,0.14007,-1.0,-1.0,45.507,1.2626,815,2763793 +1269,delete,107.037,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,235.988,1.19778,781,2627384 +1270,search,2510.3,3100.44,1157.62,0.929711,105.215,-nan,-nan,0.86851,0.139332,-1.0,-1.0,55.242,1.19901,788,2627384 +1271,delete,48.256,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,179.53,1.16272,763,2553476 +1272,search,2627.24,3089.51,1057.76,1.08033,90.1717,-nan,-nan,0.86737,0.140326,-1.0,-1.0,83.334,1.16378,769,2553476 +1273,delete,98.075,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,272.742,1.11092,727,2443067 +1274,search,2630.83,3104.95,1081.69,0.943602,100.004,-nan,-nan,0.86516,0.141377,-1.0,-1.0,69.494,1.11178,732,2443067 +1275,delete,22.57,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.779,1.10804,730,2421654 +1276,search,2545.83,3068.28,1063.37,0.942027,89.9401,-nan,-nan,0.8659,0.140918,-1.0,-1.0,26.709,1.10805,731,2421654 +1277,delete,40.245,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,81.475,1.09764,723,2391396 +1278,search,2479.67,3055.98,1051.37,0.910048,90.4225,-nan,-nan,0.8654,0.140847,-1.0,-1.0,19.58,1.09792,724,2391396 +1279,delete,21.315,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,57.374,1.09117,719,2367854 +1280,search,2559.22,3039.74,1060.02,0.905771,99.6779,-nan,-nan,0.8642,0.141876,-1.0,-1.0,18.086,1.0915,720,2367854 diff --git a/scripts/big_ann_perf_numbers/perf_debug_scan_0.15_counters.csv b/scripts/big_ann_perf_numbers/perf_debug_scan_0.15_counters.csv new file mode 100644 index 00000000..ff5c1f64 --- /dev/null +++ b/scripts/big_ann_perf_numbers/perf_debug_scan_0.15_counters.csv @@ -0,0 +1,1280 @@ +step_num,step_type,search_latency_ms,worker_partition_size,worker_scan_time_ms,worker_scan_throughput,worker_result_time_ms,measured_ipc,cache_miss_rate,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,3343.64,37.4413,563.1,0.382835,418.41,0.771449,4.85742,0.8551,0.130106,-1.0,-1.0,11.008,0.0184881,1000,38806 +3,insert,22.364,-1.0,-1.0,-1.0,-1.0,19.806,0.022655,996,49334 +4,search,2642.33,64.8478,242.19,0.677423,156.2,0.869095,22.2654,0.86686,0.128427,-1.0,-1.0,29.982,0.0226179,991,49334 +5,insert,19.66,-1.0,-1.0,-1.0,-1.0,171.169,0.0329472,982,73557 +6,search,2610.23,185.601,366.063,0.637806,144.764,1.42254,9.84325,0.92575,0.103236,-1.0,-1.0,137.704,0.0329835,983,73557 +7,insert,24.241,-1.0,-1.0,-1.0,-1.0,141.555,0.0426531,985,95766 +8,search,2605.64,234.625,294.752,0.825211,126.344,1.06839,24.9182,0.92447,0.103861,-1.0,-1.0,83.391,0.0429,989,95766 +9,insert,3.621,-1.0,-1.0,-1.0,-1.0,70.924,0.044738,989,100025 +10,search,2635.35,241.911,371.061,0.66167,137.947,1.41836,14.0481,0.92708,0.100706,-1.0,-1.0,92.459,0.0448066,995,100025 +11,insert,14.855,-1.0,-1.0,-1.0,-1.0,75.578,0.0498131,996,111417 +12,search,2707.07,277.335,468.483,0.591891,140.019,1.75245,10.7512,0.92551,0.101769,-1.0,-1.0,76.195,0.0498433,997,111417 +13,insert,19.283,-1.0,-1.0,-1.0,-1.0,77.777,0.0612885,999,137139 +14,search,2636.59,317.883,348.06,0.761631,127.324,1.44851,12.7546,0.92122,0.104957,-1.0,-1.0,18.974,0.0614097,1001,137139 +15,insert,25.797,-1.0,-1.0,-1.0,-1.0,16.905,0.069717,1003,156710 +16,search,2698.78,347.826,372.411,0.72489,137.629,1.73887,9.49277,0.91945,0.105125,-1.0,-1.0,16.094,0.069719,1005,156710 +17,insert,3.622,-1.0,-1.0,-1.0,-1.0,9.838,0.0716876,1007,161773 +18,search,2673.24,360.089,304.494,0.782145,129.382,1.40476,22.7395,0.92035,0.105235,-1.0,-1.0,17.054,0.072065,1011,161773 +19,insert,31.1,-1.0,-1.0,-1.0,-1.0,13.682,0.0788819,1015,177161 +20,search,2735.89,388.83,379.232,0.730813,124.612,1.73328,10.7717,0.9237,0.102574,-1.0,-1.0,14.512,0.0788827,1018,177161 +21,insert,336.849,-1.0,-1.0,-1.0,-1.0,204.04,0.201255,1082,448898 +22,search,2981.66,748.264,505.422,0.885998,104.243,1.87153,8.90268,0.90797,0.109463,-1.0,-1.0,67.432,0.201722,1103,448898 +23,insert,386.262,-1.0,-1.0,-1.0,-1.0,283.23,0.32894,1171,733493 +24,search,3297.87,989.892,671.849,0.850398,104.19,1.89335,6.3205,0.90064,0.113494,-1.0,-1.0,102.887,0.32979,1203,733493 +25,insert,35.669,-1.0,-1.0,-1.0,-1.0,26.802,0.347361,1210,777443 +26,search,3367.47,944.33,686.981,0.943532,102.309,2.07415,8.01136,0.89896,0.114829,-1.0,-1.0,29.189,0.347529,1218,777443 +27,insert,25.796,-1.0,-1.0,-1.0,-1.0,11.594,0.358526,1219,805394 +28,search,3436.43,953.917,732.879,0.8747,93.6469,1.99291,7.76607,0.89871,0.114554,-1.0,-1.0,23.108,0.358793,1223,805394 +29,insert,53.579,-1.0,-1.0,-1.0,-1.0,51.158,0.385772,1236,866391 +30,search,3510.98,1001.37,779.529,0.838457,95.0301,1.96025,7.48291,0.90178,0.113257,-1.0,-1.0,30.802,0.385827,1242,866391 +31,insert,453.894,-1.0,-1.0,-1.0,-1.0,330.287,0.55272,1322,1239816 +32,search,3824.11,1238.56,955.957,0.871054,94.2941,2.14795,7.71408,0.8971,0.117045,-1.0,-1.0,65.733,0.553069,1335,1239816 +33,insert,61.077,-1.0,-1.0,-1.0,-1.0,29.433,0.573909,1342,1291211 +34,search,3915.99,1244.23,1018.2,0.885409,108.255,2.15544,6.91559,0.89721,0.117937,-1.0,-1.0,21.523,0.573983,1345,1291211 +35,insert,44.337,-1.0,-1.0,-1.0,-1.0,47.193,0.585607,1359,1324978 +36,search,3950.25,1244.81,1030.4,0.866789,106.53,2.09807,8.16167,0.89696,0.1175,-1.0,-1.0,10.219,0.585607,1359,1324978 +37,insert,46.826,-1.0,-1.0,-1.0,-1.0,50.667,0.606676,1369,1371885 +38,search,3900.73,1263.54,928.406,0.916513,99.0434,2.0626,6.60548,0.89727,0.117808,-1.0,-1.0,15.693,0.606676,1370,1371885 +39,insert,8.734,-1.0,-1.0,-1.0,-1.0,11.475,0.609952,1371,1379335 +40,search,3965.23,1266.36,1042.24,0.84002,106.561,2.07557,7.68403,0.89724,0.118217,-1.0,-1.0,11.15,0.609952,1371,1379335 +41,insert,35.985,-1.0,-1.0,-1.0,-1.0,34.149,0.624036,1378,1413312 +42,search,4021.23,1272.79,1064.81,0.898335,104.657,2.13308,10.7149,0.89779,0.118505,-1.0,-1.0,10.196,0.624036,1378,1413312 +43,insert,12.111,-1.0,-1.0,-1.0,-1.0,12.928,0.628448,1379,1423411 +44,search,3987.84,1284.37,996.45,0.897737,97.177,2.07377,8.67195,0.89837,0.117447,-1.0,-1.0,11.555,0.628448,1379,1423411 +45,insert,201.309,-1.0,-1.0,-1.0,-1.0,26.144,0.643687,1386,1457559 +46,search,4010,1303.52,1040.92,0.897942,90.9448,2.13535,9.09615,0.89876,0.116768,-1.0,-1.0,12.739,0.643687,1386,1457559 +47,insert,294.876,-1.0,-1.0,-1.0,-1.0,256.013,0.747053,1426,1688593 +48,search,4246.99,1429.64,1211.66,0.880839,104.637,2.10042,8.35542,0.90176,0.116429,-1.0,-1.0,101.369,0.747412,1441,1688593 +49,insert,35.647,-1.0,-1.0,-1.0,-1.0,24.157,0.762204,1444,1723505 +50,search,4122.35,1426.84,1183.31,0.910668,113.808,2.10762,8.20371,0.90318,0.115166,-1.0,-1.0,51.665,0.762425,1451,1723505 +51,insert,337.921,-1.0,-1.0,-1.0,-1.0,301.76,0.902133,1516,2035040 +52,search,4519.27,1548.71,1310.07,0.888803,120.79,2.16496,9.48027,0.90624,0.113443,-1.0,-1.0,30.25,0.902531,1522,2035040 +53,insert,32.286,-1.0,-1.0,-1.0,-1.0,17.193,0.9201,1524,2078341 +54,search,4291.99,1566.1,1214.59,0.983559,112.032,2.17062,9.11406,0.90635,0.113561,-1.0,-1.0,14.357,0.920122,1525,2078341 +55,insert,414.422,-1.0,-1.0,-1.0,-1.0,378.968,1.06594,1589,2399905 +56,search,4656.09,1719.22,1388.49,0.955747,136.796,2.30258,7.68155,0.90836,0.114592,-1.0,-1.0,140.806,1.06696,1618,2399905 +57,insert,24.043,-1.0,-1.0,-1.0,-1.0,19.971,1.07463,1621,2419836 +58,search,4499.89,1672.88,1338.3,0.986193,125.925,2.20212,7.14915,0.90736,0.115037,-1.0,-1.0,23.35,1.07465,1625,2419836 +59,insert,189.91,-1.0,-1.0,-1.0,-1.0,125.22,1.17032,1645,2643083 +60,search,4935.21,1720.98,1434.96,0.942602,128.949,2.1497,8.73685,0.90031,0.123055,-1.0,-1.0,35.802,1.17043,1649,2643083 +61,insert,10.306,-1.0,-1.0,-1.0,-1.0,19.587,1.17387,1652,2653064 +62,search,4929.23,1716.9,1412.38,0.954052,124.956,8.52278,8.71836,0.90004,0.122916,-1.0,-1.0,12.513,1.17387,1652,2653064 +63,insert,3.187,-1.0,-1.0,-1.0,-1.0,9.73,1.17504,1652,2656118 +64,search,4516.69,1718.57,1328.51,1.04822,120.057,4.06057,8.74473,0.90016,0.122751,-1.0,-1.0,12.269,1.17505,1653,2656118 +65,insert,64.47,-1.0,-1.0,-1.0,-1.0,104.263,1.20551,1684,2723424 +66,search,4853.78,1720.22,1428.59,0.972227,124.353,2.16736,8.30377,0.90025,0.122556,-1.0,-1.0,14.435,1.20551,1684,2723424 +67,insert,44.175,-1.0,-1.0,-1.0,-1.0,81.441,1.22545,1701,2769021 +68,search,5135.36,1731.87,1498.32,0.959422,138.435,2.15068,7.8652,0.90187,0.121292,-1.0,-1.0,21.422,1.22545,1703,2769021 +69,insert,50.072,-1.0,-1.0,-1.0,-1.0,92.467,1.24732,1722,2821777 +70,search,4718.97,1733.53,1352.57,1.08966,116.895,2.11711,9.00953,0.90376,0.11984,-1.0,-1.0,26.386,1.24732,1723,2821777 +71,insert,23.904,-1.0,-1.0,-1.0,-1.0,26.357,1.25196,1725,2834921 +72,search,5102.67,1743.51,1485.33,0.93513,124.501,2.14531,7.66143,0.90416,0.119667,-1.0,-1.0,15.783,1.25196,1725,2834921 +73,insert,5.999,-1.0,-1.0,-1.0,-1.0,19.696,1.2547,1726,2842044 +74,search,5070.19,1746.59,1493.19,0.962098,141.744,2.09811,8.10587,0.90418,0.119697,-1.0,-1.0,12.176,1.2547,1726,2842044 +75,insert,18.137,-1.0,-1.0,-1.0,-1.0,21.395,1.2626,1728,2858890 +76,search,5043.96,1754.18,1507.28,0.93519,136.494,2.16475,6.54169,0.90493,0.119199,-1.0,-1.0,11.566,1.2626,1728,2858890 +77,insert,22.586,-1.0,-1.0,-1.0,-1.0,17.391,1.27306,1730,2882818 +78,search,4970.02,1764.11,1497.4,0.963509,146.672,2.23834,8.54019,0.90469,0.119578,-1.0,-1.0,17.287,1.27306,1730,2882818 +79,insert,476.204,-1.0,-1.0,-1.0,-1.0,33.669,1.30089,1735,2947806 +80,search,5231.19,1789.52,1489.09,1.01418,139.036,2.18478,8.27512,0.90501,0.119413,-1.0,-1.0,18.048,1.30089,1735,2947806 +81,insert,39.31,-1.0,-1.0,-1.0,-1.0,23.07,1.31258,1737,2976895 +82,search,5049.26,1798.55,1491.55,1.06826,140.593,2.1474,7.60697,0.90594,0.118494,-1.0,-1.0,18.058,1.31258,1737,2976895 +83,insert,16.114,-1.0,-1.0,-1.0,-1.0,30.759,1.31819,1740,2990682 +84,search,5118.52,1802.31,1544.3,0.981751,135.291,2.25565,7.38681,0.90639,0.117924,-1.0,-1.0,21.853,1.31824,1741,2990682 +85,insert,37.238,-1.0,-1.0,-1.0,-1.0,38.969,1.3335,1746,3025613 +86,search,5110.87,1811.22,1485.08,1.04874,114.62,2.15143,6.87409,0.90683,0.117967,-1.0,-1.0,24.905,1.33371,1747,3025613 +87,insert,532.086,-1.0,-1.0,-1.0,-1.0,654.492,1.53826,1855,3468632 +88,search,5496.24,1950.7,1697.82,0.986497,156.852,2.12916,7.99709,0.91167,0.112324,-1.0,-1.0,104.648,1.53851,1871,3468632 +89,insert,401.346,-1.0,-1.0,-1.0,-1.0,559.382,1.70103,1976,3831692 +90,search,5943.72,1996.96,1882.24,1.0252,172.18,2.14472,8.04168,0.91545,0.108394,-1.0,-1.0,95.251,1.70121,1987,3831692 +91,insert,35.521,-1.0,-1.0,-1.0,-1.0,59.97,1.71117,1994,3864067 +92,search,6114.88,1991.36,1848.42,1.00131,168.444,2.16376,7.86994,0.91561,0.108416,-1.0,-1.0,32.193,1.71129,1995,3864067 +93,insert,207.32,-1.0,-1.0,-1.0,-1.0,240.315,1.81579,2031,4103827 +94,search,6318.73,2053.22,1972.8,1.0405,182.822,2.59716,7.62398,0.91619,0.10706,-1.0,-1.0,40.887,1.81598,2037,4103827 +95,insert,10.008,-1.0,-1.0,-1.0,-1.0,28.395,1.82099,2039,4119898 +96,search,6194.03,2056.82,2004.49,0.987145,190.395,2.19855,6.78772,0.91625,0.107031,-1.0,-1.0,32.787,1.82099,2039,4119898 +97,insert,12.872,-1.0,-1.0,-1.0,-1.0,43.782,1.82506,2042,4129367 +98,search,6156.44,2054.65,1905.04,1.02966,160.309,2.14016,7.64524,0.91639,0.107028,-1.0,-1.0,32.582,1.82506,2042,4129367 +99,insert,10.254,-1.0,-1.0,-1.0,-1.0,31.957,1.8286,2042,4134502 +100,search,6269.72,2056.94,2015.77,0.973243,186.092,2.17079,7.43277,0.91657,0.106954,-1.0,-1.0,33.931,1.8286,2042,4134502 +101,insert,39.982,-1.0,-1.0,-1.0,-1.0,88.753,1.8413,2052,4164482 +102,search,6278.86,2063.49,1981.99,1.01025,182.027,2.15471,8.20389,0.91695,0.106522,-1.0,-1.0,37.394,1.84144,2053,4164482 +103,insert,17.307,-1.0,-1.0,-1.0,-1.0,50.679,1.84672,2056,4178601 +104,search,6265.16,2069.78,1951.69,1.01526,175.319,2.12214,7.3176,0.91727,0.106544,-1.0,-1.0,30.23,1.84672,2056,4178601 +105,insert,6.898,-1.0,-1.0,-1.0,-1.0,33.662,1.85061,2057,4187774 +106,search,6270.63,2072.77,2024.95,1.01447,177.562,2.15703,7.34658,0.91731,0.106855,-1.0,-1.0,42.152,1.85067,2058,4187774 +107,insert,16.973,-1.0,-1.0,-1.0,-1.0,44.344,1.85633,2060,4200109 +108,search,6123.58,2073.2,1896.7,1.11276,161.792,2.36266,8.51211,0.91737,0.106748,-1.0,-1.0,28.976,1.85644,2061,4200109 +109,insert,462.813,-1.0,-1.0,-1.0,-1.0,527.145,2.02013,2162,4557867 +110,search,6530.72,2117.39,2104.74,1.0288,197.263,2.14885,7.97372,0.91913,0.10597,-1.0,-1.0,101.211,2.02044,2171,4557867 +111,insert,32.82,-1.0,-1.0,-1.0,-1.0,71.928,2.03336,2178,4587252 +112,search,6416,2116.87,1936.79,1.13626,157.652,2.16537,7.58852,0.91967,0.105697,-1.0,-1.0,37.223,2.03336,2178,4587252 +113,insert,13.012,-1.0,-1.0,-1.0,-1.0,55.982,2.03803,2181,4600056 +114,search,6595.97,2119.36,2073.68,1.07972,189.415,2.11546,7.84023,0.91981,0.105686,-1.0,-1.0,34.553,2.03803,2181,4600056 +115,insert,34.968,-1.0,-1.0,-1.0,-1.0,67.464,2.04805,2187,4627422 +116,search,6704.29,2122.18,2168.41,1.01926,198.618,2.16669,7.76799,0.92041,0.105174,-1.0,-1.0,53.981,2.04847,2189,4627422 +117,insert,13.718,-1.0,-1.0,-1.0,-1.0,53.845,2.05196,2192,4639292 +118,search,6655.19,2121.82,2143.04,1.04138,208.444,2.12295,8.16661,0.9204,0.105007,-1.0,-1.0,39.124,2.05196,2192,4639292 +119,insert,6.894,-1.0,-1.0,-1.0,-1.0,36.774,2.05428,2192,4643423 +120,search,6578.79,2123.09,2111.92,1.0413,187.001,2.17971,7.36136,0.92048,0.104978,-1.0,-1.0,29.103,2.05428,2192,4643423 +121,insert,14.292,-1.0,-1.0,-1.0,-1.0,33.36,2.05997,2192,4657252 +122,search,6777.28,2130.57,2160.99,1.00088,202.948,2.18996,7.60062,0.92059,0.104705,-1.0,-1.0,31.336,2.06013,2193,4657252 +123,insert,312.978,-1.0,-1.0,-1.0,-1.0,429.263,2.1878,2245,4938477 +124,search,6917.29,2176.81,2221.89,1.02048,193.65,2.18264,7.61526,0.92134,0.104699,-1.0,-1.0,215.757,2.18954,2271,4938477 +125,insert,60.021,-1.0,-1.0,-1.0,-1.0,64.625,2.20534,2276,4980492 +126,search,7003.38,2157.72,2226.41,1.01766,206.454,2.16458,9.03005,0.92229,0.10396,-1.0,-1.0,81.609,2.20543,2281,4980492 +127,insert,60.658,-1.0,-1.0,-1.0,-1.0,115.494,2.22701,2300,5030515 +128,search,7016.39,2157.83,2209.51,1.08318,192.982,2.14825,7.85647,0.92276,0.103968,-1.0,-1.0,61.278,2.22722,2304,5030515 +129,delete,18.123,-1.0,-1.0,-1.0,-1.0,118.256,2.22527,2271,5009391 +130,search,7059.17,2151.45,2226.24,1.01281,201.325,2.12181,7.84027,0.92145,0.104919,-1.0,-1.0,25.949,2.22527,2271,5009391 +131,delete,4.212,-1.0,-1.0,-1.0,-1.0,25.202,2.22527,2271,5004093 +132,search,7025.87,2147.32,2255.33,0.996996,211.578,2.15687,7.50579,0.9215,0.104974,-1.0,-1.0,37.379,2.22527,2271,5004093 +133,delete,13.204,-1.0,-1.0,-1.0,-1.0,48.646,2.22444,2270,4991787 +134,search,7126.17,2140.82,2234.04,0.993622,200.687,2.11948,7.83604,0.9213,0.105013,-1.0,-1.0,40.859,2.22444,2270,4991787 +135,delete,34.451,-1.0,-1.0,-1.0,-1.0,72.475,2.22232,2267,4974857 +136,search,7016.59,2137.29,2148.33,1.04045,182.993,2.48524,7.93794,0.92137,0.104906,-1.0,-1.0,39.455,2.22232,2267,4974857 +137,delete,6.985,-1.0,-1.0,-1.0,-1.0,37.198,2.22232,2267,4971386 +138,search,7178.93,2135.34,2272.16,0.972406,207.63,2.1853,7.65042,0.92132,0.104948,-1.0,-1.0,38.973,2.22232,2267,4971386 +139,delete,14.092,-1.0,-1.0,-1.0,-1.0,37.073,2.22232,2267,4963487 +140,search,7163.66,2127.32,2264.75,0.973391,217.876,2.15995,11.7841,0.92114,0.105014,-1.0,-1.0,39.069,2.22232,2267,4963487 +141,delete,33.109,-1.0,-1.0,-1.0,-1.0,127.42,2.21737,2260,4941830 +142,search,7153.29,2123.29,2269.76,0.990955,220.726,2.16448,7.85508,0.92104,0.104928,-1.0,-1.0,38.234,2.21737,2260,4941830 +143,delete,21.355,-1.0,-1.0,-1.0,-1.0,108.129,2.21238,2256,4925657 +144,search,7118.81,2120.05,2252.39,0.986844,203.928,2.13921,7.77774,0.9209,0.104891,-1.0,-1.0,45.829,2.2124,2257,4925657 +145,delete,2.928,-1.0,-1.0,-1.0,-1.0,42.57,2.2121,2256,4922686 +146,search,6994.83,2118.76,2141.67,1.01598,197.506,2.16487,8.69267,0.92049,0.105265,-1.0,-1.0,37.435,2.2121,2256,4922686 +147,delete,10.281,-1.0,-1.0,-1.0,-1.0,36.44,2.2121,2256,4913636 +148,search,6887.7,2114.21,2058.2,1.10688,179.451,2.10521,7.75119,0.92053,0.105255,-1.0,-1.0,37.512,2.2121,2256,4913636 +149,delete,231.572,-1.0,-1.0,-1.0,-1.0,1414.87,2.1339,2147,4717250 +150,search,6682.84,2127.62,2046.98,1.04571,179.183,2.16299,8.59915,0.91971,0.106282,-1.0,-1.0,94.043,2.13516,2157,4717250 +151,delete,154.17,-1.0,-1.0,-1.0,-1.0,1273.24,2.06518,2056,4560269 +152,search,6507.23,2141.61,2079.53,0.98514,187.497,2.15763,8.88862,0.91875,0.107108,-1.0,-1.0,98.007,2.06623,2064,4560269 +153,delete,27.732,-1.0,-1.0,-1.0,-1.0,162.456,2.05397,2049,4521936 +154,search,6481.54,2130.82,2062.48,1.01399,184.815,2.14428,8.77076,0.91769,0.108162,-1.0,-1.0,44.362,2.05414,2051,4521936 +155,delete,19.847,-1.0,-1.0,-1.0,-1.0,102.907,2.04922,2046,4502222 +156,search,6431.13,2122.3,2047.51,0.98511,177.177,2.15809,7.23408,0.91755,0.108183,-1.0,-1.0,41.493,2.04927,2047,4502222 +157,delete,47.736,-1.0,-1.0,-1.0,-1.0,223.623,2.03523,2034,4456039 +158,search,6398.59,2108.91,2015.22,0.967369,181.31,2.17657,10.4281,0.91747,0.108479,-1.0,-1.0,50.812,2.03559,2037,4456039 +159,delete,273.004,-1.0,-1.0,-1.0,-1.0,2146.55,1.88197,1860,4143922 +160,search,5934.24,2159.5,1892.37,0.954262,170.887,2.1742,7.68506,0.91598,0.109376,-1.0,-1.0,185.51,1.88401,1887,4143922 +161,delete,50.231,-1.0,-1.0,-1.0,-1.0,123.839,1.87609,1881,4101445 +162,search,5953.01,2094.72,1877.67,0.971614,163.18,2.15354,7.97109,0.91545,0.109842,-1.0,-1.0,31.324,1.87609,1881,4101445 +163,delete,26.606,-1.0,-1.0,-1.0,-1.0,51.825,1.87447,1881,4080175 +164,search,5968.11,2077.42,1897.05,0.984801,170.173,2.15132,8.53681,0.91514,0.110002,-1.0,-1.0,29.274,1.87447,1881,4080175 +165,delete,32.609,-1.0,-1.0,-1.0,-1.0,109.855,1.86814,1876,4047313 +166,search,5904.57,2064.97,1839.3,1.00151,175.15,2.17052,8.3916,0.91509,0.109921,-1.0,-1.0,38.739,1.86819,1877,4047313 +167,delete,8.409,-1.0,-1.0,-1.0,-1.0,28.518,1.86775,1877,4040738 +168,search,5921.41,2061.71,1853.65,0.949375,165.945,2.24502,8.01764,0.91515,0.109914,-1.0,-1.0,29.728,1.86775,1877,4040738 +169,delete,21.088,-1.0,-1.0,-1.0,-1.0,51.146,1.86396,1874,4022674 +170,search,5873.3,2053.55,1832.81,0.976545,158.731,2.21558,7.44432,0.91484,0.110027,-1.0,-1.0,29.682,1.86396,1874,4022674 +171,delete,8.506,-1.0,-1.0,-1.0,-1.0,27.945,1.86357,1874,4015835 +172,search,5908.97,2049.15,1842.03,0.953863,161.732,3.18897,8.14437,0.9149,0.110019,-1.0,-1.0,29.524,1.86357,1874,4015835 +173,delete,43.605,-1.0,-1.0,-1.0,-1.0,40.984,1.85913,1873,3986424 +174,search,5880.42,2025.2,1836.88,0.940957,167.474,2.16939,8.31011,0.91415,0.110696,-1.0,-1.0,29.299,1.85913,1873,3986424 +175,delete,225.743,-1.0,-1.0,-1.0,-1.0,1031.64,1.76817,1771,3804693 +176,search,5569.3,2051.01,1744.88,0.970769,156.247,2.224,7.31876,0.91173,0.11167,-1.0,-1.0,72.567,1.7692,1776,3804693 +177,delete,17.068,-1.0,-1.0,-1.0,-1.0,85.699,1.76466,1772,3786352 +178,search,5561.93,2035.3,1727.78,0.984876,154.89,3.3266,9.19215,0.91175,0.11161,-1.0,-1.0,57.385,1.76501,1775,3786352 +179,delete,252.59,-1.0,-1.0,-1.0,-1.0,1326.71,1.64131,1615,3556994 +180,search,4931.52,2089.57,1452.86,1.07085,122.521,2.13187,8.33731,0.90383,0.11623,-1.0,-1.0,107.742,1.643,1627,3556994 +181,delete,26.204,-1.0,-1.0,-1.0,-1.0,86.457,1.63575,1623,3526824 +182,search,5103.97,2054,1594.04,0.962717,148.169,2.15728,7.93063,0.90341,0.116359,-1.0,-1.0,35.645,1.63578,1624,3526824 +183,delete,182.966,-1.0,-1.0,-1.0,-1.0,1120.95,1.54483,1493,3350245 +184,search,4622.45,2108.4,1402.98,1.0638,116.276,2.13252,7.88048,0.88762,0.134568,-1.0,-1.0,130.119,1.54707,1510,3350245 +185,delete,9.149,-1.0,-1.0,-1.0,-1.0,30.349,1.54644,1507,3340234 +186,search,4770.21,2069.54,1497.15,0.955335,132.242,2.19481,7.62293,0.89421,0.125437,-1.0,-1.0,44.865,1.54651,1510,3340234 +187,delete,192.879,-1.0,-1.0,-1.0,-1.0,362.031,1.46166,1463,3150857 +188,search,4640.37,2067.69,1458.93,0.966898,137.3,2.13962,8.2307,0.8972,0.122971,-1.0,-1.0,36.936,1.46188,1464,3150857 +189,delete,15.668,-1.0,-1.0,-1.0,-1.0,21.192,1.46159,1464,3143609 +190,search,4633.4,2054.29,1439.81,0.978528,133.097,2.19772,8.95725,0.89682,0.123066,-1.0,-1.0,21.527,1.46159,1464,3143609 +191,delete,4.539,-1.0,-1.0,-1.0,-1.0,20.495,1.46088,1464,3141038 +192,search,4628.94,2054.43,1442.13,0.970659,133.955,2.13279,8.22841,0.89674,0.123214,-1.0,-1.0,22.386,1.46088,1464,3141038 +193,delete,74.954,-1.0,-1.0,-1.0,-1.0,207.219,1.44091,1442,3081734 +194,search,4564.38,2041.03,1416.38,1.0056,131.553,2.74671,7.90809,0.8955,0.124512,-1.0,-1.0,56.637,1.44183,1445,3081734 +195,delete,47.809,-1.0,-1.0,-1.0,-1.0,193.395,1.42518,1429,3043966 +196,search,4508.67,2024.91,1400.48,0.999048,127.427,2.33406,7.86758,0.89574,0.123754,-1.0,-1.0,21.778,1.42518,1429,3043966 +197,delete,48.721,-1.0,-1.0,-1.0,-1.0,87.816,1.41246,1423,3008145 +198,search,4515.16,2012.62,1409.46,0.949411,136.273,2.15606,9.6558,0.89516,0.124258,-1.0,-1.0,38.051,1.41306,1425,3008145 +199,delete,11.635,-1.0,-1.0,-1.0,-1.0,52.326,1.41003,1422,2999384 +200,search,4486.39,2009.34,1385.96,0.969456,123.277,2.22538,8.28009,0.89493,0.124194,-1.0,-1.0,21.529,1.41003,1422,2999384 +201,delete,8.326,-1.0,-1.0,-1.0,-1.0,41.483,1.4074,1420,2993528 +202,search,4503.14,2007.69,1391.24,0.940792,132.516,2.18084,8.14495,0.89498,0.12422,-1.0,-1.0,21.711,1.4074,1420,2993528 +203,delete,18.546,-1.0,-1.0,-1.0,-1.0,19.466,1.40593,1420,2980973 +204,search,4493.94,1997.01,1399.74,0.928013,133.005,2.1573,9.09979,0.89481,0.1244,-1.0,-1.0,20.802,1.40593,1420,2980973 +205,delete,28.471,-1.0,-1.0,-1.0,-1.0,62.089,1.39933,1412,2960291 +206,search,4422.33,1985.22,1359.91,1.0322,122.608,2.161,10.1193,0.89405,0.124657,-1.0,-1.0,20.84,1.39933,1412,2960291 +207,delete,58.516,-1.0,-1.0,-1.0,-1.0,232.447,1.37681,1364,2920620 +208,search,4298.56,1997.71,1318.1,0.992786,126.293,2.14847,8.48783,0.89181,0.126233,-1.0,-1.0,19.459,1.37681,1364,2920620 +209,delete,26.851,-1.0,-1.0,-1.0,-1.0,59.408,1.36984,1358,2900501 +210,search,4266.91,1988.74,1310.61,0.980013,123.918,2.18821,9.42572,0.8908,0.126605,-1.0,-1.0,19.249,1.36984,1358,2900501 +211,delete,12.297,-1.0,-1.0,-1.0,-1.0,46.016,1.36655,1356,2891660 +212,search,4287.36,1984.83,1312.15,0.991281,129.458,2.19289,9.71508,0.89076,0.126722,-1.0,-1.0,19.458,1.36655,1356,2891660 +213,delete,18.484,-1.0,-1.0,-1.0,-1.0,48.465,1.36232,1352,2873257 +214,search,4187.95,1975.82,1254.34,1.09736,121.69,2.12177,8.11964,0.88994,0.127102,-1.0,-1.0,19.261,1.36232,1352,2873257 +215,delete,425.94,-1.0,-1.0,-1.0,-1.0,2081,1.16227,1116,2537571 +216,search,3632.37,2149.29,1168.9,0.930995,110.138,2.19071,8.85912,0.88504,0.131064,-1.0,-1.0,303.415,1.16768,1152,2537571 +217,delete,209.764,-1.0,-1.0,-1.0,-1.0,1200.97,1.05072,997,2308194 +218,search,3193.75,2170.3,1000.82,1.02597,90.5887,2.15247,8.00017,0.87882,0.135604,-1.0,-1.0,211.743,1.05422,1024,2308194 +219,delete,16.151,-1.0,-1.0,-1.0,-1.0,73.645,1.04924,1016,2285820 +220,search,3261.02,2066.39,1034.91,0.964033,95.5327,2.3101,9.31096,0.8771,0.136907,-1.0,-1.0,121.105,1.04944,1028,2285820 +221,delete,196.869,-1.0,-1.0,-1.0,-1.0,974.677,0.938243,871,2082858 +222,search,2838.43,2113.58,908.789,1.07869,81.9203,2.17687,8.54087,0.86842,0.144227,-1.0,-1.0,98.32,0.939912,881,2082858 +223,delete,10.054,-1.0,-1.0,-1.0,-1.0,54.539,0.936074,874,2071207 +224,search,2852.24,2090.14,915.73,0.889097,90.0483,2.17906,10.7037,0.86821,0.144636,-1.0,-1.0,40.417,0.936443,877,2071207 +225,delete,5.085,-1.0,-1.0,-1.0,-1.0,25.579,0.935208,876,2065400 +226,search,2800.11,2078.36,878.004,0.958665,85.8559,2.18927,9.56018,0.8677,0.144864,-1.0,-1.0,22.012,0.935313,877,2065400 +227,delete,2.74,-1.0,-1.0,-1.0,-1.0,14.52,0.935313,877,2062105 +228,search,2857.07,2073.13,900.389,0.949921,89.4051,2.24548,8.614,0.86768,0.144776,-1.0,-1.0,15.194,0.935313,877,2062105 +229,delete,19.817,-1.0,-1.0,-1.0,-1.0,52.217,0.929263,872,2039610 +230,search,2704.03,2058.16,889.762,0.989741,87.6486,2.33468,9.87168,0.86616,0.146004,-1.0,-1.0,15.604,0.929263,872,2039610 +231,delete,16.802,-1.0,-1.0,-1.0,-1.0,41.106,0.924668,868,2027795 +232,search,2803.12,2050.12,888.815,0.900543,85.3496,2.17947,8.23698,0.866,0.146053,-1.0,-1.0,15.246,0.924668,868,2027795 +233,delete,6.296,-1.0,-1.0,-1.0,-1.0,29.128,0.922931,867,2022859 +234,search,2820.48,2047.07,882.73,0.955081,87.9743,2.18152,9.20642,0.86556,0.146196,-1.0,-1.0,16.189,0.922931,867,2022859 +235,delete,11.459,-1.0,-1.0,-1.0,-1.0,32.384,0.921845,862,2016264 +236,search,2649.05,2046.05,906.29,0.950818,92.312,2.46224,9.31313,0.86441,0.147034,-1.0,-1.0,13.106,0.921845,862,2016264 +237,delete,206.463,-1.0,-1.0,-1.0,-1.0,952.584,0.803277,720,1782866 +238,search,2303.15,2188.9,799.436,0.912137,79.2275,2.24456,9.32694,0.85768,0.152504,-1.0,-1.0,150.006,0.806017,745,1782866 +239,delete,15.544,-1.0,-1.0,-1.0,-1.0,80.23,0.795133,733,1758931 +240,search,2357.68,2092.74,736.688,1.0052,73,2.17367,10.7908,0.85475,0.153855,-1.0,-1.0,57.098,0.795228,739,1758931 +241,delete,6.645,-1.0,-1.0,-1.0,-1.0,32.377,0.792553,737,1751275 +242,search,2372.94,2070.54,738.712,0.946342,67.7867,2.20752,10.1037,0.85456,0.153796,-1.0,-1.0,28.12,0.792625,739,1751275 +243,delete,16.854,-1.0,-1.0,-1.0,-1.0,51.402,0.788358,733,1733264 +244,search,2336.72,2057.52,749,0.918927,77.4291,2.18817,10.415,0.85214,0.154732,-1.0,-1.0,8.166,0.788358,733,1733264 +245,delete,7.971,-1.0,-1.0,-1.0,-1.0,16.767,0.786174,731,1722658 +246,search,2294.25,2048.83,721.519,0.965207,77.6755,2.17557,9.44945,0.85186,0.154642,-1.0,-1.0,11.927,0.786174,731,1722658 +247,delete,2.895,-1.0,-1.0,-1.0,-1.0,11.288,0.786174,731,1719601 +248,search,2268.19,2042.91,742.582,0.953413,75.8065,2.37083,9.011,0.85158,0.154755,-1.0,-1.0,11.936,0.786174,731,1719601 +249,delete,7.627,-1.0,-1.0,-1.0,-1.0,23.787,0.784841,729,1711648 +250,search,2398.95,2039.7,755.521,0.997928,78.4027,2.272,9.81759,0.85188,0.154591,-1.0,-1.0,12.187,0.784841,729,1711648 +251,delete,324.911,-1.0,-1.0,-1.0,-1.0,513.312,0.665284,615,1470624 +252,search,1995.59,2124.79,645.677,0.986468,68.7814,2.15757,9.51391,0.84743,0.157568,-1.0,-1.0,111.356,0.666953,629,1470624 +253,delete,24.469,-1.0,-1.0,-1.0,-1.0,58.503,0.659941,622,1442070 +254,search,1909.58,2042.75,616.459,0.978334,63.1977,2.14825,10.1884,0.84572,0.157951,-1.0,-1.0,7.92,0.659941,622,1442070 +255,delete,28.726,-1.0,-1.0,-1.0,-1.0,123.747,0.642208,596,1405549 +256,search,1945.83,2087.77,642.816,0.864495,72.0763,2.25833,9.64688,0.84176,0.15953,-1.0,-1.0,28.827,0.642544,599,1405549 +257,insert,12.079,-1.0,-1.0,-1.0,-1.0,8.657,0.645112,599,1416288 +258,search,1873.57,2082.04,637.869,0.927374,69.6129,2.24719,8.54538,0.84092,0.160758,-1.0,-1.0,7.779,0.645112,599,1416288 +259,insert,17.756,-1.0,-1.0,-1.0,-1.0,33.741,0.653597,603,1443010 +260,search,2030.03,2103.71,654.855,0.883785,71.522,2.28189,10.1871,0.84202,0.160199,-1.0,-1.0,10.027,0.653597,603,1443010 +261,insert,7.297,-1.0,-1.0,-1.0,-1.0,13.13,0.654982,604,1447529 +262,search,1855.29,2110.56,652.508,0.926417,74.9984,2.27443,8.97647,0.84228,0.160261,-1.0,-1.0,7.726,0.654982,604,1447529 +263,insert,152.013,-1.0,-1.0,-1.0,-1.0,151.533,0.702673,621,1569110 +264,search,2125.9,2243.58,713.107,0.943403,72.7195,2.18675,9.65041,0.84786,0.157097,-1.0,-1.0,60.793,0.703007,627,1569110 +265,insert,101.08,-1.0,-1.0,-1.0,-1.0,110.454,0.731704,643,1636119 +266,search,2205.12,2271.59,742.354,0.953802,76.8132,2.17927,8.50914,0.85161,0.155064,-1.0,-1.0,19.409,0.731706,644,1636119 +267,insert,34.725,-1.0,-1.0,-1.0,-1.0,58.695,0.744301,649,1665152 +268,search,2107.12,2293.97,754.12,0.96788,75.2651,2.15495,7.99682,0.85396,0.154407,-1.0,-1.0,23.736,0.744493,652,1665152 +269,insert,292.073,-1.0,-1.0,-1.0,-1.0,354.876,0.858188,697,1916952 +270,search,2403.92,2450.62,843.46,0.9216,84.6377,2.2077,9.56226,0.85634,0.152773,-1.0,-1.0,99.401,0.858901,713,1916952 +271,insert,2.211,-1.0,-1.0,-1.0,-1.0,10.803,0.859586,714,1920898 +272,search,2366.48,2393.48,808.799,0.96935,81.2228,2.15136,9.64716,0.85776,0.151785,-1.0,-1.0,71.869,0.859808,721,1920898 +273,insert,29.152,-1.0,-1.0,-1.0,-1.0,38.231,0.868712,726,1942757 +274,search,2479.86,2383.15,855.896,0.897697,89.3587,2.18574,7.8848,0.85876,0.150685,-1.0,-1.0,19.093,0.868758,727,1942757 +275,insert,475.216,-1.0,-1.0,-1.0,-1.0,582.341,1.04466,806,2328372 +276,search,2789.94,2562.85,1000.28,0.98127,94.2893,2.21137,10.6726,0.8655,0.146576,-1.0,-1.0,212.003,1.04621,835,2328372 +277,insert,43.11,-1.0,-1.0,-1.0,-1.0,105.886,1.06101,851,2363257 +278,search,2914.93,2454.64,1020.3,0.948617,101.755,2.1765,8.89593,0.86712,0.145417,-1.0,-1.0,112.912,1.06171,864,2363257 +279,insert,25.453,-1.0,-1.0,-1.0,-1.0,33.486,1.07092,871,2388933 +280,search,2731.22,2409.69,990.521,0.989683,92.9254,2.23177,9.1395,0.86831,0.144322,-1.0,-1.0,49.628,1.07093,875,2388933 +281,insert,379.127,-1.0,-1.0,-1.0,-1.0,276.977,1.2032,912,2684315 +282,search,2962.51,2516.8,1089.8,0.934864,106.633,2.22244,9.92197,0.8701,0.143432,-1.0,-1.0,154.927,1.20494,935,2684315 +283,insert,520.312,-1.0,-1.0,-1.0,-1.0,588.397,1.39768,1048,3109157 +284,search,3447.77,2590.69,1242.75,1.03843,114.945,2.14368,9.30139,0.88099,0.136004,-1.0,-1.0,267.02,1.39937,1088,3109157 +285,insert,13.074,-1.0,-1.0,-1.0,-1.0,40.383,1.40378,1093,3122751 +286,search,3621.89,2468.6,1247.5,1.02695,118.953,2.16076,9.10236,0.87985,0.136283,-1.0,-1.0,63.235,1.40379,1097,3122751 +287,insert,39.683,-1.0,-1.0,-1.0,-1.0,98.135,1.421,1109,3163679 +288,search,3673.23,2469.29,1256.61,1.05682,116.367,2.16274,8.76944,0.88105,0.135707,-1.0,-1.0,63.901,1.42123,1115,3163679 +289,insert,239.292,-1.0,-1.0,-1.0,-1.0,500.864,1.55676,1195,3472031 +290,search,3911.94,2537.39,1429.12,1.03242,135.382,2.17219,8.59996,0.88433,0.134192,-1.0,-1.0,207.437,1.55805,1222,3472031 +291,insert,26.277,-1.0,-1.0,-1.0,-1.0,71.039,1.56751,1236,3495577 +292,search,4012.75,2455.44,1429.12,0.966958,137.306,2.22183,9.4312,0.88595,0.133315,-1.0,-1.0,55.654,1.56756,1241,3495577 +293,insert,19.399,-1.0,-1.0,-1.0,-1.0,49.213,1.57642,1245,3515193 +294,search,4120.38,2444.8,1430.84,0.959771,136.595,2.16935,8.55833,0.88521,0.134129,-1.0,-1.0,32.527,1.57651,1249,3515193 +295,insert,237.126,-1.0,-1.0,-1.0,-1.0,396.885,1.66868,1310,3723063 +296,search,4373.24,2472.56,1514.84,0.987664,144.642,2.17865,9.83279,0.88971,0.131417,-1.0,-1.0,82.358,1.67027,1326,3723063 +297,insert,2.159,-1.0,-1.0,-1.0,-1.0,17.653,1.67175,1328,3727399 +298,search,4207.68,2448.94,1506.92,1.04082,145.439,2.17033,9.43862,0.8901,0.131163,-1.0,-1.0,39.571,1.67188,1335,3727399 +299,insert,60.11,-1.0,-1.0,-1.0,-1.0,84.189,1.69521,1355,3788002 +300,search,4403.55,2431.71,1553.98,0.982235,140.182,2.14747,9.88311,0.89229,0.128737,-1.0,-1.0,24.059,1.69532,1356,3788002 +301,insert,713.477,-1.0,-1.0,-1.0,-1.0,926.74,1.96339,1533,4370866 +302,search,5001.1,2593.69,1854.04,1.00162,177.268,2.16422,8.66389,0.91377,0.111945,-1.0,-1.0,221.328,1.96421,1571,4370866 +303,insert,58.299,-1.0,-1.0,-1.0,-1.0,112.538,1.98491,1584,4420306 +304,search,5115,2492.53,1828.48,0.979073,169.97,2.16098,9.39638,0.91312,0.112209,-1.0,-1.0,26.008,1.98491,1588,4420306 +305,insert,375.359,-1.0,-1.0,-1.0,-1.0,406.005,2.14825,1643,4789669 +306,search,5393.06,2574.91,1963.59,0.978666,179.52,2.19105,9.05689,0.91537,0.110231,-1.0,-1.0,161.34,2.15124,1663,4789669 +307,insert,21.098,-1.0,-1.0,-1.0,-1.0,52.671,2.1595,1672,4815550 +308,search,5384.23,2548.92,1980.72,0.973978,177.382,2.35076,8.74566,0.91609,0.109934,-1.0,-1.0,88.37,2.15959,1679,4815550 +309,insert,9.301,-1.0,-1.0,-1.0,-1.0,26.564,2.16191,1680,4823345 +310,search,5437.95,2538.42,1944.48,0.993976,181.333,2.18209,8.58538,0.91667,0.10956,-1.0,-1.0,43.38,2.16204,1683,4823345 +311,insert,11.631,-1.0,-1.0,-1.0,-1.0,23.945,2.1691,1684,4843493 +312,search,5549.7,2537.3,1948.75,0.983819,178.036,2.17185,8.65011,0.91617,0.110111,-1.0,-1.0,42.565,2.1691,1686,4843493 +313,insert,45.283,-1.0,-1.0,-1.0,-1.0,87.394,2.18684,1695,4885489 +314,search,5676.11,2544.87,2001.72,0.982835,183.23,2.1504,9.33695,0.9166,0.10999,-1.0,-1.0,73.874,2.18762,1704,4885489 +315,insert,7.742,-1.0,-1.0,-1.0,-1.0,17.893,2.19013,1704,4892335 +316,search,5684.22,2530.61,1996.88,1.00806,179.847,2.55152,8.69637,0.91661,0.110042,-1.0,-1.0,18.608,2.19013,1704,4892335 +317,insert,47.828,-1.0,-1.0,-1.0,-1.0,124.097,2.20582,1723,4932996 +318,search,5729.86,2521.74,2012.68,0.976576,182.355,2.20883,9.53416,0.91696,0.10981,-1.0,-1.0,22.649,2.20586,1724,4932996 +319,insert,660.978,-1.0,-1.0,-1.0,-1.0,807.641,2.42161,1850,5430684 +320,search,6215.97,2610.52,2205.29,1.01328,204.823,3.05321,8.26986,0.92124,0.106488,-1.0,-1.0,316.36,2.42604,1894,5430684 +321,insert,48.375,-1.0,-1.0,-1.0,-1.0,84.359,2.4412,1908,5471239 +322,search,6356.85,2533.55,2216.97,0.981514,207.425,2.16445,8.94242,0.92178,0.105881,-1.0,-1.0,157.595,2.44171,1917,5471239 +323,insert,123.621,-1.0,-1.0,-1.0,-1.0,202.677,2.4869,1950,5575560 +324,search,6381.44,2538.16,2274.74,0.970521,210.186,2.20896,8.85358,0.92273,0.105176,-1.0,-1.0,87.415,2.48719,1957,5575560 +325,insert,1158.77,-1.0,-1.0,-1.0,-1.0,469.445,2.71259,2032,6069499 +326,search,6671.37,2648.99,2422.04,0.969521,224.903,2.21724,8.71466,0.92545,0.103769,-1.0,-1.0,359.373,2.71346,2073,6069499 +327,insert,83.639,-1.0,-1.0,-1.0,-1.0,122.27,2.74247,2088,6139647 +328,search,6937.71,2616.29,2419.48,1.04306,221.558,2.42129,8.87681,0.92705,0.102485,-1.0,-1.0,121.022,2.74312,2100,6139647 +329,insert,10.923,-1.0,-1.0,-1.0,-1.0,32.791,2.74599,2102,6146884 +330,search,6987.47,2600.17,2445.36,1.01712,224.553,2.14537,8.4269,0.92733,0.102219,-1.0,-1.0,61.678,2.74616,2107,6146884 +331,insert,52.074,-1.0,-1.0,-1.0,-1.0,102.377,2.76355,2118,6188816 +332,search,7129.04,2602.18,2532.81,1.00669,243.526,2.14219,10.5012,0.92718,0.102344,-1.0,-1.0,59.506,2.76418,2122,6188816 +333,insert,3.359,-1.0,-1.0,-1.0,-1.0,31.732,2.76488,2123,6190810 +334,search,6979.27,2597.04,2402.16,1.09996,218.067,2.14546,8.71633,0.92742,0.102121,-1.0,-1.0,27.594,2.76488,2123,6190810 +335,insert,54.318,-1.0,-1.0,-1.0,-1.0,109.591,2.78083,2138,6230108 +336,search,7067.52,2600.33,2442.97,1.0803,221.304,2.2665,8.55772,0.92792,0.101968,-1.0,-1.0,34.8,2.7809,2139,6230108 +337,insert,26.463,-1.0,-1.0,-1.0,-1.0,68.166,2.78952,2144,6249699 +338,search,7073.12,2600.46,2441.2,1.07326,220.987,2.27178,8.28821,0.92823,0.101658,-1.0,-1.0,53.837,2.78962,2147,6249699 +339,insert,60.023,-1.0,-1.0,-1.0,-1.0,95.187,2.80547,2158,6290226 +340,search,7177.19,2603.6,2551.95,1.01223,236.846,2.173,8.5004,0.92869,0.101291,-1.0,-1.0,21.56,2.8055,2159,6290226 +341,insert,6.406,-1.0,-1.0,-1.0,-1.0,31.688,2.80948,2162,6297556 +342,search,6907.76,2600.42,2574.54,1.00255,245.02,2.18875,8.79646,0.92884,0.10121,-1.0,-1.0,38.618,2.80966,2164,6297556 +343,insert,39.77,-1.0,-1.0,-1.0,-1.0,79.514,2.82218,2172,6330843 +344,search,7109.79,2605.5,2543.23,1.00628,235.988,2.17564,8.70314,0.92874,0.101378,-1.0,-1.0,28.145,2.82218,2172,6330843 +345,insert,42.41,-1.0,-1.0,-1.0,-1.0,116.297,2.83604,2185,6370265 +346,search,7029.2,2606.69,2497.91,1.03457,226.283,2.14911,9.55573,0.9292,0.100751,-1.0,-1.0,39.154,2.83613,2187,6370265 +347,insert,475.165,-1.0,-1.0,-1.0,-1.0,945.562,3.04153,2327,6831086 +348,search,7401.28,2635.98,2660.64,1.06401,237.876,2.15236,8.61135,0.93239,0.0977587,-1.0,-1.0,241.734,3.04451,2358,6831086 +349,insert,58.136,-1.0,-1.0,-1.0,-1.0,100.766,3.06354,2368,6876154 +350,search,7815.18,2607.69,2806.16,0.988177,269.87,2.26015,8.63697,0.93287,0.097447,-1.0,-1.0,39.484,3.06377,2371,6876154 +351,insert,477.089,-1.0,-1.0,-1.0,-1.0,600.351,3.25061,2464,7288266 +352,search,8069.88,2634.61,2959.84,1.02027,283.021,2.1893,8.72282,0.93268,0.0979329,-1.0,-1.0,270.608,3.25238,2497,7288266 +353,insert,57.893,-1.0,-1.0,-1.0,-1.0,111.694,3.269,2512,7333132 +354,search,8174.84,2595.08,2964.02,0.994647,283.798,2.20741,8.63969,0.93295,0.0980115,-1.0,-1.0,70.802,3.26938,2518,7333132 +355,insert,378.692,-1.0,-1.0,-1.0,-1.0,586.842,3.42856,2613,7696838 +356,search,8500.45,2612.74,3054.05,0.990874,284.058,2.16182,8.14795,0.93432,0.0969263,-1.0,-1.0,194.683,3.43041,2639,7696838 +357,insert,26.595,-1.0,-1.0,-1.0,-1.0,49.409,3.43801,2643,7714498 +358,search,8338.06,2588.51,3062.99,1.02805,286.848,3.06089,8.63461,0.93487,0.0964041,-1.0,-1.0,57.509,3.43909,2652,7714498 +359,insert,9.184,-1.0,-1.0,-1.0,-1.0,45.021,3.4437,2659,7725755 +360,search,8552.94,2574.99,2878.64,1.1364,255.397,2.11897,9.15015,0.93477,0.096524,-1.0,-1.0,23.259,3.4437,2659,7725755 +361,insert,69.418,-1.0,-1.0,-1.0,-1.0,153.88,3.46783,2680,7789312 +362,search,8780.53,2575.79,3013.85,1.08375,271.612,2.16545,8.72537,0.93532,0.0957465,-1.0,-1.0,45.868,3.46827,2684,7789312 +363,insert,20.402,-1.0,-1.0,-1.0,-1.0,66.017,3.47275,2690,7803163 +364,search,8948.44,2569.08,3151.59,0.993175,291.7,2.18887,8.04895,0.93532,0.0955269,-1.0,-1.0,38.818,3.47309,2692,7803163 +365,insert,74.551,-1.0,-1.0,-1.0,-1.0,154.836,3.50086,2712,7863722 +366,search,9002.33,2569.38,3156.95,0.986484,302.752,2.12492,8.48667,0.93589,0.0948893,-1.0,-1.0,52.414,3.50103,2716,7863722 +367,insert,35.451,-1.0,-1.0,-1.0,-1.0,81.214,3.51314,2725,7895271 +368,search,9008.04,2567.83,3159.37,1.00794,288.498,2.14809,8.7062,0.93632,0.0944056,-1.0,-1.0,38.045,3.51315,2726,7895271 +369,insert,75.758,-1.0,-1.0,-1.0,-1.0,167.676,3.53963,2744,7961194 +370,search,9109.27,2575.97,3198.64,0.989174,301.449,2.14891,8.42396,0.93689,0.0935448,-1.0,-1.0,56.663,3.54004,2748,7961194 +371,insert,21.047,-1.0,-1.0,-1.0,-1.0,53.03,3.54531,2753,7970239 +372,search,8858.01,2569.74,2983.72,1.11587,262.503,3.06589,9.74729,0.93689,0.0935876,-1.0,-1.0,26.27,3.54531,2753,7970239 +373,insert,64.57,-1.0,-1.0,-1.0,-1.0,111.439,3.56003,2767,8010868 +374,search,8845.37,2569.67,2951.44,1.17965,249.93,2.09482,8.64276,0.93733,0.0930754,-1.0,-1.0,52.081,3.56034,2771,8010868 +375,insert,47.428,-1.0,-1.0,-1.0,-1.0,91.325,3.5726,2780,8040698 +376,search,9264.21,2566.9,3259.99,0.955315,300.855,2.16988,8.03108,0.93749,0.0930683,-1.0,-1.0,38.349,3.57273,2782,8040698 +377,insert,83.887,-1.0,-1.0,-1.0,-1.0,147.989,3.60164,2799,8104748 +378,search,9276.28,2567.38,3267.91,0.989809,305.691,2.14347,8.95744,0.93767,0.093103,-1.0,-1.0,45.697,3.60222,2802,8104748 +379,insert,6.652,-1.0,-1.0,-1.0,-1.0,24.532,3.60406,2802,8109755 +380,search,9320.73,2567.57,3279.58,0.992877,309.666,2.1128,9.56269,0.9378,0.092961,-1.0,-1.0,26.007,3.60406,2802,8109755 +381,insert,28.244,-1.0,-1.0,-1.0,-1.0,53.948,3.61153,2806,8130382 +382,search,9321.49,2569.57,3295.35,0.997692,313.372,2.60243,8.50475,0.93777,0.0931968,-1.0,-1.0,26.715,3.61153,2806,8130382 +383,insert,36.162,-1.0,-1.0,-1.0,-1.0,58.761,3.61985,2812,8153290 +384,search,9351.69,2570.37,3290.9,0.999845,304.177,2.24646,9.12021,0.9378,0.0930147,-1.0,-1.0,33.062,3.62003,2813,8153290 +385,delete,17.044,-1.0,-1.0,-1.0,-1.0,24.838,3.62003,2813,8138873 +386,search,9280.72,2567.14,3260.59,1.00195,302.043,2.28337,8.53623,0.93782,0.0929674,-1.0,-1.0,26.074,3.62003,2813,8138873 +387,delete,29.031,-1.0,-1.0,-1.0,-1.0,171.671,3.61261,2806,8115069 +388,search,9250.72,2566.41,3241.74,1.0072,292.457,2.15696,8.15193,0.93782,0.0930642,-1.0,-1.0,29.716,3.61261,2806,8115069 +389,delete,38.327,-1.0,-1.0,-1.0,-1.0,101.916,3.60976,2802,8101157 +390,search,9014.09,2560.24,3196.74,1.05458,298.127,2.16276,9.64112,0.93769,0.0932498,-1.0,-1.0,31.95,3.60999,2803,8101157 +391,delete,98.742,-1.0,-1.0,-1.0,-1.0,615.187,3.57881,2761,8014644 +392,search,9046.23,2571.82,3235.27,0.983206,302.974,2.14192,8.59784,0.93721,0.0932295,-1.0,-1.0,55.76,3.5793,2769,8014644 +393,delete,25.372,-1.0,-1.0,-1.0,-1.0,127.845,3.5726,2762,7978358 +394,search,9112.17,2555.61,3220.81,0.976934,307.595,2.14949,8.92736,0.93707,0.0934202,-1.0,-1.0,35.58,3.57281,2763,7978358 +395,delete,27.655,-1.0,-1.0,-1.0,-1.0,243.742,3.561,2752,7957094 +396,search,9100.29,2562.01,3204.3,0.970482,298.581,2.15679,9.22986,0.93654,0.0937023,-1.0,-1.0,49.903,3.56123,2754,7957094 +397,delete,153.416,-1.0,-1.0,-1.0,-1.0,844.04,3.50485,2715,7823916 +398,search,9008.13,2548.7,3161.74,0.977078,300.77,2.12993,8.71578,0.93631,0.0939026,-1.0,-1.0,110.907,3.50628,2724,7823916 +399,delete,5.444,-1.0,-1.0,-1.0,-1.0,27.28,3.50628,2724,7818430 +400,search,8933.78,2538.89,3148.3,1.01734,289.462,2.19981,9.1492,0.93625,0.0938803,-1.0,-1.0,51.834,3.50628,2726,7818430 +401,delete,17.718,-1.0,-1.0,-1.0,-1.0,43.526,3.50327,2724,7800702 +402,search,9006.37,2531.82,3146.75,1.0035,298.344,2.15859,8.56794,0.93569,0.0943286,-1.0,-1.0,38.289,3.50327,2725,7800702 +403,delete,341.149,-1.0,-1.0,-1.0,-1.0,2298.81,3.37369,2585,7517858 +404,search,8587,2563.7,3032.03,0.971629,285.496,2.15594,8.09293,0.93377,0.0959031,-1.0,-1.0,238.545,3.37793,2609,7517858 +405,delete,65.752,-1.0,-1.0,-1.0,-1.0,389.114,3.35939,2591,7451637 +406,search,8561.56,2536.65,2982.33,1.02979,273.922,2.18047,8.20061,0.93431,0.0952936,-1.0,-1.0,90.167,3.35969,2596,7451637 +407,delete,122.594,-1.0,-1.0,-1.0,-1.0,896.061,3.31242,2551,7360417 +408,search,8424.88,2542.2,2948.05,1.00764,272.193,2.15031,9.03043,0.93355,0.0960791,-1.0,-1.0,120.762,3.31401,2560,7360417 +409,delete,212.598,-1.0,-1.0,-1.0,-1.0,795.214,3.21909,2514,7131620 +410,search,8347.68,2524.17,2903.44,1.01361,283.209,2.19018,8.20904,0.93344,0.0963595,-1.0,-1.0,102.488,3.21973,2522,7131620 +411,delete,316.518,-1.0,-1.0,-1.0,-1.0,2189.21,3.09961,2380,6867617 +412,search,7944.2,2566.87,2803.17,0.967143,265.968,2.1911,8.90059,0.93331,0.0968004,-1.0,-1.0,252.826,3.10392,2411,6867617 +413,delete,16.007,-1.0,-1.0,-1.0,-1.0,71.108,3.10054,2408,6850098 +414,search,8007.49,2527.95,2793.75,0.973046,270.014,2.199,8.77855,0.93341,0.0966498,-1.0,-1.0,47.306,3.10094,2411,6850098 +415,delete,79.961,-1.0,-1.0,-1.0,-1.0,583.211,3.06758,2380,6782921 +416,search,7722.42,2528.21,2609.19,1.09466,229.022,2.16589,8.48857,0.93258,0.0972934,-1.0,-1.0,74.863,3.06832,2385,6782921 +417,delete,257.493,-1.0,-1.0,-1.0,-1.0,1367.9,2.9818,2308,6562039 +418,search,7575.24,2520.93,2594.34,1.01693,236.284,2.17532,9.08246,0.93165,0.0974473,-1.0,-1.0,185.919,2.98385,2324,6562039 +419,delete,23.194,-1.0,-1.0,-1.0,-1.0,79.132,2.98077,2321,6536285 +420,search,7514.26,2496.75,2520.31,1.09692,222.087,2.15139,8.70293,0.93201,0.0975575,-1.0,-1.0,74.801,2.98103,2324,6536285 +421,delete,21.747,-1.0,-1.0,-1.0,-1.0,170.006,2.97095,2315,6510820 +422,search,7560.43,2496.86,2564.31,1.06069,228.482,2.40558,8.86209,0.93237,0.097444,-1.0,-1.0,59.103,2.97132,2318,6510820 +423,delete,199.762,-1.0,-1.0,-1.0,-1.0,966.048,2.89478,2244,6334168 +424,search,7437.38,2502.4,2581.44,0.997804,238.606,2.16067,8.68092,0.93072,0.098534,-1.0,-1.0,136.314,2.89702,2256,6334168 +425,delete,18.531,-1.0,-1.0,-1.0,-1.0,82.728,2.89302,2254,6316443 +426,search,7439.3,2486.21,2567.43,0.987791,235.375,2.1363,9.07575,0.93095,0.0987958,-1.0,-1.0,63.558,2.89348,2257,6316443 +427,delete,63.391,-1.0,-1.0,-1.0,-1.0,68.862,2.88782,2255,6266345 +428,search,7413.69,2457.22,2541.21,0.972752,234.706,2.25231,8.92271,0.93078,0.0988302,-1.0,-1.0,36.837,2.88782,2255,6266345 +429,delete,578.757,-1.0,-1.0,-1.0,-1.0,3588.03,2.66338,1952,5883847 +430,search,6513.21,2726.33,2282.27,1.08727,200.831,2.11755,8.84706,0.92185,0.105488,-1.0,-1.0,524.061,2.6728,2029,5883847 +431,delete,77.079,-1.0,-1.0,-1.0,-1.0,388.435,2.63652,1999,5801380 +432,search,6727.47,2577.12,2365,0.995691,224.351,2.18348,9.10548,0.92033,0.107782,-1.0,-1.0,130.962,2.63714,2013,5801380 +433,delete,231.884,-1.0,-1.0,-1.0,-1.0,1439.74,2.54605,1915,5607567 +434,search,6483.88,2603.78,2296.89,0.994144,221.83,2.19264,8.68144,0.91755,0.109332,-1.0,-1.0,178.394,2.54799,1934,5607567 +435,delete,103.735,-1.0,-1.0,-1.0,-1.0,462.758,2.51488,1897,5512176 +436,search,6370.96,2581.6,2251.76,0.968345,206.561,2.16874,10.7218,0.91782,0.109203,-1.0,-1.0,95.859,2.51534,1904,5512176 +437,delete,12.697,-1.0,-1.0,-1.0,-1.0,57.613,2.51167,1901,5501141 +438,search,6396.77,2570.68,2244.42,1.00901,215.235,2.1586,8.89915,0.91777,0.109205,-1.0,-1.0,55.215,2.51191,1904,5501141 +439,delete,129.618,-1.0,-1.0,-1.0,-1.0,430.611,2.44801,1877,5369822 +440,search,6120.42,2578.05,2077.96,1.14111,178.461,2.13108,9.18204,0.91914,0.107501,-1.0,-1.0,106.147,2.44947,1885,5369822 +441,delete,32.006,-1.0,-1.0,-1.0,-1.0,133.234,2.43906,1879,5338985 +442,search,6200.72,2558.7,2135.64,1.04474,189.486,4.67728,8.7252,0.91977,0.107048,-1.0,-1.0,39.446,2.43906,1880,5338985 +443,delete,33.753,-1.0,-1.0,-1.0,-1.0,140.095,2.42389,1869,5307279 +444,search,6285.23,2560.98,2200.94,0.979808,204.708,2.49971,8.45638,0.92034,0.106458,-1.0,-1.0,27.404,2.42389,1869,5307279 +445,delete,43.244,-1.0,-1.0,-1.0,-1.0,134.994,2.41374,1861,5268382 +446,search,6111.57,2552.26,2089.77,1.05321,185.541,2.12449,8.70046,0.9198,0.106801,-1.0,-1.0,27.682,2.41374,1861,5268382 +447,delete,459.432,-1.0,-1.0,-1.0,-1.0,1991.51,2.24611,1677,4922787 +448,search,5592.46,2635.86,1938.51,1.10167,180.501,2.20981,8.4586,0.91364,0.110398,-1.0,-1.0,310.678,2.25114,1711,4922787 +449,delete,30.131,-1.0,-1.0,-1.0,-1.0,169.405,2.2402,1698,4887227 +450,search,5685.14,2577.05,2023.7,0.964678,174.163,2.11487,8.84571,0.91545,0.1089,-1.0,-1.0,108.802,2.24036,1706,4887227 +451,delete,107.494,-1.0,-1.0,-1.0,-1.0,467.829,2.19807,1660,4795082 +452,search,5593.84,2590.06,1972.74,0.991473,183.606,2.17684,9.03822,0.91532,0.108946,-1.0,-1.0,84.271,2.19914,1668,4795082 +453,delete,320.095,-1.0,-1.0,-1.0,-1.0,1801.22,2.05339,1545,4500175 +454,search,5115.68,2593.36,1753.79,1.05938,158.216,2.11067,9.12469,0.91046,0.113013,-1.0,-1.0,354.756,2.05856,1575,4500175 +455,delete,34.861,-1.0,-1.0,-1.0,-1.0,99.385,2.05011,1569,4462648 +456,search,5274.17,2537.91,1844.33,0.960378,178.156,2.17625,9.6528,0.91087,0.112478,-1.0,-1.0,185.618,2.05061,1585,4462648 +457,delete,5.418,-1.0,-1.0,-1.0,-1.0,44.443,2.04803,1583,4456761 +458,search,5220.72,2519.64,1774.72,1.01809,168.856,2.1403,9.22964,0.91125,0.112264,-1.0,-1.0,44.475,2.04803,1585,4456761 +459,delete,53.313,-1.0,-1.0,-1.0,-1.0,208.539,2.03016,1569,4419992 +460,search,5259.16,2516.24,1839.66,1.01082,164.697,2.09899,8.91067,0.91078,0.112793,-1.0,-1.0,32.957,2.03025,1570,4419992 +461,delete,6.341,-1.0,-1.0,-1.0,-1.0,20.068,2.03025,1570,4415725 +462,search,5249.63,2509.52,1826.45,0.9939,168.923,2.10075,9.39079,0.91083,0.112606,-1.0,-1.0,21.124,2.03025,1570,4415725 +463,delete,68.316,-1.0,-1.0,-1.0,-1.0,318.867,2.00716,1541,4370609 +464,search,5185.23,2537.33,1817.22,0.998811,172.244,2.18152,8.63072,0.91053,0.112584,-1.0,-1.0,57.483,2.00769,1545,4370609 +465,delete,25.78,-1.0,-1.0,-1.0,-1.0,63.821,1.99718,1539,4345244 +466,search,5177.79,2522.36,1811.3,1.01399,161.775,2.12987,8.42054,0.90872,0.114136,-1.0,-1.0,27.753,1.99718,1540,4345244 +467,delete,33.784,-1.0,-1.0,-1.0,-1.0,123.858,1.98846,1534,4319734 +468,search,5166.53,2511.93,1796.57,0.995974,172.625,2.1642,9.16951,0.9089,0.113974,-1.0,-1.0,20.349,1.98846,1534,4319734 +469,delete,21.511,-1.0,-1.0,-1.0,-1.0,93.868,1.97838,1526,4298361 +470,search,5114.79,2513.06,1775.61,1.00474,171.819,2.18925,8.89651,0.90768,0.114937,-1.0,-1.0,30.492,1.97859,1527,4298361 +471,delete,163.874,-1.0,-1.0,-1.0,-1.0,518.807,1.92156,1472,4188315 +472,search,4953.41,2535.53,1733.42,0.970177,161.183,2.12446,9.32748,0.90696,0.115027,-1.0,-1.0,112.836,1.923,1483,4188315 +473,delete,135.758,-1.0,-1.0,-1.0,-1.0,635.753,1.86554,1432,4074567 +474,search,4734.26,2525.31,1600.06,1.01734,152.174,2.1414,8.92311,0.90541,0.116332,-1.0,-1.0,126.859,1.86717,1445,4074567 +475,delete,486.122,-1.0,-1.0,-1.0,-1.0,2185.72,1.66233,1236,3691698 +476,search,4219.68,2637.52,1486.83,1.04992,136.445,2.18291,9.45118,0.89973,0.120636,-1.0,-1.0,413.417,1.67027,1284,3691698 +477,delete,43.606,-1.0,-1.0,-1.0,-1.0,190.535,1.65178,1272,3642619 +478,search,4310.45,2530.6,1503.15,0.980387,144.483,2.17805,8.76673,0.90001,0.121018,-1.0,-1.0,86.562,1.65248,1279,3642619 +479,delete,247.511,-1.0,-1.0,-1.0,-1.0,1582.97,1.55083,1164,3421368 +480,search,4007.91,2621.57,1424.68,0.959664,135.367,2.19831,8.72782,0.89505,0.130279,-1.0,-1.0,454.712,1.55957,1196,3421368 +481,delete,48.171,-1.0,-1.0,-1.0,-1.0,184.009,1.54174,1180,3378034 +482,search,3979.5,2529.27,1382.16,0.992741,131.785,2.14481,9.53647,0.89508,0.128705,-1.0,-1.0,201.372,1.54221,1187,3378034 +483,delete,336.186,-1.0,-1.0,-1.0,-1.0,1416.2,1.40325,1039,3115682 +484,search,3600.85,2638.18,1284.21,0.922017,128.774,2.19202,9.00298,0.89062,0.129891,-1.0,-1.0,323.222,1.40984,1072,3115682 +485,delete,19.948,-1.0,-1.0,-1.0,-1.0,48.105,1.40616,1069,3094629 +486,search,3668.12,2553.51,1295.73,1.02142,114.966,2.16189,10.9024,0.89195,0.129493,-1.0,-1.0,110.212,1.40638,1078,3094629 +487,delete,13.873,-1.0,-1.0,-1.0,-1.0,20.187,1.40569,1078,3086025 +488,search,3681.12,2524.88,1281.35,1.00226,125.178,2.20716,9.39144,0.89172,0.129609,-1.0,-1.0,21.998,1.40569,1078,3086025 +489,delete,52.758,-1.0,-1.0,-1.0,-1.0,150.25,1.39261,1068,3052105 +490,search,3649.02,2519.43,1275.48,1.03481,120.082,2.1119,8.091,0.89158,0.129224,-1.0,-1.0,54.492,1.39305,1071,3052105 +491,delete,15.722,-1.0,-1.0,-1.0,-1.0,48.248,1.38873,1068,3038974 +492,search,3648.5,2512.09,1283.85,0.987192,116.804,2.23156,8.72093,0.89117,0.129775,-1.0,-1.0,21.448,1.38873,1068,3038974 +493,delete,186.259,-1.0,-1.0,-1.0,-1.0,629.889,1.31941,1003,2911063 +494,search,3455.01,2564.22,1230.21,0.926686,108.416,2.14302,8.90165,0.88846,0.132009,-1.0,-1.0,89.893,1.32032,1010,2911063 +495,delete,28.382,-1.0,-1.0,-1.0,-1.0,112.754,1.31082,1000,2888785 +496,search,3430.38,2544.73,1224.82,0.984651,112.421,2.16216,9.07756,0.88804,0.131771,-1.0,-1.0,69.425,1.31125,1005,2888785 +497,delete,81.96,-1.0,-1.0,-1.0,-1.0,168.947,1.28934,987,2829718 +498,search,3383.02,2518.51,1191.55,0.939508,107.298,2.14187,7.71341,0.88672,0.132265,-1.0,-1.0,18.877,1.28934,987,2829718 +499,delete,22.841,-1.0,-1.0,-1.0,-1.0,42.399,1.28572,984,2816082 +500,search,3367.17,2511.98,1174,0.957327,117.912,2.20515,8.97271,0.88601,0.132832,-1.0,-1.0,18.086,1.28572,984,2816082 +501,delete,28.434,-1.0,-1.0,-1.0,-1.0,110.919,1.2768,975,2790880 +502,search,3230.54,2512.42,1105.96,1.08023,101.218,2.19402,9.90639,0.88581,0.132754,-1.0,-1.0,28.411,1.27698,976,2790880 +503,delete,19.576,-1.0,-1.0,-1.0,-1.0,30.515,1.27541,975,2774459 +504,search,3348.5,2498.14,1168.89,1.00288,103.05,2.1837,9.06869,0.88539,0.132951,-1.0,-1.0,18.067,1.27541,975,2774459 +505,delete,82.032,-1.0,-1.0,-1.0,-1.0,183.198,1.24704,955,2714683 +506,search,3280.94,2493.53,1151.97,0.9809,104.712,2.1232,10.2477,0.8838,0.133331,-1.0,-1.0,28.081,1.24711,956,2714683 +507,delete,54.231,-1.0,-1.0,-1.0,-1.0,122.388,1.23432,945,2680986 +508,search,3138.22,2487.34,1053.76,1.12662,91.1182,2.18904,8.66887,0.88317,0.133689,-1.0,-1.0,70.407,1.23515,951,2680986 +509,delete,23.241,-1.0,-1.0,-1.0,-1.0,123.171,1.22514,944,2662966 +510,search,3103.48,2478.31,1011.52,1.18829,96.3774,2.19827,12.3337,0.88324,0.133528,-1.0,-1.0,37.405,1.22548,946,2662966 +511,delete,26.07,-1.0,-1.0,-1.0,-1.0,54.972,1.22074,942,2642855 +512,search,3078.9,2460,995.15,1.16601,93.2664,2.14374,11.4085,0.88249,0.134501,-1.0,-1.0,16.886,1.22074,942,2642855 +513,insert,433.765,-1.0,-1.0,-1.0,-1.0,359.656,1.32573,973,2906838 +514,search,3355,2593.48,1200.44,0.977786,106.36,2.18338,11.0565,0.8846,0.133063,-1.0,-1.0,140.813,1.32593,985,2906838 +515,insert,7.785,-1.0,-1.0,-1.0,-1.0,45.797,1.32775,988,2913746 +516,search,3398.68,2568.72,1211.35,0.958387,107.009,2.10656,8.71371,0.88575,0.132443,-1.0,-1.0,17.796,1.32775,988,2913746 +517,insert,34.991,-1.0,-1.0,-1.0,-1.0,70.436,1.33458,997,2936862 +518,search,3433.66,2562.13,1224.48,0.981601,111.116,3.37818,8.96953,0.8863,0.131943,-1.0,-1.0,25.088,1.33468,998,2936862 +519,insert,122.368,-1.0,-1.0,-1.0,-1.0,142.075,1.36572,1007,3009732 +520,search,3478.76,2598.48,1246.27,0.954578,114.351,2.66438,8.88152,0.88705,0.132185,-1.0,-1.0,107.148,1.36589,1016,3009732 +521,insert,26.371,-1.0,-1.0,-1.0,-1.0,53.889,1.37147,1020,3025906 +522,search,3523.58,2584.23,1254.98,0.968291,116.471,2.16449,10.3859,0.88773,0.131781,-1.0,-1.0,48.113,1.37169,1024,3025906 +523,insert,57.76,-1.0,-1.0,-1.0,-1.0,106.28,1.38627,1034,3061899 +524,search,3552.29,2577.16,1257.66,1.00677,115.799,2.18049,10.8237,0.88832,0.131624,-1.0,-1.0,41.054,1.38632,1036,3061899 +525,insert,64.523,-1.0,-1.0,-1.0,-1.0,120.778,1.40198,1049,3101209 +526,search,3622.17,2575.56,1289.07,0.971519,117.635,2.66163,10.9425,0.88872,0.131419,-1.0,-1.0,31.666,1.40212,1051,3101209 +527,insert,738.044,-1.0,-1.0,-1.0,-1.0,872.441,1.58454,1172,3522334 +528,search,4025.58,2660.5,1458.73,0.945779,131.533,2.21129,9.00364,0.89622,0.125253,-1.0,-1.0,197.728,1.58697,1198,3522334 +529,insert,404.524,-1.0,-1.0,-1.0,-1.0,461.736,1.70442,1267,3797294 +530,search,4369.49,2688.17,1584.25,0.997461,149.367,2.16807,9.12073,0.90228,0.121865,-1.0,-1.0,138.449,1.70576,1281,3797294 +531,insert,106.363,-1.0,-1.0,-1.0,-1.0,164.276,1.72972,1299,3853781 +532,search,4444,2661.78,1590.48,1.0065,141.391,2.18883,9.09593,0.90226,0.121877,-1.0,-1.0,38.762,1.72981,1302,3853781 +533,insert,27.331,-1.0,-1.0,-1.0,-1.0,50.009,1.73654,1306,3873352 +534,search,4489.29,2661.32,1614.35,1.00901,150.409,2.19009,7.45947,0.90259,0.121664,-1.0,-1.0,18.125,1.73654,1306,3873352 +535,insert,31.017,-1.0,-1.0,-1.0,-1.0,66.631,1.74367,1313,3893308 +536,search,4401.39,2665.9,1525.86,1.05776,139.114,2.16064,8.78021,0.90293,0.121386,-1.0,-1.0,18.312,1.74367,1313,3893308 +537,insert,9.645,-1.0,-1.0,-1.0,-1.0,26.436,1.74655,1314,3901271 +538,search,4426.81,2666.57,1552.69,1.10564,142.318,2.17928,8.16269,0.90352,0.120951,-1.0,-1.0,50.016,1.74703,1318,3901271 +539,insert,33.615,-1.0,-1.0,-1.0,-1.0,52.19,1.75489,1323,3925164 +540,search,4556.89,2663.68,1643.11,0.994465,150.182,2.43607,8.25252,0.90348,0.120964,-1.0,-1.0,26.279,1.7549,1324,3925164 +541,insert,585.599,-1.0,-1.0,-1.0,-1.0,712.248,1.93387,1415,4320915 +542,search,4948.6,2770.72,1821.46,0.974694,175.161,3.39312,9.24032,0.90771,0.117638,-1.0,-1.0,327.478,1.93604,1460,4320915 +543,insert,59.679,-1.0,-1.0,-1.0,-1.0,112.541,1.95133,1475,4360814 +544,search,5056.77,2662.82,1830.39,0.989268,165.082,2.16786,9.20311,0.90782,0.117671,-1.0,-1.0,66.887,1.95148,1481,4360814 +545,insert,15.899,-1.0,-1.0,-1.0,-1.0,27.485,1.95554,1482,4370220 +546,search,4969.38,2652.38,1735.57,1.06767,155.105,2.14467,8.24008,0.90787,0.117579,-1.0,-1.0,24.673,1.9556,1483,4370220 +547,insert,235.775,-1.0,-1.0,-1.0,-1.0,328.354,2.01923,1539,4510675 +548,search,5246.73,2641.55,1885.42,0.998097,175.329,2.17084,8.20406,0.90997,0.115241,-1.0,-1.0,30.37,2.01927,1541,4510675 +549,insert,328.416,-1.0,-1.0,-1.0,-1.0,335.408,2.11719,1573,4744845 +550,search,5381.25,2686.7,1950.39,0.993686,173.21,2.16529,9.41825,0.90697,0.119394,-1.0,-1.0,163.723,2.11787,1584,4744845 +551,insert,32.276,-1.0,-1.0,-1.0,-1.0,71.025,2.13085,1588,4771433 +552,search,5327.74,2669.73,1874.76,1.08096,164.574,2.15018,8.70849,0.90719,0.118748,-1.0,-1.0,108.408,2.13123,1597,4771433 +553,insert,14.922,-1.0,-1.0,-1.0,-1.0,25.523,2.13442,1598,4782909 +554,search,5459.82,2657.68,1980.61,0.996333,182.971,2.17078,9.6716,0.9075,0.118357,-1.0,-1.0,56.253,2.13449,1601,4782909 +555,insert,645.509,-1.0,-1.0,-1.0,-1.0,815.576,2.31425,1745,5180322 +556,search,5936.61,2671.99,2149.78,1.08668,197.547,2.20235,9.53937,0.91313,0.113199,-1.0,-1.0,128.759,2.31485,1760,5180322 +557,insert,25.987,-1.0,-1.0,-1.0,-1.0,46.916,2.32078,1764,5203220 +558,search,5990.66,2654.69,2145.34,0.996794,197.71,2.19084,8.81969,0.91355,0.113124,-1.0,-1.0,43.258,2.32094,1767,5203220 +559,insert,7.168,-1.0,-1.0,-1.0,-1.0,39.318,2.32334,1769,5210154 +560,search,5972.77,2652.06,2117.66,1.04523,192.486,2.84343,7.88232,0.91373,0.112989,-1.0,-1.0,21.033,2.32334,1769,5210154 +561,insert,6.445,-1.0,-1.0,-1.0,-1.0,32.194,2.32496,1771,5215426 +562,search,6038.32,2650.64,2155.19,1.001,195.267,2.1564,9.48249,0.91377,0.113003,-1.0,-1.0,25.071,2.32496,1771,5215426 +563,insert,67.151,-1.0,-1.0,-1.0,-1.0,134.15,2.33873,1788,5252726 +564,search,6000.88,2644.21,2117.18,1.05586,187.176,2.21187,8.96971,0.91406,0.112674,-1.0,-1.0,52.216,2.33913,1792,5252726 +565,insert,305.171,-1.0,-1.0,-1.0,-1.0,360.666,2.42467,1824,5447274 +566,search,6137,2673.8,2161.11,1.04923,196.512,2.37787,8.96204,0.91434,0.112786,-1.0,-1.0,92.861,2.42534,1833,5447274 +567,insert,20.613,-1.0,-1.0,-1.0,-1.0,33.86,2.4323,1835,5463135 +568,search,6250.89,2663.19,2247.17,0.979041,209.712,2.13323,9.59511,0.91436,0.112741,-1.0,-1.0,58.353,2.43244,1839,5463135 +569,insert,23.163,-1.0,-1.0,-1.0,-1.0,60.002,2.439,1844,5478168 +570,search,6142.4,2659.16,2145.31,1.03441,191.484,2.14522,9.53366,0.91443,0.112757,-1.0,-1.0,23.289,2.439,1844,5478168 +571,insert,20.421,-1.0,-1.0,-1.0,-1.0,40.736,2.44516,1845,5492309 +572,search,6169.53,2662.04,2163.23,1.0333,193.304,2.15925,8.9191,0.91412,0.113075,-1.0,-1.0,22.457,2.44516,1845,5492309 +573,insert,91.304,-1.0,-1.0,-1.0,-1.0,119.988,2.46673,1858,5554599 +574,search,6339.62,2673.64,2281.46,1.00131,211.839,2.14418,8.67172,0.91515,0.111996,-1.0,-1.0,29.075,2.46675,1859,5554599 +575,insert,85.612,-1.0,-1.0,-1.0,-1.0,144.95,2.48883,1873,5604500 +576,search,6401.62,2680.31,2313.07,0.953921,214.822,2.17235,8.55493,0.91571,0.111582,-1.0,-1.0,45.57,2.48905,1877,5604500 +577,insert,51.996,-1.0,-1.0,-1.0,-1.0,106.091,2.50209,1893,5638834 +578,search,6340.92,2670.6,2238.16,1.0245,204.018,2.21994,8.81228,0.91593,0.111148,-1.0,-1.0,36.663,2.50238,1895,5638834 +579,insert,970.645,-1.0,-1.0,-1.0,-1.0,1254.31,2.78281,2087,6229778 +580,search,7010.14,2707.36,2503.13,1.03351,227.013,2.134,9.56808,0.92018,0.108348,-1.0,-1.0,111.855,2.78343,2100,6229778 +581,insert,42.809,-1.0,-1.0,-1.0,-1.0,115.241,2.79737,2112,6261240 +582,search,7146.12,2688.49,2590.95,0.985253,238.028,2.16376,8.87744,0.92023,0.108255,-1.0,-1.0,26.71,2.79737,2112,6261240 +583,insert,26.382,-1.0,-1.0,-1.0,-1.0,84.089,2.8057,2120,6283315 +584,search,7143.1,2687.45,2559.76,1.03084,232.881,2.14413,8.57555,0.92056,0.107933,-1.0,-1.0,60.198,2.80618,2124,6283315 +585,insert,359.395,-1.0,-1.0,-1.0,-1.0,491.914,2.93355,2181,6556101 +586,search,7189.06,2709.44,2497.04,1.11188,209.043,2.14751,8.83756,0.92199,0.107289,-1.0,-1.0,230.718,2.93579,2207,6556101 +587,insert,42.007,-1.0,-1.0,-1.0,-1.0,108.886,2.94733,2218,6587187 +588,search,7433.53,2672.29,2649.06,1.00221,238.734,2.16602,8.51224,0.92235,0.106976,-1.0,-1.0,73.504,2.94755,2223,6587187 +589,insert,71.603,-1.0,-1.0,-1.0,-1.0,104.259,2.96329,2235,6631526 +590,search,7565.79,2666.99,2730.09,0.992804,252.133,2.1562,8.65332,0.92285,0.10641,-1.0,-1.0,46.458,2.96358,2237,6631526 +591,insert,673.777,-1.0,-1.0,-1.0,-1.0,808.101,3.16818,2368,7108942 +592,search,8077.3,2729.8,2961.74,0.971779,279.272,2.16564,8.10575,0.92709,0.102706,-1.0,-1.0,195.522,3.16951,2393,7108942 +593,insert,252.832,-1.0,-1.0,-1.0,-1.0,328.295,3.25389,2436,7303976 +594,search,8305.95,2728.65,3025.86,0.975828,292.503,2.17261,9.11149,0.92836,0.102405,-1.0,-1.0,174.051,3.25566,2456,7303976 +595,insert,5.586,-1.0,-1.0,-1.0,-1.0,38.826,3.25701,2458,7309244 +596,search,8335.32,2701.51,3022.24,0.986652,279.55,2.15694,9.30662,0.92849,0.102245,-1.0,-1.0,80.36,3.2575,2464,7309244 +597,insert,325.964,-1.0,-1.0,-1.0,-1.0,434.044,3.363,2522,7542477 +598,search,8612.21,2724.73,3143.07,0.976487,300.085,2.16198,7.83561,0.9312,0.0999456,-1.0,-1.0,242.481,3.36624,2557,7542477 +599,insert,139.095,-1.0,-1.0,-1.0,-1.0,276.951,3.40765,2599,7654438 +600,search,8792.24,2683.99,3172.23,0.988879,292.758,2.18313,8.75488,0.93147,0.09954,-1.0,-1.0,82.131,3.40876,2608,7654438 +601,insert,83.612,-1.0,-1.0,-1.0,-1.0,152.116,3.43745,2627,7717936 +602,search,8872.86,2677.28,3188.44,0.966072,300.319,2.16466,8.11468,0.93235,0.0987201,-1.0,-1.0,73.057,3.43783,2634,7717936 +603,insert,6.679,-1.0,-1.0,-1.0,-1.0,25.852,3.43945,2634,7723521 +604,search,8853.14,2671.57,3177.48,1.03,290.894,2.14781,8.71958,0.93241,0.0987031,-1.0,-1.0,30.127,3.43945,2634,7723521 +605,insert,72.29,-1.0,-1.0,-1.0,-1.0,101.428,3.45583,2644,7767404 +606,search,8913.69,2678.7,3199.25,1.02127,298.382,2.15545,8.85685,0.9326,0.0987912,-1.0,-1.0,29.007,3.45583,2644,7767404 +607,insert,100.632,-1.0,-1.0,-1.0,-1.0,54.277,3.4767,2647,7826115 +608,search,8966.05,2686.91,3233.76,0.999251,306.515,2.15309,8.99679,0.93235,0.0989124,-1.0,-1.0,42.415,3.47684,2649,7826115 +609,insert,312.103,-1.0,-1.0,-1.0,-1.0,418.003,3.56603,2702,8036351 +610,search,9182.17,2695.73,3321.43,0.960355,315.761,2.20229,8.40403,0.93352,0.0977004,-1.0,-1.0,79.293,3.56629,2708,8036351 +611,insert,33.641,-1.0,-1.0,-1.0,-1.0,71.245,3.57559,2715,8060364 +612,search,9195.1,2688.82,3325,1.02786,310.155,2.18487,8.12103,0.93369,0.0976991,-1.0,-1.0,61.746,3.57576,2719,8060364 +613,insert,278.1,-1.0,-1.0,-1.0,-1.0,357.693,3.66294,2766,8256083 +614,search,9351.19,2700.68,3394.58,1.0007,317.92,2.14298,8.52057,0.93475,0.0965308,-1.0,-1.0,122.95,3.66437,2782,8256083 +615,insert,458.732,-1.0,-1.0,-1.0,-1.0,635.828,3.83282,2863,8625707 +616,search,9735.77,2714.72,3541.31,0.985341,332.138,2.88095,8.40871,0.93665,0.0945862,-1.0,-1.0,314.562,3.83537,2908,8625707 +617,insert,398.677,-1.0,-1.0,-1.0,-1.0,667.992,3.97915,2987,8949227 +618,search,10054.2,2686.41,3630.84,0.991558,340.455,2.38865,9.91561,0.93821,0.093056,-1.0,-1.0,327.396,3.98174,3031,8949227 +619,insert,284.561,-1.0,-1.0,-1.0,-1.0,418.617,4.08384,3090,9195551 +620,search,10305.7,2646.76,3671.65,0.998419,334.098,2.11931,8.82791,0.93925,0.0918332,-1.0,-1.0,200.031,4.08546,3110,9195551 +621,insert,47.078,-1.0,-1.0,-1.0,-1.0,79.83,4.10007,3118,9232372 +622,search,10421.4,2632.2,3705.75,0.988474,347.065,3.25307,7.96301,0.93946,0.0915174,-1.0,-1.0,90.594,4.10057,3123,9232372 +623,insert,25.892,-1.0,-1.0,-1.0,-1.0,60.837,4.1094,3128,9253561 +624,search,10528.3,2634.3,3764.71,0.97012,365.783,2.14437,8.70782,0.93961,0.0912735,-1.0,-1.0,77.404,4.10992,3135,9253561 +625,insert,99.417,-1.0,-1.0,-1.0,-1.0,185.105,4.14274,3154,9331230 +626,search,10599,2632.77,3782.08,1.01203,362.956,2.16451,8.92612,0.94001,0.090979,-1.0,-1.0,66.317,4.14342,3160,9331230 +627,insert,73.587,-1.0,-1.0,-1.0,-1.0,107.822,4.16176,3172,9374063 +628,search,10605.5,2629.68,3778.53,0.972963,357.274,2.65072,8.34092,0.94006,0.0909517,-1.0,-1.0,30.466,4.16176,3172,9374063 +629,insert,24.024,-1.0,-1.0,-1.0,-1.0,55.241,4.16726,3175,9387598 +630,search,10648,2630.88,3807.3,0.973403,364.099,2.13903,9.1745,0.94027,0.0907873,-1.0,-1.0,45.986,4.16742,3178,9387598 +631,insert,64.901,-1.0,-1.0,-1.0,-1.0,62.062,4.18109,3182,9421724 +632,search,10626.6,2629.75,3795.08,0.992216,354.967,2.38727,9.3418,0.94021,0.0908149,-1.0,-1.0,41.217,4.18118,3183,9421724 +633,insert,64.067,-1.0,-1.0,-1.0,-1.0,132.568,4.19669,3198,9461430 +634,search,10662.9,2632.25,3783.07,1.03107,345.474,2.16934,8.37784,0.94048,0.0905006,-1.0,-1.0,35.258,4.19669,3198,9461430 +635,insert,47.574,-1.0,-1.0,-1.0,-1.0,103.612,4.20258,3206,9479469 +636,search,10720.1,2630.59,3807.59,1.00901,355.728,2.13828,8.0844,0.94064,0.0904735,-1.0,-1.0,39.814,4.20264,3207,9479469 +637,insert,692.741,-1.0,-1.0,-1.0,-1.0,921.281,4.38872,3344,9885997 +638,search,11235,2642.49,4031.88,0.976575,386.688,2.16984,9.44998,0.94272,0.0884278,-1.0,-1.0,195.445,4.39215,3369,9885997 +639,insert,570.863,-1.0,-1.0,-1.0,-1.0,868.025,4.57013,3506,10292043 +640,search,11725.9,2629.8,4195.15,1.0085,393.803,2.15365,8.41465,0.94395,0.0876046,-1.0,-1.0,171.212,4.57199,3525,10292043 +641,delete,217.222,-1.0,-1.0,-1.0,-1.0,1881.19,4.49006,3451,10096584 +642,search,11550,2619.94,4128.4,0.970531,390.553,2.16584,8.66097,0.94314,0.088411,-1.0,-1.0,144.121,4.49216,3465,10096584 +643,delete,12.654,-1.0,-1.0,-1.0,-1.0,62.604,4.49093,3464,10084840 +644,search,11537.4,2607.87,4095.56,0.978474,375.847,2.1575,8.60995,0.94321,0.0883537,-1.0,-1.0,72.065,4.49145,3465,10084840 +645,delete,25.487,-1.0,-1.0,-1.0,-1.0,35.084,4.49145,3465,10069802 +646,search,11506,2601.88,4054.89,1.01509,366.664,2.13837,9.24514,0.94323,0.0883342,-1.0,-1.0,39.863,4.49145,3465,10069802 +647,delete,143.678,-1.0,-1.0,-1.0,-1.0,879.967,4.45564,3429,9990953 +648,search,11478.4,2606.11,4058.56,1.00749,385.302,2.15261,9.03574,0.94257,0.0885648,-1.0,-1.0,93.59,4.45623,3434,9990953 +649,delete,64.173,-1.0,-1.0,-1.0,-1.0,157.236,4.45179,3430,9951625 +650,search,11404.3,2595.44,4007.49,1.01943,378.702,2.15109,8.7427,0.94269,0.0886288,-1.0,-1.0,38.251,4.45179,3430,9951625 +651,delete,39.354,-1.0,-1.0,-1.0,-1.0,258.091,4.4419,3421,9927065 +652,search,10922.7,2599.31,3629.15,1.13513,301.883,2.08717,9.02227,0.94255,0.0888328,-1.0,-1.0,51.041,4.44204,3422,9927065 +653,delete,133.739,-1.0,-1.0,-1.0,-1.0,829.222,4.39925,3382,9831098 +654,search,10805,2599.58,3597.05,1.1459,296.216,2.09977,9.11409,0.94218,0.0888652,-1.0,-1.0,77.376,4.39983,3386,9831098 +655,delete,492.602,-1.0,-1.0,-1.0,-1.0,2909.46,4.24922,3227,9462172 +656,search,10588,2617.22,3637.82,1.04922,321.151,2.11876,10.0076,0.94031,0.0911645,-1.0,-1.0,320.995,4.25337,3260,9462172 +657,delete,251.429,-1.0,-1.0,-1.0,-1.0,880.589,4.16541,3164,9218902 +658,search,10335.7,2590.39,3549.66,1.06798,316.814,2.16572,9.47313,0.93761,0.0930735,-1.0,-1.0,104.285,4.16622,3173,9218902 +659,delete,174.642,-1.0,-1.0,-1.0,-1.0,865.158,4.11201,3125,9106546 +660,search,10312.7,2582.13,3593.95,1.04751,330.117,2.15571,9.17991,0.93701,0.0937447,-1.0,-1.0,142.668,4.11372,3136,9106546 +661,delete,68.539,-1.0,-1.0,-1.0,-1.0,199.925,4.10462,3130,9059776 +662,search,10248.1,2567.96,3511.56,1.0846,321.175,2.14406,8.52,0.93712,0.0934899,-1.0,-1.0,50.719,4.10483,3131,9059776 +663,delete,114.363,-1.0,-1.0,-1.0,-1.0,543.977,4.07693,3104,8990616 +664,search,10109.2,2564.51,3453.09,1.08002,309.673,2.13565,9.21014,0.9362,0.0942932,-1.0,-1.0,71.515,4.07767,3107,8990616 +665,delete,70.445,-1.0,-1.0,-1.0,-1.0,499.959,4.05066,3088,8937437 +666,search,10085.4,2563.26,3446.81,1.07232,309.805,2.1412,8.78845,0.93594,0.094591,-1.0,-1.0,54.399,4.05098,3090,8937437 +667,delete,225.995,-1.0,-1.0,-1.0,-1.0,1363.64,3.96971,3020,8766390 +668,search,9908.23,2564.47,3428.96,1.066,311.73,2.25289,9.05846,0.93511,0.0957063,-1.0,-1.0,197.159,3.97177,3035,8766390 +669,delete,266.828,-1.0,-1.0,-1.0,-1.0,2227.9,3.86155,2928,8532008 +670,search,9723.93,2567.7,3403.14,1.00038,312.172,2.14948,9.353,0.93439,0.0956478,-1.0,-1.0,258.65,3.86538,2953,8532008 +671,delete,50.775,-1.0,-1.0,-1.0,-1.0,223.431,3.85156,2940,8489178 +672,search,9759.74,2540.61,3416.77,0.977619,320.538,2.15648,9.6547,0.93453,0.0960209,-1.0,-1.0,89.089,3.85156,2944,8489178 +673,delete,89.68,-1.0,-1.0,-1.0,-1.0,390.338,3.82508,2922,8421866 +674,search,9698.31,2537.14,3383,0.983541,327.57,2.14177,9.34259,0.93423,0.0960861,-1.0,-1.0,64.637,3.82541,2925,8421866 +675,delete,196.046,-1.0,-1.0,-1.0,-1.0,926.449,3.77328,2870,8288319 +676,search,9541.33,2548.28,3338.13,1.00609,318.688,2.26628,9.02737,0.93415,0.0963306,-1.0,-1.0,162.613,3.77638,2876,8288319 +677,delete,220.222,-1.0,-1.0,-1.0,-1.0,640.038,3.69903,2840,8072197 +678,search,9271.78,2517.75,3139.8,1.04398,280.423,2.15358,9.22276,0.93658,0.093414,-1.0,-1.0,107.809,3.69965,2846,8072197 +679,delete,36.598,-1.0,-1.0,-1.0,-1.0,406.688,3.68265,2834,8036947 +680,search,9279.51,2514.07,3180.32,1.01138,286.983,2.33273,8.91618,0.93668,0.0932087,-1.0,-1.0,86.958,3.68341,2838,8036947 +681,delete,10.746,-1.0,-1.0,-1.0,-1.0,30.08,3.68312,2838,8026197 +682,search,9369.62,2509.86,3230,0.979416,296.899,2.15341,8.19871,0.93672,0.0931063,-1.0,-1.0,31.528,3.68312,2838,8026197 +683,delete,423.607,-1.0,-1.0,-1.0,-1.0,2021.87,3.56823,2705,7744229 +684,search,8761.33,2548.29,2948.22,1.06166,259.162,2.11819,9.0634,0.93658,0.0938199,-1.0,-1.0,192.433,3.57046,2725,7744229 +685,delete,303.567,-1.0,-1.0,-1.0,-1.0,1413.53,3.48065,2622,7541066 +686,search,8614.69,2561.92,2966.48,1.00638,272.286,2.10152,9.11044,0.93378,0.0972359,-1.0,-1.0,284.174,3.48368,2653,7541066 +687,delete,21.027,-1.0,-1.0,-1.0,-1.0,127.42,3.47826,2649,7522190 +688,search,8813.22,2522.3,3075.89,0.968207,295.715,2.15422,9.35403,0.93329,0.0974451,-1.0,-1.0,65.678,3.47893,2652,7522190 +689,delete,157.58,-1.0,-1.0,-1.0,-1.0,614.061,3.41949,2621,7409400 +690,search,8499.61,2513.22,2873.31,1.06243,255.757,2.09349,8.711,0.93258,0.0977958,-1.0,-1.0,84.78,3.42032,2626,7409400 +691,delete,50.988,-1.0,-1.0,-1.0,-1.0,88.251,3.41248,2622,7375648 +692,search,8571.33,2497.81,2935.48,1.05422,257.041,2.13216,8.99167,0.9327,0.0976918,-1.0,-1.0,48.993,3.41276,2624,7375648 +693,delete,220.197,-1.0,-1.0,-1.0,-1.0,521.984,3.34718,2597,7222068 +694,search,8361.82,2471.79,2772.67,1.07994,245.71,2.11012,9.36002,0.93278,0.0977682,-1.0,-1.0,99.422,3.34765,2603,7222068 +695,delete,31.012,-1.0,-1.0,-1.0,-1.0,236.763,3.33466,2595,7196331 +696,search,8539.62,2476.31,2939.74,0.987576,279.549,2.16114,9.22251,0.93332,0.0968277,-1.0,-1.0,67.082,3.33522,2599,7196331 +697,delete,35.124,-1.0,-1.0,-1.0,-1.0,181.247,3.32179,2592,7170716 +698,search,8491.82,2466.37,2931.93,0.98286,255.792,2.12845,8.30874,0.93347,0.0967656,-1.0,-1.0,42.428,3.32185,2593,7170716 +699,delete,22.27,-1.0,-1.0,-1.0,-1.0,120.42,3.31296,2588,7155255 +700,search,8437.25,2468.64,2876.35,1.02535,250.644,2.09744,9.6107,0.93395,0.0961656,-1.0,-1.0,49.21,3.31317,2590,7155255 +701,delete,94.179,-1.0,-1.0,-1.0,-1.0,190.212,3.30145,2582,7097733 +702,search,8487.65,2449.21,2906.54,0.979857,271.122,2.13783,9.05043,0.93351,0.0965103,-1.0,-1.0,41.417,3.30148,2583,7097733 +703,delete,294.996,-1.0,-1.0,-1.0,-1.0,1077.59,3.21184,2507,6924172 +704,search,8241.23,2459.89,2805.19,1.01214,264.488,2.156,9.32177,0.93325,0.0968531,-1.0,-1.0,143.608,3.21344,2518,6924172 +705,delete,43.311,-1.0,-1.0,-1.0,-1.0,121.27,3.20573,2510,6896276 +706,search,8272.01,2447.12,2848.82,0.98678,257.536,2.0872,9.05113,0.93325,0.0969047,-1.0,-1.0,49.708,3.20614,2512,6896276 +707,delete,770.86,-1.0,-1.0,-1.0,-1.0,4423.63,2.90875,2179,6385507 +708,search,7323.23,2663.77,2591.88,1.02386,236.149,2.32893,10.8842,0.93259,0.0991764,-1.0,-1.0,855.046,2.92513,2306,6385507 +709,delete,169.245,-1.0,-1.0,-1.0,-1.0,1225.85,2.83029,2214,6207658 +710,search,7330.57,2482.73,2518.57,0.967773,240.693,2.18859,8.42454,0.92884,0.102458,-1.0,-1.0,279.91,2.83442,2251,6207658 +711,delete,43.517,-1.0,-1.0,-1.0,-1.0,234.583,2.81646,2239,6162720 +712,search,7340.87,2420.24,2486.39,0.996043,230.237,2.151,8.00985,0.92811,0.103226,-1.0,-1.0,76.145,2.81724,2244,6162720 +713,delete,310.967,-1.0,-1.0,-1.0,-1.0,1327.89,2.7073,2164,5927803 +714,search,7087.9,2415.37,2407.72,1.00872,221.98,2.12471,8.32557,0.92549,0.105311,-1.0,-1.0,130.291,2.70867,2172,5927803 +715,delete,38.052,-1.0,-1.0,-1.0,-1.0,180.886,2.69466,2162,5895666 +716,search,6991.26,2404.87,2326.35,1.04192,204.491,2.09679,9.60724,0.9256,0.105179,-1.0,-1.0,36.096,2.69466,2162,5895666 +717,delete,51.937,-1.0,-1.0,-1.0,-1.0,177.051,2.68577,2152,5868774 +718,search,6958.39,2403.54,2295.12,1.04755,210.338,2.15991,9.69897,0.92537,0.105385,-1.0,-1.0,46.256,2.68599,2153,5868774 +719,delete,546.644,-1.0,-1.0,-1.0,-1.0,2489.14,2.52302,1942,5533540 +720,search,6529.63,2545.82,2291.33,0.960884,203.136,2.1748,9.11051,0.92251,0.107521,-1.0,-1.0,365.576,2.52871,1993,5533540 +721,delete,136.918,-1.0,-1.0,-1.0,-1.0,589.301,2.47968,1951,5418387 +722,search,6313.11,2459.83,2093.98,1.08699,189.866,2.12931,11.3094,0.92,0.109969,-1.0,-1.0,254.852,2.48212,1975,5418387 +723,delete,24.256,-1.0,-1.0,-1.0,-1.0,114.906,2.47683,1970,5398291 +724,search,6499.03,2405.03,2212.9,1.00655,216.852,2.17662,8.44342,0.91903,0.110604,-1.0,-1.0,70.927,2.47687,1973,5398291 +725,delete,168.09,-1.0,-1.0,-1.0,-1.0,833.541,2.4181,1914,5273546 +726,search,6206.13,2409.75,2143.72,1.01629,202.07,2.17903,8.85903,0.91776,0.11143,-1.0,-1.0,165.484,2.42019,1926,5273546 +727,delete,106.547,-1.0,-1.0,-1.0,-1.0,705.732,2.37913,1871,5180937 +728,search,5987.57,2437.21,2035.88,1.04677,188.216,2.17244,10.0585,0.91778,0.111042,-1.0,-1.0,112.879,2.38067,1886,5180937 +729,delete,98.49,-1.0,-1.0,-1.0,-1.0,441.748,2.33803,1843,5079524 +730,search,5961.07,2412.43,1996.04,1.03888,179.14,2.13621,10.9199,0.91676,0.111472,-1.0,-1.0,102.964,2.33901,1853,5079524 +731,delete,71.16,-1.0,-1.0,-1.0,-1.0,173.136,2.32191,1839,5019265 +732,search,5892.07,2387.1,2032.71,0.990122,192.446,2.15616,8.39025,0.91605,0.112112,-1.0,-1.0,65.98,2.32231,1843,5019265 +733,delete,91.719,-1.0,-1.0,-1.0,-1.0,258.616,2.29701,1825,4955565 +734,search,5918.27,2369,1962.45,0.990495,170.453,2.10577,8.35529,0.91573,0.112638,-1.0,-1.0,49.465,2.29729,1827,4955565 +735,delete,207.504,-1.0,-1.0,-1.0,-1.0,489.5,2.21729,1781,4776948 +736,search,5690.81,2360.02,1974.58,0.962436,169.751,2.09688,8.66426,0.91907,0.10937,-1.0,-1.0,111.802,2.21768,1791,4776948 +737,delete,146.222,-1.0,-1.0,-1.0,-1.0,578.797,2.16602,1753,4654026 +738,search,5516.17,2333.59,1806.6,1.08377,157.606,2.14175,8.39516,0.91809,0.110547,-1.0,-1.0,81.951,2.16649,1758,4654026 +739,delete,63.077,-1.0,-1.0,-1.0,-1.0,305.425,2.13489,1735,4589888 +740,search,5527.75,2328.18,1895.96,1.01374,159.442,2.12288,8.44423,0.91898,0.10927,-1.0,-1.0,100.376,2.13562,1741,4589888 +741,delete,216.607,-1.0,-1.0,-1.0,-1.0,542.953,2.05317,1676,4430862 +742,search,4393.29,2320.06,1690.48,1.10295,150.758,2.09739,9.23153,0.91759,0.11134,-1.0,-1.0,97.638,2.05565,1688,4430862 +743,delete,143.843,-1.0,-1.0,-1.0,-1.0,901.494,1.95924,1593,4237049 +744,search,4182.46,2344.86,1633.16,1.09573,139.44,2.06261,9.15419,0.91498,0.113146,-1.0,-1.0,165.507,1.96451,1614,4237049 +745,delete,196.294,-1.0,-1.0,-1.0,-1.0,961.166,1.82602,1491,4000195 +746,search,3959.94,2405.54,1551.3,1.14883,136.373,2.09702,9.26397,0.91275,0.113781,-1.0,-1.0,125.607,1.82822,1511,4000195 +747,delete,146.417,-1.0,-1.0,-1.0,-1.0,979.573,1.73233,1428,3803954 +748,search,4641.76,2395.51,1628.12,0.967005,142.968,2.07421,8.77338,0.90972,0.115958,-1.0,-1.0,141.806,1.73536,1443,3803954 +749,delete,64.079,-1.0,-1.0,-1.0,-1.0,245.964,1.70945,1420,3733214 +750,search,4517.19,2366.07,1606.09,0.932806,140.255,3.11736,8.38133,0.90955,0.115926,-1.0,-1.0,68.946,1.7099,1424,3733214 +751,delete,35.593,-1.0,-1.0,-1.0,-1.0,192.814,1.69574,1411,3709324 +752,search,4575.75,2359.56,1504.68,1.05583,132.77,2.15057,9.19086,0.90816,0.117002,-1.0,-1.0,50.277,1.69641,1414,3709324 +753,delete,87.92,-1.0,-1.0,-1.0,-1.0,402.125,1.66383,1381,3649123 +754,search,4570.57,2367.67,1555.22,0.958252,139.306,2.2067,9.18156,0.90736,0.11776,-1.0,-1.0,63.34,1.66466,1383,3649123 +755,delete,34.734,-1.0,-1.0,-1.0,-1.0,100.853,1.65854,1377,3621475 +756,search,4538.62,2355.55,1550.82,0.988006,133.716,3.79801,8.99181,0.90691,0.117519,-1.0,-1.0,60.599,1.65858,1380,3621475 +757,delete,37.511,-1.0,-1.0,-1.0,-1.0,109.797,1.64633,1372,3594479 +758,search,4541.37,2353.06,1541.35,0.977136,135.613,2.12489,8.51057,0.90675,0.117428,-1.0,-1.0,45.365,1.64662,1374,3594479 +759,delete,35.712,-1.0,-1.0,-1.0,-1.0,195.134,1.63305,1363,3569055 +760,search,4521.69,2354.61,1536.41,1.00348,132.752,2.14005,8.65962,0.90649,0.117891,-1.0,-1.0,30.33,1.63328,1364,3569055 +761,delete,52.884,-1.0,-1.0,-1.0,-1.0,113.431,1.62164,1355,3527896 +762,search,4457.12,2337.14,1494.3,0.984344,133.963,2.13178,8.54237,0.9058,0.1182,-1.0,-1.0,32.444,1.62169,1356,3527896 +763,delete,25.334,-1.0,-1.0,-1.0,-1.0,63.336,1.61659,1352,3509970 +764,search,4453.36,2329.93,1497.99,0.984101,131.573,2.17914,8.65347,0.9053,0.118781,-1.0,-1.0,18.952,1.61659,1352,3509970 +765,delete,528.504,-1.0,-1.0,-1.0,-1.0,1494.91,1.44401,1180,3187549 +766,search,4004.44,2445.18,1384.94,1.03991,122.55,2.0888,8.40654,0.90273,0.120385,-1.0,-1.0,207.735,1.44733,1205,3187549 +767,delete,506.207,-1.0,-1.0,-1.0,-1.0,1696.06,1.26088,1003,2822906 +768,search,3444.26,2536.94,1218.05,1.00733,110.852,2.18949,8.70168,0.89251,0.129839,-1.0,-1.0,269.421,1.26505,1032,2822906 +769,insert,17.612,-1.0,-1.0,-1.0,-1.0,18.041,1.26807,1032,2832899 +770,search,3505.46,2455.93,1213.77,1.03552,115.27,2.17112,9.40813,0.89385,0.128311,-1.0,-1.0,52.501,1.26838,1036,2832899 +771,insert,58.416,-1.0,-1.0,-1.0,-1.0,83.115,1.28185,1042,2863523 +772,search,3526.28,2459.32,1219.41,0.975331,114.389,2.16565,8.24167,0.89461,0.127618,-1.0,-1.0,30.144,1.28197,1043,2863523 +773,insert,111.976,-1.0,-1.0,-1.0,-1.0,149.212,1.30614,1059,2925169 +774,search,3592.07,2472.92,1249.54,1.03236,117.635,2.18723,9.92345,0.89599,0.126794,-1.0,-1.0,41.904,1.30614,1061,2925169 +775,insert,64.701,-1.0,-1.0,-1.0,-1.0,77.786,1.32189,1066,2961356 +776,search,3624.82,2487.52,1262.03,0.953143,119.279,2.18679,9.34526,0.89632,0.126877,-1.0,-1.0,41.228,1.32189,1068,2961356 +777,insert,63.519,-1.0,-1.0,-1.0,-1.0,86.73,1.33483,1074,2998678 +778,search,3668.15,2510.46,1279.59,0.964936,116.244,2.21624,8.3352,0.89797,0.125656,-1.0,-1.0,51.597,1.3351,1077,2998678 +779,insert,27.916,-1.0,-1.0,-1.0,-1.0,18.739,1.33949,1077,3011024 +780,search,3671.14,2511.22,1292.64,0.97732,117.373,2.17306,8.61488,0.89802,0.125914,-1.0,-1.0,20.665,1.33949,1077,3011024 +781,insert,14.892,-1.0,-1.0,-1.0,-1.0,43.439,1.34246,1079,3019515 +782,search,3616.02,2517.37,1227.37,1.03054,110.475,2.14348,9.65412,0.8981,0.125595,-1.0,-1.0,20.613,1.34246,1079,3019515 +783,insert,136.093,-1.0,-1.0,-1.0,-1.0,161.593,1.36702,1094,3082298 +784,search,3673.96,2538.59,1246.84,1.04347,110.722,2.17387,9.88538,0.89918,0.124243,-1.0,-1.0,48.762,1.36728,1097,3082298 +785,insert,62.323,-1.0,-1.0,-1.0,-1.0,117.481,1.38435,1105,3122090 +786,search,3694.12,2543.56,1248.12,1.08824,116.8,2.12064,10.8697,0.89982,0.123859,-1.0,-1.0,59.359,1.38435,1109,3122090 +787,insert,58.159,-1.0,-1.0,-1.0,-1.0,85.218,1.39554,1115,3140294 +788,search,3839.43,2542.41,1344.33,0.981757,127.619,2.27771,9.78827,0.90094,0.122791,-1.0,-1.0,42.494,1.39598,1118,3140294 +789,insert,15.939,-1.0,-1.0,-1.0,-1.0,25.139,1.39945,1119,3147819 +790,search,3826.81,2537.3,1350.6,0.933947,126.71,2.19263,8.85316,0.90049,0.123267,-1.0,-1.0,15.728,1.39945,1119,3147819 +791,insert,73.209,-1.0,-1.0,-1.0,-1.0,45.301,1.41151,1124,3180668 +792,search,3834.44,2550.96,1355.17,0.936811,124.63,2.2415,8.92197,0.90042,0.123442,-1.0,-1.0,15.744,1.41151,1124,3180668 +793,insert,2.563,-1.0,-1.0,-1.0,-1.0,13.519,1.41199,1124,3183235 +794,search,3837.39,2551.36,1355.82,0.917216,130.752,2.12023,10.3016,0.90051,0.123361,-1.0,-1.0,15.632,1.41199,1124,3183235 +795,insert,134.892,-1.0,-1.0,-1.0,-1.0,150.758,1.43847,1138,3242089 +796,search,3906.04,2575.38,1398.28,0.955863,122.703,2.20563,8.96474,0.9014,0.122944,-1.0,-1.0,36.679,1.43854,1140,3242089 +797,insert,8.161,-1.0,-1.0,-1.0,-1.0,23.755,1.43989,1141,3247192 +798,search,3928.78,2572.38,1395.88,0.958371,129.154,2.15228,8.65152,0.90203,0.12223,-1.0,-1.0,44.568,1.44016,1145,3247192 +799,insert,17.563,-1.0,-1.0,-1.0,-1.0,45.255,1.44411,1148,3254626 +800,search,3947.15,2556.49,1404.08,0.994113,120.737,2.10781,8.81798,0.90204,0.122409,-1.0,-1.0,15.695,1.44411,1148,3254626 +801,insert,46.355,-1.0,-1.0,-1.0,-1.0,61.149,1.45218,1153,3274953 +802,search,3947.87,2562.35,1405.49,0.934495,126.138,2.15328,9.01176,0.90183,0.122771,-1.0,-1.0,15.582,1.45218,1153,3274953 +803,insert,1571.69,-1.0,-1.0,-1.0,-1.0,1511.68,1.81391,1330,4032688 +804,search,4604.62,2804.67,1701.44,1.03278,152.817,2.11187,9.39099,0.9105,0.116725,-1.0,-1.0,595.308,1.81524,1400,4032688 +805,insert,28.241,-1.0,-1.0,-1.0,-1.0,17.128,1.82143,1400,4050220 +806,search,4814.69,2656.55,1739.78,0.997531,168.922,2.19636,8.59914,0.91107,0.116534,-1.0,-1.0,248.53,1.82235,1429,4050220 +807,insert,24.222,-1.0,-1.0,-1.0,-1.0,85.659,1.82991,1435,4067940 +808,search,4792.69,2592.49,1661.82,1.03612,153.012,2.23214,8.37958,0.91069,0.11653,-1.0,-1.0,45.406,1.83015,1438,4067940 +809,insert,48.298,-1.0,-1.0,-1.0,-1.0,73.882,1.84167,1445,4099369 +810,search,4873.95,2590.15,1720.29,1.01356,151.595,2.11195,9.19462,0.91106,0.11629,-1.0,-1.0,28.873,1.84167,1446,4099369 +811,insert,51.987,-1.0,-1.0,-1.0,-1.0,119.98,1.85319,1461,4129509 +812,search,4850.7,2584.02,1660.54,1.11221,146.611,2.63082,8.34537,0.9117,0.115432,-1.0,-1.0,45.771,1.85337,1464,4129509 +813,insert,80.452,-1.0,-1.0,-1.0,-1.0,120.901,1.86931,1480,4178669 +814,search,4958.24,2576.59,1732.68,1.03784,147.417,2.10567,9.00932,0.9128,0.113954,-1.0,-1.0,21.727,1.86931,1480,4178669 +815,insert,6.325,-1.0,-1.0,-1.0,-1.0,29.252,1.87079,1481,4180960 +816,search,5030.28,2574.87,1791.79,0.967974,154.665,2.13881,8.32997,0.9128,0.113954,-1.0,-1.0,21.843,1.87079,1481,4180960 +817,insert,119.982,-1.0,-1.0,-1.0,-1.0,189.561,1.89657,1501,4255359 +818,search,5101.15,2585.54,1808.81,0.977562,172.144,2.18547,8.59546,0.91374,0.113281,-1.0,-1.0,73.563,1.89731,1506,4255359 +819,insert,42.263,-1.0,-1.0,-1.0,-1.0,43.815,1.90324,1510,4272706 +820,search,5103.41,2579.96,1820.44,0.952022,162.435,2.0876,7.76473,0.9137,0.113434,-1.0,-1.0,20.792,1.90324,1510,4272706 +821,insert,25.012,-1.0,-1.0,-1.0,-1.0,50.386,1.91008,1513,4287851 +822,search,5122.38,2581.87,1823.96,0.946946,164.253,2.25072,7.53821,0.91356,0.113642,-1.0,-1.0,21.224,1.91008,1513,4287851 +823,insert,61.886,-1.0,-1.0,-1.0,-1.0,107.717,1.92645,1522,4322788 +824,search,5181.58,2589.54,1859.34,0.949637,168.795,2.11,8.36082,0.91368,0.113936,-1.0,-1.0,56.892,1.92691,1527,4322788 +825,insert,139.831,-1.0,-1.0,-1.0,-1.0,184.299,1.95914,1544,4396408 +826,search,5246.6,2588.67,1878.39,0.965653,164.65,4.81543,10.1274,0.91402,0.113636,-1.0,-1.0,66.323,1.95949,1549,4396408 +827,insert,35.802,-1.0,-1.0,-1.0,-1.0,46.572,1.96753,1552,4415721 +828,search,5233.55,2581.37,1858.53,1.00091,167.841,2.20301,8.20314,0.91366,0.114035,-1.0,-1.0,35.418,1.96798,1553,4415721 +829,insert,806.177,-1.0,-1.0,-1.0,-1.0,885.818,2.15704,1665,4832079 +830,search,5633.51,2653.41,2028.79,1.0014,183.066,2.19041,8.9726,0.91761,0.110602,-1.0,-1.0,342.398,2.16131,1703,4832079 +831,insert,50.291,-1.0,-1.0,-1.0,-1.0,91.489,2.17342,1712,4864525 +832,search,5644.9,2592.88,1945.91,1.10379,177.906,5.44506,9.71494,0.91786,0.110218,-1.0,-1.0,53.207,2.17399,1716,4864525 +833,insert,17.153,-1.0,-1.0,-1.0,-1.0,20.695,2.17703,1716,4874443 +834,search,5808.2,2592.67,2069.72,0.952381,181.668,2.10044,8.0333,0.91834,0.10994,-1.0,-1.0,35.94,2.17719,1718,4874443 +835,insert,110.174,-1.0,-1.0,-1.0,-1.0,178.357,2.2039,1739,4938267 +836,search,5728.54,2601.54,1958.63,1.08941,180.292,2.15451,12.0281,0.91846,0.109625,-1.0,-1.0,37.269,2.20425,1741,4938267 +837,insert,31.238,-1.0,-1.0,-1.0,-1.0,63.43,2.21092,1746,4961147 +838,search,5844.27,2603,2037.59,1.05866,184.211,2.56812,9.53002,0.91874,0.109448,-1.0,-1.0,21.453,2.21092,1746,4961147 +839,insert,116.026,-1.0,-1.0,-1.0,-1.0,187.125,2.2405,1767,5033740 +840,search,5961.82,2605.56,2108.22,1.01212,191.605,2.17005,8.40737,0.91984,0.108449,-1.0,-1.0,62.072,2.24091,1772,5033740 +841,insert,73.014,-1.0,-1.0,-1.0,-1.0,127.089,2.25956,1783,5078736 +842,search,5983.06,2601.45,2093.05,1.04934,183.37,2.14393,9.75288,0.92052,0.107805,-1.0,-1.0,43.668,2.25977,1785,5078736 +843,insert,694.797,-1.0,-1.0,-1.0,-1.0,838.052,2.43822,1896,5467376 +844,search,6423.41,2650.14,2282.98,1.02644,213.763,2.12684,9.76595,0.9234,0.105781,-1.0,-1.0,310.756,2.44091,1933,5467376 +845,insert,595.487,-1.0,-1.0,-1.0,-1.0,691.484,2.5979,2037,5815547 +846,search,6795.14,2621.44,2387.59,1.04159,215.752,2.49045,9.03916,0.92644,0.103448,-1.0,-1.0,241.643,2.59908,2063,5815547 +847,insert,9.235,-1.0,-1.0,-1.0,-1.0,33.24,2.60124,2064,5821782 +848,search,6900.05,2590.44,2433.21,1.05296,226.879,2.16568,10.4506,0.92696,0.103061,-1.0,-1.0,59.31,2.60134,2068,5821782 +849,insert,2.921,-1.0,-1.0,-1.0,-1.0,41.223,2.6019,2070,5825594 +850,search,6874.47,2584.7,2399,1.0589,221.044,2.78213,9.45366,0.92688,0.103147,-1.0,-1.0,27.306,2.6019,2070,5825594 +851,insert,15.187,-1.0,-1.0,-1.0,-1.0,53.268,2.60505,2074,5834880 +852,search,6984.49,2583.65,2464.8,0.990393,232.568,2.29277,7.97404,0.92702,0.102979,-1.0,-1.0,27.983,2.60505,2074,5834880 +853,insert,56.074,-1.0,-1.0,-1.0,-1.0,80.625,2.61632,2079,5860274 +854,search,6980.14,2588.05,2475.53,0.976324,227.597,2.15583,8.40663,0.92693,0.103108,-1.0,-1.0,27.473,2.61632,2079,5860274 +855,insert,11.832,-1.0,-1.0,-1.0,-1.0,34.254,2.61779,2081,5866064 +856,search,6741.63,2587.73,2280.86,1.12088,197.732,2.11537,9.56784,0.92717,0.102769,-1.0,-1.0,52.241,2.61815,2085,5866064 +857,insert,32.953,-1.0,-1.0,-1.0,-1.0,68.011,2.62414,2091,5884456 +858,search,7006.11,2582.69,2490.09,0.990908,222.217,2.10133,7.91699,0.92714,0.102807,-1.0,-1.0,38.102,2.62442,2092,5884456 +859,insert,56.681,-1.0,-1.0,-1.0,-1.0,65.766,2.63573,2097,5916786 +860,search,6910.03,2592.07,2379.14,1.05547,216.256,2.14157,9.98933,0.92738,0.102649,-1.0,-1.0,35.497,2.63574,2098,5916786 +861,insert,10.479,-1.0,-1.0,-1.0,-1.0,51.274,2.63729,2101,5923467 +862,search,6874.97,2589.77,2350.07,1.08012,215.127,2.44953,9.26134,0.92757,0.102579,-1.0,-1.0,27.245,2.63729,2101,5923467 +863,insert,118.781,-1.0,-1.0,-1.0,-1.0,176.594,2.66916,2118,5997844 +864,search,6879.33,2593.16,2335.47,1.1558,205.055,2.39275,9.03191,0.92637,0.104312,-1.0,-1.0,44.36,2.66932,2119,5997844 +865,insert,18.654,-1.0,-1.0,-1.0,-1.0,27.145,2.67305,2119,6009561 +866,search,6947.28,2595.67,2378.95,1.13225,214.625,2.13328,9.62793,0.92638,0.104308,-1.0,-1.0,30.433,2.67305,2119,6009561 +867,insert,70.41,-1.0,-1.0,-1.0,-1.0,125.743,2.68927,2129,6052876 +868,search,6927.54,2605.94,2336.62,1.1077,200.949,2.1015,9.26238,0.92688,0.104044,-1.0,-1.0,67.829,2.68978,2133,6052876 +869,insert,9.989,-1.0,-1.0,-1.0,-1.0,37.933,2.69176,2134,6059234 +870,search,6882.92,2600.06,2305.44,1.14373,198.106,2.08888,9.75529,0.92687,0.104041,-1.0,-1.0,43.584,2.69195,2135,6059234 +871,insert,69.024,-1.0,-1.0,-1.0,-1.0,111.232,2.70992,2143,6098225 +872,search,7210.42,2602.21,2578.84,1.01053,232.628,2.17582,8.4778,0.92702,0.103725,-1.0,-1.0,37.197,2.71,2144,6098225 +873,insert,65.179,-1.0,-1.0,-1.0,-1.0,87.025,2.72549,2150,6137308 +874,search,7252.95,2603.88,2588.88,0.999143,234.714,2.12785,7.95521,0.92719,0.103475,-1.0,-1.0,42.324,2.72569,2152,6137308 +875,insert,25.58,-1.0,-1.0,-1.0,-1.0,38.947,2.72992,2153,6154089 +876,search,7094.49,2605.78,2450.28,1.09012,227.882,2.10859,9.46428,0.92701,0.103597,-1.0,-1.0,28.987,2.72992,2153,6154089 +877,insert,6.158,-1.0,-1.0,-1.0,-1.0,26.886,2.73091,2153,6156061 +878,search,7120.75,2608.12,2474.17,1.10041,226.958,2.12908,9.2531,0.92709,0.10353,-1.0,-1.0,36.012,2.73095,2154,6156061 +879,insert,562.956,-1.0,-1.0,-1.0,-1.0,594.66,2.8621,2221,6446004 +880,search,7508.39,2631.64,2692.66,0.969374,241.273,2.0631,8.3864,0.92888,0.101857,-1.0,-1.0,196.059,2.86366,2245,6446004 +881,insert,844.309,-1.0,-1.0,-1.0,-1.0,968.21,3.10037,2384,6969764 +882,search,8034.49,2666.25,2905.51,1.00702,264.076,2.1424,8.09975,0.9321,0.0985104,-1.0,-1.0,365.227,3.10301,2430,6969764 +883,insert,643.589,-1.0,-1.0,-1.0,-1.0,865.165,3.30183,2573,7422501 +884,search,8611.83,2633.97,3071.48,0.969789,283.335,2.17227,8.15463,0.93487,0.0961859,-1.0,-1.0,312.509,3.30501,2618,7422501 +885,insert,13.991,-1.0,-1.0,-1.0,-1.0,51.27,3.30843,2622,7432053 +886,search,8768.17,2578.29,3103.99,0.982919,291.242,2.40385,8.23415,0.93513,0.0958773,-1.0,-1.0,46.6,3.30876,2625,7432053 +887,insert,547.31,-1.0,-1.0,-1.0,-1.0,617.78,3.46342,2715,7789886 +888,search,9017.43,2607.2,3184.98,1.0212,289.458,2.84615,9.38352,0.93664,0.0952044,-1.0,-1.0,242.089,3.46619,2750,7789886 +889,insert,616.641,-1.0,-1.0,-1.0,-1.0,813.239,3.65219,2875,8210883 +890,search,9497.97,2593.45,3311.29,1.02917,313.19,2.12109,9.51896,0.9385,0.0935871,-1.0,-1.0,305.161,3.65539,2920,8210883 +891,insert,47.956,-1.0,-1.0,-1.0,-1.0,91.073,3.6665,2929,8240504 +892,search,9575.71,2551.06,3295.83,1.03929,298.796,2.10904,11.667,0.93873,0.0934654,-1.0,-1.0,59.11,3.66677,2933,8240504 +893,insert,77.635,-1.0,-1.0,-1.0,-1.0,120.078,3.69017,2947,8299693 +894,search,9684.89,2553.59,3372.75,1.01418,299.739,2.10211,8.95255,0.93902,0.093178,-1.0,-1.0,42.807,3.69037,2948,8299693 +895,insert,67.473,-1.0,-1.0,-1.0,-1.0,112.215,3.70878,2962,8349126 +896,search,9734.52,2555.73,3347.45,1.0434,300.607,2.13023,9.60421,0.93941,0.0929208,-1.0,-1.0,76.784,3.70938,2966,8349126 +897,delete,113.982,-1.0,-1.0,-1.0,-1.0,503.336,3.67618,2941,8268842 +898,search,9507.04,2555.21,3204.76,1.08963,299.963,2.15244,12.4099,0.93908,0.092828,-1.0,-1.0,83.746,3.67695,2947,8268842 +899,delete,24.281,-1.0,-1.0,-1.0,-1.0,92.145,3.67338,2944,8250527 +900,search,9679.58,2548.24,3343.19,1.04889,309.666,2.15202,9.85786,0.93895,0.0930777,-1.0,-1.0,37.93,3.67339,2945,8250527 +901,delete,95.952,-1.0,-1.0,-1.0,-1.0,358.421,3.65439,2925,8185565 +902,search,9603.85,2540.15,3317.52,1.02964,304.412,2.10006,10.3516,0.93847,0.0932699,-1.0,-1.0,65.08,3.65487,2929,8185565 +903,delete,71.664,-1.0,-1.0,-1.0,-1.0,228.145,3.62931,2909,8126915 +904,search,9653.68,2533.28,3373.67,0.982307,316.391,2.16563,8.51769,0.93768,0.0936311,-1.0,-1.0,40.244,3.62931,2910,8126915 +905,delete,39.955,-1.0,-1.0,-1.0,-1.0,81.693,3.62675,2908,8098533 +906,search,9583.99,2524.85,3320.84,0.966789,301.78,2.08501,10.0696,0.93763,0.0936821,-1.0,-1.0,30.525,3.62675,2908,8098533 +907,delete,30.343,-1.0,-1.0,-1.0,-1.0,266.772,3.6154,2898,8074833 +908,search,9283.61,2525.8,3097.61,1.13641,281.817,2.15393,12.3518,0.93739,0.0939467,-1.0,-1.0,40.314,3.61562,2899,8074833 +909,delete,54.036,-1.0,-1.0,-1.0,-1.0,143.86,3.61022,2894,8035274 +910,search,9561.29,2517.18,3310.32,0.989511,306.135,2.16848,9.09616,0.93735,0.0938224,-1.0,-1.0,36.338,3.61022,2894,8035274 +911,delete,161.687,-1.0,-1.0,-1.0,-1.0,373.989,3.5899,2876,7957223 +912,search,9477.13,2502.58,3288.56,0.981954,306.028,2.14487,8.4529,0.93687,0.0940842,-1.0,-1.0,44.899,3.59014,2877,7957223 +913,delete,56.491,-1.0,-1.0,-1.0,-1.0,264.866,3.57219,2860,7917554 +914,search,9406.46,2503.38,3233.08,1.0304,294.75,2.10831,8.90828,0.93568,0.0948162,-1.0,-1.0,31.888,3.57219,2860,7917554 +915,delete,97.891,-1.0,-1.0,-1.0,-1.0,310.228,3.5507,2838,7861503 +916,search,9216.28,2503.48,3104.54,1.05182,286.941,2.15585,12.0368,0.93541,0.0952913,-1.0,-1.0,53.028,3.55098,2840,7861503 +917,delete,26.02,-1.0,-1.0,-1.0,-1.0,173.781,3.5444,2835,7844487 +918,search,9376.17,2499.05,3242.82,1.00922,316.851,2.1572,8.91296,0.93546,0.0951831,-1.0,-1.0,30.911,3.5444,2835,7844487 +919,delete,42.456,-1.0,-1.0,-1.0,-1.0,76.947,3.542,2832,7814993 +920,search,9348.09,2491.42,3234.13,0.980941,293.146,2.14237,8.63673,0.93542,0.0954277,-1.0,-1.0,29.223,3.542,2832,7814993 +921,delete,28.659,-1.0,-1.0,-1.0,-1.0,223.154,3.53304,2824,7793855 +922,search,9145.91,2490.67,3085.81,1.0453,280.872,2.79058,10.3801,0.9354,0.0955608,-1.0,-1.0,39.569,3.53329,2825,7793855 +923,delete,70.28,-1.0,-1.0,-1.0,-1.0,310.948,3.51627,2809,7736054 +924,search,9210.13,2482.78,3190.49,0.987862,271.355,2.11137,8.86392,0.93526,0.0954327,-1.0,-1.0,32.338,3.51627,2809,7736054 +925,delete,221.824,-1.0,-1.0,-1.0,-1.0,1077.12,3.44017,2732,7587942 +926,search,8945.69,2504.66,3029.5,1.03033,279.729,2.16343,9.4577,0.9339,0.0967681,-1.0,-1.0,196.563,3.44312,2752,7587942 +927,delete,27.877,-1.0,-1.0,-1.0,-1.0,75.24,3.43871,2749,7565670 +928,search,9062.44,2479.27,3134.52,0.973515,283.861,2.09302,8.62869,0.93383,0.097396,-1.0,-1.0,53.795,3.4388,2751,7565670 +929,delete,59.098,-1.0,-1.0,-1.0,-1.0,97.815,3.43471,2748,7529948 +930,search,9063.39,2467.75,3112.37,0.98447,305.364,2.15393,8.80166,0.93379,0.0974094,-1.0,-1.0,29.976,3.43471,2748,7529948 +931,delete,703.656,-1.0,-1.0,-1.0,-1.0,4820.39,3.17546,2433,7016700 +932,search,8172.31,2586.97,2901.53,0.969308,263.776,7.22249,8.65226,0.93141,0.0990332,-1.0,-1.0,664.817,3.18972,2525,7016700 +933,delete,26.011,-1.0,-1.0,-1.0,-1.0,193.808,3.17519,2514,6980979 +934,search,8294.9,2485.32,2850.41,1.00199,272.195,2.15272,9.32905,0.93262,0.098409,-1.0,-1.0,106.627,3.17591,2524,6980979 +935,delete,27.121,-1.0,-1.0,-1.0,-1.0,187.734,3.16332,2515,6954250 +936,search,8265.54,2474.91,2840.13,0.966953,269.199,2.16643,8.9057,0.93227,0.0986312,-1.0,-1.0,39.445,3.16345,2516,6954250 +937,delete,30.956,-1.0,-1.0,-1.0,-1.0,142.504,3.15392,2509,6924596 +938,search,8256.26,2470.97,2830.53,1.01086,256.603,2.13887,8.84571,0.93174,0.0990439,-1.0,-1.0,25.845,3.15392,2509,6924596 +939,delete,159.699,-1.0,-1.0,-1.0,-1.0,560.26,3.11735,2466,6784831 +940,search,8085.68,2457.08,2765.81,1.00675,257.547,2.17347,8.03399,0.93159,0.0996093,-1.0,-1.0,62.199,3.11772,2469,6784831 +941,delete,74.418,-1.0,-1.0,-1.0,-1.0,30.451,3.1155,2470,6745819 +942,search,8083.08,2431.45,2740.56,1.00385,253.336,2.15204,8.57742,0.93124,0.100536,-1.0,-1.0,50.373,3.1159,2472,6745819 +943,delete,8.334,-1.0,-1.0,-1.0,-1.0,25.93,3.1159,2472,6740944 +944,search,8133.49,2426.91,2771.55,0.971693,253.139,2.09462,8.76006,0.93115,0.100623,-1.0,-1.0,26.715,3.1159,2472,6740944 +945,delete,166.553,-1.0,-1.0,-1.0,-1.0,1136.39,3.04945,2404,6627202 +946,search,7858.65,2454.88,2675.75,0.977419,254.063,2.07366,10.0868,0.92983,0.10139,-1.0,-1.0,171.097,3.05153,2419,6627202 +947,delete,28.639,-1.0,-1.0,-1.0,-1.0,37.935,3.04836,2418,6607194 +948,search,7598.13,2431.87,2412.4,1.14952,225.157,3.56689,11.8697,0.9298,0.101671,-1.0,-1.0,101.912,3.04937,2427,6607194 +949,delete,64.914,-1.0,-1.0,-1.0,-1.0,301.032,3.02317,2406,6551065 +950,search,7725.28,2420.74,2573.1,1.04463,233.474,2.08324,9.52213,0.92929,0.102007,-1.0,-1.0,25.447,3.02317,2406,6551065 +951,delete,45.436,-1.0,-1.0,-1.0,-1.0,115.312,3.00211,2393,6506798 +952,search,7582.82,2420.83,2468.98,1.08379,213.057,2.08363,8.71755,0.92985,0.101673,-1.0,-1.0,35.404,3.00235,2394,6506798 +953,delete,67.839,-1.0,-1.0,-1.0,-1.0,190.402,2.9909,2387,6459301 +954,search,7479.5,2410.19,2386.58,1.15029,201.541,2.10645,8.4605,0.92999,0.10153,-1.0,-1.0,23.995,2.9909,2387,6459301 +955,delete,21.532,-1.0,-1.0,-1.0,-1.0,49.752,2.98324,2383,6440069 +956,search,7773.18,2408.74,2645.14,0.97434,229.746,2.09908,8.82179,0.9302,0.101074,-1.0,-1.0,24.279,2.98324,2383,6440069 +957,delete,455.421,-1.0,-1.0,-1.0,-1.0,2379.63,2.81542,2215,6132470 +958,search,7315.83,2465.35,2511.79,0.980667,223.734,2.04892,7.88782,0.92721,0.103445,-1.0,-1.0,215.133,2.81908,2238,6132470 +959,delete,58.437,-1.0,-1.0,-1.0,-1.0,139.178,2.80382,2228,6086241 +960,search,7293.72,2427.36,2444.96,1.02149,222.584,2.13002,8.78306,0.92654,0.104136,-1.0,-1.0,109.898,2.80477,2234,6086241 +961,delete,36.964,-1.0,-1.0,-1.0,-1.0,65.429,2.80193,2232,6064734 +962,search,7317.82,2413.39,2475.98,1.00098,230.055,2.1376,8.94282,0.92619,0.104623,-1.0,-1.0,54.396,2.80229,2233,6064734 +963,delete,177.392,-1.0,-1.0,-1.0,-1.0,886.726,2.74451,2167,5967543 +964,search,7187.15,2457.87,2460.03,0.981298,233.158,3.47587,8.92146,0.92645,0.104291,-1.0,-1.0,193.197,2.74719,2187,5967543 +965,delete,81.673,-1.0,-1.0,-1.0,-1.0,397.307,2.72014,2159,5906381 +966,search,6880.3,2427.53,2241.55,1.08394,201.272,2.13477,8.85683,0.92514,0.105538,-1.0,-1.0,94.673,2.72106,2168,5906381 +967,delete,71.844,-1.0,-1.0,-1.0,-1.0,310.222,2.6992,2151,5851849 +968,search,6914.76,2406.64,2276.86,1.03926,199.299,2.13339,8.60781,0.92445,0.105885,-1.0,-1.0,51.366,2.69963,2152,5851849 +969,delete,85.998,-1.0,-1.0,-1.0,-1.0,282.66,2.66998,2130,5783023 +970,search,6800.53,2404.11,2213.46,1.09707,201.807,2.14962,10.0333,0.92329,0.10657,-1.0,-1.0,44.842,2.67009,2131,5783023 +971,delete,403.349,-1.0,-1.0,-1.0,-1.0,1901.7,2.53246,1986,5496517 +972,search,6574.36,2446.67,2248.81,0.970083,215.045,2.44927,8.88685,0.92089,0.108505,-1.0,-1.0,299.562,2.53759,2018,5496517 +973,delete,326.769,-1.0,-1.0,-1.0,-1.0,1617.37,2.41478,1893,5237981 +974,search,6254.95,2437.55,2143.61,0.991296,184.823,2.07679,9.30479,0.91968,0.10864,-1.0,-1.0,295.929,2.42015,1925,5237981 +975,delete,184.472,-1.0,-1.0,-1.0,-1.0,756.521,2.34675,1851,5099794 +976,search,5953.19,2424.84,1945.29,1.06609,169.512,2.07543,10.3646,0.91898,0.110054,-1.0,-1.0,230.727,2.34899,1883,5099794 +977,delete,64.912,-1.0,-1.0,-1.0,-1.0,263.363,2.32166,1863,5032210 +978,search,6098.84,2356.09,2038.98,0.984188,193.439,2.17381,8.81301,0.91723,0.111182,-1.0,-1.0,69.252,2.32209,1869,5032210 +979,delete,14.658,-1.0,-1.0,-1.0,-1.0,50.701,2.31822,1866,5020211 +980,search,6089.39,2343.89,2043.7,1.03479,178.499,2.14918,8.31699,0.91694,0.111491,-1.0,-1.0,32.279,2.31822,1867,5020211 +981,delete,159.117,-1.0,-1.0,-1.0,-1.0,507.235,2.25942,1824,4902291 +982,search,5790.67,2338.69,1832.81,1.05637,170.422,2.13178,11.3395,0.91516,0.112856,-1.0,-1.0,81.993,2.26055,1829,4902291 +983,delete,41.458,-1.0,-1.0,-1.0,-1.0,73.944,2.25424,1826,4872577 +984,search,5970.58,2320.68,1987.08,0.97459,168.3,2.12399,9.50216,0.91518,0.113043,-1.0,-1.0,46.395,2.25453,1827,4872577 +985,delete,62.02,-1.0,-1.0,-1.0,-1.0,120.76,2.24582,1820,4836730 +986,search,5930.56,2306.06,1960.97,0.967085,182.285,2.15022,9.04172,0.91523,0.112917,-1.0,-1.0,28.463,2.24582,1820,4836730 +987,delete,72.16,-1.0,-1.0,-1.0,-1.0,97.685,2.23808,1814,4797164 +988,search,5881.55,2287.61,1933.59,0.967159,171.008,2.99881,9.20977,0.91478,0.113479,-1.0,-1.0,35.984,2.2381,1815,4797164 +989,delete,28.471,-1.0,-1.0,-1.0,-1.0,25.566,2.23648,1815,4784789 +990,search,5915.75,2277.65,1940.05,0.983346,187.87,2.17224,10.1118,0.9147,0.11361,-1.0,-1.0,26.468,2.23648,1815,4784789 +991,delete,108.447,-1.0,-1.0,-1.0,-1.0,486.794,2.20023,1788,4705420 +992,search,5847.56,2299.62,1928.79,0.982741,180.378,2.18204,9.63207,0.91618,0.112638,-1.0,-1.0,167.041,2.20167,1800,4705420 +993,delete,122.605,-1.0,-1.0,-1.0,-1.0,331.787,2.16464,1776,4621807 +994,search,5744.71,2262.77,1883.18,0.962904,176.23,2.16856,8.6393,0.91625,0.111981,-1.0,-1.0,132.133,2.16523,1787,4621807 +995,delete,99.108,-1.0,-1.0,-1.0,-1.0,458.878,2.12694,1754,4555849 +996,search,5678.34,2254.08,1855.43,0.933782,169.061,2.16634,9.59748,0.91567,0.112207,-1.0,-1.0,54.848,2.12736,1757,4555849 +997,delete,41.391,-1.0,-1.0,-1.0,-1.0,201.322,2.11429,1747,4526975 +998,search,5660.76,2253.91,1860.81,0.978483,159.036,2.1236,10.0042,0.9155,0.112424,-1.0,-1.0,37.132,2.11431,1748,4526975 +999,delete,164.186,-1.0,-1.0,-1.0,-1.0,679.749,2.05021,1696,4408448 +1000,search,5479.82,2256.2,1792.1,0.985374,164.435,2.16198,9.03923,0.91456,0.113243,-1.0,-1.0,82.35,2.05093,1702,4408448 +1001,delete,165.494,-1.0,-1.0,-1.0,-1.0,779.022,1.98935,1642,4303257 +1002,search,5329.91,2272.21,1754.29,0.947696,160.095,2.1983,8.71151,0.9128,0.113822,-1.0,-1.0,173.27,1.99162,1655,4303257 +1003,delete,47.434,-1.0,-1.0,-1.0,-1.0,190.488,1.97219,1640,4259551 +1004,search,5308.62,2257.24,1737.46,0.982959,163.518,2.17215,9.23829,0.91282,0.113775,-1.0,-1.0,45.099,1.97242,1642,4259551 +1005,delete,23.624,-1.0,-1.0,-1.0,-1.0,85.329,1.9653,1635,4241434 +1006,search,5288.57,2254.71,1738.32,0.953329,152.811,2.08458,8.86623,0.91218,0.114099,-1.0,-1.0,43.926,1.96577,1637,4241434 +1007,delete,254.472,-1.0,-1.0,-1.0,-1.0,1160.66,1.87084,1557,4052722 +1008,search,5041,2267.41,1667.52,0.999924,143.141,2.12399,8.91067,0.90908,0.116165,-1.0,-1.0,175.333,1.873,1569,4052722 +1009,delete,439.79,-1.0,-1.0,-1.0,-1.0,1453.93,1.71546,1430,3738432 +1010,search,4670.73,2293.11,1541.36,0.961886,144.437,4.18394,9.1892,0.90605,0.117974,-1.0,-1.0,285.349,1.71884,1456,3738432 +1011,delete,519.349,-1.0,-1.0,-1.0,-1.0,1959.24,1.52775,1250,3372539 +1012,search,4169.14,2408.15,1421.37,0.989617,125.784,2.19794,8.71775,0.90373,0.119678,-1.0,-1.0,296.014,1.53217,1282,3372539 +1013,delete,11.018,-1.0,-1.0,-1.0,-1.0,39.82,1.52883,1279,3361432 +1014,search,4222.51,2325,1409.23,0.987095,135.221,2.16512,8.92302,0.90359,0.118927,-1.0,-1.0,81.006,1.52939,1287,3361432 +1015,delete,380.022,-1.0,-1.0,-1.0,-1.0,1367.93,1.39475,1154,3091692 +1016,search,3886.22,2383.05,1313.91,0.982664,128.47,2.1619,9.61531,0.89805,0.122684,-1.0,-1.0,269.932,1.39925,1180,3091692 +1017,delete,456.549,-1.0,-1.0,-1.0,-1.0,1735.43,1.2454,1023,2775418 +1018,search,3485.69,2453.53,1209.48,0.958636,108.646,2.0751,8.59573,0.89524,0.125042,-1.0,-1.0,326.348,1.24908,1052,2775418 +1019,delete,23.52,-1.0,-1.0,-1.0,-1.0,75.283,1.24173,1047,2754073 +1020,search,3537.19,2380.4,1207.81,0.989858,106.666,2.1191,9.29027,0.89607,0.123999,-1.0,-1.0,79.645,1.24197,1052,2754073 +1021,delete,160.346,-1.0,-1.0,-1.0,-1.0,420.006,1.19763,1012,2638917 +1022,search,3383.68,2353.68,1154.58,0.927435,96.4403,2.10822,8.76759,0.89366,0.125958,-1.0,-1.0,82.79,1.19839,1017,2638917 +1023,delete,129.102,-1.0,-1.0,-1.0,-1.0,378.463,1.15802,974,2556494 +1024,search,3252.65,2364.87,1095,0.97622,100.706,2.15354,8.49478,0.89172,0.12808,-1.0,-1.0,67.484,1.15874,979,2556494 +1025,insert,70.982,-1.0,-1.0,-1.0,-1.0,49.379,1.17257,982,2594934 +1026,search,3307.41,2381.77,1139.05,0.948358,100.244,2.1716,8.87987,0.89262,0.126615,-1.0,-1.0,26.616,1.17262,983,2594934 +1027,insert,357.542,-1.0,-1.0,-1.0,-1.0,441.381,1.25613,1024,2802105 +1028,search,3460.59,2449.18,1213.18,1.01213,106.66,2.15436,8.12585,0.89324,0.126664,-1.0,-1.0,176.598,1.25642,1037,2802105 +1029,insert,748.03,-1.0,-1.0,-1.0,-1.0,565.535,1.43043,1105,3203975 +1030,search,3756.45,2688.28,1309.92,1.09038,118.153,3.16189,10.8905,0.89985,0.122289,-1.0,-1.0,167.845,1.43127,1123,3203975 +1031,insert,1051.4,-1.0,-1.0,-1.0,-1.0,1006,1.71754,1208,3830566 +1032,search,4311.63,2906.95,1641.8,1.00407,146.529,2.18995,9.26587,0.90486,0.119863,-1.0,-1.0,566.958,1.72141,1245,3830566 +1033,insert,757.677,-1.0,-1.0,-1.0,-1.0,960.11,1.91434,1361,4252669 +1034,search,4797.43,2845.95,1798.01,0.973044,156.654,2.45099,8.77997,0.91018,0.116006,-1.0,-1.0,425.791,1.91769,1408,4252669 +1035,insert,385.67,-1.0,-1.0,-1.0,-1.0,515.842,2.03318,1464,4512107 +1036,search,5092.35,2759.78,1873.34,0.987862,171.515,2.17578,9.91988,0.91318,0.114569,-1.0,-1.0,337.894,2.03559,1502,4512107 +1037,insert,20.039,-1.0,-1.0,-1.0,-1.0,50.267,2.04066,1506,4525815 +1038,search,5108.37,2688.8,1833.35,1.02038,166.922,2.15361,10.1935,0.91387,0.114067,-1.0,-1.0,136.401,2.04122,1519,4525815 +1039,insert,120.076,-1.0,-1.0,-1.0,-1.0,197.199,2.07218,1546,4599215 +1040,search,5137.48,2668.39,1795.76,1.10545,159.668,2.13542,8.99564,0.91484,0.113772,-1.0,-1.0,91.346,2.07264,1553,4599215 +1041,insert,11.844,-1.0,-1.0,-1.0,-1.0,49.736,2.07679,1556,4609851 +1042,search,5169.23,2657.31,1793.92,1.10066,161.525,2.55066,8.93456,0.91544,0.113261,-1.0,-1.0,36.144,2.07685,1558,4609851 +1043,insert,26.006,-1.0,-1.0,-1.0,-1.0,59.988,2.08127,1563,4627411 +1044,search,5288.68,2653.14,1876.22,1.03623,171.743,2.18378,9.80853,0.91564,0.113293,-1.0,-1.0,22.324,2.08127,1563,4627411 +1045,insert,75.188,-1.0,-1.0,-1.0,-1.0,115.804,2.09567,1576,4670607 +1046,search,5183.6,2654.34,1767.34,1.13296,152.314,2.47736,8.42621,0.91577,0.113191,-1.0,-1.0,28.341,2.09568,1577,4670607 +1047,insert,5.927,-1.0,-1.0,-1.0,-1.0,45.952,2.09695,1580,4675386 +1048,search,5408.94,2649.29,1947.18,0.951148,184.179,2.17995,9.79801,0.91598,0.112965,-1.0,-1.0,21.724,2.09695,1580,4675386 +1049,insert,22.42,-1.0,-1.0,-1.0,-1.0,42.099,2.10226,1582,4689998 +1050,search,5317.58,2656.05,1865.31,1.03289,170.949,2.12752,9.01948,0.9159,0.113006,-1.0,-1.0,21.048,2.10226,1582,4689998 +1051,insert,172.223,-1.0,-1.0,-1.0,-1.0,242.717,2.13989,1611,4783971 +1052,search,5486.64,2664.47,1974.69,0.992819,187.583,2.17076,9.37321,0.91606,0.113238,-1.0,-1.0,30.909,2.14014,1612,4783971 +1053,insert,84.685,-1.0,-1.0,-1.0,-1.0,106.621,2.15864,1624,4826219 +1054,search,5352.63,2666.93,1837.88,1.08908,162.512,2.15898,8.87809,0.91668,0.113034,-1.0,-1.0,30.262,2.15868,1625,4826219 +1055,insert,61.496,-1.0,-1.0,-1.0,-1.0,74.949,2.17052,1631,4862355 +1056,search,5577.97,2677.96,2025.13,0.957979,189.307,2.18807,8.88059,0.91681,0.112868,-1.0,-1.0,39.329,2.17075,1633,4862355 +1057,insert,13.048,-1.0,-1.0,-1.0,-1.0,35.394,2.17339,1635,4869564 +1058,search,5521.91,2676.96,1974.6,1.01911,180.596,2.23117,8.92504,0.91718,0.112731,-1.0,-1.0,28.907,2.17348,1636,4869564 +1059,insert,54.879,-1.0,-1.0,-1.0,-1.0,124.157,2.18546,1648,4898570 +1060,search,5606.73,2674.51,2020.95,1.03519,184.316,2.1591,9.48229,0.91756,0.112769,-1.0,-1.0,51.519,2.18578,1651,4898570 +1061,insert,18.545,-1.0,-1.0,-1.0,-1.0,34.963,2.1897,1653,4907501 +1062,search,5599.73,2675.1,2012.29,1.0047,188.233,2.16606,9.13353,0.91757,0.112748,-1.0,-1.0,21.675,2.1897,1653,4907501 +1063,insert,15.611,-1.0,-1.0,-1.0,-1.0,75.316,2.19396,1658,4916386 +1064,search,5536.24,2670.83,1944.22,1.04899,173.546,2.23326,7.87974,0.91776,0.112591,-1.0,-1.0,21.06,2.19396,1658,4916386 +1065,insert,524.305,-1.0,-1.0,-1.0,-1.0,624.48,2.31571,1738,5194839 +1066,search,5926.44,2691.32,2136.94,1.05362,199.14,2.11876,9.03377,0.91942,0.111235,-1.0,-1.0,156.618,2.31717,1757,5194839 +1067,insert,11.048,-1.0,-1.0,-1.0,-1.0,35.164,2.31943,1759,5200182 +1068,search,5890.67,2658.86,2086.66,1.05261,186.973,2.11988,8.41673,0.91961,0.110946,-1.0,-1.0,21.648,2.31943,1759,5200182 +1069,insert,65.208,-1.0,-1.0,-1.0,-1.0,97.746,2.33378,1775,5230883 +1070,search,6024.66,2642.75,2153.7,0.991604,199.785,2.21342,8.38735,0.92029,0.110243,-1.0,-1.0,32.198,2.33386,1776,5230883 +1071,insert,37.407,-1.0,-1.0,-1.0,-1.0,59.763,2.34337,1780,5255659 +1072,search,5945.9,2645.97,2070.46,1.09011,185.298,2.1323,8.90863,0.92057,0.11024,-1.0,-1.0,61.533,2.34384,1783,5255659 +1073,insert,61.363,-1.0,-1.0,-1.0,-1.0,65.795,2.35649,1788,5281693 +1074,search,6091.82,2649.04,2186.74,0.967611,212.116,2.29063,8.0776,0.9209,0.109909,-1.0,-1.0,45.466,2.35674,1790,5281693 +1075,insert,20.409,-1.0,-1.0,-1.0,-1.0,60.451,2.35949,1796,5290081 +1076,search,5977.35,2639.48,2090.48,1.06928,186.611,2.23902,8.98065,0.92123,0.109677,-1.0,-1.0,36.635,2.35972,1797,5290081 +1077,insert,46.099,-1.0,-1.0,-1.0,-1.0,54.253,2.37068,1800,5316166 +1078,search,6129.51,2648.09,2211.84,1.00017,199.074,2.12705,9.06837,0.92163,0.109206,-1.0,-1.0,57.938,2.37087,1804,5316166 +1079,insert,41.225,-1.0,-1.0,-1.0,-1.0,78.868,2.37819,1813,5335531 +1080,search,6136.34,2637.81,2214.38,0.971636,193.812,2.08289,9.02444,0.92135,0.109476,-1.0,-1.0,47.63,2.37839,1816,5335531 +1081,insert,674.299,-1.0,-1.0,-1.0,-1.0,719.305,2.53318,1907,5678781 +1082,search,6501.22,2674.91,2341.85,0.997725,223.092,2.19952,9.62032,0.92321,0.107267,-1.0,-1.0,228.032,2.53591,1936,5678781 +1083,insert,1.024,-1.0,-1.0,-1.0,-1.0,22.801,2.53599,1936,5679848 +1084,search,6495.39,2627.57,2298.21,1.0401,205.763,2.19417,9.09606,0.92299,0.107495,-1.0,-1.0,44.03,2.5361,1938,5679848 +1085,insert,39.544,-1.0,-1.0,-1.0,-1.0,77.429,2.54541,1945,5702205 +1086,search,6420.8,2630.14,2226.77,1.09099,200.631,2.12346,8.32705,0.92328,0.107108,-1.0,-1.0,31.972,2.54554,1946,5702205 +1087,insert,23.967,-1.0,-1.0,-1.0,-1.0,48.99,2.55021,1949,5715514 +1088,search,6527.58,2634.74,2308.09,1.05041,211.425,2.15132,9.28873,0.92348,0.106943,-1.0,-1.0,36.268,2.55046,1950,5715514 +1089,insert,770.695,-1.0,-1.0,-1.0,-1.0,879.72,2.72597,2078,6103499 +1090,search,7018.17,2669.29,2522.93,0.951755,234.513,2.18788,8.70995,0.92616,0.105068,-1.0,-1.0,161.845,2.72708,2097,6103499 +1091,insert,76.844,-1.0,-1.0,-1.0,-1.0,159.16,2.74858,2120,6152505 +1092,search,7137.62,2636.87,2553.38,0.984325,234.169,2.15808,9.1544,0.92678,0.104682,-1.0,-1.0,41.66,2.74871,2122,6152505 +1093,insert,25.096,-1.0,-1.0,-1.0,-1.0,34.059,2.75573,2123,6171104 +1094,search,7074.45,2633.72,2502.48,1.02516,223.272,2.17948,8.00192,0.92665,0.104647,-1.0,-1.0,27.988,2.75573,2123,6171104 +1095,insert,595.092,-1.0,-1.0,-1.0,-1.0,788.238,2.92678,2231,6557850 +1096,search,7457.53,2675.24,2621.73,1.08888,234.495,2.54726,8.98933,0.92863,0.102998,-1.0,-1.0,302.12,2.93019,2270,6557850 +1097,insert,24.743,-1.0,-1.0,-1.0,-1.0,83.221,2.93761,2277,6579289 +1098,search,7312.1,2619,2460.63,1.16866,210.406,2.10031,8.91564,0.92903,0.102764,-1.0,-1.0,104.75,2.93859,2287,6579289 +1099,insert,21.799,-1.0,-1.0,-1.0,-1.0,87.441,2.94442,2296,6591442 +1100,search,7373.04,2597.74,2452.6,1.18831,206.144,2.11862,9.67549,0.92897,0.102949,-1.0,-1.0,62.081,2.94486,2299,6591442 +1101,insert,71.259,-1.0,-1.0,-1.0,-1.0,199.04,2.96468,2323,6633739 +1102,search,7629.97,2585.27,2623.58,1.08019,234.171,2.15826,9.6027,0.92969,0.102378,-1.0,-1.0,50.081,2.96469,2325,6633739 +1103,insert,151.046,-1.0,-1.0,-1.0,-1.0,254.646,3.0053,2362,6724357 +1104,search,7824.06,2583.56,2740.2,1.00107,251.866,2.16299,8.68735,0.93079,0.101431,-1.0,-1.0,53.332,3.0053,2367,6724357 +1105,insert,5.322,-1.0,-1.0,-1.0,-1.0,31.498,3.00625,2368,6727059 +1106,search,7799.3,2573.74,2707.38,1.03761,244.067,2.13891,9.18781,0.9307,0.10145,-1.0,-1.0,25.154,3.00625,2368,6727059 +1107,insert,374.835,-1.0,-1.0,-1.0,-1.0,453.206,3.10227,2433,6950869 +1108,search,8061.72,2593.23,2829.72,1.02484,260.216,2.11517,8.91722,0.93158,0.100629,-1.0,-1.0,73.586,3.10254,2439,6950869 +1109,insert,9.047,-1.0,-1.0,-1.0,-1.0,24.166,3.10413,2439,6958173 +1110,search,8035.8,2591.08,2789.42,1.03697,251.319,2.17687,8.87623,0.93166,0.100628,-1.0,-1.0,26.138,3.10413,2439,6958173 +1111,insert,74.307,-1.0,-1.0,-1.0,-1.0,141.91,3.11847,2458,6995273 +1112,search,8109.11,2587.31,2829.46,1.05476,254.93,2.31832,8.8224,0.93181,0.100158,-1.0,-1.0,57.454,3.11881,2462,6995273 +1113,insert,81.207,-1.0,-1.0,-1.0,-1.0,110.267,3.13875,2475,7047862 +1114,search,8227.49,2585.32,2903.33,0.98516,264.455,2.1635,8.72425,0.93215,0.100128,-1.0,-1.0,75.467,3.13917,2480,7047862 +1115,insert,108.876,-1.0,-1.0,-1.0,-1.0,157.615,3.16479,2498,7110727 +1116,search,8265.84,2588.63,2905.43,1.01667,267.892,2.1452,9.31463,0.93261,0.099842,-1.0,-1.0,48.156,3.16493,2499,7110727 +1117,insert,31.147,-1.0,-1.0,-1.0,-1.0,47.292,3.17173,2502,7125591 +1118,search,8348.05,2591.86,2959.33,0.972743,274.021,2.26774,9.44944,0.93263,0.0996389,-1.0,-1.0,39.234,3.17191,2503,7125591 +1119,insert,21.109,-1.0,-1.0,-1.0,-1.0,25.36,3.1762,2503,7134442 +1120,search,8322.62,2595.7,2920.23,0.996233,270.302,2.15025,8.38976,0.9327,0.099598,-1.0,-1.0,28.101,3.1762,2503,7134442 +1121,insert,48.96,-1.0,-1.0,-1.0,-1.0,84.413,3.18699,2509,7162109 +1122,search,8309.44,2598.74,2906.81,1.02333,265.355,2.15523,8.75217,0.93313,0.099248,-1.0,-1.0,47.748,3.18722,2512,7162109 +1123,insert,33.149,-1.0,-1.0,-1.0,-1.0,58.422,3.19383,2516,7179509 +1124,search,8257.33,2594.43,2855.83,1.06318,256.102,2.12271,9.14633,0.93323,0.0994766,-1.0,-1.0,27.001,3.19383,2516,7179509 +1125,insert,12.134,-1.0,-1.0,-1.0,-1.0,35.894,3.19531,2517,7186372 +1126,search,8384.15,2595.5,2973.25,1.00888,274.72,2.15303,8.77167,0.93317,0.0995134,-1.0,-1.0,27.106,3.19531,2517,7186372 +1127,insert,81.164,-1.0,-1.0,-1.0,-1.0,120.974,3.21424,2527,7231450 +1128,search,8449.17,2602.29,2993.9,1.02349,275.465,2.41981,8.68664,0.93341,0.0992027,-1.0,-1.0,50.695,3.21431,2530,7231450 +1129,insert,24.325,-1.0,-1.0,-1.0,-1.0,50.464,3.2181,2533,7243215 +1130,search,8428.27,2599.81,2983.01,1.02412,273.939,2.14069,8.19819,0.93322,0.0993687,-1.0,-1.0,26.493,3.2181,2533,7243215 +1131,insert,67.543,-1.0,-1.0,-1.0,-1.0,115.404,3.22947,2543,7276107 +1132,search,8479.75,2599.53,3006.24,1.02199,273.602,2.66186,8.01478,0.93364,0.0989482,-1.0,-1.0,50.033,3.22975,2546,7276107 +1133,insert,30.026,-1.0,-1.0,-1.0,-1.0,86.305,3.23697,2556,7292577 +1134,search,8491.8,2590.05,2995.22,1.01039,272.615,2.13527,8.43897,0.93345,0.099196,-1.0,-1.0,40.685,3.23706,2557,7292577 +1135,insert,15.87,-1.0,-1.0,-1.0,-1.0,44.669,3.24022,2559,7299867 +1136,search,8529.86,2589.51,3007.32,1.00594,278.653,2.48836,8.70124,0.93346,0.099183,-1.0,-1.0,29.896,3.24022,2559,7299867 +1137,insert,56.251,-1.0,-1.0,-1.0,-1.0,80.555,3.2504,2564,7323342 +1138,search,8592.42,2593.08,3038.9,0.984682,286.428,2.18722,8.53579,0.93354,0.0991463,-1.0,-1.0,48.088,3.25084,2566,7323342 +1139,insert,4.8,-1.0,-1.0,-1.0,-1.0,37.439,3.25172,2567,7325844 +1140,search,8616.55,2590.15,3039.69,1.00264,287.025,2.15103,8.82263,0.93354,0.0991866,-1.0,-1.0,46.703,3.25193,2569,7325844 +1141,insert,423.832,-1.0,-1.0,-1.0,-1.0,426.05,3.34547,2617,7540761 +1142,search,8778.75,2613.08,3120.94,1.01143,297.671,2.18466,8.55391,0.93396,0.0991054,-1.0,-1.0,138.292,3.34652,2631,7540761 +1143,insert,36.093,-1.0,-1.0,-1.0,-1.0,38.842,3.35341,2633,7561926 +1144,search,8788.22,2602.1,3121.4,0.982615,287.378,2.15802,8.15982,0.93393,0.0992191,-1.0,-1.0,37.27,3.35353,2634,7561926 +1145,insert,19.02,-1.0,-1.0,-1.0,-1.0,56.644,3.35803,2638,7571861 +1146,search,8807.89,2601.59,3122.14,0.98747,289.121,2.31683,9.95924,0.93405,0.0991074,-1.0,-1.0,27.304,3.35803,2638,7571861 +1147,insert,39.269,-1.0,-1.0,-1.0,-1.0,66.198,3.36543,2643,7590881 +1148,search,8795.11,2604.76,3110.14,1.02546,283.791,2.76206,8.36969,0.93432,0.0990081,-1.0,-1.0,47.576,3.36584,2646,7590881 +1149,insert,9.048,-1.0,-1.0,-1.0,-1.0,30.329,3.36792,2647,7595507 +1150,search,8838.3,2601.16,3123.82,1.02953,283.12,2.1243,8.21915,0.93439,0.0988965,-1.0,-1.0,53.458,3.36809,2649,7595507 +1151,insert,10.664,-1.0,-1.0,-1.0,-1.0,34.696,3.3701,2650,7601664 +1152,search,8832.41,2600.04,3131.21,0.970351,286.02,2.18963,8.21748,0.9343,0.098989,-1.0,-1.0,30.235,3.3701,2650,7601664 +1153,delete,68.866,-1.0,-1.0,-1.0,-1.0,204.117,3.36002,2643,7566707 +1154,search,8850.05,2597.13,3136.14,0.98935,298.083,2.15297,9.78273,0.93438,0.0987966,-1.0,-1.0,56.484,3.36028,2644,7566707 +1155,delete,236.348,-1.0,-1.0,-1.0,-1.0,1083.99,3.29543,2591,7396192 +1156,search,8680.54,2592.99,3080.4,0.962204,288.968,2.18683,8.93514,0.93412,0.098891,-1.0,-1.0,118.133,3.29736,2599,7396192 +1157,delete,418.416,-1.0,-1.0,-1.0,-1.0,2002.67,3.17507,2469,7079041 +1158,search,8303.25,2574.97,2903.55,0.966921,276.495,2.60265,9.10669,0.93211,0.100272,-1.0,-1.0,252.003,3.17828,2498,7079041 +1159,delete,536.547,-1.0,-1.0,-1.0,-1.0,1484.84,2.97457,2380,6597115 +1160,search,7774.59,2492.97,2624.93,1.07038,240.469,2.26569,8.47282,0.92999,0.100221,-1.0,-1.0,187.36,2.97698,2397,6597115 +1161,delete,337.732,-1.0,-1.0,-1.0,-1.0,2166.29,2.84818,2258,6308902 +1162,search,7520.77,2508.79,2608.03,0.995052,245.47,2.18565,8.6061,0.92864,0.101373,-1.0,-1.0,335.975,2.85287,2295,6308902 +1163,delete,136.688,-1.0,-1.0,-1.0,-1.0,817.633,2.78895,2243,6164514 +1164,search,7320.98,2476.86,2481.19,1.03219,225.982,2.14087,9.42489,0.92886,0.100853,-1.0,-1.0,128.804,2.79054,2254,6164514 +1165,delete,44.036,-1.0,-1.0,-1.0,-1.0,227.807,2.77722,2243,6133412 +1166,search,7386.5,2464.41,2534.17,0.980176,235.754,2.15865,8.63389,0.92845,0.1016,-1.0,-1.0,50.829,2.77729,2244,6133412 +1167,delete,118.686,-1.0,-1.0,-1.0,-1.0,276.188,2.76201,2230,6057218 +1168,search,7345.74,2442.93,2517.31,0.989311,235.722,2.16717,9.31349,0.92803,0.101511,-1.0,-1.0,66.616,2.76242,2233,6057218 +1169,delete,31.984,-1.0,-1.0,-1.0,-1.0,186.599,2.74694,2220,6028053 +1170,search,7353.55,2440.47,2518.85,0.986848,242.159,2.21377,8.11927,0.92571,0.103154,-1.0,-1.0,62.332,2.74772,2223,6028053 +1171,delete,41.888,-1.0,-1.0,-1.0,-1.0,112.36,2.74254,2219,6001880 +1172,search,7299.88,2432.86,2480.59,1.00035,236.5,2.17068,8.5249,0.92544,0.1035,-1.0,-1.0,50.935,2.74281,2221,6001880 +1173,delete,64.641,-1.0,-1.0,-1.0,-1.0,202.7,2.73092,2212,5957180 +1174,search,7295.25,2417.84,2474.82,0.976915,239.802,2.22869,8.81107,0.92548,0.103461,-1.0,-1.0,43.914,2.73101,2213,5957180 +1175,delete,25.011,-1.0,-1.0,-1.0,-1.0,56.766,2.72957,2212,5939326 +1176,search,7280.88,2410.33,2466.99,0.993166,235.49,2.15326,8.75915,0.92553,0.103741,-1.0,-1.0,32.33,2.72957,2212,5939326 +1177,delete,18.597,-1.0,-1.0,-1.0,-1.0,105.262,2.72489,2209,5924145 +1178,search,7100.12,2408.54,2333.83,1.04793,212.658,2.33259,8.88717,0.92562,0.103731,-1.0,-1.0,42.945,2.72519,2210,5924145 +1179,delete,87.385,-1.0,-1.0,-1.0,-1.0,403.625,2.69823,2186,5863334 +1180,search,7182.42,2405.01,2434.49,0.984646,226.327,2.13463,8.11194,0.92469,0.10493,-1.0,-1.0,51.371,2.69848,2188,5863334 +1181,delete,52.775,-1.0,-1.0,-1.0,-1.0,51.987,2.69584,2186,5822703 +1182,search,7145.48,2386.21,2414.78,0.99347,232.556,2.18994,8.57101,0.92444,0.10523,-1.0,-1.0,31.713,2.69584,2186,5822703 +1183,delete,61.266,-1.0,-1.0,-1.0,-1.0,212.639,2.68118,2175,5775807 +1184,search,7111.96,2374.78,2394.73,0.976828,221.976,2.14321,8.4456,0.92415,0.105565,-1.0,-1.0,40.86,2.68137,2176,5775807 +1185,delete,30.17,-1.0,-1.0,-1.0,-1.0,138.09,2.67242,2169,5750627 +1186,search,7071.92,2369.37,2372.85,0.965385,230.739,2.11185,8.80743,0.9239,0.105678,-1.0,-1.0,40.156,2.67246,2170,5750627 +1187,delete,199.831,-1.0,-1.0,-1.0,-1.0,1214.58,2.599,2097,5601207 +1188,search,6718.89,2374.97,2204.14,1.03937,202.028,2.12529,9.25849,0.92341,0.105775,-1.0,-1.0,135.505,2.60014,2107,5601207 +1189,delete,17.813,-1.0,-1.0,-1.0,-1.0,62.353,2.59614,2104,5586425 +1190,search,6858.76,2364.43,2295.25,1.01699,217.64,2.16081,8.04666,0.92319,0.106168,-1.0,-1.0,32.775,2.59614,2104,5586425 +1191,delete,25.151,-1.0,-1.0,-1.0,-1.0,50.628,2.5947,2103,5574023 +1192,search,6862.22,2361.23,2290.05,1.00615,213.94,2.14885,8.88988,0.92308,0.106178,-1.0,-1.0,32.004,2.5947,2103,5574023 +1193,delete,316.215,-1.0,-1.0,-1.0,-1.0,1417.33,2.48682,2010,5356157 +1194,search,6550.67,2372.81,2183.28,0.999023,200.438,2.13031,7.7997,0.92139,0.106661,-1.0,-1.0,155.197,2.48885,2023,5356157 +1195,delete,19.081,-1.0,-1.0,-1.0,-1.0,56.022,2.48626,2021,5340404 +1196,search,6606.53,2350.83,2211.54,0.992503,206.99,2.13492,8.47843,0.92184,0.106419,-1.0,-1.0,29.326,2.48626,2021,5340404 +1197,delete,69.986,-1.0,-1.0,-1.0,-1.0,65.151,2.47975,2018,5293259 +1198,search,6565.85,2325.21,2198.47,0.968174,196.12,2.14395,8.81075,0.92156,0.10691,-1.0,-1.0,37.636,2.47983,2019,5293259 +1199,delete,19.934,-1.0,-1.0,-1.0,-1.0,70.466,2.47603,2016,5279787 +1200,search,6563.81,2323.13,2177.66,0.978823,208.533,2.14924,8.15585,0.92156,0.106676,-1.0,-1.0,28.816,2.47603,2016,5279787 +1201,delete,99.59,-1.0,-1.0,-1.0,-1.0,396.232,2.44703,1991,5216567 +1202,search,6473.48,2322.65,2155.26,1.01051,198.632,2.13605,7.92637,0.92084,0.106698,-1.0,-1.0,48.864,2.44735,1993,5216567 +1203,delete,30.309,-1.0,-1.0,-1.0,-1.0,34.234,2.44509,1992,5196947 +1204,search,6298.73,2310.83,1995.33,1.04849,184.537,2.15693,11.1989,0.92095,0.106983,-1.0,-1.0,48.505,2.4455,1994,5196947 +1205,delete,35.38,-1.0,-1.0,-1.0,-1.0,70.188,2.43607,1990,5168447 +1206,search,6458.18,2302.29,2139.23,0.996488,201.078,2.16621,8.11527,0.92114,0.106771,-1.0,-1.0,27.393,2.43607,1990,5168447 +1207,delete,32.69,-1.0,-1.0,-1.0,-1.0,124.936,2.42835,1983,5146927 +1208,search,6403.32,2297.06,2118.6,0.991606,191.864,2.17245,8.41306,0.92037,0.106848,-1.0,-1.0,36.666,2.42842,1984,5146927 +1209,delete,397.097,-1.0,-1.0,-1.0,-1.0,1232.16,2.29078,1890,4887612 +1210,search,6112.64,2294.33,2005.04,0.993505,184.3,2.15564,8.57613,0.91901,0.107401,-1.0,-1.0,95.233,2.29168,1897,4887612 +1211,delete,6.832,-1.0,-1.0,-1.0,-1.0,25.215,2.29117,1897,4882301 +1212,search,6091.61,2284.64,1993.29,1.04392,166.646,2.13751,8.17661,0.9195,0.106914,-1.0,-1.0,25.678,2.29117,1897,4882301 +1213,delete,154.299,-1.0,-1.0,-1.0,-1.0,620.913,2.23123,1849,4780915 +1214,search,6005.87,2290.74,1981.86,1.01102,183.571,3.25375,8.26584,0.91774,0.10801,-1.0,-1.0,111.457,2.23248,1857,4780915 +1215,delete,20.173,-1.0,-1.0,-1.0,-1.0,37.595,2.22995,1856,4765751 +1216,search,6011.65,2277.24,1965.44,0.97134,180.597,2.13647,8.65659,0.91789,0.108124,-1.0,-1.0,28.595,2.22995,1856,4765751 +1217,delete,521.421,-1.0,-1.0,-1.0,-1.0,2111.75,2.05037,1665,4452306 +1218,search,5489.22,2367.15,1858.73,0.962896,163.702,2.31055,8.63721,0.91326,0.112271,-1.0,-1.0,239.942,2.0534,1684,4452306 +1219,delete,133.451,-1.0,-1.0,-1.0,-1.0,629.542,1.9967,1619,4362045 +1220,search,5291.67,2391.74,1770.63,1.02857,167.954,3.02156,9.10938,0.9122,0.112757,-1.0,-1.0,148.189,1.99781,1633,4362045 +1221,delete,50.447,-1.0,-1.0,-1.0,-1.0,61.356,1.99188,1631,4326917 +1222,search,5364.05,2352.7,1806.49,0.99497,173.191,2.19386,8.96917,0.91226,0.112442,-1.0,-1.0,36.859,1.99219,1632,4326917 +1223,delete,300.099,-1.0,-1.0,-1.0,-1.0,916.258,1.89274,1546,4109781 +1224,search,5006.23,2359.62,1633.66,1.04319,152.707,2.1842,8.94606,0.90936,0.114209,-1.0,-1.0,171.852,1.89507,1562,4109781 +1225,delete,36.257,-1.0,-1.0,-1.0,-1.0,171.235,1.88099,1551,4075879 +1226,search,5083.97,2333.11,1714.77,0.98236,165.533,2.18253,9.51206,0.90875,0.114565,-1.0,-1.0,54.417,1.8813,1554,4075879 +1227,delete,100.602,-1.0,-1.0,-1.0,-1.0,583.007,1.84235,1515,4006503 +1228,search,4866.05,2350.25,1567.47,1.03092,138.5,2.13518,9.30138,0.90876,0.114978,-1.0,-1.0,82.301,1.84321,1521,4006503 +1229,delete,125.394,-1.0,-1.0,-1.0,-1.0,640.612,1.79824,1474,3922625 +1230,search,4877.85,2368.24,1650.27,0.955768,154.114,2.15594,8.6413,0.90851,0.114635,-1.0,-1.0,126.253,1.79979,1484,3922625 +1231,delete,99.44,-1.0,-1.0,-1.0,-1.0,431.177,1.75941,1430,3844860 +1232,search,4751.07,2406.62,1631.78,0.963039,143.653,2.46565,8.26658,0.9063,0.116154,-1.0,-1.0,123.343,1.76072,1441,3844860 +1233,delete,13.486,-1.0,-1.0,-1.0,-1.0,58.084,1.75619,1438,3831414 +1234,search,4653.62,2377.47,1533.62,1.0262,139.457,2.16057,8.34852,0.90584,0.116784,-1.0,-1.0,44.237,1.75641,1441,3831414 +1235,delete,195.811,-1.0,-1.0,-1.0,-1.0,754.52,1.68922,1381,3684234 +1236,search,4425.03,2382.03,1411.41,1.08131,130.183,2.1329,9.38378,0.90446,0.11824,-1.0,-1.0,102.78,1.69005,1391,3684234 +1237,delete,16.329,-1.0,-1.0,-1.0,-1.0,35.409,1.68712,1389,3669074 +1238,search,4576.62,2352.8,1557.91,0.936967,134.43,2.19682,8.31286,0.90425,0.118618,-1.0,-1.0,36.442,1.6873,1391,3669074 +1239,delete,66.4,-1.0,-1.0,-1.0,-1.0,142.62,1.67246,1379,3620984 +1240,search,4511.13,2330.83,1513.76,0.971475,138.883,2.22228,7.95777,0.90428,0.118788,-1.0,-1.0,18.428,1.67246,1379,3620984 +1241,delete,46.85,-1.0,-1.0,-1.0,-1.0,106.309,1.66395,1372,3580113 +1242,search,4480.58,2310.22,1498.17,0.966252,144.443,2.15409,9.31357,0.90353,0.119294,-1.0,-1.0,29.14,1.66397,1373,3580113 +1243,delete,64.744,-1.0,-1.0,-1.0,-1.0,279.193,1.64026,1350,3533909 +1244,search,4362.27,2309.77,1413.16,1.07115,131.279,2.32109,8.46927,0.90272,0.120204,-1.0,-1.0,76.601,1.64112,1355,3533909 +1245,delete,18.439,-1.0,-1.0,-1.0,-1.0,75.435,1.63507,1350,3519590 +1246,search,4363.68,2300.24,1412.06,1.07546,129.085,2.14697,8.53118,0.90267,0.120432,-1.0,-1.0,25.593,1.63517,1351,3519590 +1247,delete,80.222,-1.0,-1.0,-1.0,-1.0,187.151,1.60103,1320,3452537 +1248,search,4349.76,2317.74,1468.17,0.972276,126.791,2.11869,8.63661,0.90378,0.119014,-1.0,-1.0,63.737,1.60146,1326,3452537 +1249,delete,43.841,-1.0,-1.0,-1.0,-1.0,136.683,1.58855,1317,3416711 +1250,search,4336.98,2296.07,1442.15,1.01008,135.092,2.20769,9.48661,0.90293,0.119684,-1.0,-1.0,19.881,1.58855,1317,3416711 +1251,delete,51.841,-1.0,-1.0,-1.0,-1.0,111.999,1.57763,1309,3387653 +1252,search,4224.94,2286.67,1356.92,1.04901,124.342,3.34634,9.24213,0.90222,0.12002,-1.0,-1.0,41.987,1.57791,1311,3387653 +1253,delete,28.731,-1.0,-1.0,-1.0,-1.0,90.291,1.5696,1306,3372157 +1254,search,4278.74,2283.63,1408.9,1.01278,139.848,2.21069,8.49129,0.90185,0.120228,-1.0,-1.0,60.064,1.57001,1309,3372157 +1255,delete,177.912,-1.0,-1.0,-1.0,-1.0,241.897,1.51516,1269,3258215 +1256,search,4144.53,2278.21,1357.54,0.973876,115.543,2.13518,8.00427,0.89954,0.121339,-1.0,-1.0,55.068,1.51557,1272,3258215 +1257,delete,54.933,-1.0,-1.0,-1.0,-1.0,141.113,1.49705,1261,3223657 +1258,search,4157.8,2270.64,1378.03,0.991776,133.691,2.19949,9.26506,0.89845,0.122,-1.0,-1.0,42.57,1.49762,1263,3223657 +1259,delete,76.94,-1.0,-1.0,-1.0,-1.0,250.689,1.47133,1242,3174982 +1260,search,4095.87,2268.54,1350.1,1.00226,126.191,2.20993,8.80446,0.89753,0.122548,-1.0,-1.0,42.954,1.47156,1244,3174982 +1261,delete,26.807,-1.0,-1.0,-1.0,-1.0,40.723,1.46808,1242,3157038 +1262,search,4097.29,2253.26,1357.05,0.926059,115.132,2.15699,9.06416,0.89733,0.12236,-1.0,-1.0,30.647,1.46836,1243,3157038 +1263,delete,149.843,-1.0,-1.0,-1.0,-1.0,562.257,1.41519,1204,3058847 +1264,search,3972.09,2264.58,1324.71,0.968248,113.942,2.1321,9.04302,0.89618,0.122633,-1.0,-1.0,100.77,1.4165,1211,3058847 +1265,delete,406.985,-1.0,-1.0,-1.0,-1.0,729.207,1.29469,1091,2833531 +1266,search,3652.77,2315.97,1236.08,1.01118,112.115,2.9355,8.69796,0.89078,0.127057,-1.0,-1.0,170.448,1.29688,1106,2833531 +1267,delete,83.33,-1.0,-1.0,-1.0,-1.0,202.824,1.27274,1091,2763793 +1268,search,3551.21,2251.44,1150.54,1.02867,99.2539,2.12604,8.52954,0.89016,0.127259,-1.0,-1.0,45.289,1.27301,1093,2763793 +1269,delete,226.998,-1.0,-1.0,-1.0,-1.0,452.722,1.20637,1050,2627384 +1270,search,3499.6,2268.77,1160.19,0.939111,118.578,2.2162,8.89959,0.89077,0.126459,-1.0,-1.0,79.849,1.20738,1056,2627384 +1271,delete,94.466,-1.0,-1.0,-1.0,-1.0,357.027,1.17388,1026,2553476 +1272,search,3363.25,2261.5,1110.01,0.979234,99.7522,2.51756,7.98769,0.89049,0.126738,-1.0,-1.0,82.731,1.17454,1032,2553476 +1273,delete,168.322,-1.0,-1.0,-1.0,-1.0,437.141,1.11887,986,2443067 +1274,search,3257.61,2261.06,1083.46,0.94722,105.22,2.20769,9.19572,0.88802,0.12879,-1.0,-1.0,97.403,1.12011,993,2443067 +1275,delete,25.389,-1.0,-1.0,-1.0,-1.0,74.769,1.11408,988,2421654 +1276,search,3276.58,2240.05,1097.34,0.944785,93.662,2.22772,6.83604,0.88821,0.128379,-1.0,-1.0,18.462,1.11408,988,2421654 +1277,delete,40.105,-1.0,-1.0,-1.0,-1.0,87.168,1.10525,980,2391396 +1278,search,3254.76,2227.85,1085.88,0.942998,93.0007,2.13983,7.99082,0.88808,0.12828,-1.0,-1.0,18.699,1.10525,980,2391396 +1279,delete,29.042,-1.0,-1.0,-1.0,-1.0,45.529,1.10224,977,2367854 +1280,search,3211.46,2212.09,1066.97,0.918439,94.3906,2.1755,9.79768,0.8877,0.12839,-1.0,-1.0,18.377,1.10224,977,2367854 diff --git a/scripts/big_ann_perf_numbers/perf_debug_scan_0.15_fine_grain_analyze.csv b/scripts/big_ann_perf_numbers/perf_debug_scan_0.15_fine_grain_analyze.csv new file mode 100644 index 00000000..c98f38c7 --- /dev/null +++ b/scripts/big_ann_perf_numbers/perf_debug_scan_0.15_fine_grain_analyze.csv @@ -0,0 +1,11 @@ +step_num,step_type,latency_ms,worker_partition_size,worker_scan_time_ms,worker_scan_throughput,worker_result_time_ms,measured_ipc,cache_miss_rate,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,2820.94,37.4413,340.143,3.62873,398.433,-nan,-nan,0.8551,0.130106,-1.0,-1.0,6.414,0.0184881,1000,38806 +3,insert,10.982,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.309,0.022655,996,49334 +4,search,2945.64,64.8478,374.049,3.25734,508.487,-nan,-nan,0.86686,0.128427,-1.0,-1.0,15.332,0.0226615,993,49334 +5,insert,17.938,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,162.509,0.032935,982,73557 +6,search,3501.53,185.758,686.185,3.07149,778.034,-nan,-nan,0.92586,0.10314,-1.0,-1.0,126.549,0.0330859,984,73557 +7,insert,30.113,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,123.09,0.0427351,986,95766 +8,search,3735.66,233.069,813.092,2.83407,881.965,-nan,-nan,0.92464,0.103834,-1.0,-1.0,32.819,0.0433149,992,95766 +9,insert,6.21,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,6.485,0.0451264,992,100025 +10,search,3843.37,236.809,818.361,2.62811,975.12,-nan,-nan,0.92487,0.102107,-1.0,-1.0,19.174,0.0451705,997,100025 +11,insert,8.018,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.338,0.0496899,999,111417 diff --git a/scripts/big_ann_perf_numbers/perf_debug_scan_0.15_no_aps_refinment_wma_delete.csv b/scripts/big_ann_perf_numbers/perf_debug_scan_0.15_no_aps_refinment_wma_delete.csv new file mode 100644 index 00000000..5da7f512 --- /dev/null +++ b/scripts/big_ann_perf_numbers/perf_debug_scan_0.15_no_aps_refinment_wma_delete.csv @@ -0,0 +1,76 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,3501.12,0.8551,0.130106,-1.0,-1.0,7.247,0.0184881,1000,38806 +3,insert,21.437,-1.0,-1.0,-1.0,-1.0,11.994,0.0226489,997,49334 +4,search,2653.89,0.86623,0.128786,-1.0,-1.0,10.61,0.0226285,996,49334 +5,insert,16.162,-1.0,-1.0,-1.0,-1.0,163.62,0.0332675,987,73557 +6,search,2526.55,0.92564,0.103451,-1.0,-1.0,109.996,0.0333764,990,73557 +7,insert,23.905,-1.0,-1.0,-1.0,-1.0,107.795,0.043033,992,95766 +8,search,2533.96,0.92473,0.103666,-1.0,-1.0,89.323,0.0432141,995,95766 +9,insert,4.134,-1.0,-1.0,-1.0,-1.0,78.416,0.0448731,995,100025 +10,search,2553.3,0.92668,0.100654,-1.0,-1.0,83.164,0.0448873,997,100025 +11,insert,9.317,-1.0,-1.0,-1.0,-1.0,84.32,0.0497899,999,111417 +12,search,2556.17,0.92582,0.10188,-1.0,-1.0,82.133,0.0497899,999,111417 +13,insert,18.923,-1.0,-1.0,-1.0,-1.0,82.91,0.0609009,1000,137139 +14,search,2607.65,0.92274,0.103788,-1.0,-1.0,16.785,0.0611229,1001,137139 +15,insert,16.964,-1.0,-1.0,-1.0,-1.0,8.969,0.0699797,1002,156710 +16,search,2617.5,0.92112,0.104544,-1.0,-1.0,16.214,0.0699822,1004,156710 +17,insert,3.751,-1.0,-1.0,-1.0,-1.0,8.187,0.0719859,1005,161773 +18,search,2628.55,0.92122,0.104633,-1.0,-1.0,17.882,0.0722066,1009,161773 +19,insert,30.402,-1.0,-1.0,-1.0,-1.0,10.545,0.079079,1011,177161 +20,search,2688.58,0.92442,0.101885,-1.0,-1.0,14.093,0.0790826,1014,177161 +21,insert,357.587,-1.0,-1.0,-1.0,-1.0,170.596,0.200924,1060,448898 +22,search,2967.15,0.91162,0.106615,-1.0,-1.0,51.614,0.201311,1073,448898 +23,insert,352.27,-1.0,-1.0,-1.0,-1.0,254.625,0.329156,1128,733493 +24,search,3238.16,0.90717,0.10925,-1.0,-1.0,61.716,0.329255,1140,733493 +25,insert,36.016,-1.0,-1.0,-1.0,-1.0,20.868,0.346707,1144,777443 +26,search,3280.1,0.90784,0.108781,-1.0,-1.0,26.4,0.346754,1148,777443 +27,insert,35.986,-1.0,-1.0,-1.0,-1.0,16.827,0.357795,1149,805394 +28,search,3339.07,0.90791,0.108498,-1.0,-1.0,18.188,0.358026,1151,805394 +29,insert,60.352,-1.0,-1.0,-1.0,-1.0,48.879,0.385613,1160,866391 +30,search,3390.57,0.91036,0.107782,-1.0,-1.0,23.151,0.385612,1162,866391 +31,insert,457.127,-1.0,-1.0,-1.0,-1.0,331.116,0.554438,1231,1239816 +32,search,3743.85,0.90345,0.113196,-1.0,-1.0,56.109,0.55473,1241,1239816 +33,insert,44.652,-1.0,-1.0,-1.0,-1.0,34.628,0.574174,1249,1291211 +34,search,3793.6,0.9031,0.113418,-1.0,-1.0,9.607,0.574174,1249,1291211 +35,insert,30.328,-1.0,-1.0,-1.0,-1.0,23.274,0.586072,1253,1324978 +36,search,3796.75,0.90438,0.112151,-1.0,-1.0,9.496,0.586072,1253,1324978 +37,insert,42.266,-1.0,-1.0,-1.0,-1.0,25.146,0.605461,1256,1371885 +38,search,3897.39,0.90477,0.111023,-1.0,-1.0,18.871,0.605461,1257,1371885 +39,insert,14.817,-1.0,-1.0,-1.0,-1.0,13.322,0.608401,1258,1379335 +40,search,3869.86,0.90455,0.111586,-1.0,-1.0,10.388,0.608401,1258,1379335 +41,insert,55.458,-1.0,-1.0,-1.0,-1.0,32.478,0.624165,1263,1413312 +42,search,3907.2,0.90539,0.111602,-1.0,-1.0,16.726,0.62419,1264,1413312 +43,insert,12.375,-1.0,-1.0,-1.0,-1.0,9.321,0.627996,1264,1423411 +44,search,3908.39,0.90555,0.111557,-1.0,-1.0,9.803,0.627996,1264,1423411 +45,insert,210.399,-1.0,-1.0,-1.0,-1.0,37.258,0.642859,1272,1457559 +46,search,3943.63,0.90588,0.111224,-1.0,-1.0,9.4,0.642859,1272,1457559 +47,insert,268.08,-1.0,-1.0,-1.0,-1.0,260.586,0.747977,1309,1688593 +48,search,4162.47,0.9073,0.111488,-1.0,-1.0,73.613,0.74809,1316,1688593 +49,insert,36.419,-1.0,-1.0,-1.0,-1.0,33.041,0.763532,1321,1723505 +50,search,4195.21,0.90795,0.111388,-1.0,-1.0,35.026,0.763612,1324,1723505 +51,insert,324.511,-1.0,-1.0,-1.0,-1.0,281.385,0.904526,1374,2035040 +52,search,4524.53,0.91046,0.109908,-1.0,-1.0,62.904,0.904558,1381,2035040 +53,insert,50.771,-1.0,-1.0,-1.0,-1.0,35.12,0.921856,1385,2078341 +54,search,4525.94,0.91036,0.109832,-1.0,-1.0,17.299,0.922113,1386,2078341 +55,insert,396.289,-1.0,-1.0,-1.0,-1.0,400.726,1.06875,1440,2399905 +56,search,4907.83,0.91181,0.110055,-1.0,-1.0,133.135,1.06927,1461,2399905 +57,insert,26.12,-1.0,-1.0,-1.0,-1.0,27.496,1.07686,1464,2419836 +58,search,4651.68,0.91077,0.110844,-1.0,-1.0,51.209,1.0769,1470,2419836 +59,insert,232.29,-1.0,-1.0,-1.0,-1.0,109.939,1.1754,1483,2643083 +60,search,5000.83,0.90391,0.118459,-1.0,-1.0,34.102,1.17575,1487,2643083 +61,insert,12.219,-1.0,-1.0,-1.0,-1.0,15.476,1.18003,1488,2653064 +62,search,4923.67,0.90386,0.118465,-1.0,-1.0,17.959,1.18012,1489,2653064 +63,insert,3.41,-1.0,-1.0,-1.0,-1.0,10.734,1.18123,1489,2656118 +64,search,4742.82,0.9039,0.11847,-1.0,-1.0,11.125,1.18123,1489,2656118 +65,insert,72.96,-1.0,-1.0,-1.0,-1.0,94.552,1.20956,1508,2723424 +66,search,5103.51,0.90554,0.11719,-1.0,-1.0,11.279,1.20956,1508,2723424 +67,insert,46.114,-1.0,-1.0,-1.0,-1.0,78.987,1.23164,1523,2769021 +68,search,5131.05,0.90653,0.116421,-1.0,-1.0,24.94,1.23164,1526,2769021 +69,insert,50.266,-1.0,-1.0,-1.0,-1.0,57.509,1.25403,1535,2821777 +70,search,4863.58,0.90844,0.114447,-1.0,-1.0,23.343,1.25403,1536,2821777 +71,insert,25.548,-1.0,-1.0,-1.0,-1.0,35.735,1.26041,1540,2834921 +72,search,5273.28,0.90898,0.114243,-1.0,-1.0,11.676,1.26041,1540,2834921 +73,insert,6.716,-1.0,-1.0,-1.0,-1.0,36.706,1.26336,1544,2842044 +74,search,4944.68,0.90877,0.114226,-1.0,-1.0,12.623,1.26336,1544,2842044 +75,insert,25.716,-1.0,-1.0,-1.0,-1.0,24.623,1.26947,1546,2858890 +76,search,5254.27,0.90899,0.114142,-1.0,-1.0,12.304,1.26947,1546,2858890 diff --git a/scripts/big_ann_perf_numbers/perf_debug_scan_0.15_vary_batch_size.csv b/scripts/big_ann_perf_numbers/perf_debug_scan_0.15_vary_batch_size.csv new file mode 100644 index 00000000..c385b5b5 --- /dev/null +++ b/scripts/big_ann_perf_numbers/perf_debug_scan_0.15_vary_batch_size.csv @@ -0,0 +1,2101 @@ +batch_size,step_num,step_type,search_latency_ms,worker_partition_size,worker_scan_time_ms,worker_scan_throughput,worker_result_time_ms,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +1,2,search,3785,37.3093,123.379,27.6391,113.241,0.8551,0.130106,-1.0,-1.0,14.867,0.0184881,1000,38806 +1,3,insert,22.387,-1.0,-1.0,-1.0,-1.0,19.853,0.022655,996,49334 +1,4,search,3325.33,66.3453,498.773,15.8079,134.22,0.86686,0.128427,-1.0,-1.0,14.166,0.0226469,994,49334 +1,5,insert,17.132,-1.0,-1.0,-1.0,-1.0,178.984,0.0330329,986,73557 +1,6,search,2890.47,195.078,506.466,36.266,69.9005,0.92469,0.104049,-1.0,-1.0,126.666,0.0331418,987,73557 +1,7,insert,23.973,-1.0,-1.0,-1.0,-1.0,123.843,0.0425911,988,95766 +1,8,search,3110.59,247.911,652.613,34.9803,85.6366,0.92524,0.103379,-1.0,-1.0,20.529,0.0431778,991,95766 +1,9,insert,3.424,-1.0,-1.0,-1.0,-1.0,5.196,0.0448706,991,100025 +1,10,search,2884.13,255.196,571.217,36.791,61.7057,0.92675,0.10077,-1.0,-1.0,15.297,0.0448853,993,100025 +1,11,insert,9.188,-1.0,-1.0,-1.0,-1.0,8.919,0.049778,994,111417 +1,12,search,2938.43,297.254,608.654,39.9052,56.7331,0.92675,0.101492,-1.0,-1.0,6.847,0.049778,994,111417 +1,13,insert,18.772,-1.0,-1.0,-1.0,-1.0,8.226,0.0609625,995,137139 +1,14,search,3020.59,341.923,685.157,40.5329,63.1383,0.92349,0.103267,-1.0,-1.0,6.957,0.0609625,995,137139 +1,15,insert,14.916,-1.0,-1.0,-1.0,-1.0,6.711,0.0694257,995,156710 +1,16,search,3104.07,382.93,752.239,41.562,59.1598,0.92282,0.103406,-1.0,-1.0,9.077,0.0694257,995,156710 +1,17,insert,8.896,-1.0,-1.0,-1.0,-1.0,11.447,0.0717203,996,161773 +1,18,search,3161.34,402.544,821.252,40.8878,60.7769,0.92393,0.102988,-1.0,-1.0,7.068,0.0717203,996,161773 +1,19,insert,30.933,-1.0,-1.0,-1.0,-1.0,10.81,0.0786942,998,177161 +1,20,search,3227.24,445.222,884.604,41.1437,65.1267,0.92707,0.100522,-1.0,-1.0,6.789,0.0786942,998,177161 +1,21,insert,314.978,-1.0,-1.0,-1.0,-1.0,114.326,0.199732,1021,448898 +1,22,search,4248.28,964.817,1849.48,43.8901,77.0617,0.91928,0.101702,-1.0,-1.0,23.899,0.199731,1025,448898 +1,23,insert,366.365,-1.0,-1.0,-1.0,-1.0,242.808,0.328243,1061,733493 +1,24,search,5223.19,1355.54,2695.07,44.089,87.3084,0.91399,0.104408,-1.0,-1.0,85.314,0.328353,1074,733493 +1,25,insert,45.219,-1.0,-1.0,-1.0,-1.0,17.364,0.345328,1076,777443 +1,26,search,5245.3,1342.13,2710.74,44.9455,86.0274,0.91329,0.104989,-1.0,-1.0,24.434,0.345389,1078,777443 +1,27,insert,35.048,-1.0,-1.0,-1.0,-1.0,16.123,0.357044,1079,805394 +1,28,search,5288.02,1364.22,2781.47,44.2439,75.1515,0.91349,0.104892,-1.0,-1.0,10.499,0.357044,1079,805394 +1,29,insert,56.958,-1.0,-1.0,-1.0,-1.0,41.567,0.384589,1083,866391 +1,30,search,5548.82,1446.01,2984.79,43.7879,82.3114,0.91582,0.104342,-1.0,-1.0,25.692,0.384588,1085,866391 +1,31,insert,422.614,-1.0,-1.0,-1.0,-1.0,369.032,0.552831,1137,1239816 +1,32,search,6484.85,1754.01,3793.11,44.7031,95.442,0.90782,0.11124,-1.0,-1.0,98.829,0.553106,1145,1239816 +1,33,insert,48.956,-1.0,-1.0,-1.0,-1.0,26.499,0.572978,1147,1291211 +1,34,search,6065.62,1766.98,3830.63,43.4618,133.003,0.90787,0.110891,-1.0,-1.0,30.445,0.573056,1151,1291211 +1,35,insert,17.035,-1.0,-1.0,-1.0,-1.0,13.122,0.58561,1154,1324978 +1,36,search,6158.2,1805.8,3984.08,42.4427,142.275,0.90906,0.109653,-1.0,-1.0,16.141,0.585611,1156,1324978 +1,37,insert,22.374,-1.0,-1.0,-1.0,-1.0,15.824,0.606138,1159,1371885 +1,38,search,6214.9,1832.85,4103.54,42.1518,121.88,0.90837,0.109156,-1.0,-1.0,9.333,0.606138,1160,1371885 +1,39,insert,4.037,-1.0,-1.0,-1.0,-1.0,5.494,0.609136,1160,1379335 +1,40,search,6230.58,1838.05,4078.69,42.4024,138.418,0.90914,0.109277,-1.0,-1.0,5.65,0.609136,1160,1379335 +1,41,insert,18.99,-1.0,-1.0,-1.0,-1.0,24.053,0.624517,1167,1413312 +1,42,search,6290.95,1843.49,4104.89,42.5103,144.977,0.9092,0.109146,-1.0,-1.0,6.847,0.624517,1167,1413312 +1,43,insert,8.684,-1.0,-1.0,-1.0,-1.0,5.847,0.628827,1167,1423411 +1,44,search,6309.64,1860.65,4146.07,42.6457,147.334,0.90928,0.10923,-1.0,-1.0,9.242,0.628926,1168,1423411 +1,45,insert,122.838,-1.0,-1.0,-1.0,-1.0,23.429,0.644478,1174,1457559 +1,46,search,6409.69,1864.54,4236.68,42.2779,131.543,0.90922,0.108792,-1.0,-1.0,6.054,0.644478,1174,1457559 +1,47,insert,152.893,-1.0,-1.0,-1.0,-1.0,166.458,0.749589,1207,1688593 +1,48,search,6849.46,1993.82,4611.11,43.1549,143.331,0.91021,0.11023,-1.0,-1.0,45.272,0.749588,1213,1688593 +1,49,insert,23.842,-1.0,-1.0,-1.0,-1.0,19.125,0.76327,1217,1723505 +1,50,search,7514.42,1998.37,4690.69,43.6195,126.381,0.91018,0.110305,-1.0,-1.0,48.201,0.76327,1222,1723505 +1,51,insert,344.024,-1.0,-1.0,-1.0,-1.0,281.148,0.903601,1260,2035040 +1,52,search,8223.79,2179.1,5222.98,44.0729,133.694,0.91288,0.108774,-1.0,-1.0,45.215,0.903645,1266,2035040 +1,53,insert,47.757,-1.0,-1.0,-1.0,-1.0,43.854,0.919307,1271,2078341 +1,54,search,8576.35,2186.31,5404.67,43.383,176.64,0.91218,0.10951,-1.0,-1.0,16.129,0.919307,1272,2078341 +1,55,insert,341.624,-1.0,-1.0,-1.0,-1.0,460.007,1.06855,1324,2399905 +1,56,search,8978.33,2346.21,5802.4,44.668,131.651,0.91284,0.110422,-1.0,-1.0,113.359,1.06854,1339,2399905 +1,57,insert,21.295,-1.0,-1.0,-1.0,-1.0,29.739,1.07722,1342,2419836 +1,58,search,9327.29,2301.44,5997,42.8207,203.886,0.91248,0.110721,-1.0,-1.0,57.387,1.07727,1349,2419836 +1,59,insert,199.379,-1.0,-1.0,-1.0,-1.0,133.02,1.17411,1359,2643083 +1,60,search,10012.5,2354.41,6530.65,41.4958,232.145,0.90538,0.118027,-1.0,-1.0,37.425,1.17435,1363,2643083 +1,61,insert,9.867,-1.0,-1.0,-1.0,-1.0,19.637,1.17818,1365,2653064 +1,62,search,9832.52,2352.24,6385.87,42.6507,219.92,0.90556,0.117893,-1.0,-1.0,12.892,1.17818,1365,2653064 +1,63,insert,6.054,-1.0,-1.0,-1.0,-1.0,10.762,1.17953,1365,2656118 +1,64,search,10192.3,2354.98,6697.38,41.5597,251.818,0.9056,0.117788,-1.0,-1.0,11.898,1.17953,1365,2656118 +1,65,insert,98.426,-1.0,-1.0,-1.0,-1.0,94.471,1.20713,1379,2723424 +1,66,search,10039.7,2388.06,6549.52,42.8488,228.75,0.90587,0.116739,-1.0,-1.0,10.542,1.20713,1379,2723424 +1,67,insert,49.952,-1.0,-1.0,-1.0,-1.0,74.185,1.22813,1389,2769021 +1,68,search,10173.6,2412.89,6650.6,42.7387,230.066,0.90719,0.115754,-1.0,-1.0,21.209,1.22813,1391,2769021 +1,69,insert,58.218,-1.0,-1.0,-1.0,-1.0,94.162,1.253,1405,2821777 +1,70,search,10516,2409.48,6925.21,42.0111,239.788,0.9089,0.114001,-1.0,-1.0,20.128,1.253,1406,2821777 +1,71,insert,18.77,-1.0,-1.0,-1.0,-1.0,34.657,1.25883,1410,2834921 +1,72,search,10271.6,2409.78,6702.08,42.9864,236.982,0.90928,0.113768,-1.0,-1.0,10.665,1.25883,1410,2834921 +1,73,insert,8.635,-1.0,-1.0,-1.0,-1.0,16.66,1.26198,1411,2842044 +1,74,search,9740.84,2412.95,6254.09,45.6657,159.176,0.90928,0.113847,-1.0,-1.0,10.689,1.26198,1411,2842044 +1,75,insert,19.928,-1.0,-1.0,-1.0,-1.0,15.047,1.26802,1412,2858890 +1,76,search,10518.9,2424.08,6924.59,41.9049,253.551,0.9093,0.113967,-1.0,-1.0,12.707,1.26802,1412,2858890 +1,77,insert,54.962,-1.0,-1.0,-1.0,-1.0,10.745,1.27732,1412,2882818 +1,78,search,9844.91,2438.56,6487.59,44.8724,157.39,0.90886,0.114312,-1.0,-1.0,11.612,1.27732,1412,2882818 +1,79,insert,492.447,-1.0,-1.0,-1.0,-1.0,15.727,1.30319,1413,2947806 +1,80,search,9812.94,2473.56,6475.73,45.6723,148.727,0.90878,0.11491,-1.0,-1.0,11.012,1.30319,1413,2947806 +1,81,insert,38.534,-1.0,-1.0,-1.0,-1.0,34.242,1.31603,1417,2976895 +1,82,search,10782.6,2481.72,7169.91,42.2771,250.094,0.91008,0.114578,-1.0,-1.0,10.872,1.31603,1417,2976895 +1,83,insert,16.428,-1.0,-1.0,-1.0,-1.0,21.647,1.32033,1419,2990682 +1,84,search,10481.1,2486.87,6933.22,42.8404,234.899,0.91031,0.114278,-1.0,-1.0,10.855,1.32033,1419,2990682 +1,85,insert,36.714,-1.0,-1.0,-1.0,-1.0,66.408,1.33517,1428,3025613 +1,86,search,10958.9,2487.36,7264.06,41.5189,256.016,0.91121,0.113506,-1.0,-1.0,12.382,1.33517,1428,3025613 +1,87,insert,596.553,-1.0,-1.0,-1.0,-1.0,771.492,1.54412,1525,3468632 +1,88,search,10710.7,2625.43,6950.78,49.4382,152.406,0.91465,0.110027,-1.0,-1.0,102.949,1.54412,1536,3468632 +1,89,insert,388.993,-1.0,-1.0,-1.0,-1.0,546.679,1.70299,1621,3831692 +1,90,search,11460.6,2676.72,7457.92,49.8344,167.728,0.91857,0.105217,-1.0,-1.0,83.643,1.70312,1631,3831692 +1,91,insert,36.012,-1.0,-1.0,-1.0,-1.0,42.801,1.71484,1636,3864067 +1,92,search,11502.3,2666.19,7461.36,49.929,168.55,0.91854,0.104939,-1.0,-1.0,18.883,1.71484,1637,3864067 +1,93,insert,260.794,-1.0,-1.0,-1.0,-1.0,216.863,1.81984,1667,4103827 +1,94,search,11974.2,2730.51,7854.67,49.5306,169.955,0.91917,0.104363,-1.0,-1.0,43.275,1.82031,1671,4103827 +1,95,insert,15.929,-1.0,-1.0,-1.0,-1.0,25.121,1.82549,1673,4119898 +1,96,search,11664.1,2731.89,7601.26,51.2247,157.338,0.91886,0.104689,-1.0,-1.0,13.494,1.82549,1673,4119898 +1,97,insert,10.72,-1.0,-1.0,-1.0,-1.0,11.919,1.82885,1673,4129367 +1,98,search,11634.6,2738.02,7562.77,51.4114,157.134,0.91903,0.104409,-1.0,-1.0,22.273,1.82904,1674,4129367 +1,99,insert,7.491,-1.0,-1.0,-1.0,-1.0,35.034,1.83165,1677,4134502 +1,100,search,11260.7,2731.02,7110.69,53.8225,184.328,0.91916,0.104433,-1.0,-1.0,15.279,1.83165,1677,4134502 +1,101,insert,43.514,-1.0,-1.0,-1.0,-1.0,46.428,1.84224,1681,4164482 +1,102,search,11354.1,2738.92,7203.75,53.8262,182.13,0.9194,0.104258,-1.0,-1.0,36.416,1.84262,1683,4164482 +1,103,insert,23.342,-1.0,-1.0,-1.0,-1.0,28.387,1.84878,1685,4178601 +1,104,search,11405,2738.08,7285.04,53.7341,188.22,0.91927,0.104453,-1.0,-1.0,14.105,1.84878,1685,4178601 +1,105,insert,12.676,-1.0,-1.0,-1.0,-1.0,13.952,1.8525,1685,4187774 +1,106,search,11425.8,2742.9,7310.29,53.4763,186.545,0.9192,0.104682,-1.0,-1.0,14.422,1.8525,1685,4187774 +1,107,insert,23.042,-1.0,-1.0,-1.0,-1.0,14.452,1.85734,1685,4200109 +1,108,search,11383,2747.93,7264.48,53.8799,191.977,0.91913,0.104803,-1.0,-1.0,14.726,1.85734,1685,4200109 +1,109,insert,503.849,-1.0,-1.0,-1.0,-1.0,545.69,2.01627,1754,4557867 +1,110,search,12218.1,2831.26,7884.1,53.4149,196.997,0.92093,0.103273,-1.0,-1.0,59.129,2.01635,1759,4557867 +1,111,insert,31.183,-1.0,-1.0,-1.0,-1.0,57.71,2.02804,1766,4587252 +1,112,search,12158.6,2827.1,7805.16,54.0157,203.579,0.92092,0.10365,-1.0,-1.0,26.72,2.02804,1767,4587252 +1,113,insert,15.282,-1.0,-1.0,-1.0,-1.0,30.25,2.03355,1769,4600056 +1,114,search,12152.9,2828.49,7777.04,53.9462,201.596,0.92123,0.103552,-1.0,-1.0,17.603,2.03355,1769,4600056 +1,115,insert,34.56,-1.0,-1.0,-1.0,-1.0,81.217,2.04379,1777,4627422 +1,116,search,11978.3,2830.5,7549.56,55.465,205.494,0.9215,0.103574,-1.0,-1.0,20.763,2.04379,1777,4627422 +1,117,insert,15.109,-1.0,-1.0,-1.0,-1.0,36.899,2.04898,1779,4639292 +1,118,search,12016.9,2830.85,7655.57,55.2289,197.973,0.92157,0.103415,-1.0,-1.0,20.554,2.04898,1779,4639292 +1,119,insert,7.973,-1.0,-1.0,-1.0,-1.0,18.857,2.05149,1779,4643423 +1,120,search,12069.6,2833.13,7713.78,55.1761,205.377,0.92159,0.103442,-1.0,-1.0,19.858,2.05149,1779,4643423 +1,121,insert,19.077,-1.0,-1.0,-1.0,-1.0,43.392,2.05663,1781,4657252 +1,122,search,12116.2,2839.72,7708.93,55.6766,213.701,0.92204,0.103013,-1.0,-1.0,32.465,2.05676,1782,4657252 +1,123,insert,419.067,-1.0,-1.0,-1.0,-1.0,480.673,2.18611,1833,4938477 +1,124,search,12465.8,2889.27,8168.1,54.9565,208.725,0.92286,0.10317,-1.0,-1.0,96.912,2.1872,1846,4938477 +1,125,insert,30.611,-1.0,-1.0,-1.0,-1.0,28.761,2.20207,1850,4980492 +1,126,search,12378.7,2880.28,8152.51,54.9982,217.429,0.92341,0.102812,-1.0,-1.0,79.455,2.2024,1856,4980492 +1,127,insert,65.364,-1.0,-1.0,-1.0,-1.0,121.569,2.22632,1870,5030515 +1,128,search,12348.5,2875.3,7859.23,56.4721,240.335,0.92417,0.102472,-1.0,-1.0,24.283,2.22632,1870,5030515 +1,129,delete,36.478,-1.0,-1.0,-1.0,-1.0,86.525,2.22469,1841,5009391 +1,130,search,12111.7,2862.32,7771.58,56.4938,240.538,0.92291,0.103255,-1.0,-1.0,23.811,2.22469,1841,5009391 +1,131,delete,11.548,-1.0,-1.0,-1.0,-1.0,23.061,2.22469,1841,5004093 +1,132,search,12099.6,2859.86,7734.39,56.7888,247.63,0.92283,0.103196,-1.0,-1.0,23.387,2.22469,1841,5004093 +1,133,delete,21.076,-1.0,-1.0,-1.0,-1.0,21.575,2.22466,1841,4991787 +1,134,search,12120.1,2846.14,7765.53,56.7113,242.325,0.92254,0.103569,-1.0,-1.0,22.749,2.22466,1841,4991787 +1,135,delete,26.94,-1.0,-1.0,-1.0,-1.0,58.538,2.22255,1838,4974857 +1,136,search,12020.7,2838.41,7668.07,56.8352,244.183,0.92233,0.103418,-1.0,-1.0,22.886,2.22255,1838,4974857 +1,137,delete,5.193,-1.0,-1.0,-1.0,-1.0,22.099,2.22255,1838,4971386 +1,138,search,12031.9,2835.79,7699.79,56.6013,228.429,0.92228,0.103551,-1.0,-1.0,22.53,2.22255,1838,4971386 +1,139,delete,9.713,-1.0,-1.0,-1.0,-1.0,21.825,2.22255,1838,4963487 +1,140,search,11927,2829.35,7574.33,56.8145,240.635,0.92224,0.103487,-1.0,-1.0,22.12,2.22255,1838,4963487 +1,141,delete,27.769,-1.0,-1.0,-1.0,-1.0,66.696,2.21859,1832,4941830 +1,142,search,11882,2821.04,7554.4,56.6701,222.259,0.92197,0.103563,-1.0,-1.0,25.841,2.21859,1832,4941830 +1,143,delete,39.792,-1.0,-1.0,-1.0,-1.0,65.033,2.2149,1829,4925657 +1,144,search,11910.3,2815.94,7570.8,56.86,241.551,0.92213,0.103521,-1.0,-1.0,23.662,2.2149,1829,4925657 +1,145,delete,5.403,-1.0,-1.0,-1.0,-1.0,22.059,2.2149,1829,4922686 +1,146,search,11918.2,2812.87,7581.51,56.8352,236.277,0.92198,0.103692,-1.0,-1.0,23.144,2.2149,1829,4922686 +1,147,delete,16.425,-1.0,-1.0,-1.0,-1.0,22.267,2.2149,1829,4913636 +1,148,search,11927.4,2805.55,7634.91,56.5437,221.554,0.92185,0.103767,-1.0,-1.0,22.954,2.2149,1829,4913636 +1,149,delete,240.376,-1.0,-1.0,-1.0,-1.0,737.648,2.13462,1751,4717250 +1,150,search,11279.2,2806.58,7273.29,56.3604,226.08,0.92119,0.103966,-1.0,-1.0,73.98,2.13549,1758,4717250 +1,151,delete,150.739,-1.0,-1.0,-1.0,-1.0,976.145,2.06652,1683,4560269 +2,2,search,2693.59,37.3093,118.243,33.621,56.2831,0.8551,0.130106,-1.0,-1.0,10.738,0.0184881,1000,38806 +2,3,insert,12.852,-1.0,-1.0,-1.0,-1.0,14.997,0.0226681,998,49334 +2,4,search,2740.54,66.7645,216.519,29.109,71.1175,0.86613,0.128797,-1.0,-1.0,12.267,0.0226611,997,49334 +2,5,insert,24.602,-1.0,-1.0,-1.0,-1.0,172.348,0.0329766,993,73557 +2,6,search,2920.69,193.743,485.51,37.5235,73.9561,0.92433,0.104255,-1.0,-1.0,111.999,0.0330819,995,73557 +2,7,insert,31.824,-1.0,-1.0,-1.0,-1.0,102.906,0.0426107,995,95766 +2,8,search,3067.69,246.253,604.228,38.0433,82.8645,0.92582,0.103612,-1.0,-1.0,90.335,0.0428012,997,95766 +2,9,insert,3.562,-1.0,-1.0,-1.0,-1.0,77.996,0.0445267,997,100025 +2,10,search,3181.5,255.431,703.457,33.8131,86.5186,0.92703,0.100737,-1.0,-1.0,80.067,0.0445267,997,100025 +2,11,insert,12.959,-1.0,-1.0,-1.0,-1.0,83.044,0.0498592,998,111417 +2,12,search,3131.1,301.481,711.53,39.0233,76.1398,0.92731,0.100772,-1.0,-1.0,81.149,0.0498592,998,111417 +2,13,insert,24.282,-1.0,-1.0,-1.0,-1.0,80.176,0.061027,998,137139 +2,14,search,3224.77,347.374,813.671,38.7823,77.9262,0.92407,0.103186,-1.0,-1.0,19.31,0.0610735,999,137139 +2,15,insert,24.03,-1.0,-1.0,-1.0,-1.0,10.527,0.0698186,1000,156710 +2,16,search,3407.07,382.315,879.493,36.8138,96.1963,0.92236,0.104082,-1.0,-1.0,19.38,0.0698219,1002,156710 +2,17,insert,5.414,-1.0,-1.0,-1.0,-1.0,10.8,0.0717574,1003,161773 +2,18,search,3372.1,397.142,852.074,38.5,93.8068,0.9226,0.103835,-1.0,-1.0,20.724,0.0720553,1007,161773 +2,19,insert,32.923,-1.0,-1.0,-1.0,-1.0,9.911,0.0786767,1008,177161 +2,20,search,3486.93,432.221,934.291,38.3532,99.7491,0.92598,0.100475,-1.0,-1.0,13.964,0.0786865,1010,177161 +2,21,insert,325.44,-1.0,-1.0,-1.0,-1.0,147.799,0.201104,1044,448898 +2,22,search,4517.68,896.136,1881.14,41.1039,117.432,0.91446,0.104467,-1.0,-1.0,41.228,0.201332,1054,448898 +2,23,insert,374.091,-1.0,-1.0,-1.0,-1.0,262.331,0.328658,1110,733493 +2,24,search,5501.58,1184.26,2695.2,41.127,134.214,0.90701,0.108066,-1.0,-1.0,77.494,0.328925,1123,733493 +2,25,insert,35.336,-1.0,-1.0,-1.0,-1.0,31.249,0.345993,1127,777443 +2,26,search,5669.21,1177.24,2650.57,41.734,141.019,0.90771,0.107577,-1.0,-1.0,24.881,0.346027,1129,777443 +2,27,insert,47.015,-1.0,-1.0,-1.0,-1.0,25.65,0.357907,1132,805394 +2,28,search,5672.26,1192.73,2814.07,41.4023,133.542,0.90763,0.108027,-1.0,-1.0,12.975,0.357914,1133,805394 +2,29,insert,68.267,-1.0,-1.0,-1.0,-1.0,57.946,0.385733,1144,866391 +2,30,search,5785.96,1241.86,2897.96,41.5407,134.085,0.9087,0.109043,-1.0,-1.0,27.03,0.385754,1147,866391 +2,31,insert,449.087,-1.0,-1.0,-1.0,-1.0,343.852,0.553944,1209,1239816 +2,32,search,6837.46,1513.27,3767.59,42.0494,165.172,0.90265,0.113124,-1.0,-1.0,88.955,0.553953,1217,1239816 +2,33,insert,50.549,-1.0,-1.0,-1.0,-1.0,26.338,0.573596,1221,1291211 +2,34,search,6806.24,1526.36,3746.12,42.5053,151.136,0.90287,0.112714,-1.0,-1.0,49.885,0.573606,1224,1291211 +2,35,insert,33.195,-1.0,-1.0,-1.0,-1.0,31.44,0.585225,1230,1324978 +2,36,search,6945.65,1551.71,3834.48,42.6607,148.921,0.90399,0.112372,-1.0,-1.0,53.639,0.585234,1234,1324978 +2,37,insert,67.41,-1.0,-1.0,-1.0,-1.0,44.878,0.605375,1241,1371885 +2,38,search,7062.01,1564.86,3962.01,42.5376,150.66,0.90363,0.112762,-1.0,-1.0,15.782,0.605375,1242,1371885 +2,39,insert,9.348,-1.0,-1.0,-1.0,-1.0,15.499,0.608745,1244,1379335 +2,40,search,6849.8,1562.69,3783.64,43.1409,153.353,0.9032,0.113324,-1.0,-1.0,9.812,0.608745,1244,1379335 +2,41,insert,49.812,-1.0,-1.0,-1.0,-1.0,24.519,0.624458,1247,1413312 +2,42,search,6965.62,1588.12,3854.94,43.4796,153.557,0.90416,0.113393,-1.0,-1.0,9.98,0.624458,1247,1413312 +2,43,insert,9.939,-1.0,-1.0,-1.0,-1.0,12.987,0.627111,1248,1423411 +2,44,search,7206.64,1599.71,4017.09,41.7536,174.271,0.90417,0.113371,-1.0,-1.0,9.694,0.627111,1248,1423411 +2,45,insert,217.246,-1.0,-1.0,-1.0,-1.0,32.8,0.644387,1254,1457559 +2,46,search,6983.44,1617.25,3865.84,43.9745,138.715,0.90527,0.112404,-1.0,-1.0,11.962,0.644387,1254,1457559 +2,47,insert,321.031,-1.0,-1.0,-1.0,-1.0,251.386,0.748311,1288,1688593 +2,48,search,8182.65,1755.36,4854.24,41.8663,193.871,0.90615,0.113797,-1.0,-1.0,103.553,0.748398,1297,1688593 +2,49,insert,49.265,-1.0,-1.0,-1.0,-1.0,26.261,0.762867,1300,1723505 +2,50,search,8180.66,1763.99,4833.36,41.6412,198.474,0.90775,0.112454,-1.0,-1.0,41.852,0.762867,1303,1723505 +2,51,insert,366.805,-1.0,-1.0,-1.0,-1.0,234.55,0.901317,1341,2035040 +2,52,search,8731.97,1950.21,5323.24,41.9332,203.019,0.91007,0.111707,-1.0,-1.0,43.734,0.901342,1346,2035040 +2,53,insert,44.175,-1.0,-1.0,-1.0,-1.0,27.763,0.919668,1349,2078341 +2,54,search,8775.63,1967.64,5295.41,41.81,216.702,0.91003,0.111613,-1.0,-1.0,12.773,0.919668,1349,2078341 +2,55,insert,406.671,-1.0,-1.0,-1.0,-1.0,433.95,1.06736,1405,2399905 +2,56,search,9541.8,2123.65,6015.5,42.1028,237.576,0.91051,0.112346,-1.0,-1.0,146.722,1.06781,1425,2399905 +2,57,insert,26.869,-1.0,-1.0,-1.0,-1.0,24.34,1.07534,1428,2419836 +2,58,search,9463.73,2068.54,5875.04,43.4426,216.588,0.91039,0.112441,-1.0,-1.0,35.542,1.07534,1430,2419836 +2,59,insert,244.893,-1.0,-1.0,-1.0,-1.0,183.822,1.17537,1447,2643083 +2,60,search,9837.96,2126,6181.49,42.6422,222.013,0.90453,0.118971,-1.0,-1.0,47.297,1.17539,1452,2643083 +2,61,insert,13.908,-1.0,-1.0,-1.0,-1.0,11.008,1.18001,1452,2653064 +2,62,search,9524.35,2122.91,5961.1,43.4819,214.458,0.90357,0.119918,-1.0,-1.0,17.606,1.18001,1452,2653064 +2,63,insert,4.286,-1.0,-1.0,-1.0,-1.0,12.346,1.18096,1452,2656118 +2,64,search,10208.1,2125.25,6436.3,41.3493,263.158,0.90378,0.119743,-1.0,-1.0,12.749,1.18096,1452,2656118 +2,65,insert,96.667,-1.0,-1.0,-1.0,-1.0,95.514,1.20953,1469,2723424 +2,66,search,10418.5,2151.43,6605.56,42.0996,268.614,0.90515,0.118597,-1.0,-1.0,11.854,1.20953,1469,2723424 +2,67,insert,58.71,-1.0,-1.0,-1.0,-1.0,105.438,1.23187,1486,2769021 +2,68,search,10125.7,2157.26,6416.3,43.2596,229.024,0.90598,0.11749,-1.0,-1.0,19.032,1.23186,1487,2769021 +2,69,insert,59.606,-1.0,-1.0,-1.0,-1.0,71.285,1.25385,1496,2821777 +2,70,search,10278.2,2173.76,6495.31,42.672,245.153,0.90789,0.115809,-1.0,-1.0,20.076,1.25385,1497,2821777 +2,71,insert,19.627,-1.0,-1.0,-1.0,-1.0,31.304,1.25943,1500,2834921 +2,72,search,10706.9,2182.26,6821.04,41.4029,269.086,0.9087,0.115093,-1.0,-1.0,16.304,1.25943,1501,2834921 +2,73,insert,9.077,-1.0,-1.0,-1.0,-1.0,11.118,1.26323,1501,2842044 +2,74,search,10638.5,2183.98,6752.34,41.6559,255.768,0.90865,0.11511,-1.0,-1.0,12.954,1.26323,1501,2842044 +2,75,insert,27.074,-1.0,-1.0,-1.0,-1.0,27.216,1.26918,1503,2858890 +2,76,search,10464.8,2192.33,6700.52,42.6117,232.054,0.90865,0.114788,-1.0,-1.0,22.55,1.26939,1504,2858890 +2,77,insert,43.909,-1.0,-1.0,-1.0,-1.0,24.497,1.27774,1506,2882818 +2,78,search,10563,2198.01,6786.69,42.3209,237.463,0.90813,0.115319,-1.0,-1.0,12.627,1.27774,1506,2882818 +2,79,insert,447.347,-1.0,-1.0,-1.0,-1.0,18.206,1.30506,1508,2947806 +2,80,search,10217.1,2227.32,6492.74,43.6831,228.57,0.90813,0.115769,-1.0,-1.0,12.153,1.30506,1508,2947806 +2,81,insert,42.973,-1.0,-1.0,-1.0,-1.0,27.051,1.31812,1510,2976895 +2,82,search,10192.8,2239.14,6520.05,43.7236,218.772,0.90903,0.115044,-1.0,-1.0,12.405,1.31812,1510,2976895 +2,83,insert,17.137,-1.0,-1.0,-1.0,-1.0,11.534,1.32351,1510,2990682 +2,84,search,10318.4,2250.37,6637.17,42.7786,213.924,0.90956,0.114619,-1.0,-1.0,11.789,1.32351,1510,2990682 +2,85,insert,37.426,-1.0,-1.0,-1.0,-1.0,11.642,1.33752,1510,3025613 +2,86,search,10074.1,2274.35,6411.84,44.3303,226.063,0.91042,0.114187,-1.0,-1.0,13.788,1.33752,1510,3025613 +2,87,insert,572.751,-1.0,-1.0,-1.0,-1.0,667.37,1.54243,1602,3468632 +2,88,search,11787.3,2421.35,7776.13,43.274,256.83,0.91413,0.110686,-1.0,-1.0,74.753,1.54243,1611,3468632 +2,89,insert,407.292,-1.0,-1.0,-1.0,-1.0,429.672,1.69953,1676,3831692 +2,90,search,10634.8,2542.81,6687.01,54.0863,176.149,0.91866,0.10478,-1.0,-1.0,78.924,1.70011,1684,3831692 +2,91,insert,36.935,-1.0,-1.0,-1.0,-1.0,44.861,1.71202,1689,3864067 +2,92,search,10929.5,2543.26,6772.59,53.6595,174.411,0.91899,0.104408,-1.0,-1.0,13.942,1.71202,1689,3864067 +2,93,insert,276.752,-1.0,-1.0,-1.0,-1.0,281.679,1.81766,1724,4103827 +2,94,search,11260.1,2606.64,7037.48,54.0505,181.927,0.91969,0.103577,-1.0,-1.0,64.073,1.81829,1730,4103827 +2,95,insert,19.084,-1.0,-1.0,-1.0,-1.0,35.414,1.82569,1734,4119898 +2,96,search,11424.2,2597.52,7139.05,53.3747,168.802,0.91974,0.10332,-1.0,-1.0,16.072,1.82569,1734,4119898 +2,97,insert,11.412,-1.0,-1.0,-1.0,-1.0,34.564,1.82998,1737,4129367 +2,98,search,11266.8,2598.66,7014.69,53.7849,178.826,0.91966,0.103387,-1.0,-1.0,17.537,1.82998,1737,4129367 +2,99,insert,10.316,-1.0,-1.0,-1.0,-1.0,23.819,1.83231,1738,4134502 +2,100,search,11401.8,2599.34,7177.21,53.2721,172.405,0.9197,0.103309,-1.0,-1.0,18.116,1.83231,1738,4134502 +2,101,insert,49.351,-1.0,-1.0,-1.0,-1.0,70.102,1.8437,1745,4164482 +2,102,search,11436.7,2601.34,7137.01,54.0505,183.575,0.9198,0.10352,-1.0,-1.0,17.679,1.8437,1745,4164482 +2,103,insert,23.575,-1.0,-1.0,-1.0,-1.0,31.346,1.84911,1747,4178601 +2,104,search,11463.2,2606.7,7144.46,54.2878,186.698,0.9202,0.103087,-1.0,-1.0,26.683,1.84931,1748,4178601 +2,105,insert,12.216,-1.0,-1.0,-1.0,-1.0,32.195,1.85251,1750,4187774 +2,106,search,11372.8,2603.94,7113.22,54.1556,185.488,0.92008,0.103526,-1.0,-1.0,17.916,1.85251,1750,4187774 +2,107,insert,23.481,-1.0,-1.0,-1.0,-1.0,16.803,1.85739,1750,4200109 +2,108,search,11431.4,2608.5,7153.59,53.952,182.358,0.92005,0.103416,-1.0,-1.0,17.439,1.85739,1750,4200109 +2,109,insert,481.08,-1.0,-1.0,-1.0,-1.0,547.819,2.01829,1823,4557867 +2,110,search,12200.5,2688.19,7644.45,53.7306,191.237,0.92215,0.102318,-1.0,-1.0,88.391,2.01923,1829,4557867 +2,111,insert,30.535,-1.0,-1.0,-1.0,-1.0,42.292,2.03045,1832,4587252 +2,112,search,12178.3,2692.28,7671.85,54.0805,193.848,0.92241,0.102185,-1.0,-1.0,29.44,2.03045,1833,4587252 +2,113,insert,14.32,-1.0,-1.0,-1.0,-1.0,51.794,2.03508,1837,4600056 +2,114,search,12406.5,2688.98,7851.06,53.5583,200.165,0.92256,0.102187,-1.0,-1.0,22.852,2.03508,1837,4600056 +2,115,insert,50.323,-1.0,-1.0,-1.0,-1.0,65.709,2.04552,1843,4627422 +2,116,search,12383.6,2693.51,7823.61,53.8058,206.31,0.92329,0.101976,-1.0,-1.0,22.305,2.04552,1843,4627422 +2,117,insert,12.132,-1.0,-1.0,-1.0,-1.0,21.024,2.0479,1843,4639292 +2,118,search,11826.3,2699.04,7294.55,56.9307,221.395,0.92321,0.101882,-1.0,-1.0,21.964,2.0479,1843,4639292 +2,119,insert,4.109,-1.0,-1.0,-1.0,-1.0,20.943,2.04896,1843,4643423 +2,120,search,11797.5,2701.16,7271.53,57.0946,225.91,0.92325,0.101889,-1.0,-1.0,22.022,2.04896,1843,4643423 +2,121,insert,26.669,-1.0,-1.0,-1.0,-1.0,55.5,2.05586,1847,4657252 +2,122,search,11847.6,2703.08,7273.14,57.1211,223.5,0.92376,0.101188,-1.0,-1.0,23.131,2.05586,1847,4657252 +2,123,insert,403.119,-1.0,-1.0,-1.0,-1.0,476.935,2.18421,1895,4938477 +2,124,search,12439.8,2750.45,7773.38,56.3693,214.242,0.9238,0.101873,-1.0,-1.0,145.141,2.18469,1905,4938477 +2,125,insert,67.431,-1.0,-1.0,-1.0,-1.0,59.486,2.20267,1911,4980492 +2,126,search,12460,2738.61,7752.11,56.6027,228.182,0.92372,0.102101,-1.0,-1.0,80.723,2.20268,1916,4980492 +2,127,insert,64.02,-1.0,-1.0,-1.0,-1.0,71.923,2.22396,1924,5030515 +2,128,search,12574.4,2755.95,7823.46,56.7546,232.915,0.92492,0.101283,-1.0,-1.0,31.854,2.22396,1925,5030515 +2,129,delete,23.453,-1.0,-1.0,-1.0,-1.0,95.556,2.22208,1893,5009391 +2,130,search,12276.7,2740.92,7605.33,57.0149,231.267,0.92364,0.102078,-1.0,-1.0,25.223,2.22208,1893,5009391 +2,131,delete,7.058,-1.0,-1.0,-1.0,-1.0,24.654,2.22208,1893,5004093 +2,132,search,12313.6,2738.6,7676.58,56.8081,221.224,0.92357,0.102063,-1.0,-1.0,25.022,2.22208,1893,5004093 +2,133,delete,13.181,-1.0,-1.0,-1.0,-1.0,23.35,2.22208,1893,4991787 +2,134,search,12219.3,2725.52,7563.12,56.82,224.397,0.92347,0.102221,-1.0,-1.0,24.682,2.22208,1893,4991787 +2,135,delete,18.699,-1.0,-1.0,-1.0,-1.0,62.36,2.21919,1890,4974857 +2,136,search,12165.2,2718.43,7498.65,56.8339,217.357,0.9236,0.102087,-1.0,-1.0,27.633,2.21919,1890,4974857 +2,137,delete,8.16,-1.0,-1.0,-1.0,-1.0,25.259,2.21919,1890,4971386 +2,138,search,12211.2,2715.9,7537.85,57.0112,226.076,0.92344,0.102222,-1.0,-1.0,26.394,2.21919,1890,4971386 +2,139,delete,17.729,-1.0,-1.0,-1.0,-1.0,24.645,2.21919,1890,4963487 +2,140,search,12199.9,2709.86,7548.66,56.9793,224.968,0.92328,0.102168,-1.0,-1.0,25.179,2.21919,1890,4963487 +2,141,delete,37.799,-1.0,-1.0,-1.0,-1.0,105.777,2.21347,1883,4941830 +2,142,search,12218.2,2702.88,7570.19,56.7141,231.637,0.92312,0.102346,-1.0,-1.0,24.942,2.21347,1883,4941830 +2,143,delete,25.64,-1.0,-1.0,-1.0,-1.0,84.257,2.21031,1880,4925657 +2,144,search,12168.4,2698.33,7520.38,56.6389,228.274,0.92321,0.102264,-1.0,-1.0,25.932,2.21031,1880,4925657 +2,145,delete,4.028,-1.0,-1.0,-1.0,-1.0,35.832,2.2095,1878,4922686 +2,146,search,12100.9,2698.19,7463.31,56.7152,230.021,0.92293,0.102602,-1.0,-1.0,24.909,2.2095,1878,4922686 +2,147,delete,11.364,-1.0,-1.0,-1.0,-1.0,24.055,2.2095,1878,4913636 +2,148,search,12065,2691.18,7358.57,56.7867,227.498,0.92289,0.102605,-1.0,-1.0,26.303,2.2095,1878,4913636 +2,149,delete,239.812,-1.0,-1.0,-1.0,-1.0,1080.75,2.13338,1799,4717250 +2,150,search,11539.2,2687.18,7106.93,56.6346,216.873,0.92205,0.103563,-1.0,-1.0,60.656,2.13392,1803,4717250 +2,151,delete,180.977,-1.0,-1.0,-1.0,-1.0,1079.82,2.06492,1724,4560269 +4,2,search,2566.29,37.3142,238.232,4.5843,70.2033,0.8551,0.130106,-1.0,-1.0,7.675,0.0184881,1000,38806 +4,3,insert,10.248,-1.0,-1.0,-1.0,-1.0,6.376,0.0226644,1000,49334 +4,4,search,2802.96,65.7492,310.22,4.85697,71.6462,0.86604,0.12886,-1.0,-1.0,7.211,0.0226644,1000,49334 +4,5,insert,27.002,-1.0,-1.0,-1.0,-1.0,169.617,0.0330953,997,73557 +4,6,search,2891.25,191.294,470.737,7.59158,65.4699,0.92386,0.105036,-1.0,-1.0,124.65,0.0330953,997,73557 +4,7,insert,28.335,-1.0,-1.0,-1.0,-1.0,102.482,0.0427388,997,95766 +4,8,search,2884.2,246.513,521.621,9.25031,58.3556,0.92559,0.103709,-1.0,-1.0,83.136,0.0429143,998,95766 +4,9,insert,4.117,-1.0,-1.0,-1.0,-1.0,74.836,0.0447637,998,100025 +4,10,search,2960.38,257.725,558.203,8.99203,65.3267,0.92806,0.0998639,-1.0,-1.0,76.171,0.0447637,998,100025 +4,11,insert,10.895,-1.0,-1.0,-1.0,-1.0,82.004,0.0496597,999,111417 +4,12,search,3008.08,302.85,616.17,9.51679,65.6407,0.92812,0.10013,-1.0,-1.0,19.381,0.0500379,1000,111417 +4,13,insert,24.477,-1.0,-1.0,-1.0,-1.0,6.841,0.0611825,1000,137139 +4,14,search,3007.5,345.115,605.527,10.4247,65.2688,0.92499,0.10227,-1.0,-1.0,18.886,0.0611939,1002,137139 +4,15,insert,22.595,-1.0,-1.0,-1.0,-1.0,6.822,0.0699043,1002,156710 +4,16,search,3129.95,380.852,705.22,10.0784,67.8908,0.92274,0.103392,-1.0,-1.0,7.096,0.0699043,1002,156710 +4,17,insert,5.045,-1.0,-1.0,-1.0,-1.0,6.559,0.0715465,1002,161773 +4,18,search,3133.31,400.814,709.681,10.4246,68.172,0.9242,0.102652,-1.0,-1.0,7.241,0.0715465,1002,161773 +4,19,insert,36.729,-1.0,-1.0,-1.0,-1.0,6.764,0.0781136,1002,177161 +4,20,search,3066.15,446.577,730.09,11.5957,59.8781,0.92842,0.0993882,-1.0,-1.0,7.178,0.0781136,1002,177161 +4,21,insert,335.286,-1.0,-1.0,-1.0,-1.0,95.322,0.199618,1017,448898 +4,22,search,3925.14,992.028,1502.96,13.0063,85.0196,0.92146,0.100464,-1.0,-1.0,14.521,0.199617,1018,448898 +4,23,insert,388.222,-1.0,-1.0,-1.0,-1.0,238.123,0.327875,1050,733493 +4,24,search,4549.79,1419.79,2042.95,14.4904,101.434,0.91739,0.102601,-1.0,-1.0,72.345,0.327872,1061,733493 +4,25,insert,39.18,-1.0,-1.0,-1.0,-1.0,24.355,0.3443,1064,777443 +4,26,search,4580.95,1421.97,2011.93,15.2209,100.163,0.91738,0.102935,-1.0,-1.0,9.154,0.3443,1064,777443 +4,27,insert,40.667,-1.0,-1.0,-1.0,-1.0,16.257,0.356074,1065,805394 +4,28,search,4979.88,1450.09,2379.78,13.2298,105.818,0.91789,0.102706,-1.0,-1.0,9.619,0.356074,1065,805394 +4,29,insert,96.239,-1.0,-1.0,-1.0,-1.0,42.047,0.383832,1069,866391 +4,30,search,4914.06,1527.32,2301.64,14.4741,103.109,0.91956,0.10247,-1.0,-1.0,22.783,0.383831,1071,866391 +4,31,insert,487.495,-1.0,-1.0,-1.0,-1.0,369.243,0.55599,1120,1239816 +4,32,search,5891.7,1853.05,3135.35,13.634,130.313,0.91241,0.108427,-1.0,-1.0,89.197,0.556021,1126,1239816 +4,33,insert,56.137,-1.0,-1.0,-1.0,-1.0,13.991,0.576749,1127,1291211 +4,34,search,5924.86,1869.03,3018.52,14.605,130.563,0.91202,0.108497,-1.0,-1.0,62.804,0.576765,1131,1291211 +4,35,insert,29.13,-1.0,-1.0,-1.0,-1.0,24.562,0.58573,1134,1324978 +4,36,search,5783.45,1910.03,3001.87,14.6358,128.274,0.91335,0.107448,-1.0,-1.0,33.429,0.58573,1137,1324978 +4,37,insert,50.629,-1.0,-1.0,-1.0,-1.0,41.728,0.605563,1142,1371885 +4,38,search,5820.57,1923.15,3000.79,15.1064,128.272,0.9123,0.108101,-1.0,-1.0,21.971,0.605563,1143,1371885 +4,39,insert,12.346,-1.0,-1.0,-1.0,-1.0,8.701,0.608722,1143,1379335 +4,40,search,5850.56,1929.73,3014.37,14.8093,123.809,0.91247,0.108415,-1.0,-1.0,9.303,0.608722,1143,1379335 +4,41,insert,45.651,-1.0,-1.0,-1.0,-1.0,30.85,0.625451,1147,1413312 +4,42,search,6016.92,1949.46,3183.06,15.0029,133.109,0.91313,0.108296,-1.0,-1.0,10.389,0.625451,1147,1413312 +4,43,insert,16.623,-1.0,-1.0,-1.0,-1.0,33.743,0.630251,1151,1423411 +4,44,search,6106.34,1945.45,3272.22,14.3544,131.016,0.91279,0.107714,-1.0,-1.0,9.866,0.630251,1151,1423411 +4,45,insert,230.086,-1.0,-1.0,-1.0,-1.0,57.726,0.646294,1159,1457559 +4,46,search,6210.26,1946.6,3345.54,14.0433,135.26,0.91244,0.107128,-1.0,-1.0,9.225,0.646294,1159,1457559 +4,47,insert,288.891,-1.0,-1.0,-1.0,-1.0,261.122,0.750285,1184,1688593 +4,48,search,6648.85,2095.94,3694.28,14.1521,157.026,0.9128,0.108191,-1.0,-1.0,75.253,0.750284,1190,1688593 +4,49,insert,37.925,-1.0,-1.0,-1.0,-1.0,29.338,0.7648,1193,1723505 +4,50,search,6709.54,2104.28,3744.55,14.011,157.827,0.91347,0.108022,-1.0,-1.0,33.684,0.7648,1196,1723505 +4,51,insert,326.457,-1.0,-1.0,-1.0,-1.0,240.729,0.902044,1229,2035040 +4,52,search,7060.44,2310.82,4014.15,14.7269,162.456,0.91649,0.106604,-1.0,-1.0,34.789,0.902043,1233,2035040 +4,53,insert,41.155,-1.0,-1.0,-1.0,-1.0,60.449,0.919696,1241,2078341 +4,54,search,7130.09,2318.11,4021.39,14.7339,166.892,0.91654,0.106798,-1.0,-1.0,10.362,0.919696,1241,2078341 +4,55,insert,346.436,-1.0,-1.0,-1.0,-1.0,535.517,1.06898,1301,2399905 +4,56,search,8106.35,2442.15,4532.92,14.8157,195.781,0.91505,0.109318,-1.0,-1.0,101.978,1.06898,1313,2399905 +4,57,insert,27.644,-1.0,-1.0,-1.0,-1.0,32.115,1.07747,1317,2419836 +4,58,search,8093.38,2399.77,4716.97,14.6695,200.695,0.914,0.110185,-1.0,-1.0,35.86,1.07748,1319,2419836 +4,59,insert,273.915,-1.0,-1.0,-1.0,-1.0,201.44,1.17784,1333,2643083 +4,60,search,8324.58,2466.32,4909.67,14.3752,198.045,0.908,0.116845,-1.0,-1.0,72.026,1.17812,1337,2643083 +4,61,insert,11.371,-1.0,-1.0,-1.0,-1.0,9.375,1.18161,1337,2653064 +4,62,search,8169.68,2468.87,4798.47,14.4991,195.809,0.90816,0.11719,-1.0,-1.0,18.106,1.18161,1338,2653064 +4,63,insert,4.351,-1.0,-1.0,-1.0,-1.0,8.93,1.18318,1338,2656118 +4,64,search,8019.86,2468.07,4642.3,14.8322,196.218,0.90806,0.117376,-1.0,-1.0,11.146,1.18318,1338,2656118 +4,65,insert,94.745,-1.0,-1.0,-1.0,-1.0,142.695,1.21176,1356,2723424 +4,66,search,8139.33,2483.74,4717.94,14.9903,201.225,0.90904,0.115995,-1.0,-1.0,11.513,1.21176,1356,2723424 +4,67,insert,60.847,-1.0,-1.0,-1.0,-1.0,107.753,1.23233,1370,2769021 +4,68,search,8367.51,2481.38,4908.01,14.9139,209.155,0.90964,0.115255,-1.0,-1.0,23.19,1.23233,1372,2769021 +4,69,insert,59.313,-1.0,-1.0,-1.0,-1.0,92.026,1.25357,1384,2821777 +4,70,search,8458.36,2489.87,4961.7,14.8529,210.198,0.9107,0.114342,-1.0,-1.0,10.64,1.25357,1384,2821777 +4,71,insert,17.965,-1.0,-1.0,-1.0,-1.0,29.962,1.25907,1387,2834921 +4,72,search,8663.89,2495.5,5160.22,14.3619,207.755,0.91115,0.113886,-1.0,-1.0,16.292,1.25907,1388,2834921 +4,73,insert,6.608,-1.0,-1.0,-1.0,-1.0,16.627,1.26136,1389,2842044 +4,74,search,8342.94,2495.79,4835.3,14.9625,202.948,0.91106,0.114024,-1.0,-1.0,10.41,1.26136,1389,2842044 +4,75,insert,20.488,-1.0,-1.0,-1.0,-1.0,26.793,1.26775,1392,2858890 +4,76,search,8235.45,2500.82,4402.1,16.0388,189.168,0.91078,0.114212,-1.0,-1.0,10.949,1.26775,1392,2858890 +4,77,insert,35.709,-1.0,-1.0,-1.0,-1.0,16.585,1.2782,1393,2882818 +4,78,search,8336.71,2511.85,4563.78,15.2496,206.434,0.91042,0.114581,-1.0,-1.0,11.17,1.2782,1393,2882818 +4,79,insert,498.6,-1.0,-1.0,-1.0,-1.0,22.095,1.30497,1395,2947806 +4,80,search,8699.87,2547.39,4868.51,15.3436,207.687,0.91073,0.114524,-1.0,-1.0,10.329,1.30497,1395,2947806 +4,81,insert,37.258,-1.0,-1.0,-1.0,-1.0,37.49,1.31772,1398,2976895 +4,82,search,9031.45,2560.11,5346.9,14.5212,211.305,0.91186,0.113613,-1.0,-1.0,10.785,1.31772,1398,2976895 +4,83,insert,17.215,-1.0,-1.0,-1.0,-1.0,29.255,1.32397,1401,2990682 +4,84,search,8698.64,2564.17,5160.24,14.926,217.952,0.91213,0.113052,-1.0,-1.0,27.503,1.3241,1403,2990682 +4,85,insert,40.366,-1.0,-1.0,-1.0,-1.0,43.901,1.3388,1408,3025613 +4,86,search,7992.03,2561.48,4531.81,16.48,165.321,0.91251,0.113056,-1.0,-1.0,10.888,1.3388,1408,3025613 +4,87,insert,518.285,-1.0,-1.0,-1.0,-1.0,620.42,1.54234,1484,3468632 +4,88,search,8905.44,2747.53,5102.53,16.419,187.805,0.91575,0.109845,-1.0,-1.0,104.286,1.54234,1494,3468632 +4,89,insert,386.155,-1.0,-1.0,-1.0,-1.0,571.522,1.70706,1572,3831692 +4,90,search,9294.62,2803.08,5407.93,16.4803,214.279,0.91957,0.105115,-1.0,-1.0,78.412,1.70718,1582,3831692 +4,91,insert,29.712,-1.0,-1.0,-1.0,-1.0,37.975,1.71776,1586,3864067 +4,92,search,9646,2794.12,5471.13,16.4653,211.152,0.91985,0.104892,-1.0,-1.0,13.075,1.71776,1586,3864067 +4,93,insert,291.266,-1.0,-1.0,-1.0,-1.0,298.629,1.82319,1626,4103827 +4,94,search,9950.11,2834.38,5768.71,16.5091,216.605,0.91994,0.103918,-1.0,-1.0,37.678,1.82319,1629,4103827 +4,95,insert,19.15,-1.0,-1.0,-1.0,-1.0,38.03,1.82947,1633,4119898 +4,96,search,10089.6,2833.05,5878.14,16.3276,215.884,0.92017,0.104034,-1.0,-1.0,22.873,1.82979,1634,4119898 +4,97,insert,10.365,-1.0,-1.0,-1.0,-1.0,24.355,1.83317,1636,4129367 +4,98,search,9871.05,2833.05,5837.05,16.4445,216.945,0.92051,0.103658,-1.0,-1.0,13.189,1.83317,1636,4129367 +4,99,insert,7.157,-1.0,-1.0,-1.0,-1.0,11.326,1.83523,1636,4134502 +4,100,search,9523.71,2837.2,5517.75,17.5498,199.69,0.9205,0.10376,-1.0,-1.0,12.735,1.83523,1636,4134502 +4,101,insert,43.891,-1.0,-1.0,-1.0,-1.0,51.902,1.84728,1641,4164482 +4,102,search,9989.25,2847.51,5920.1,16.2491,218.19,0.92097,0.103232,-1.0,-1.0,12.341,1.84728,1641,4164482 +4,103,insert,20.393,-1.0,-1.0,-1.0,-1.0,26.827,1.85411,1643,4178601 +4,104,search,9923.56,2852.69,5834.73,16.3525,217.987,0.92096,0.103366,-1.0,-1.0,13.223,1.85411,1643,4178601 +4,105,insert,11.448,-1.0,-1.0,-1.0,-1.0,38.715,1.85785,1646,4187774 +4,106,search,10015.8,2849.58,5909.68,16.3511,222.718,0.92068,0.103931,-1.0,-1.0,13.065,1.85785,1646,4187774 +4,107,insert,26.991,-1.0,-1.0,-1.0,-1.0,32.367,1.86437,1648,4200109 +4,108,search,10140.6,2853.16,6040.07,16.1953,221.267,0.92098,0.103761,-1.0,-1.0,23.594,1.8645,1649,4200109 +4,109,insert,500.383,-1.0,-1.0,-1.0,-1.0,553.049,2.01995,1719,4557867 +4,110,search,11118.5,2937.63,6488.63,16.2248,227.182,0.92273,0.102313,-1.0,-1.0,80.634,2.02024,1726,4557867 +4,111,insert,37.255,-1.0,-1.0,-1.0,-1.0,64.063,2.03314,1733,4587252 +4,112,search,10789.6,2928.15,6518.6,16.1689,227.581,0.92292,0.102371,-1.0,-1.0,15.758,2.03314,1733,4587252 +4,113,insert,14.985,-1.0,-1.0,-1.0,-1.0,30.651,2.03841,1735,4600056 +4,114,search,10672.8,2932.74,6345.94,16.6535,234.389,0.92329,0.10226,-1.0,-1.0,15.781,2.03841,1735,4600056 +4,115,insert,34.543,-1.0,-1.0,-1.0,-1.0,48.59,2.04878,1740,4627422 +4,116,search,10667.8,2941.48,6337.99,16.5898,228.903,0.92406,0.101699,-1.0,-1.0,15.529,2.04878,1740,4627422 +4,117,insert,14.421,-1.0,-1.0,-1.0,-1.0,27.056,2.05381,1742,4639292 +4,118,search,10633.7,2943.1,6290.66,16.7204,232.647,0.9239,0.10191,-1.0,-1.0,16.512,2.05381,1742,4639292 +4,119,insert,5.823,-1.0,-1.0,-1.0,-1.0,14.184,2.05555,1742,4643423 +4,120,search,10984.2,2945.4,6413.99,16.5301,241.605,0.92396,0.101984,-1.0,-1.0,15.87,2.05555,1742,4643423 +4,121,insert,19.58,-1.0,-1.0,-1.0,-1.0,47.02,2.0614,1746,4657252 +4,122,search,10428.3,2942.02,6141.82,17.4511,227.605,0.92389,0.102066,-1.0,-1.0,16.422,2.0614,1746,4657252 +4,123,insert,378.145,-1.0,-1.0,-1.0,-1.0,471.747,2.18998,1792,4938477 +4,124,search,11252.7,2991.21,6737.42,16.4344,245.472,0.92413,0.102572,-1.0,-1.0,152.307,2.19049,1801,4938477 +4,125,insert,64.19,-1.0,-1.0,-1.0,-1.0,65.3,2.20863,1808,4980492 +4,126,search,11263,2981.27,6767.18,16.5973,249.533,0.92484,0.10241,-1.0,-1.0,86.617,2.20882,1812,4980492 +4,127,insert,66.2,-1.0,-1.0,-1.0,-1.0,102.143,2.23154,1825,5030515 +4,128,search,11329,2978.53,6797.47,16.584,248.705,0.92532,0.102263,-1.0,-1.0,64.464,2.23154,1828,5030515 +4,129,delete,17.666,-1.0,-1.0,-1.0,-1.0,85.275,2.22982,1798,5009391 +4,130,search,11546.2,2962.66,6576.32,16.6187,241.732,0.92432,0.10286,-1.0,-1.0,38.741,2.22982,1799,5009391 +4,131,delete,9.418,-1.0,-1.0,-1.0,-1.0,20.439,2.22982,1799,5004093 +4,132,search,11620.6,2959.46,6651.79,16.4689,243.702,0.92428,0.102898,-1.0,-1.0,21.087,2.22982,1799,5004093 +4,133,delete,20.651,-1.0,-1.0,-1.0,-1.0,33.743,2.22833,1798,4991787 +4,134,search,11621.5,2947.87,6666.93,16.5131,240.319,0.92416,0.102938,-1.0,-1.0,21.043,2.22833,1798,4991787 +4,135,delete,25.116,-1.0,-1.0,-1.0,-1.0,41.128,2.22658,1796,4974857 +4,136,search,11656.5,2938.13,6702.51,16.4483,240.048,0.92428,0.102694,-1.0,-1.0,20.828,2.22658,1796,4974857 +4,137,delete,6.089,-1.0,-1.0,-1.0,-1.0,19.502,2.22658,1796,4971386 +4,138,search,11673.9,2935.15,6731.16,16.3914,242.778,0.92422,0.10267,-1.0,-1.0,20.179,2.22658,1796,4971386 +4,139,delete,11.98,-1.0,-1.0,-1.0,-1.0,21.772,2.22658,1796,4963487 +4,140,search,11871.4,2928.25,6648.24,16.4898,235.353,0.92419,0.102669,-1.0,-1.0,20.558,2.22658,1796,4963487 +4,141,delete,25.619,-1.0,-1.0,-1.0,-1.0,58.409,2.22301,1791,4941830 +4,142,search,11435.2,2917.42,6490.98,16.7802,224.737,0.92395,0.103092,-1.0,-1.0,19.761,2.22301,1791,4941830 +4,143,delete,17.216,-1.0,-1.0,-1.0,-1.0,60.039,2.21927,1788,4925657 +4,144,search,11686.7,2911.04,6482.81,16.4551,233.184,0.92397,0.103139,-1.0,-1.0,21.355,2.21927,1788,4925657 +4,145,delete,6.865,-1.0,-1.0,-1.0,-1.0,26.229,2.21881,1787,4922686 +4,146,search,11688.2,2910.09,6453.07,16.9285,217.503,0.92379,0.103392,-1.0,-1.0,20.576,2.21881,1787,4922686 +4,147,delete,13.426,-1.0,-1.0,-1.0,-1.0,19.767,2.21881,1787,4913636 +4,148,search,11472.7,2902.45,6543.42,16.8217,214.832,0.92375,0.103449,-1.0,-1.0,21.242,2.21881,1787,4913636 +4,149,delete,243.73,-1.0,-1.0,-1.0,-1.0,1072.59,2.1391,1707,4717250 +4,150,search,10855.4,2904.5,6167.59,16.9427,212.359,0.92255,0.104352,-1.0,-1.0,52.93,2.14012,1711,4717250 +4,151,delete,162.667,-1.0,-1.0,-1.0,-1.0,938.761,2.07097,1640,4560269 +8,2,search,2872.91,37.32,332.077,2.04143,129.967,0.8551,0.130106,-1.0,-1.0,7.463,0.0184881,1000,38806 +8,3,insert,11.925,-1.0,-1.0,-1.0,-1.0,14.718,0.0226681,998,49334 +8,4,search,2731.74,66.6883,222.064,3.8079,93.4078,0.86613,0.128797,-1.0,-1.0,13.162,0.0226611,997,49334 +8,5,insert,25.182,-1.0,-1.0,-1.0,-1.0,174.253,0.0329766,993,73557 +8,6,search,2646.1,193.295,325.122,6.06004,56.0027,0.92433,0.104255,-1.0,-1.0,111.805,0.0330819,995,73557 +8,7,insert,31.91,-1.0,-1.0,-1.0,-1.0,104.099,0.0426107,995,95766 +8,8,search,2745.52,245.53,386.325,6.35336,69.1233,0.92582,0.103612,-1.0,-1.0,90.744,0.0428012,997,95766 +8,9,insert,4.544,-1.0,-1.0,-1.0,-1.0,78.425,0.0445267,997,100025 +8,10,search,2754.37,254.798,390.632,6.54646,69.2487,0.92703,0.100737,-1.0,-1.0,79.781,0.0445267,997,100025 +8,11,insert,13.651,-1.0,-1.0,-1.0,-1.0,80.228,0.0497772,997,111417 +8,12,search,2791.65,301.907,431.768,6.96195,65.594,0.92784,0.100473,-1.0,-1.0,18.801,0.049818,998,111417 +8,13,insert,27.375,-1.0,-1.0,-1.0,-1.0,7.23,0.0610837,998,137139 +8,14,search,2805.39,343.746,454.958,7.05071,73.8489,0.92376,0.103456,-1.0,-1.0,18.362,0.0610898,1000,137139 +8,15,insert,23.186,-1.0,-1.0,-1.0,-1.0,7.083,0.0698161,1000,156710 +8,16,search,2852.72,379.776,499.958,7.28536,73.1593,0.92221,0.104397,-1.0,-1.0,7.549,0.0698161,1000,156710 +8,17,insert,6.055,-1.0,-1.0,-1.0,-1.0,14.957,0.07189,1002,161773 +8,18,search,2917.57,396.368,526.687,7.39429,72.1474,0.92306,0.103575,-1.0,-1.0,8.022,0.07189,1002,161773 +8,19,insert,42.115,-1.0,-1.0,-1.0,-1.0,7.379,0.0785567,1002,177161 +8,20,search,2906.16,442.002,550.537,7.67054,72.8881,0.9275,0.0997837,-1.0,-1.0,7.628,0.0785567,1002,177161 +8,21,insert,386.86,-1.0,-1.0,-1.0,-1.0,122.182,0.199676,1026,448898 +8,22,search,3342.61,946.371,1024.64,9.31362,72.7231,0.91757,0.102984,-1.0,-1.0,15.438,0.199676,1027,448898 +8,23,insert,404.437,-1.0,-1.0,-1.0,-1.0,268.248,0.328139,1072,733493 +8,24,search,3934.89,1317.55,1476.01,9.73431,94.8916,0.9124,0.10614,-1.0,-1.0,66.474,0.328138,1081,733493 +8,25,insert,46.237,-1.0,-1.0,-1.0,-1.0,26.342,0.345341,1084,777443 +8,26,search,3929,1327.77,1441.51,10.1069,88.0319,0.91363,0.105454,-1.0,-1.0,24.088,0.345427,1086,777443 +8,27,insert,39.192,-1.0,-1.0,-1.0,-1.0,19.588,0.35769,1087,805394 +8,28,search,4020.02,1348.37,1519.02,9.86643,88.7745,0.91437,0.105238,-1.0,-1.0,17.875,0.357691,1088,805394 +8,29,insert,83.741,-1.0,-1.0,-1.0,-1.0,52.794,0.384731,1093,866391 +8,30,search,4074.75,1422.53,1559.58,10.0372,88.9072,0.91596,0.104891,-1.0,-1.0,28.852,0.384731,1095,866391 +8,31,insert,492.946,-1.0,-1.0,-1.0,-1.0,359.53,0.55411,1150,1239816 +8,32,search,4691.21,1731.94,2029.16,10.0925,110.475,0.90942,0.109454,-1.0,-1.0,91.991,0.554467,1156,1239816 +8,33,insert,55.474,-1.0,-1.0,-1.0,-1.0,31.17,0.574903,1161,1291211 +8,34,search,4529.2,1734.06,1857.56,10.9023,99.4259,0.90938,0.108679,-1.0,-1.0,60.736,0.574919,1165,1291211 +8,35,insert,36.462,-1.0,-1.0,-1.0,-1.0,33.936,0.587117,1170,1324978 +8,36,search,4537.59,1764.32,1856.06,11.1478,93.7796,0.90946,0.108328,-1.0,-1.0,27.319,0.587121,1172,1324978 +8,37,insert,53.207,-1.0,-1.0,-1.0,-1.0,33.071,0.607443,1176,1371885 +8,38,search,4696.47,1786.54,1980.43,10.676,100.323,0.90903,0.108624,-1.0,-1.0,9.177,0.607443,1176,1371885 +8,39,insert,10.236,-1.0,-1.0,-1.0,-1.0,8.713,0.611104,1176,1379335 +8,40,search,4882.73,1795.24,2139.18,10.1108,114.046,0.90955,0.108391,-1.0,-1.0,11.57,0.611104,1176,1379335 +8,41,insert,62.94,-1.0,-1.0,-1.0,-1.0,43.096,0.625089,1181,1413312 +8,42,search,4985.02,1816.16,2248.28,10.1237,119.784,0.91024,0.108455,-1.0,-1.0,9.786,0.625089,1181,1413312 +8,43,insert,12.549,-1.0,-1.0,-1.0,-1.0,14.971,0.628586,1182,1423411 +8,44,search,4885.57,1828.88,2135.91,10.5541,110.344,0.91039,0.108723,-1.0,-1.0,9.625,0.628586,1182,1423411 +8,45,insert,236.554,-1.0,-1.0,-1.0,-1.0,19.238,0.643369,1184,1457559 +8,46,search,4820.59,1857.91,2108.99,10.9321,104.578,0.91083,0.107889,-1.0,-1.0,10.151,0.643369,1184,1457559 +8,47,insert,315.558,-1.0,-1.0,-1.0,-1.0,266.957,0.748453,1214,1688593 +8,48,search,5276.87,2008.65,2425.74,10.5893,125.096,0.91184,0.109063,-1.0,-1.0,112.529,0.749047,1224,1688593 +8,49,insert,46.191,-1.0,-1.0,-1.0,-1.0,44.844,0.764861,1229,1723505 +8,50,search,5362.73,1997.34,2493.34,10.2681,125.844,0.91226,0.108495,-1.0,-1.0,42.11,0.764861,1232,1723505 +8,51,insert,360.349,-1.0,-1.0,-1.0,-1.0,264.065,0.90579,1270,2035040 +8,52,search,5582.88,2179.53,2624.75,10.859,136.152,0.91427,0.10776,-1.0,-1.0,66.725,0.905862,1278,2035040 +8,53,insert,42.018,-1.0,-1.0,-1.0,-1.0,34.363,0.922294,1281,2078341 +8,54,search,5731.04,2189.53,2747.2,10.6508,137.392,0.91436,0.10751,-1.0,-1.0,17.354,0.922312,1282,2078341 +8,55,insert,361.98,-1.0,-1.0,-1.0,-1.0,441.474,1.07139,1331,2399905 +8,56,search,6414.05,2369.55,3285.77,10.242,165.657,0.91483,0.108346,-1.0,-1.0,115.847,1.07154,1345,2399905 +8,57,insert,22.422,-1.0,-1.0,-1.0,-1.0,15.79,1.07951,1346,2419836 +8,58,search,6094.31,2332.43,2963.42,11.0179,146.059,0.91338,0.109028,-1.0,-1.0,46.279,1.07952,1350,2419836 +8,59,insert,229.794,-1.0,-1.0,-1.0,-1.0,219.048,1.17622,1370,2643083 +8,60,search,6307.53,2368.4,3103.57,10.8425,154.444,0.90767,0.11597,-1.0,-1.0,51.574,1.17659,1376,2643083 +8,61,insert,11.848,-1.0,-1.0,-1.0,-1.0,15.858,1.18143,1377,2653064 +8,62,search,6480.86,2362.07,3221.66,10.6086,160.225,0.90785,0.115693,-1.0,-1.0,12.238,1.18143,1377,2653064 +8,63,insert,4.678,-1.0,-1.0,-1.0,-1.0,19.182,1.18228,1378,2656118 +8,64,search,6476.85,2360.66,3234.55,11.0404,161.23,0.90783,0.11549,-1.0,-1.0,12.836,1.18228,1378,2656118 +8,65,insert,101.278,-1.0,-1.0,-1.0,-1.0,101.775,1.20876,1393,2723424 +8,66,search,6533.26,2391.13,3255.78,10.9442,167.971,0.90889,0.114082,-1.0,-1.0,11.982,1.20876,1393,2723424 +8,67,insert,52.79,-1.0,-1.0,-1.0,-1.0,88.826,1.22792,1405,2769021 +8,68,search,6718.23,2401.6,3401.14,10.7149,172.558,0.90971,0.113439,-1.0,-1.0,16.682,1.22797,1406,2769021 +8,69,insert,55.935,-1.0,-1.0,-1.0,-1.0,61.883,1.24984,1414,2821777 +8,70,search,6883.22,2423.2,3523.6,10.4966,178.704,0.91105,0.112088,-1.0,-1.0,22.917,1.24986,1416,2821777 +8,71,insert,16.738,-1.0,-1.0,-1.0,-1.0,43.313,1.25527,1421,2834921 +8,72,search,6587.63,2418.81,3275.29,11.311,158.933,0.91137,0.112008,-1.0,-1.0,17.293,1.25527,1422,2834921 +8,73,insert,9.732,-1.0,-1.0,-1.0,-1.0,23.061,1.25869,1424,2842044 +8,74,search,6227.22,2415.78,2960.2,12.0174,134.286,0.91118,0.112292,-1.0,-1.0,11.398,1.25869,1424,2842044 +8,75,insert,23.81,-1.0,-1.0,-1.0,-1.0,10.98,1.26632,1424,2858890 +8,76,search,6909.47,2430.04,3528.52,10.3209,177.169,0.91155,0.112034,-1.0,-1.0,13.498,1.26632,1424,2858890 +8,77,insert,49.301,-1.0,-1.0,-1.0,-1.0,11.989,1.27694,1424,2882818 +8,78,search,6747.93,2443.85,3395.28,10.7852,170.274,0.9113,0.112371,-1.0,-1.0,12.123,1.27694,1424,2882818 +8,79,insert,523.23,-1.0,-1.0,-1.0,-1.0,30.404,1.30532,1427,2947806 +8,80,search,6991.46,2472.74,3609.79,10.757,184.848,0.9113,0.11246,-1.0,-1.0,11.632,1.30532,1427,2947806 +8,81,insert,45.808,-1.0,-1.0,-1.0,-1.0,67.811,1.32034,1434,2976895 +8,82,search,6948.85,2478,3549.45,10.7966,183.056,0.9124,0.112079,-1.0,-1.0,11.261,1.32034,1434,2976895 +8,83,insert,17.817,-1.0,-1.0,-1.0,-1.0,31.097,1.32678,1437,2990682 +8,84,search,6903.81,2483.79,3513.76,10.7774,176.925,0.91258,0.111729,-1.0,-1.0,12.077,1.32678,1437,2990682 +8,85,insert,43.356,-1.0,-1.0,-1.0,-1.0,48.937,1.34097,1442,3025613 +8,86,search,7030.35,2493.69,3623.56,10.6591,181.194,0.91311,0.111704,-1.0,-1.0,20.859,1.34123,1443,3025613 +8,87,insert,535.974,-1.0,-1.0,-1.0,-1.0,714.305,1.54435,1535,3468632 +8,88,search,7540.62,2630.86,3941.45,10.8957,191.175,0.91552,0.10882,-1.0,-1.0,122.17,1.54434,1550,3468632 +8,89,insert,401.243,-1.0,-1.0,-1.0,-1.0,480.238,1.70513,1619,3831692 +8,90,search,7955.03,2715.68,4171.86,11.0058,200.987,0.91908,0.104677,-1.0,-1.0,58.245,1.70543,1626,3831692 +8,91,insert,30.704,-1.0,-1.0,-1.0,-1.0,58.261,1.71718,1634,3864067 +8,92,search,8155.65,2699.91,4315.3,10.681,213.36,0.91899,0.104523,-1.0,-1.0,24.951,1.7172,1635,3864067 +8,93,insert,304.334,-1.0,-1.0,-1.0,-1.0,245.158,1.82158,1668,4103827 +8,94,search,8455.4,2760.05,4530.72,10.9233,225.026,0.91909,0.104192,-1.0,-1.0,39.228,1.82164,1671,4103827 +8,95,insert,22.159,-1.0,-1.0,-1.0,-1.0,19.841,1.82847,1672,4119898 +8,96,search,8801.65,2767.62,4845.67,10.5941,246.17,0.91931,0.103976,-1.0,-1.0,14.792,1.82847,1672,4119898 +8,97,insert,12.148,-1.0,-1.0,-1.0,-1.0,25.468,1.831,1673,4129367 +8,98,search,8526.33,2772.02,4582.32,10.8524,227.82,0.91939,0.103944,-1.0,-1.0,13.439,1.831,1673,4129367 +8,99,insert,9.745,-1.0,-1.0,-1.0,-1.0,13.543,1.83288,1673,4134502 +8,100,search,8265.51,2775.87,4365.23,11.2359,208.956,0.91947,0.103948,-1.0,-1.0,13.865,1.83288,1673,4134502 +8,101,insert,37.562,-1.0,-1.0,-1.0,-1.0,61.231,1.84276,1679,4164482 +8,102,search,7970.57,2784.43,4088.17,11.9717,188.969,0.92012,0.103314,-1.0,-1.0,13.379,1.84276,1679,4164482 +8,103,insert,19.63,-1.0,-1.0,-1.0,-1.0,26.336,1.84934,1681,4178601 +8,104,search,8303.45,2790.37,4361.96,11.1525,213.626,0.92046,0.103276,-1.0,-1.0,15.231,1.84934,1681,4178601 +8,105,insert,11.453,-1.0,-1.0,-1.0,-1.0,31.951,1.85321,1683,4187774 +8,106,search,8860.65,2790.32,4854.79,10.3265,240.829,0.92027,0.103706,-1.0,-1.0,28.25,1.85323,1684,4187774 +8,107,insert,30.005,-1.0,-1.0,-1.0,-1.0,34.364,1.85858,1686,4200109 +8,108,search,8871.14,2791.85,4878.83,10.5445,247.916,0.92014,0.103767,-1.0,-1.0,16.125,1.85858,1686,4200109 +8,109,insert,522.963,-1.0,-1.0,-1.0,-1.0,590.254,2.02021,1758,4557867 +8,110,search,9007.13,2868.13,4865.86,11.0401,242.864,0.92195,0.101653,-1.0,-1.0,68.616,2.02073,1764,4557867 +8,111,insert,31.507,-1.0,-1.0,-1.0,-1.0,58.338,2.032,1770,4587252 +8,112,search,9426.71,2861.64,5227.94,10.5488,259.084,0.92236,0.101484,-1.0,-1.0,18.571,2.032,1770,4587252 +8,113,insert,14.962,-1.0,-1.0,-1.0,-1.0,25.213,2.0368,1771,4600056 +8,114,search,9163.83,2866.13,4983.75,10.8102,248.202,0.92245,0.101389,-1.0,-1.0,19.323,2.0368,1771,4600056 +8,115,insert,42.225,-1.0,-1.0,-1.0,-1.0,60.48,2.04926,1777,4627422 +8,116,search,8629.68,2868.08,4502.16,11.7284,209.727,0.92297,0.101065,-1.0,-1.0,18.215,2.04926,1777,4627422 +8,117,insert,14.434,-1.0,-1.0,-1.0,-1.0,25.775,2.05439,1778,4639292 +8,118,search,9097.09,2872.99,4900.82,10.8086,237.234,0.92307,0.101124,-1.0,-1.0,19.829,2.05439,1778,4639292 +8,119,insert,4.601,-1.0,-1.0,-1.0,-1.0,18.505,2.05563,1778,4643423 +8,120,search,9435.74,2875.63,5198.17,10.557,257.417,0.92309,0.101098,-1.0,-1.0,19.975,2.05563,1778,4643423 +8,121,insert,20.465,-1.0,-1.0,-1.0,-1.0,46.315,2.06152,1781,4657252 +8,122,search,9340.35,2879,5130.83,10.7615,254.158,0.92329,0.100783,-1.0,-1.0,19.815,2.06152,1781,4657252 +8,123,insert,408.429,-1.0,-1.0,-1.0,-1.0,474.403,2.19199,1828,4938477 +8,124,search,9732.51,2923.75,5402.53,10.755,270.272,0.92334,0.101597,-1.0,-1.0,156.069,2.19279,1839,4938477 +8,125,insert,58.031,-1.0,-1.0,-1.0,-1.0,102.392,2.20915,1851,4980492 +8,126,search,9597.11,2901.53,5235.79,11.0823,251.41,0.92378,0.101354,-1.0,-1.0,95.473,2.20918,1856,4980492 +8,127,insert,61.052,-1.0,-1.0,-1.0,-1.0,65.737,2.22746,1863,5030515 +8,128,search,9013.01,2914.98,4680.55,12.201,209.48,0.92462,0.100902,-1.0,-1.0,31.754,2.22749,1864,5030515 +8,129,delete,19.129,-1.0,-1.0,-1.0,-1.0,91.186,2.22572,1833,5009391 +8,130,search,9411.28,2901.08,5100.96,10.9075,253.918,0.92306,0.102244,-1.0,-1.0,22.486,2.22572,1833,5009391 +8,131,delete,4.717,-1.0,-1.0,-1.0,-1.0,22.105,2.22572,1833,5004093 +8,132,search,9460.87,2898.4,5141.78,10.8288,246.934,0.92304,0.102236,-1.0,-1.0,23.319,2.22572,1833,5004093 +8,133,delete,18.499,-1.0,-1.0,-1.0,-1.0,21.806,2.22572,1833,4991787 +8,134,search,9115.2,2884.04,4819.04,11.6074,228.4,0.92293,0.102309,-1.0,-1.0,23.492,2.22572,1833,4991787 +8,135,delete,25.503,-1.0,-1.0,-1.0,-1.0,57.127,2.223,1830,4974857 +8,136,search,9560.7,2876.42,5207.54,10.8087,261.639,0.92309,0.102112,-1.0,-1.0,23.492,2.223,1830,4974857 +8,137,delete,5.02,-1.0,-1.0,-1.0,-1.0,21.987,2.223,1830,4971386 +8,138,search,9673.84,2873.55,5328,10.7434,267.252,0.92295,0.102268,-1.0,-1.0,22.99,2.223,1830,4971386 +8,139,delete,12.338,-1.0,-1.0,-1.0,-1.0,21.926,2.223,1830,4963487 +8,140,search,9625.76,2866.25,5271.62,10.7711,257.944,0.92282,0.102317,-1.0,-1.0,23.174,2.223,1830,4963487 +8,141,delete,28.28,-1.0,-1.0,-1.0,-1.0,71.714,2.21967,1825,4941830 +8,142,search,9537.66,2856.02,5209.27,10.8065,256.165,0.92252,0.102558,-1.0,-1.0,23.622,2.21967,1825,4941830 +8,143,delete,21.684,-1.0,-1.0,-1.0,-1.0,78.954,2.21512,1822,4925657 +8,144,search,8835.97,2851.92,4568.97,11.9084,212.227,0.92274,0.102235,-1.0,-1.0,22.745,2.21512,1822,4925657 +8,145,delete,2.918,-1.0,-1.0,-1.0,-1.0,26.68,2.21484,1821,4922686 +8,146,search,9604.57,2849.67,5051.16,10.8808,244.744,0.92265,0.102138,-1.0,-1.0,22.571,2.21484,1821,4922686 +8,147,delete,10.186,-1.0,-1.0,-1.0,-1.0,21.782,2.2146,1821,4913636 +8,148,search,9378.56,2842.06,4839.28,11.208,231.987,0.92261,0.102209,-1.0,-1.0,23.933,2.2146,1821,4913636 +8,149,delete,234.634,-1.0,-1.0,-1.0,-1.0,1074.1,2.13663,1741,4717250 +8,150,search,9321.51,2847.49,4938.46,10.8649,247.768,0.9219,0.103039,-1.0,-1.0,68.771,2.13715,1746,4717250 +8,151,delete,188.33,-1.0,-1.0,-1.0,-1.0,925.633,2.06948,1672,4560269 +16,2,search,2646.28,37.3196,160.794,2.32472,102.764,0.8551,0.130106,-1.0,-1.0,7.887,0.0184881,1000,38806 +16,3,insert,14.569,-1.0,-1.0,-1.0,-1.0,15.578,0.0226681,998,49334 +16,4,search,2613.88,66.6444,159.881,2.94427,102.904,0.86613,0.128797,-1.0,-1.0,13.98,0.0226611,997,49334 +16,5,insert,34.125,-1.0,-1.0,-1.0,-1.0,174.437,0.0329766,993,73557 +16,6,search,2604.61,193.073,266.948,4.15128,68.3108,0.92433,0.104255,-1.0,-1.0,112.606,0.0330819,995,73557 +16,7,insert,36.112,-1.0,-1.0,-1.0,-1.0,106.48,0.0426107,995,95766 +16,8,search,2655.74,245.079,293.537,4.64327,71.1985,0.92582,0.103612,-1.0,-1.0,95.831,0.0428012,997,95766 +16,9,insert,5.755,-1.0,-1.0,-1.0,-1.0,78.984,0.0445267,997,100025 +16,10,search,2637.37,254.375,295.299,4.72495,72.7382,0.92703,0.100737,-1.0,-1.0,80.289,0.0445267,997,100025 +16,11,insert,15.686,-1.0,-1.0,-1.0,-1.0,80.728,0.0497772,997,111417 +16,12,search,2673,301.372,328.841,4.96063,70.2381,0.92784,0.100473,-1.0,-1.0,82.329,0.0497772,997,111417 +16,13,insert,31.956,-1.0,-1.0,-1.0,-1.0,81.183,0.0610515,997,137139 +16,14,search,2676.21,347.51,358.514,5.19002,71.1586,0.92449,0.102927,-1.0,-1.0,20.848,0.061098,998,137139 +16,15,insert,30.374,-1.0,-1.0,-1.0,-1.0,7.818,0.0698157,998,156710 +16,16,search,2671.51,383.909,365.2,5.34038,71.938,0.92241,0.104278,-1.0,-1.0,19.734,0.0698182,1000,156710 +16,17,insert,7.069,-1.0,-1.0,-1.0,-1.0,7.121,0.0717762,1000,161773 +16,18,search,2779.65,400.184,386.568,5.1967,78.8566,0.92371,0.103298,-1.0,-1.0,16.876,0.0719569,1002,161773 +16,19,insert,40.709,-1.0,-1.0,-1.0,-1.0,7.721,0.0785469,1002,177161 +16,20,search,2769.4,441.39,416.522,5.38885,74.3241,0.92719,0.0999856,-1.0,-1.0,7.667,0.0785469,1002,177161 +16,21,insert,393.069,-1.0,-1.0,-1.0,-1.0,161.109,0.201067,1037,448898 +16,22,search,3082.7,906.422,742.932,6.49531,66.4159,0.91597,0.103468,-1.0,-1.0,31.805,0.201067,1042,448898 +16,23,insert,427.335,-1.0,-1.0,-1.0,-1.0,244.652,0.32776,1084,733493 +16,24,search,3493.95,1278.47,1063.25,6.78262,76.0798,0.91152,0.106469,-1.0,-1.0,74.589,0.327759,1094,733493 +16,25,insert,47.007,-1.0,-1.0,-1.0,-1.0,36.015,0.345775,1099,777443 +16,26,search,3568.53,1272.02,1075.36,6.81113,82.0328,0.91116,0.106938,-1.0,-1.0,25.994,0.345815,1101,777443 +16,27,insert,49.724,-1.0,-1.0,-1.0,-1.0,20.003,0.358047,1102,805394 +16,28,search,3586.43,1290.23,1111.03,6.8628,74.0825,0.91158,0.106722,-1.0,-1.0,21.402,0.358055,1103,805394 +16,29,insert,95.383,-1.0,-1.0,-1.0,-1.0,62.011,0.385238,1111,866391 +16,30,search,3647.15,1349.79,1150.14,6.88785,81.5473,0.91278,0.107318,-1.0,-1.0,28.286,0.385238,1113,866391 +16,31,insert,512.212,-1.0,-1.0,-1.0,-1.0,347.804,0.551645,1165,1239816 +16,32,search,3919.85,1663.86,1322.31,7.62769,82.9215,0.90573,0.111989,-1.0,-1.0,106.617,0.551893,1173,1239816 +16,33,insert,58.332,-1.0,-1.0,-1.0,-1.0,18.943,0.571773,1175,1291211 +16,34,search,4111.29,1675.38,1470.04,7.08703,96.4712,0.90682,0.110791,-1.0,-1.0,63.436,0.57179,1179,1291211 +16,35,insert,42.158,-1.0,-1.0,-1.0,-1.0,41.985,0.584151,1185,1324978 +16,36,search,4151.12,1705.25,1457.76,7.20558,97.0179,0.90805,0.110128,-1.0,-1.0,19.078,0.584151,1186,1324978 +16,37,insert,63.154,-1.0,-1.0,-1.0,-1.0,30.805,0.604236,1189,1371885 +16,38,search,4149.11,1733.6,1465.17,7.29541,95.6378,0.90811,0.110169,-1.0,-1.0,17.891,0.604236,1190,1371885 +16,39,insert,11.253,-1.0,-1.0,-1.0,-1.0,9.024,0.607442,1190,1379335 +16,40,search,4200.04,1741.72,1529.06,7.09852,94.994,0.90842,0.110616,-1.0,-1.0,10.557,0.607442,1190,1379335 +16,41,insert,50.753,-1.0,-1.0,-1.0,-1.0,48.677,0.623707,1196,1413312 +16,42,search,4270.26,1756.94,1561.38,7.08434,95.8333,0.90912,0.11016,-1.0,-1.0,9.76,0.623707,1196,1413312 +16,43,insert,13.355,-1.0,-1.0,-1.0,-1.0,25.353,0.627725,1199,1423411 +16,44,search,4273.46,1759.71,1563.04,7.09458,94.9439,0.90864,0.110747,-1.0,-1.0,9.978,0.627725,1199,1423411 +16,45,insert,227.337,-1.0,-1.0,-1.0,-1.0,33.31,0.643337,1204,1457559 +16,46,search,4268.12,1777.23,1544.13,7.26637,98.5551,0.90879,0.1103,-1.0,-1.0,9.463,0.643337,1204,1457559 +16,47,insert,319.649,-1.0,-1.0,-1.0,-1.0,260.7,0.749984,1234,1688593 +16,48,search,4571,1920.94,1763.5,7.14922,108.388,0.91037,0.110203,-1.0,-1.0,91.242,0.749984,1241,1688593 +16,49,insert,38.203,-1.0,-1.0,-1.0,-1.0,30.119,0.763725,1244,1723505 +16,50,search,4465.04,1926.95,1657.16,7.52931,98.5739,0.9102,0.110561,-1.0,-1.0,33.777,0.763724,1246,1723505 +16,51,insert,350.366,-1.0,-1.0,-1.0,-1.0,244.552,0.902309,1283,2035040 +16,52,search,4779.79,2128.2,1878.13,7.6784,114.429,0.91354,0.107937,-1.0,-1.0,49.461,0.902328,1289,2035040 +16,53,insert,47.122,-1.0,-1.0,-1.0,-1.0,31.219,0.920683,1292,2078341 +16,54,search,4921.21,2136.35,1985.69,7.45249,119.14,0.91301,0.108384,-1.0,-1.0,11.971,0.920683,1292,2078341 +16,55,insert,387.549,-1.0,-1.0,-1.0,-1.0,425.797,1.06954,1341,2399905 +16,56,search,5294.88,2310.69,2232.41,7.60286,135.579,0.91379,0.108663,-1.0,-1.0,151.621,1.07019,1357,2399905 +16,57,insert,33.408,-1.0,-1.0,-1.0,-1.0,27.544,1.08007,1360,2419836 +16,58,search,5492.34,2256.73,2366.24,7.13082,149.928,0.91285,0.10969,-1.0,-1.0,60.067,1.08009,1365,2419836 +16,59,insert,260.516,-1.0,-1.0,-1.0,-1.0,168.153,1.17785,1380,2643083 +16,60,search,5603.92,2307.7,2451.14,7.18871,147.154,0.90692,0.116472,-1.0,-1.0,96.817,1.17831,1390,2643083 +16,61,insert,12.024,-1.0,-1.0,-1.0,-1.0,16.797,1.18227,1391,2653064 +16,62,search,5559.57,2283.73,2388.41,7.24766,145.581,0.90629,0.1167,-1.0,-1.0,23.719,1.18227,1392,2653064 +16,63,insert,4.876,-1.0,-1.0,-1.0,-1.0,10.451,1.18381,1392,2656118 +16,64,search,5348.94,2285.01,2205.93,7.65576,132.067,0.90622,0.116862,-1.0,-1.0,12.408,1.18381,1392,2656118 +16,65,insert,83.013,-1.0,-1.0,-1.0,-1.0,109.389,1.2119,1410,2723424 +16,66,search,5651.54,2306.06,2424.19,7.22369,142.89,0.90657,0.116307,-1.0,-1.0,16.938,1.21217,1411,2723424 +16,67,insert,46.477,-1.0,-1.0,-1.0,-1.0,67.923,1.23089,1420,2769021 +16,68,search,5520.9,2333.74,2283.19,7.65861,136.652,0.90814,0.115123,-1.0,-1.0,10.79,1.23089,1420,2769021 +16,69,insert,56.324,-1.0,-1.0,-1.0,-1.0,86.646,1.25379,1431,2821777 +16,70,search,5489.19,2350.69,2255.87,7.76707,132.902,0.90999,0.113131,-1.0,-1.0,11.091,1.25379,1431,2821777 +16,71,insert,14.748,-1.0,-1.0,-1.0,-1.0,44.564,1.25882,1436,2834921 +16,72,search,5560.97,2353.17,2302.87,7.71845,140.115,0.91037,0.112893,-1.0,-1.0,11.229,1.25882,1436,2834921 +16,73,insert,8.431,-1.0,-1.0,-1.0,-1.0,16.937,1.26192,1437,2842044 +16,74,search,5740.54,2356.28,2465.98,7.28877,148.998,0.91026,0.11299,-1.0,-1.0,12.455,1.26192,1437,2842044 +16,75,insert,25.02,-1.0,-1.0,-1.0,-1.0,27.411,1.26779,1439,2858890 +16,76,search,5623.68,2366.31,2365.43,7.77662,142.205,0.91064,0.112609,-1.0,-1.0,13.65,1.26779,1439,2858890 +16,77,insert,53.44,-1.0,-1.0,-1.0,-1.0,25.988,1.27756,1441,2882818 +16,78,search,5710.34,2374.06,2425.39,7.73178,150.658,0.91066,0.1127,-1.0,-1.0,18.267,1.27772,1442,2882818 +16,79,insert,529.433,-1.0,-1.0,-1.0,-1.0,18.395,1.3048,1443,2947806 +16,80,search,5698.14,2405.07,2420.98,7.69986,145.609,0.91032,0.11305,-1.0,-1.0,12.254,1.3048,1443,2947806 +16,81,insert,44.341,-1.0,-1.0,-1.0,-1.0,33.765,1.31831,1446,2976895 +16,82,search,5544.93,2415.92,2287.66,8.0336,135.661,0.91137,0.112543,-1.0,-1.0,18.622,1.31837,1447,2976895 +16,83,insert,16.742,-1.0,-1.0,-1.0,-1.0,18.286,1.32442,1448,2990682 +16,84,search,5771.32,2421.78,2470.09,7.6312,148.867,0.91197,0.111953,-1.0,-1.0,11.875,1.32442,1448,2990682 +16,85,insert,41.815,-1.0,-1.0,-1.0,-1.0,37.321,1.33817,1451,3025613 +16,86,search,6022.72,2439.18,2685.98,7.18223,165.698,0.91231,0.111853,-1.0,-1.0,11.734,1.33817,1451,3025613 +16,87,insert,558.638,-1.0,-1.0,-1.0,-1.0,738.592,1.5446,1551,3468632 +16,88,search,6339.56,2557.79,2836.07,7.66008,169.24,0.9151,0.109525,-1.0,-1.0,97.25,1.5446,1563,3468632 +16,89,insert,383.234,-1.0,-1.0,-1.0,-1.0,421.384,1.70273,1628,3831692 +16,90,search,6775.95,2671.81,3045.3,7.4872,183.257,0.91973,0.104333,-1.0,-1.0,42.972,1.70273,1632,3831692 +16,91,insert,31.369,-1.0,-1.0,-1.0,-1.0,35.238,1.71361,1636,3864067 +16,92,search,6821.96,2679.08,3093.25,7.50437,185.615,0.91978,0.104169,-1.0,-1.0,12.808,1.71361,1636,3864067 +16,93,insert,248.71,-1.0,-1.0,-1.0,-1.0,269.95,1.81862,1677,4103827 +16,94,search,6786.36,2724.41,3003.62,7.97399,170.156,0.92005,0.103618,-1.0,-1.0,42.81,1.81911,1681,4103827 +16,95,insert,15.571,-1.0,-1.0,-1.0,-1.0,18.186,1.82435,1682,4119898 +16,96,search,6747.11,2723.21,2948.48,8.12453,170.109,0.92034,0.103466,-1.0,-1.0,14.831,1.82435,1682,4119898 +16,97,insert,14.049,-1.0,-1.0,-1.0,-1.0,19.081,1.82818,1683,4129367 +16,98,search,7359.44,2726.66,3506.3,7.1764,207.235,0.92044,0.103295,-1.0,-1.0,15.468,1.82818,1683,4129367 +16,99,insert,9.266,-1.0,-1.0,-1.0,-1.0,14.64,1.83003,1683,4134502 +16,100,search,7018.25,2730.79,3178.1,7.72565,192.969,0.92058,0.103141,-1.0,-1.0,14.987,1.83003,1683,4134502 +16,101,insert,43.84,-1.0,-1.0,-1.0,-1.0,35.794,1.84016,1686,4164482 +16,102,search,6917.33,2742.51,3101.26,7.9446,183.184,0.92068,0.103217,-1.0,-1.0,14.853,1.84016,1686,4164482 +16,103,insert,20.451,-1.0,-1.0,-1.0,-1.0,40.056,1.84645,1689,4178601 +16,104,search,7228.34,2744.11,3354.13,7.56212,202.496,0.92104,0.102877,-1.0,-1.0,14.595,1.84645,1689,4178601 +16,105,insert,9.026,-1.0,-1.0,-1.0,-1.0,22.506,1.84903,1690,4187774 +16,106,search,7158.12,2748.22,3296.35,7.68553,194.795,0.92111,0.103017,-1.0,-1.0,14.906,1.84903,1690,4187774 +16,107,insert,21.069,-1.0,-1.0,-1.0,-1.0,36.82,1.85373,1692,4200109 +16,108,search,7287.82,2751.47,3409.43,7.37056,208.106,0.92099,0.10299,-1.0,-1.0,22.6,1.85398,1693,4200109 +16,109,insert,475.73,-1.0,-1.0,-1.0,-1.0,594.121,2.01915,1763,4557867 +16,110,search,7505.8,2827.61,3459.93,7.61127,199.752,0.92274,0.101843,-1.0,-1.0,91.793,2.02012,1770,4557867 +16,111,insert,29.019,-1.0,-1.0,-1.0,-1.0,70.915,2.0313,1779,4587252 +16,112,search,7581.39,2818.44,3504.89,7.65254,208.001,0.92282,0.101759,-1.0,-1.0,19.607,2.0313,1779,4587252 +16,113,insert,14.538,-1.0,-1.0,-1.0,-1.0,34.388,2.03604,1781,4600056 +16,114,search,7790.34,2820.49,3667.44,7.50148,219.793,0.92325,0.101476,-1.0,-1.0,28.613,2.03607,1782,4600056 +16,115,insert,46.69,-1.0,-1.0,-1.0,-1.0,50.454,2.04695,1786,4627422 +16,116,search,7742.46,2830.42,3638.92,7.58418,216.157,0.92344,0.101387,-1.0,-1.0,19.879,2.04695,1786,4627422 +16,117,insert,20.029,-1.0,-1.0,-1.0,-1.0,31.508,2.05196,1787,4639292 +16,118,search,7760.73,2835.66,3643.32,7.65253,217.354,0.92393,0.10079,-1.0,-1.0,29.884,2.05216,1788,4639292 +16,119,insert,6.867,-1.0,-1.0,-1.0,-1.0,18.565,2.05405,1788,4643423 +16,120,search,7797.08,2834.06,3699.21,7.52746,222.098,0.92386,0.101026,-1.0,-1.0,20.271,2.05405,1788,4643423 +16,121,insert,20.558,-1.0,-1.0,-1.0,-1.0,19.201,2.05966,1788,4657252 +16,122,search,7804.72,2843.32,3711.29,7.47564,218.253,0.92413,0.100752,-1.0,-1.0,20.311,2.05966,1788,4657252 +16,123,insert,390.164,-1.0,-1.0,-1.0,-1.0,541.223,2.18968,1842,4938477 +16,124,search,7729.9,2877.61,3542.77,8.01018,196.048,0.924,0.101579,-1.0,-1.0,138.927,2.18981,1850,4938477 +16,125,insert,53.705,-1.0,-1.0,-1.0,-1.0,80.121,2.20515,1858,4980492 +16,126,search,8208.6,2876.69,3919.44,7.25018,230.075,0.92462,0.101259,-1.0,-1.0,128.233,2.20555,1866,4980492 +16,127,insert,59.555,-1.0,-1.0,-1.0,-1.0,124.617,2.22656,1883,5030515 +16,128,search,8215.08,2852.82,3876.54,7.53119,226.983,0.92484,0.101517,-1.0,-1.0,42.871,2.22656,1884,5030515 +16,129,delete,26.562,-1.0,-1.0,-1.0,-1.0,95.357,2.22477,1852,5009391 +16,130,search,7947.48,2839.95,3697.05,7.75345,220.525,0.92369,0.102122,-1.0,-1.0,25.624,2.22477,1852,5009391 +16,131,delete,8.556,-1.0,-1.0,-1.0,-1.0,23.916,2.22477,1852,5004093 +16,132,search,7722.7,2836.65,3480.34,8.01132,203.071,0.92368,0.102026,-1.0,-1.0,24.748,2.22477,1852,5004093 +16,133,delete,14.776,-1.0,-1.0,-1.0,-1.0,23.155,2.22477,1852,4991787 +16,134,search,8088.4,2823.33,3815.97,7.54235,226.125,0.92348,0.102249,-1.0,-1.0,24.671,2.22477,1852,4991787 +16,135,delete,25.035,-1.0,-1.0,-1.0,-1.0,62.744,2.22148,1849,4974857 +16,136,search,7820.34,2816.34,3593.25,7.86617,206.763,0.92383,0.101948,-1.0,-1.0,24.99,2.22148,1849,4974857 +16,137,delete,5.122,-1.0,-1.0,-1.0,-1.0,23.347,2.22148,1849,4971386 +16,138,search,7701.16,2813.54,3448.92,8.05285,200.577,0.92377,0.102075,-1.0,-1.0,24.384,2.22148,1849,4971386 +16,139,delete,8.836,-1.0,-1.0,-1.0,-1.0,24.34,2.22148,1849,4963487 +16,140,search,7680.33,2807.31,3464.36,8.03639,199.023,0.92371,0.102061,-1.0,-1.0,24.46,2.22148,1849,4963487 +16,141,delete,24.208,-1.0,-1.0,-1.0,-1.0,91.028,2.21645,1843,4941830 +16,142,search,7948.07,2798.62,3715.41,7.4674,222.97,0.92346,0.102376,-1.0,-1.0,24.356,2.21645,1843,4941830 +16,143,delete,18.547,-1.0,-1.0,-1.0,-1.0,70.514,2.21272,1840,4925657 +16,144,search,7936.96,2793.74,3672.62,7.49025,215.329,0.92347,0.102358,-1.0,-1.0,24.3,2.21272,1840,4925657 +16,145,delete,3.144,-1.0,-1.0,-1.0,-1.0,28.91,2.2126,1839,4922686 +16,146,search,7879.54,2792.17,3643.22,7.48754,215.255,0.92316,0.102619,-1.0,-1.0,24.781,2.2126,1839,4922686 +16,147,delete,8.787,-1.0,-1.0,-1.0,-1.0,23.404,2.2126,1839,4913636 +16,148,search,7708.04,2784.58,3501.45,7.71997,201.013,0.92307,0.102636,-1.0,-1.0,24.06,2.2126,1839,4913636 +16,149,delete,245.551,-1.0,-1.0,-1.0,-1.0,1159.16,2.12996,1755,4717250 +16,150,search,7509.31,2794.1,3442.17,7.58149,205.721,0.92247,0.104006,-1.0,-1.0,48.272,2.13058,1758,4717250 +16,151,delete,169.885,-1.0,-1.0,-1.0,-1.0,907.976,2.06241,1687,4560269 +32,2,search,2599.15,37.3245,137.518,2.05555,145.33,0.8551,0.130106,-1.0,-1.0,7.807,0.0184881,1000,38806 +32,3,insert,16.466,-1.0,-1.0,-1.0,-1.0,15.625,0.0226681,998,49334 +32,4,search,2627.96,66.5585,137.808,2.35615,142.371,0.86613,0.128797,-1.0,-1.0,14.461,0.0226611,997,49334 +32,5,insert,34.935,-1.0,-1.0,-1.0,-1.0,176.276,0.0329766,993,73557 +32,6,search,2613.1,192.225,215.273,3.29354,92.8759,0.92433,0.104255,-1.0,-1.0,106.377,0.0329766,993,73557 +32,7,insert,41.685,-1.0,-1.0,-1.0,-1.0,103.164,0.0425054,993,95766 +32,8,search,2582.74,247.251,240.507,3.42631,96.8515,0.9262,0.103389,-1.0,-1.0,87.324,0.042614,994,95766 +32,9,insert,6.115,-1.0,-1.0,-1.0,-1.0,77.886,0.0443284,994,100025 +32,10,search,2607.19,258.65,241.855,3.43259,101.206,0.92954,0.0985602,-1.0,-1.0,80.099,0.0443284,994,100025 +32,11,insert,18.587,-1.0,-1.0,-1.0,-1.0,80.554,0.0497523,994,111417 +32,12,search,2611.92,306.036,271.119,3.52729,96.3119,0.93027,0.0988801,-1.0,-1.0,20.622,0.0498144,995,111417 +32,13,insert,37.787,-1.0,-1.0,-1.0,-1.0,7.277,0.0611217,995,137139 +32,14,search,2674.45,349.111,291.975,3.41849,109.403,0.92593,0.101996,-1.0,-1.0,20.475,0.061131,997,137139 +32,15,insert,34.652,-1.0,-1.0,-1.0,-1.0,7.392,0.0697407,997,156710 +32,16,search,2663.42,384.787,307.9,3.58703,96.6557,0.92334,0.103711,-1.0,-1.0,7.765,0.0697407,997,156710 +32,17,insert,7.943,-1.0,-1.0,-1.0,-1.0,7.144,0.071544,997,161773 +32,18,search,2682.51,406.081,323.102,3.58594,101.801,0.92497,0.102835,-1.0,-1.0,7.918,0.071544,997,161773 +32,19,insert,45.301,-1.0,-1.0,-1.0,-1.0,7.092,0.0782662,997,177161 +32,20,search,2695.09,451.935,343.362,3.67672,100.064,0.92892,0.0991464,-1.0,-1.0,7.753,0.0782662,997,177161 +32,21,insert,429.99,-1.0,-1.0,-1.0,-1.0,111.451,0.199605,1012,448898 +32,22,search,2889.6,997.456,648.193,4.21537,73.1439,0.92131,0.100373,-1.0,-1.0,17.648,0.199605,1013,448898 +32,23,insert,456.168,-1.0,-1.0,-1.0,-1.0,235.996,0.32773,1041,733493 +32,24,search,3249.32,1455.35,947.741,4.34214,91.1731,0.91819,0.101904,-1.0,-1.0,73.474,0.327728,1049,733493 +32,25,insert,50.041,-1.0,-1.0,-1.0,-1.0,29.043,0.34431,1052,777443 +32,26,search,3235.59,1451.63,929.517,4.37148,81.3696,0.91778,0.102617,-1.0,-1.0,27.541,0.344429,1055,777443 +32,27,insert,39.999,-1.0,-1.0,-1.0,-1.0,16.611,0.356493,1056,805394 +32,28,search,3234.01,1464.13,905.766,4.59362,75.0954,0.91795,0.102405,-1.0,-1.0,9.018,0.356493,1056,805394 +32,29,insert,79.636,-1.0,-1.0,-1.0,-1.0,43.493,0.383726,1059,866391 +32,30,search,3314.41,1549.54,994.87,4.45809,86.9381,0.91927,0.102159,-1.0,-1.0,21.959,0.383726,1060,866391 +32,31,insert,490.527,-1.0,-1.0,-1.0,-1.0,367.233,0.555082,1108,1239816 +32,32,search,3781.33,1874.89,1301.34,4.36104,111.53,0.91233,0.107674,-1.0,-1.0,104.767,0.555111,1115,1239816 +32,33,insert,65.577,-1.0,-1.0,-1.0,-1.0,25.298,0.57604,1117,1291211 +32,34,search,3806.39,1890.49,1329.63,4.40896,106.441,0.91192,0.108396,-1.0,-1.0,69.747,0.576056,1121,1291211 +32,35,insert,49.753,-1.0,-1.0,-1.0,-1.0,51.405,0.588046,1127,1324978 +32,36,search,3772.49,1912.47,1276.76,4.63068,99.7529,0.91222,0.10816,-1.0,-1.0,43.054,0.588048,1131,1324978 +32,37,insert,61.654,-1.0,-1.0,-1.0,-1.0,26.828,0.607285,1133,1371885 +32,38,search,3887.15,1937.76,1378.16,4.40796,103.556,0.91058,0.108864,-1.0,-1.0,16.963,0.607285,1134,1371885 +32,39,insert,13.177,-1.0,-1.0,-1.0,-1.0,9.908,0.610575,1134,1379335 +32,40,search,3910.05,1943.48,1373.71,4.42952,106.59,0.91147,0.108628,-1.0,-1.0,10.652,0.610575,1134,1379335 +32,41,insert,62.288,-1.0,-1.0,-1.0,-1.0,54.384,0.62614,1140,1413312 +32,42,search,3948.78,1954.1,1401.81,4.40728,115.702,0.91178,0.109014,-1.0,-1.0,10.022,0.62614,1140,1413312 +32,43,insert,15.009,-1.0,-1.0,-1.0,-1.0,9.423,0.629338,1140,1423411 +32,44,search,3930.67,1971.13,1398.35,4.432,109.846,0.91217,0.108523,-1.0,-1.0,10.636,0.629338,1140,1423411 +32,45,insert,227.738,-1.0,-1.0,-1.0,-1.0,23.157,0.643618,1142,1457559 +32,46,search,3953.28,1997.5,1420.62,4.41212,107.515,0.91266,0.107608,-1.0,-1.0,10.738,0.643618,1142,1457559 +32,47,insert,344.305,-1.0,-1.0,-1.0,-1.0,312.76,0.748328,1173,1688593 +32,48,search,4074.37,2125.22,1468.13,4.63507,108.418,0.91136,0.109784,-1.0,-1.0,102.804,0.748326,1180,1688593 +32,49,insert,49.722,-1.0,-1.0,-1.0,-1.0,45.492,0.764662,1185,1723505 +32,50,search,4057.12,2117.38,1423.6,4.70802,108.185,0.91181,0.109462,-1.0,-1.0,60.981,0.764661,1190,1723505 +32,51,insert,374.875,-1.0,-1.0,-1.0,-1.0,247.489,0.904898,1223,2035040 +32,52,search,4480.15,2324.51,1746.15,4.45385,135.223,0.91498,0.107963,-1.0,-1.0,57.57,0.905006,1230,2035040 +32,53,insert,48.043,-1.0,-1.0,-1.0,-1.0,36.1,0.923389,1234,2078341 +32,54,search,4522.34,2333.15,1759.77,4.46151,134.963,0.91534,0.107645,-1.0,-1.0,9.94,0.923389,1234,2078341 +32,55,insert,352.015,-1.0,-1.0,-1.0,-1.0,482.272,1.07305,1289,2399905 +32,56,search,4802.63,2465.38,1961,4.47486,140.932,0.91451,0.110065,-1.0,-1.0,103.744,1.07305,1300,2399905 +32,57,insert,23.257,-1.0,-1.0,-1.0,-1.0,27.416,1.08127,1303,2419836 +32,58,search,4926.35,2434.18,1998.79,4.41544,147.17,0.91435,0.110377,-1.0,-1.0,55.742,1.08128,1307,2419836 +32,59,insert,261.739,-1.0,-1.0,-1.0,-1.0,244.663,1.17969,1330,2643083 +32,60,search,4784.73,2462.14,1821.67,4.96349,136.243,0.90825,0.117149,-1.0,-1.0,41.768,1.17977,1334,2643083 +32,61,insert,11.967,-1.0,-1.0,-1.0,-1.0,10.147,1.18231,1334,2653064 +32,62,search,5097.52,2458.95,2082.44,4.47541,167.177,0.90855,0.117077,-1.0,-1.0,10.959,1.18231,1334,2653064 +32,63,insert,4.76,-1.0,-1.0,-1.0,-1.0,10.171,1.18346,1334,2656118 +32,64,search,4712.86,2461.46,1762.22,5.12754,130.132,0.90863,0.116945,-1.0,-1.0,11.951,1.18346,1334,2656118 +32,65,insert,102.236,-1.0,-1.0,-1.0,-1.0,129.854,1.21094,1351,2723424 +32,66,search,4741.11,2477.93,1751.55,5.19898,124.814,0.90897,0.116223,-1.0,-1.0,10.363,1.21094,1351,2723424 +32,67,insert,52.443,-1.0,-1.0,-1.0,-1.0,89.019,1.23151,1363,2769021 +32,68,search,5146.23,2493.73,2091.46,4.4726,155.839,0.91038,0.114854,-1.0,-1.0,22.04,1.23151,1365,2769021 +32,69,insert,62.473,-1.0,-1.0,-1.0,-1.0,72.087,1.25454,1373,2821777 +32,70,search,5177.23,2510.27,2114.24,4.4628,155.792,0.91147,0.11357,-1.0,-1.0,21.263,1.25454,1374,2821777 +32,71,insert,15.938,-1.0,-1.0,-1.0,-1.0,49.644,1.2603,1380,2834921 +32,72,search,5023.58,2503.71,1940.5,4.78457,135.07,0.91209,0.112961,-1.0,-1.0,12.723,1.2603,1380,2834921 +32,73,insert,9.246,-1.0,-1.0,-1.0,-1.0,28.346,1.263,1382,2842044 +32,74,search,5018.61,2502.89,1964.83,4.74747,135.676,0.91189,0.113165,-1.0,-1.0,11.916,1.263,1382,2842044 +32,75,insert,27.767,-1.0,-1.0,-1.0,-1.0,46.402,1.27109,1386,2858890 +32,76,search,4919.72,2504.64,1870.6,5.03758,132.23,0.91186,0.113287,-1.0,-1.0,21.896,1.27134,1387,2858890 +32,77,insert,36.036,-1.0,-1.0,-1.0,-1.0,30.118,1.28071,1390,2882818 +32,78,search,5111.01,2507.56,2007.43,4.79805,145.181,0.91169,0.113496,-1.0,-1.0,20.472,1.28085,1391,2882818 +32,79,insert,509.921,-1.0,-1.0,-1.0,-1.0,20.958,1.3085,1393,2947806 +32,80,search,5168.3,2541.17,2068.25,4.78536,150.361,0.91148,0.113434,-1.0,-1.0,12.239,1.3085,1393,2947806 +32,81,insert,41.732,-1.0,-1.0,-1.0,-1.0,37.488,1.31964,1396,2976895 +32,82,search,5352.36,2555.09,2218.22,4.53075,161.852,0.91284,0.112522,-1.0,-1.0,11.751,1.31964,1396,2976895 +32,83,insert,22.037,-1.0,-1.0,-1.0,-1.0,28.293,1.32663,1398,2990682 +32,84,search,5067.64,2563.24,1968.97,4.94971,138.722,0.91301,0.112255,-1.0,-1.0,11.819,1.32663,1398,2990682 +32,85,insert,43.023,-1.0,-1.0,-1.0,-1.0,28.975,1.34015,1400,3025613 +32,86,search,5308.33,2583.96,2176.26,4.59166,153.811,0.91432,0.111409,-1.0,-1.0,10.987,1.34015,1400,3025613 +32,87,insert,551.364,-1.0,-1.0,-1.0,-1.0,690.61,1.5435,1489,3468632 +32,88,search,5857.78,2731.79,2517.74,4.49193,177.659,0.91693,0.108862,-1.0,-1.0,121.07,1.5435,1503,3468632 +32,89,insert,389.907,-1.0,-1.0,-1.0,-1.0,522.361,1.70464,1583,3831692 +32,90,search,5891.97,2777.19,2379.63,4.90476,179.767,0.91993,0.105206,-1.0,-1.0,88.843,1.70484,1593,3831692 +32,91,insert,33.171,-1.0,-1.0,-1.0,-1.0,58.143,1.71752,1601,3864067 +32,92,search,5979.17,2755.37,2419.44,4.91327,167.911,0.92008,0.105241,-1.0,-1.0,22.957,1.71763,1602,3864067 +32,93,insert,285.987,-1.0,-1.0,-1.0,-1.0,256.082,1.81873,1634,4103827 +32,94,search,6112.24,2821.78,2494.99,4.96229,182.23,0.92056,0.104707,-1.0,-1.0,34.848,1.81898,1637,4103827 +32,95,insert,15.503,-1.0,-1.0,-1.0,-1.0,25.035,1.82434,1639,4119898 +32,96,search,6168.4,2821.39,2534.73,4.92604,184.211,0.92053,0.104817,-1.0,-1.0,12.872,1.82434,1639,4119898 +32,97,insert,12.55,-1.0,-1.0,-1.0,-1.0,25.892,1.82883,1641,4129367 +32,98,search,6388.64,2823.2,2729.2,4.69383,195.325,0.92089,0.10462,-1.0,-1.0,13.437,1.82883,1641,4129367 +32,99,insert,7.63,-1.0,-1.0,-1.0,-1.0,19.612,1.83068,1642,4134502 +32,100,search,6236.83,2825.09,2598.25,4.90864,186.198,0.92088,0.104631,-1.0,-1.0,13.641,1.83068,1642,4134502 +32,101,insert,41.908,-1.0,-1.0,-1.0,-1.0,54.443,1.84314,1648,4164482 +32,102,search,6210.66,2830.83,2536.26,5.05231,184.124,0.92089,0.104696,-1.0,-1.0,13.12,1.84314,1648,4164482 +32,103,insert,22.164,-1.0,-1.0,-1.0,-1.0,30.863,1.84967,1650,4178601 +32,104,search,6100.57,2836,2453.1,5.06245,186.126,0.92073,0.104873,-1.0,-1.0,13.47,1.84967,1650,4178601 +32,105,insert,10.33,-1.0,-1.0,-1.0,-1.0,12.323,1.85245,1650,4187774 +32,106,search,6577.07,2841.1,2881.92,4.57833,211.112,0.921,0.104902,-1.0,-1.0,13.26,1.85245,1650,4187774 +32,107,insert,24.569,-1.0,-1.0,-1.0,-1.0,12.732,1.85802,1650,4200109 +32,108,search,6334.66,2846.34,2674.33,4.7373,200.32,0.92096,0.104836,-1.0,-1.0,12.995,1.85802,1650,4200109 +32,109,insert,480.493,-1.0,-1.0,-1.0,-1.0,614.198,2.02461,1731,4557867 +32,110,search,6705.44,2894.55,2856.14,4.73759,198.711,0.92221,0.103473,-1.0,-1.0,53.152,2.02464,1736,4557867 +32,111,insert,29.129,-1.0,-1.0,-1.0,-1.0,47.835,2.03593,1741,4587252 +32,112,search,6903.01,2894.66,2972.94,4.63044,209.828,0.92282,0.103067,-1.0,-1.0,25.818,2.03597,1742,4587252 +32,113,insert,13.009,-1.0,-1.0,-1.0,-1.0,23.823,2.04103,1743,4600056 +32,114,search,6760.53,2897.42,2881.8,4.76055,200.467,0.92279,0.103218,-1.0,-1.0,17.831,2.04103,1743,4600056 +32,115,insert,37.651,-1.0,-1.0,-1.0,-1.0,39.307,2.05134,1746,4627422 +32,116,search,6791.85,2912.05,2905.79,4.78149,207.477,0.92302,0.103146,-1.0,-1.0,18.304,2.05134,1746,4627422 +32,117,insert,12.864,-1.0,-1.0,-1.0,-1.0,26.414,2.05516,1747,4639292 +32,118,search,6980.54,2916.11,3048.33,4.66403,217.828,0.92337,0.102706,-1.0,-1.0,18.245,2.05516,1747,4639292 +32,119,insert,6.365,-1.0,-1.0,-1.0,-1.0,17.292,2.05687,1747,4643423 +32,120,search,7165.76,2917.98,3240.57,4.43832,228.887,0.92345,0.102684,-1.0,-1.0,18.826,2.05687,1747,4643423 +32,121,insert,18.022,-1.0,-1.0,-1.0,-1.0,26.976,2.06238,1748,4657252 +32,122,search,6781.9,2925.18,2883.43,4.86447,197.726,0.92367,0.102644,-1.0,-1.0,19.051,2.06238,1748,4657252 +32,123,insert,373.755,-1.0,-1.0,-1.0,-1.0,461.743,2.1879,1791,4938477 +32,124,search,7385.03,2986.53,3350,4.49559,238.204,0.92435,0.102825,-1.0,-1.0,151.971,2.18851,1802,4938477 +32,125,insert,62.108,-1.0,-1.0,-1.0,-1.0,94.538,2.20647,1813,4980492 +32,126,search,7436.53,2967.16,3354.13,4.49473,232.51,0.92439,0.10301,-1.0,-1.0,50.007,2.20681,1816,4980492 +32,127,insert,58.812,-1.0,-1.0,-1.0,-1.0,105.269,2.22753,1828,5030515 +32,128,search,7412.12,2962.95,3292.38,4.469,224.935,0.92505,0.102382,-1.0,-1.0,43.026,2.2277,1830,5030515 +32,129,delete,29.283,-1.0,-1.0,-1.0,-1.0,87.677,2.22608,1801,5009391 +32,130,search,7320.74,2947.77,3261.66,4.43298,227.742,0.92378,0.103338,-1.0,-1.0,23.209,2.22608,1801,5009391 +32,131,delete,8.148,-1.0,-1.0,-1.0,-1.0,21.495,2.22608,1801,5004093 +32,132,search,7065.68,2944.8,3026.37,4.8042,214.271,0.92374,0.103506,-1.0,-1.0,23.316,2.22608,1801,5004093 +32,133,delete,18.49,-1.0,-1.0,-1.0,-1.0,38.119,2.22465,1800,4991787 +32,134,search,7094.82,2932.84,3055.44,4.75779,221.056,0.92371,0.103607,-1.0,-1.0,22.414,2.22465,1800,4991787 +32,135,delete,23.12,-1.0,-1.0,-1.0,-1.0,49.523,2.22234,1798,4974857 +32,136,search,7156.63,2924.28,3117.37,4.66818,220.155,0.92347,0.103572,-1.0,-1.0,22.464,2.22234,1798,4974857 +32,137,delete,6.595,-1.0,-1.0,-1.0,-1.0,20.962,2.22234,1798,4971386 +32,138,search,7225.06,2920.85,3184.26,4.57692,224.688,0.92341,0.103626,-1.0,-1.0,21.86,2.22234,1798,4971386 +32,139,delete,10.847,-1.0,-1.0,-1.0,-1.0,21.69,2.22234,1798,4963487 +32,140,search,7130,2913.46,3077.49,4.69119,225.883,0.92334,0.10374,-1.0,-1.0,22.114,2.22234,1798,4963487 +32,141,delete,27.41,-1.0,-1.0,-1.0,-1.0,66.745,2.21837,1793,4941830 +32,142,search,7181.4,2903.59,3142.74,4.56458,220.061,0.92315,0.103854,-1.0,-1.0,22.876,2.21837,1793,4941830 +32,143,delete,21.419,-1.0,-1.0,-1.0,-1.0,72.829,2.21437,1790,4925657 +32,144,search,6903.93,2898.02,2893.78,4.87989,201.182,0.92324,0.103736,-1.0,-1.0,21.938,2.21437,1790,4925657 +32,145,delete,3.191,-1.0,-1.0,-1.0,-1.0,28.894,2.21377,1789,4922686 +32,146,search,6930.06,2897.27,2920.77,4.80801,200.654,0.92309,0.103918,-1.0,-1.0,21.58,2.21377,1789,4922686 +32,147,delete,9.314,-1.0,-1.0,-1.0,-1.0,20.911,2.21377,1789,4913636 +32,148,search,6966.02,2889.44,2949.78,4.79149,201.401,0.92311,0.103935,-1.0,-1.0,22.793,2.21377,1789,4913636 +32,149,delete,236.835,-1.0,-1.0,-1.0,-1.0,1090.42,2.13176,1704,4717250 +32,150,search,6763.5,2910.02,2920.83,4.69217,204.291,0.92204,0.104698,-1.0,-1.0,101.482,2.1335,1714,4717250 +32,151,delete,166.618,-1.0,-1.0,-1.0,-1.0,965.723,2.06444,1639,4560269 +64,2,search,2626.54,37.3338,157.931,1.81926,145.912,0.8551,0.130106,-1.0,-1.0,8.01,0.0184881,1000,38806 +64,3,insert,18.741,-1.0,-1.0,-1.0,-1.0,15.725,0.0226681,998,49334 +64,4,search,2594.21,66.3942,122.972,1.86496,137.089,0.86613,0.128797,-1.0,-1.0,14.256,0.0226611,997,49334 +64,5,insert,39.042,-1.0,-1.0,-1.0,-1.0,172.777,0.032979,994,73557 +64,6,search,2566.55,190.088,193.23,2.40647,101.882,0.92449,0.104144,-1.0,-1.0,105.28,0.032979,994,73557 +64,7,insert,44.54,-1.0,-1.0,-1.0,-1.0,103.249,0.0425732,994,95766 +64,8,search,2562.18,244.304,215.647,2.42346,101.197,0.92654,0.103026,-1.0,-1.0,96.08,0.0427523,996,95766 +64,9,insert,7.37,-1.0,-1.0,-1.0,-1.0,78.037,0.044525,996,100025 +64,10,search,2575.18,253.874,221.306,2.35147,102.972,0.92803,0.0997921,-1.0,-1.0,79.695,0.044525,996,100025 +64,11,insert,19.118,-1.0,-1.0,-1.0,-1.0,80.238,0.049798,996,111417 +64,12,search,2604.66,300.709,244.029,2.38548,100.375,0.92891,0.0999099,-1.0,-1.0,21.586,0.0498425,997,111417 +64,13,insert,38.835,-1.0,-1.0,-1.0,-1.0,7.38,0.0608291,997,137139 +64,14,search,2631.97,342.548,262.26,2.28254,102.83,0.92411,0.103417,-1.0,-1.0,21.674,0.0608373,999,137139 +64,15,insert,35.691,-1.0,-1.0,-1.0,-1.0,7.332,0.0696721,999,156710 +64,16,search,2721.14,379.057,279.223,2.33664,103.049,0.92194,0.104881,-1.0,-1.0,8.071,0.0696721,999,156710 +64,17,insert,8.804,-1.0,-1.0,-1.0,-1.0,7.098,0.0715856,999,161773 +64,18,search,2639.29,400.355,288.807,2.34008,98.4303,0.92332,0.103959,-1.0,-1.0,8.064,0.0715856,999,161773 +64,19,insert,45.099,-1.0,-1.0,-1.0,-1.0,7.12,0.0782744,999,177161 +64,20,search,2655.04,446.26,307.722,2.35572,95.6592,0.92759,0.100352,-1.0,-1.0,7.872,0.0782744,999,177161 +64,21,insert,443.576,-1.0,-1.0,-1.0,-1.0,124.861,0.199902,1018,448898 +64,22,search,2852.39,961.397,566.215,2.59674,79.3837,0.91872,0.101788,-1.0,-1.0,35.13,0.199993,1022,448898 +64,23,insert,460.691,-1.0,-1.0,-1.0,-1.0,259.602,0.328074,1053,733493 +64,24,search,3066.49,1390.87,782.442,2.68344,77.5245,0.91622,0.103371,-1.0,-1.0,78.15,0.328117,1062,733493 +64,25,insert,50.668,-1.0,-1.0,-1.0,-1.0,23.164,0.344285,1064,777443 +64,26,search,3148.99,1400.32,819.546,2.6369,84.5871,0.91637,0.103615,-1.0,-1.0,9.676,0.344285,1064,777443 +64,27,insert,41.782,-1.0,-1.0,-1.0,-1.0,17.789,0.356204,1065,805394 +64,28,search,3157.17,1428.33,849.198,2.63435,77.996,0.91692,0.103149,-1.0,-1.0,10.106,0.356204,1065,805394 +64,29,insert,82.701,-1.0,-1.0,-1.0,-1.0,48.727,0.383698,1069,866391 +64,30,search,3243.73,1506.54,900.829,2.61476,84.946,0.91896,0.102205,-1.0,-1.0,29.156,0.383698,1071,866391 +64,31,insert,513.96,-1.0,-1.0,-1.0,-1.0,335.386,0.552921,1115,1239816 +64,32,search,3598.75,1843.84,1135.49,2.61469,99.6489,0.91092,0.107936,-1.0,-1.0,96.476,0.553407,1122,1239816 +64,33,insert,61.787,-1.0,-1.0,-1.0,-1.0,33.413,0.573868,1126,1291211 +64,34,search,3585.62,1849.01,1134.71,2.65708,88.5778,0.91006,0.108664,-1.0,-1.0,62.882,0.573882,1130,1291211 +64,35,insert,40.761,-1.0,-1.0,-1.0,-1.0,34.036,0.586277,1134,1324978 +64,36,search,3712.72,1884.07,1200.13,2.59181,95.4756,0.91107,0.108028,-1.0,-1.0,42.471,0.586279,1137,1324978 +64,37,insert,67.803,-1.0,-1.0,-1.0,-1.0,26.197,0.604752,1139,1371885 +64,38,search,3730.95,1912.15,1239.69,2.60032,95.8629,0.91006,0.108267,-1.0,-1.0,10.964,0.604752,1139,1371885 +64,39,insert,15.303,-1.0,-1.0,-1.0,-1.0,10.194,0.609275,1139,1379335 +64,40,search,3729.19,1919.78,1238.29,2.59377,97.6537,0.9105,0.107984,-1.0,-1.0,10.983,0.609275,1139,1379335 +64,41,insert,57.236,-1.0,-1.0,-1.0,-1.0,48.924,0.624207,1145,1413312 +64,42,search,3930.17,1932.81,1249.87,2.60046,99.6635,0.91102,0.107985,-1.0,-1.0,10.681,0.624207,1145,1413312 +64,43,insert,22.665,-1.0,-1.0,-1.0,-1.0,15.673,0.628627,1146,1423411 +64,44,search,3885.09,1947.56,1254.09,2.61554,99.1629,0.91106,0.107969,-1.0,-1.0,10.256,0.628627,1146,1423411 +64,45,insert,226.847,-1.0,-1.0,-1.0,-1.0,34.818,0.644894,1150,1457559 +64,46,search,3760.51,1964.56,1239.46,2.686,99.1448,0.91177,0.107316,-1.0,-1.0,10.315,0.644894,1150,1457559 +64,47,insert,317.506,-1.0,-1.0,-1.0,-1.0,264.848,0.748132,1178,1688593 +64,48,search,3980.84,2108.44,1380.86,2.63049,107.889,0.91133,0.109385,-1.0,-1.0,110.34,0.74835,1187,1688593 +64,49,insert,48.873,-1.0,-1.0,-1.0,-1.0,66.261,0.76353,1195,1723505 +64,50,search,4015.77,2082.94,1386.14,2.62056,112.372,0.91207,0.108626,-1.0,-1.0,46.321,0.76353,1198,1723505 +64,51,insert,410.519,-1.0,-1.0,-1.0,-1.0,289.158,0.904273,1238,2035040 +64,52,search,4270.79,2266.8,1545.67,2.63651,119.882,0.9137,0.108734,-1.0,-1.0,65.429,0.904361,1246,2035040 +64,53,insert,49.301,-1.0,-1.0,-1.0,-1.0,16.945,0.920874,1247,2078341 +64,54,search,4220.9,2280.27,1464.39,2.8638,109.179,0.91404,0.108533,-1.0,-1.0,9.934,0.920874,1247,2078341 +64,55,insert,375.888,-1.0,-1.0,-1.0,-1.0,456.454,1.06931,1297,2399905 +64,56,search,4613.54,2439.32,1732.05,2.59923,132.804,0.91369,0.110102,-1.0,-1.0,105.217,1.06931,1307,2399905 +64,57,insert,23.96,-1.0,-1.0,-1.0,-1.0,25.279,1.0777,1310,2419836 +64,58,search,4586.64,2407.26,1689.46,2.70535,129.619,0.91319,0.110856,-1.0,-1.0,48.253,1.07771,1314,2419836 +64,59,insert,237.718,-1.0,-1.0,-1.0,-1.0,199.74,1.17583,1333,2643083 +64,60,search,4713.57,2457.41,1763.97,2.63271,138.297,0.90689,0.117823,-1.0,-1.0,48.552,1.1759,1335,2643083 +64,61,insert,10.891,-1.0,-1.0,-1.0,-1.0,10.366,1.17948,1335,2653064 +64,62,search,4784.12,2464.92,1811.23,2.58567,153.289,0.90746,0.117809,-1.0,-1.0,18.506,1.17949,1336,2653064 +64,63,insert,4.756,-1.0,-1.0,-1.0,-1.0,10.292,1.18057,1336,2656118 +64,64,search,4840.05,2464.19,1864.4,2.56628,164.521,0.90761,0.117555,-1.0,-1.0,11.879,1.18057,1336,2656118 +64,65,insert,121.704,-1.0,-1.0,-1.0,-1.0,148.988,1.21204,1357,2723424 +64,66,search,4663.62,2468.13,1671.83,2.92842,139.751,0.90775,0.116817,-1.0,-1.0,11.031,1.21204,1357,2723424 +64,67,insert,59.91,-1.0,-1.0,-1.0,-1.0,78.777,1.234,1367,2769021 +64,68,search,4917.55,2493.27,1888.63,2.61351,145.428,0.90907,0.11564,-1.0,-1.0,21.68,1.234,1369,2769021 +64,69,insert,57.066,-1.0,-1.0,-1.0,-1.0,82.344,1.25611,1379,2821777 +64,70,search,4964.19,2495.61,1904.1,2.64779,155.12,0.91,0.114672,-1.0,-1.0,20.658,1.25611,1380,2821777 +64,71,insert,17.729,-1.0,-1.0,-1.0,-1.0,36.07,1.26203,1384,2834921 +64,72,search,4873.27,2499.39,1818.64,2.80486,143.96,0.91027,0.114485,-1.0,-1.0,11.103,1.26203,1384,2834921 +64,73,insert,8.046,-1.0,-1.0,-1.0,-1.0,10.355,1.26404,1384,2842044 +64,74,search,4662.47,2505.65,1638.91,3.06525,120.376,0.91029,0.11452,-1.0,-1.0,11.078,1.26404,1384,2842044 +64,75,insert,28.554,-1.0,-1.0,-1.0,-1.0,25.112,1.27211,1386,2858890 +64,76,search,4980.15,2512.95,1917.29,2.6076,156.148,0.91015,0.11481,-1.0,-1.0,11.072,1.27211,1386,2858890 +64,77,insert,35.684,-1.0,-1.0,-1.0,-1.0,10.399,1.28091,1386,2882818 +64,78,search,4976.02,2527.06,1917.67,2.6283,156.33,0.90983,0.115032,-1.0,-1.0,10.928,1.28091,1386,2882818 +64,79,insert,460.971,-1.0,-1.0,-1.0,-1.0,32.498,1.30751,1390,2947806 +64,80,search,4750.48,2548.78,1697.25,3.02889,129.785,0.90985,0.114626,-1.0,-1.0,11.002,1.30751,1390,2947806 +64,81,insert,37.064,-1.0,-1.0,-1.0,-1.0,52.572,1.31993,1396,2976895 +64,82,search,5027.33,2554.56,1934.29,2.64605,154.159,0.91102,0.114083,-1.0,-1.0,11.436,1.31993,1396,2976895 +64,83,insert,17.578,-1.0,-1.0,-1.0,-1.0,37.354,1.32588,1400,2990682 +64,84,search,5064.99,2558.65,1951.53,2.62494,145.832,0.91144,0.113844,-1.0,-1.0,13.093,1.32588,1400,2990682 +64,85,insert,44.917,-1.0,-1.0,-1.0,-1.0,61.536,1.34102,1405,3025613 +64,86,search,4990.9,2576.22,1875.01,2.76205,140.681,0.91205,0.112905,-1.0,-1.0,14.145,1.34102,1405,3025613 +64,87,insert,574.055,-1.0,-1.0,-1.0,-1.0,691.309,1.5448,1494,3468632 +64,88,search,5387.75,2720.13,2091.58,2.78009,159.432,0.91491,0.109854,-1.0,-1.0,124.351,1.54479,1509,3468632 +64,89,insert,404.361,-1.0,-1.0,-1.0,-1.0,508.891,1.70607,1581,3831692 +64,90,search,5702.04,2779.78,2214.29,2.88679,168.557,0.91865,0.105792,-1.0,-1.0,50.516,1.7064,1586,3831692 +64,91,insert,32.196,-1.0,-1.0,-1.0,-1.0,40.277,1.71659,1590,3864067 +64,92,search,5957.23,2787.03,2437.6,2.58567,189.506,0.91865,0.10565,-1.0,-1.0,12.305,1.71659,1590,3864067 +64,93,insert,275.036,-1.0,-1.0,-1.0,-1.0,294.332,1.8221,1629,4103827 +64,94,search,5888.04,2840.72,2306.88,2.86596,172.855,0.91955,0.10462,-1.0,-1.0,37.238,1.82234,1632,4103827 +64,95,insert,16.997,-1.0,-1.0,-1.0,-1.0,25.793,1.82804,1634,4119898 +64,96,search,5916.66,2842.41,2324.77,2.88803,171.48,0.91973,0.104353,-1.0,-1.0,15.759,1.82804,1634,4119898 +64,97,insert,10.766,-1.0,-1.0,-1.0,-1.0,25.968,1.83127,1636,4129367 +64,98,search,6117.93,2842.66,2512.49,2.63016,190.134,0.91951,0.104303,-1.0,-1.0,12.607,1.83127,1636,4129367 +64,99,insert,6.656,-1.0,-1.0,-1.0,-1.0,31.873,1.8335,1639,4134502 +64,100,search,5923.55,2837.85,2354.92,2.83582,181.23,0.91934,0.104498,-1.0,-1.0,13.298,1.8335,1639,4134502 +64,101,insert,37.469,-1.0,-1.0,-1.0,-1.0,93.62,1.84511,1649,4164482 +64,102,search,6199.86,2832.1,2549.99,2.60515,193.826,0.91973,0.104439,-1.0,-1.0,13.19,1.84511,1649,4164482 +64,103,insert,15.824,-1.0,-1.0,-1.0,-1.0,55.454,1.85108,1655,4178601 +64,104,search,5889.07,2825.08,2251.31,2.90414,171.495,0.91975,0.104487,-1.0,-1.0,26.986,1.85145,1657,4178601 +64,105,insert,9.428,-1.0,-1.0,-1.0,-1.0,19.004,1.85467,1658,4187774 +64,106,search,6067.49,2825.07,2425.57,2.72635,181.892,0.91962,0.104749,-1.0,-1.0,14.758,1.85467,1658,4187774 +64,107,insert,24.768,-1.0,-1.0,-1.0,-1.0,34.64,1.8593,1660,4200109 +64,108,search,5942.58,2827.34,2284.97,2.9041,175.873,0.91962,0.104921,-1.0,-1.0,24.053,1.85956,1661,4200109 +64,109,insert,485.176,-1.0,-1.0,-1.0,-1.0,558.126,2.02228,1730,4557867 +64,110,search,6637.35,2909.7,2782.8,2.57447,225.816,0.92101,0.103273,-1.0,-1.0,33.788,2.02255,1732,4557867 +64,111,insert,29.064,-1.0,-1.0,-1.0,-1.0,55.614,2.03407,1738,4587252 +64,112,search,6260.58,2919.14,2424.25,2.95996,181.822,0.92132,0.103141,-1.0,-1.0,16.518,2.03407,1738,4587252 +64,113,insert,12.372,-1.0,-1.0,-1.0,-1.0,30.665,2.03837,1740,4600056 +64,114,search,6602.94,2924.35,2718.56,2.68409,211.139,0.92172,0.102893,-1.0,-1.0,24.827,2.0386,1741,4600056 +64,115,insert,31.006,-1.0,-1.0,-1.0,-1.0,44.215,2.04678,1745,4627422 +64,116,search,6646.24,2932.06,2798.54,2.56826,219.97,0.92204,0.102585,-1.0,-1.0,17.215,2.04678,1745,4627422 +64,117,insert,16.372,-1.0,-1.0,-1.0,-1.0,22.83,2.0526,1746,4639292 +64,118,search,6355.5,2936.91,2512.25,2.87826,189.021,0.92214,0.102435,-1.0,-1.0,17.556,2.0526,1746,4639292 +64,119,insert,6.316,-1.0,-1.0,-1.0,-1.0,24.198,2.05443,1747,4643423 +64,120,search,6707.79,2938.27,2795.43,2.64371,221.212,0.92254,0.10212,-1.0,-1.0,25.173,2.05469,1748,4643423 +64,121,insert,17.557,-1.0,-1.0,-1.0,-1.0,57.198,2.05992,1753,4657252 +64,122,search,6716.27,2934.77,2835.79,2.54687,222.412,0.9223,0.102201,-1.0,-1.0,18.247,2.05992,1753,4657252 +64,123,insert,380.504,-1.0,-1.0,-1.0,-1.0,463.51,2.18724,1797,4938477 +64,124,search,6815.01,2987.69,2828.75,2.69943,224.598,0.92269,0.10321,-1.0,-1.0,134.393,2.18771,1805,4938477 +64,125,insert,57.278,-1.0,-1.0,-1.0,-1.0,92.241,2.20649,1815,4980492 +64,126,search,6944.53,2977.75,2900.57,2.58266,220.596,0.92346,0.102659,-1.0,-1.0,109.982,2.20652,1820,4980492 +64,127,insert,65.698,-1.0,-1.0,-1.0,-1.0,116.296,2.23159,1834,5030515 +64,128,search,7028.62,2967.04,2931,2.59624,225.599,0.92426,0.102161,-1.0,-1.0,64.747,2.23163,1837,5030515 +64,129,delete,26.443,-1.0,-1.0,-1.0,-1.0,91.197,2.22978,1806,5009391 +64,130,search,6861.19,2952.42,2852.46,2.63984,228.079,0.92288,0.102948,-1.0,-1.0,21.99,2.22978,1806,5009391 +64,131,delete,6.792,-1.0,-1.0,-1.0,-1.0,20.697,2.22978,1806,5004093 +64,132,search,6628.08,2949.46,2640.26,2.87616,210.861,0.9229,0.10289,-1.0,-1.0,22.589,2.22978,1806,5004093 +64,133,delete,15.767,-1.0,-1.0,-1.0,-1.0,20.134,2.22978,1806,4991787 +64,134,search,6865.4,2933.85,2877.2,2.62521,226.831,0.92276,0.103186,-1.0,-1.0,21.775,2.22978,1806,4991787 +64,135,delete,22.055,-1.0,-1.0,-1.0,-1.0,47.219,2.22742,1804,4974857 +64,136,search,6982.47,2925.65,2956.19,2.55784,234.508,0.92274,0.102976,-1.0,-1.0,22.335,2.22742,1804,4974857 +64,137,delete,6.377,-1.0,-1.0,-1.0,-1.0,20.403,2.22742,1804,4971386 +64,138,search,6647.41,2922.99,2650.07,2.83395,203.169,0.9227,0.102915,-1.0,-1.0,21.666,2.22742,1804,4971386 +64,139,delete,10.796,-1.0,-1.0,-1.0,-1.0,21.549,2.22742,1804,4963487 +64,140,search,6676.66,2916.5,2690.98,2.81427,206.833,0.92261,0.103047,-1.0,-1.0,21.254,2.22742,1804,4963487 +64,141,delete,25.754,-1.0,-1.0,-1.0,-1.0,88.26,2.22259,1799,4941830 +64,142,search,6847.06,2907.89,2839.11,2.58996,219.557,0.9224,0.103361,-1.0,-1.0,21.465,2.22259,1799,4941830 +64,143,delete,16.859,-1.0,-1.0,-1.0,-1.0,71.392,2.21822,1795,4925657 +64,144,search,6206.4,2903.07,2268.98,3.26355,167.899,0.92263,0.103238,-1.0,-1.0,24.347,2.21822,1795,4925657 +64,145,delete,5.46,-1.0,-1.0,-1.0,-1.0,20.909,2.21822,1795,4922686 +64,146,search,6569.64,2900.63,2581.43,2.88622,199.478,0.92252,0.103286,-1.0,-1.0,23.414,2.21822,1795,4922686 +64,147,delete,14.775,-1.0,-1.0,-1.0,-1.0,20.692,2.21822,1795,4913636 +64,148,search,6844.56,2892.29,2835.33,2.57608,222.499,0.92231,0.103517,-1.0,-1.0,23.082,2.21822,1795,4913636 +64,149,delete,234.826,-1.0,-1.0,-1.0,-1.0,938.215,2.13896,1726,4717250 +64,150,search,6560.45,2870.52,2686.87,2.60014,221.504,0.92092,0.104763,-1.0,-1.0,50.944,2.13959,1729,4717250 +64,151,delete,180.865,-1.0,-1.0,-1.0,-1.0,882.738,2.0694,1662,4560269 +128,2,search,2672.92,37.3734,193.408,1.57112,146.09,0.8551,0.130106,-1.0,-1.0,8.273,0.0184881,1000,38806 +128,3,insert,19.778,-1.0,-1.0,-1.0,-1.0,12.527,0.0226428,999,49334 +128,4,search,2577.16,66.2194,114.734,1.56425,137.43,0.86574,0.128987,-1.0,-1.0,14.508,0.0226534,998,49334 +128,5,insert,38.547,-1.0,-1.0,-1.0,-1.0,170.823,0.0329117,995,73557 +128,6,search,2556.42,187.295,189.044,1.91824,100.373,0.92412,0.104554,-1.0,-1.0,106.033,0.0329117,995,73557 +128,7,insert,44.506,-1.0,-1.0,-1.0,-1.0,103.567,0.042543,995,95766 +128,8,search,2588.87,240.494,203.781,1.75569,104.853,0.92577,0.103649,-1.0,-1.0,91.689,0.0427935,997,95766 +128,9,insert,7.264,-1.0,-1.0,-1.0,-1.0,78.597,0.0446825,997,100025 +128,10,search,2571.7,250.307,204.436,1.78213,98.4934,0.92722,0.100238,-1.0,-1.0,80.141,0.0446825,997,100025 +128,11,insert,19.346,-1.0,-1.0,-1.0,-1.0,80.695,0.0499327,997,111417 +128,12,search,2587.18,295.418,227.1,1.75309,98.531,0.92777,0.100455,-1.0,-1.0,82.699,0.0499327,997,111417 +128,13,insert,39.192,-1.0,-1.0,-1.0,-1.0,81.02,0.0609344,997,137139 +128,14,search,2607.11,340.65,244.88,1.7373,97.6542,0.9243,0.102713,-1.0,-1.0,22.357,0.0610009,998,137139 +128,15,insert,33.217,-1.0,-1.0,-1.0,-1.0,7.481,0.0696093,998,156710 +128,16,search,2728.37,376.497,266.477,1.56422,114.592,0.9225,0.103704,-1.0,-1.0,20.464,0.0696162,1000,156710 +128,17,insert,9.048,-1.0,-1.0,-1.0,-1.0,7.315,0.0716815,1000,161773 +128,18,search,2732.98,392.205,276.734,1.5835,111.851,0.92366,0.102827,-1.0,-1.0,7.88,0.0716815,1000,161773 +128,19,insert,44.099,-1.0,-1.0,-1.0,-1.0,7.187,0.0784176,1000,177161 +128,20,search,2719.71,437.328,298.856,1.60736,99.0863,0.92789,0.0995028,-1.0,-1.0,8.126,0.0784176,1000,177161 +128,21,insert,440.718,-1.0,-1.0,-1.0,-1.0,117.706,0.200022,1019,448898 +128,22,search,2843.27,957.76,547.629,1.58212,86.4297,0.91922,0.101238,-1.0,-1.0,15.914,0.200021,1020,448898 +128,23,insert,462.344,-1.0,-1.0,-1.0,-1.0,270.342,0.328597,1057,733493 +128,24,search,3055.96,1347.73,748.178,1.60548,81.3464,0.91455,0.103761,-1.0,-1.0,76.181,0.328595,1066,733493 +128,25,insert,51.247,-1.0,-1.0,-1.0,-1.0,22.264,0.345498,1068,777443 +128,26,search,3159.81,1361.1,787.472,1.56722,84.5448,0.91515,0.103898,-1.0,-1.0,10.008,0.345498,1068,777443 +128,27,insert,42.327,-1.0,-1.0,-1.0,-1.0,16.808,0.357461,1069,805394 +128,28,search,3135.43,1387.96,818.094,1.57461,76.0259,0.91631,0.103092,-1.0,-1.0,10.714,0.357461,1069,805394 +128,29,insert,83.756,-1.0,-1.0,-1.0,-1.0,44.067,0.385205,1073,866391 +128,30,search,3247.34,1465.03,857.04,1.57942,81.5404,0.91711,0.103313,-1.0,-1.0,19.549,0.385205,1074,866391 +128,31,insert,490.293,-1.0,-1.0,-1.0,-1.0,374.179,0.551257,1119,1239816 +128,32,search,3468.09,1819.27,1017.75,1.63485,93.6226,0.91062,0.107705,-1.0,-1.0,95.024,0.551731,1125,1239816 +128,33,insert,72.548,-1.0,-1.0,-1.0,-1.0,27.9,0.574347,1128,1291211 +128,34,search,3631.86,1838.72,1128.5,1.52663,108.611,0.91159,0.107419,-1.0,-1.0,67.711,0.574363,1132,1291211 +128,35,insert,58.072,-1.0,-1.0,-1.0,-1.0,48.49,0.587698,1138,1324978 +128,36,search,3647.91,1864.39,1151.51,1.54002,109.336,0.91216,0.106088,-1.0,-1.0,44.172,0.587822,1142,1324978 +128,37,insert,70.171,-1.0,-1.0,-1.0,-1.0,36.307,0.607045,1145,1371885 +128,38,search,3660.8,1881.28,1167.42,1.53074,100.707,0.91112,0.106482,-1.0,-1.0,16.776,0.607045,1146,1371885 +128,39,insert,10.05,-1.0,-1.0,-1.0,-1.0,9.774,0.609242,1146,1379335 +128,40,search,3568.35,1884.38,1071.17,1.70063,94.0265,0.91113,0.107087,-1.0,-1.0,11.165,0.609242,1146,1379335 +128,41,insert,65.889,-1.0,-1.0,-1.0,-1.0,46.712,0.62536,1151,1413312 +128,42,search,3709.76,1900.02,1174.61,1.56063,109.564,0.9115,0.107412,-1.0,-1.0,10.085,0.62536,1151,1413312 +128,43,insert,17.864,-1.0,-1.0,-1.0,-1.0,27.933,0.629531,1154,1423411 +128,44,search,3720.37,1905.36,1186.06,1.52132,110.368,0.91175,0.107427,-1.0,-1.0,10.578,0.629531,1154,1423411 +128,45,insert,234.891,-1.0,-1.0,-1.0,-1.0,47.866,0.642682,1160,1457559 +128,46,search,3758.61,1917.02,1194.5,1.50288,110.1,0.91241,0.106509,-1.0,-1.0,10.125,0.642682,1160,1457559 +128,47,insert,357.856,-1.0,-1.0,-1.0,-1.0,299.407,0.749851,1192,1688593 +128,48,search,3876.12,2042.75,1264.3,1.58569,109.413,0.911,0.109397,-1.0,-1.0,94.367,0.74999,1199,1688593 +128,49,insert,49.951,-1.0,-1.0,-1.0,-1.0,27.559,0.764228,1201,1723505 +128,50,search,3867.46,2052.05,1231.36,1.63839,102.87,0.91241,0.10851,-1.0,-1.0,34.011,0.764229,1203,1723505 +128,51,insert,396.871,-1.0,-1.0,-1.0,-1.0,226.529,0.902507,1233,2035040 +128,52,search,3967.37,2275.67,1299.85,1.73018,104.569,0.91476,0.107578,-1.0,-1.0,50.752,0.902566,1238,2035040 +128,53,insert,57.107,-1.0,-1.0,-1.0,-1.0,36.495,0.921763,1241,2078341 +128,54,search,4134.24,2290.9,1406.42,1.63922,112.607,0.91496,0.107858,-1.0,-1.0,17.827,0.921977,1242,2078341 +128,55,insert,375.758,-1.0,-1.0,-1.0,-1.0,441.649,1.07026,1292,2399905 +128,56,search,4275.94,2444.12,1457.7,1.71297,112.143,0.91459,0.108633,-1.0,-1.0,93.156,1.07026,1300,2399905 +128,57,insert,24.301,-1.0,-1.0,-1.0,-1.0,27.733,1.07801,1303,2419836 +128,58,search,4522.63,2429.77,1659.22,1.54172,131.006,0.91462,0.108049,-1.0,-1.0,65.763,1.07802,1309,2419836 +128,59,insert,238.196,-1.0,-1.0,-1.0,-1.0,246.929,1.17777,1335,2643083 +128,60,search,4689.56,2451.35,1737.72,1.53919,136.991,0.90933,0.114786,-1.0,-1.0,71.702,1.17783,1339,2643083 +128,61,insert,18.728,-1.0,-1.0,-1.0,-1.0,10.54,1.18183,1339,2653064 +128,62,search,4399.76,2448.45,1482.53,1.79632,116.341,0.90872,0.115477,-1.0,-1.0,11.557,1.18183,1339,2653064 +128,63,insert,4.994,-1.0,-1.0,-1.0,-1.0,10.679,1.18277,1339,2656118 +128,64,search,4300.7,2450.54,1406.54,1.9001,111.818,0.90895,0.115223,-1.0,-1.0,10.843,1.18277,1339,2656118 +128,65,insert,96.497,-1.0,-1.0,-1.0,-1.0,118.642,1.21013,1355,2723424 +128,66,search,4344.74,2481.23,1414.44,1.92827,107.8,0.90971,0.115128,-1.0,-1.0,10.815,1.21013,1355,2723424 +128,67,insert,50.475,-1.0,-1.0,-1.0,-1.0,83.197,1.23159,1367,2769021 +128,68,search,4865.18,2501.05,1837.8,1.48516,152.946,0.91084,0.114243,-1.0,-1.0,22.388,1.23159,1369,2769021 +128,69,insert,71.992,-1.0,-1.0,-1.0,-1.0,91.249,1.25528,1380,2821777 +128,70,search,4818.08,2507.25,1837.02,1.48939,161.161,0.91229,0.112735,-1.0,-1.0,29.664,1.25528,1382,2821777 +128,71,insert,20.39,-1.0,-1.0,-1.0,-1.0,42.327,1.2608,1387,2834921 +128,72,search,4760.19,2496.71,1710.85,1.65207,139.021,0.91219,0.112676,-1.0,-1.0,11.215,1.2608,1387,2834921 +128,73,insert,10.428,-1.0,-1.0,-1.0,-1.0,39.387,1.26407,1390,2842044 +128,74,search,4632.54,2496.01,1610.51,1.7445,124.843,0.91198,0.112816,-1.0,-1.0,11.676,1.26407,1390,2842044 +128,75,insert,24.51,-1.0,-1.0,-1.0,-1.0,33.275,1.27168,1393,2858890 +128,76,search,4716.86,2502.38,1678.82,1.64033,137.458,0.91211,0.11258,-1.0,-1.0,11.768,1.27168,1393,2858890 +128,77,insert,35.895,-1.0,-1.0,-1.0,-1.0,22.862,1.28123,1395,2882818 +128,78,search,4736.67,2509.86,1649.89,1.74866,133.845,0.91192,0.113269,-1.0,-1.0,11.145,1.28123,1395,2882818 +128,79,insert,522.997,-1.0,-1.0,-1.0,-1.0,28.77,1.3082,1398,2947806 +128,80,search,4742.67,2535.28,1696.37,1.69836,133.443,0.91112,0.114186,-1.0,-1.0,11.044,1.3082,1398,2947806 +128,81,insert,40.581,-1.0,-1.0,-1.0,-1.0,26.973,1.32114,1400,2976895 +128,82,search,4717.08,2552.33,1642.97,1.7715,128.507,0.91269,0.113262,-1.0,-1.0,13.845,1.32114,1400,2976895 +128,83,insert,21.89,-1.0,-1.0,-1.0,-1.0,43.234,1.32735,1404,2990682 +128,84,search,4978.83,2545.85,1881.72,1.52826,169.901,0.91255,0.113483,-1.0,-1.0,12.03,1.32735,1404,2990682 +128,85,insert,39.963,-1.0,-1.0,-1.0,-1.0,36.838,1.34118,1407,3025613 +128,86,search,5027.54,2567.03,1894.99,1.50228,171.252,0.91364,0.112676,-1.0,-1.0,11.94,1.34118,1407,3025613 +128,87,insert,583.273,-1.0,-1.0,-1.0,-1.0,713.119,1.5438,1496,3468632 +128,88,search,5274.38,2714.3,1987.12,1.65221,162.617,0.91636,0.109635,-1.0,-1.0,133.633,1.54406,1511,3468632 +128,89,insert,407.613,-1.0,-1.0,-1.0,-1.0,525.221,1.70293,1589,3831692 +128,90,search,5671.03,2765.8,2162.66,1.61618,181.652,0.92028,0.105347,-1.0,-1.0,53.068,1.70303,1594,3831692 +128,91,insert,34.169,-1.0,-1.0,-1.0,-1.0,78.722,1.71425,1604,3864067 +128,92,search,5821.99,2747.5,2286.43,1.48336,187.114,0.91968,0.105539,-1.0,-1.0,12.755,1.71425,1604,3864067 +128,93,insert,276.199,-1.0,-1.0,-1.0,-1.0,267.595,1.81715,1637,4103827 +128,94,search,5678.44,2813.67,2103.71,1.72851,170.906,0.91939,0.105292,-1.0,-1.0,31.991,1.81715,1639,4103827 +128,95,insert,18.536,-1.0,-1.0,-1.0,-1.0,38.089,1.82414,1643,4119898 +128,96,search,6057.54,2810.38,2421.27,1.47388,211.26,0.9196,0.105084,-1.0,-1.0,12.83,1.82414,1643,4119898 +128,97,insert,10.326,-1.0,-1.0,-1.0,-1.0,12.189,1.82646,1643,4129367 +128,98,search,5702.45,2816.05,2137.41,1.67699,170.89,0.91974,0.105019,-1.0,-1.0,13.07,1.82646,1643,4129367 +128,99,insert,8.41,-1.0,-1.0,-1.0,-1.0,12.688,1.8286,1643,4134502 +128,100,search,5987.11,2820.8,2389.7,1.4877,197.015,0.91982,0.104903,-1.0,-1.0,13.215,1.8286,1643,4134502 +128,101,insert,40.264,-1.0,-1.0,-1.0,-1.0,49.839,1.83958,1647,4164482 +128,102,search,5694.63,2833.56,2086.72,1.73857,167.614,0.92025,0.104626,-1.0,-1.0,14.783,1.83958,1647,4164482 +128,103,insert,22.082,-1.0,-1.0,-1.0,-1.0,12.923,1.8457,1647,4178601 +128,104,search,5749.82,2845.16,2148.51,1.68581,176.209,0.92047,0.104456,-1.0,-1.0,14.423,1.8457,1647,4178601 +128,105,insert,16.612,-1.0,-1.0,-1.0,-1.0,32.099,1.85116,1649,4187774 +128,106,search,5413.62,2847.28,1870.44,1.94771,143.915,0.92032,0.104794,-1.0,-1.0,13.717,1.85116,1649,4187774 +128,107,insert,27.985,-1.0,-1.0,-1.0,-1.0,14.34,1.8579,1649,4200109 +128,108,search,6076.41,2853.33,2452.8,1.45184,219.078,0.92012,0.104965,-1.0,-1.0,13.389,1.8579,1649,4200109 +128,109,insert,508.097,-1.0,-1.0,-1.0,-1.0,566.208,2.0223,1719,4557867 +128,110,search,6250.29,2931.18,2450.76,1.59353,202.399,0.92161,0.103518,-1.0,-1.0,71.441,2.02237,1725,4557867 +128,111,insert,32.616,-1.0,-1.0,-1.0,-1.0,77.828,2.03297,1734,4587252 +128,112,search,6527.12,2917.35,2662.05,1.47734,233.423,0.92226,0.103215,-1.0,-1.0,16.59,2.03297,1734,4587252 +128,113,insert,13.929,-1.0,-1.0,-1.0,-1.0,45.537,2.03783,1738,4600056 +128,114,search,6084.71,2916.91,2295.75,1.7641,183.922,0.92221,0.103193,-1.0,-1.0,17.617,2.03783,1738,4600056 +128,115,insert,38.522,-1.0,-1.0,-1.0,-1.0,84.571,2.05028,1748,4627422 +128,116,search,6440.85,2910.52,2565.7,1.51956,217.169,0.92258,0.102955,-1.0,-1.0,17.666,2.05028,1748,4627422 +128,117,insert,16.603,-1.0,-1.0,-1.0,-1.0,42.956,2.05526,1751,4639292 +128,118,search,6181.35,2911.34,2348.08,1.73787,184.706,0.92262,0.102713,-1.0,-1.0,18.672,2.05526,1751,4639292 +128,119,insert,6.167,-1.0,-1.0,-1.0,-1.0,17.446,2.05682,1751,4643423 +128,120,search,6285.1,2913.87,2457.09,1.65481,196.518,0.92262,0.102801,-1.0,-1.0,18.54,2.05682,1751,4643423 +128,121,insert,15.847,-1.0,-1.0,-1.0,-1.0,54.095,2.0624,1756,4657252 +128,122,search,6507.64,2909.48,2618.13,1.51251,215.915,0.92286,0.102655,-1.0,-1.0,17.725,2.0624,1756,4657252 +128,123,insert,364.47,-1.0,-1.0,-1.0,-1.0,444.934,2.18895,1798,4938477 +128,124,search,6696.12,2972.57,2724.92,1.48439,225.272,0.92318,0.102883,-1.0,-1.0,122.379,2.18966,1806,4938477 +128,125,insert,54.468,-1.0,-1.0,-1.0,-1.0,68.089,2.20631,1813,4980492 +128,126,search,6790.38,2965.59,2769.86,1.48993,230.4,0.92326,0.102501,-1.0,-1.0,75.712,2.20633,1815,4980492 +128,127,insert,60.263,-1.0,-1.0,-1.0,-1.0,95.113,2.22727,1825,5030515 +128,128,search,6626.47,2978.65,2597.07,1.63349,212.357,0.92397,0.102067,-1.0,-1.0,49.162,2.22728,1827,5030515 +128,129,delete,22.025,-1.0,-1.0,-1.0,-1.0,89.334,2.22556,1796,5009391 +128,130,search,6474.79,2966.61,2512.51,1.66005,210.533,0.92287,0.102821,-1.0,-1.0,22.5,2.22556,1796,5009391 +128,131,delete,7.617,-1.0,-1.0,-1.0,-1.0,20.786,2.22556,1796,5004093 +128,132,search,6760.28,2963.63,2780.61,1.4696,232.463,0.92289,0.102829,-1.0,-1.0,21.653,2.22556,1796,5004093 +128,133,delete,15.215,-1.0,-1.0,-1.0,-1.0,20.319,2.22556,1796,4991787 +128,134,search,6748.44,2949.28,2765.75,1.48629,224.709,0.92269,0.102958,-1.0,-1.0,21.504,2.22556,1796,4991787 +128,135,delete,20.041,-1.0,-1.0,-1.0,-1.0,47.122,2.22298,1794,4974857 +128,136,search,6701.61,2940.97,2719.4,1.47086,225.483,0.92294,0.102725,-1.0,-1.0,29.69,2.22322,1795,4974857 +128,137,delete,3.645,-1.0,-1.0,-1.0,-1.0,20.462,2.22322,1795,4971386 +128,138,search,6661.84,2935.76,2683.09,1.532,222.81,0.92286,0.102908,-1.0,-1.0,21.85,2.22322,1795,4971386 +128,139,delete,11.537,-1.0,-1.0,-1.0,-1.0,21.169,2.22322,1795,4963487 +128,140,search,6743.66,2928.45,2752.26,1.4665,238.529,0.92279,0.102956,-1.0,-1.0,21.319,2.22322,1795,4963487 +128,141,delete,26.521,-1.0,-1.0,-1.0,-1.0,74.554,2.21841,1789,4941830 +128,142,search,6465.21,2920.1,2513.18,1.61547,208.331,0.92273,0.102956,-1.0,-1.0,21.831,2.21841,1789,4941830 +128,143,delete,18.795,-1.0,-1.0,-1.0,-1.0,53.512,2.21435,1786,4925657 +128,144,search,6464.88,2913.74,2530.09,1.61155,204.737,0.92248,0.103227,-1.0,-1.0,21.384,2.21435,1786,4925657 +128,145,delete,2.652,-1.0,-1.0,-1.0,-1.0,20.083,2.21435,1786,4922686 +128,146,search,6643.56,2911.38,2664.46,1.49956,226.949,0.9224,0.103458,-1.0,-1.0,22.681,2.21435,1786,4922686 +128,147,delete,16.747,-1.0,-1.0,-1.0,-1.0,20.977,2.21435,1786,4913636 +128,148,search,6312.67,2904.26,2374.72,1.69628,193.394,0.92221,0.103657,-1.0,-1.0,21.821,2.21435,1786,4913636 +128,149,delete,236.893,-1.0,-1.0,-1.0,-1.0,1040.08,2.13458,1710,4717250 +128,150,search,6054.93,2897.64,2303.56,1.73004,185.692,0.92174,0.104141,-1.0,-1.0,48.316,2.13515,1713,4717250 +128,151,delete,164.27,-1.0,-1.0,-1.0,-1.0,1020.29,2.06202,1632,4560269 +256,2,search,2866.33,37.4413,330.337,1.38984,158.664,0.8551,0.130106,-1.0,-1.0,8.102,0.0184881,1000,38806 +256,3,insert,20.283,-1.0,-1.0,-1.0,-1.0,16.146,0.0226681,998,49334 +256,4,search,2651.24,65.2517,113.564,1.3625,157.538,0.86613,0.128797,-1.0,-1.0,14.509,0.0226611,997,49334 +256,5,insert,40.119,-1.0,-1.0,-1.0,-1.0,175.679,0.0329766,993,73557 +256,6,search,2630.64,184.092,193.766,1.61817,109.375,0.92433,0.104255,-1.0,-1.0,105.528,0.0329766,993,73557 +256,7,insert,44.735,-1.0,-1.0,-1.0,-1.0,103.687,0.0425054,993,95766 +256,8,search,2663.43,234.618,202.5,1.43228,122.105,0.9262,0.103389,-1.0,-1.0,92.553,0.0426968,995,95766 +256,9,insert,6.946,-1.0,-1.0,-1.0,-1.0,78.873,0.0444112,995,100025 +256,10,search,2645.79,242.678,201.303,1.41079,120.474,0.92898,0.0992193,-1.0,-1.0,80.217,0.0444112,995,100025 +256,11,insert,19.58,-1.0,-1.0,-1.0,-1.0,80.938,0.0498874,995,111417 +256,12,search,2592.93,287.737,228.644,1.37054,114.145,0.92977,0.0991206,-1.0,-1.0,22.504,0.0499416,996,111417 +256,13,insert,39.701,-1.0,-1.0,-1.0,-1.0,7.54,0.061167,996,137139 +256,14,search,2646.73,329.176,246.49,1.26251,124.809,0.92541,0.102488,-1.0,-1.0,21.691,0.0611767,998,137139 +256,15,insert,33.622,-1.0,-1.0,-1.0,-1.0,7.52,0.0697745,998,156710 +256,16,search,2736.8,363.55,265.618,1.17019,122.11,0.92281,0.104133,-1.0,-1.0,8.082,0.0697745,998,156710 +256,17,insert,9.018,-1.0,-1.0,-1.0,-1.0,7.422,0.0715571,998,161773 +256,18,search,2722.54,383.144,280.243,1.22124,117.091,0.92457,0.102932,-1.0,-1.0,8.016,0.0715571,998,161773 +256,19,insert,43.901,-1.0,-1.0,-1.0,-1.0,7.708,0.0780545,998,177161 +256,20,search,2731.35,426.895,296.71,1.15218,115.739,0.92857,0.0994008,-1.0,-1.0,7.988,0.0780545,998,177161 +256,21,insert,434.263,-1.0,-1.0,-1.0,-1.0,130.134,0.2,1020,448898 +256,22,search,2989.65,920.288,555.602,1.09872,100.624,0.91983,0.101275,-1.0,-1.0,24.346,0.199999,1022,448898 +256,23,insert,461.042,-1.0,-1.0,-1.0,-1.0,253.765,0.328127,1055,733493 +256,24,search,3149,1326.46,796.222,1.01771,96.2053,0.91588,0.104384,-1.0,-1.0,84.882,0.328126,1066,733493 +256,25,insert,52.962,-1.0,-1.0,-1.0,-1.0,23.132,0.344823,1068,777443 +256,26,search,3257.76,1323.09,791.535,1.02197,101.93,0.91588,0.104126,-1.0,-1.0,14.648,0.344825,1069,777443 +256,27,insert,41.55,-1.0,-1.0,-1.0,-1.0,17.491,0.356823,1070,805394 +256,28,search,3007.11,1346.48,686.649,1.25006,60.8945,0.91612,0.103925,-1.0,-1.0,10.542,0.356823,1070,805394 +256,29,insert,80.746,-1.0,-1.0,-1.0,-1.0,49.045,0.38411,1074,866391 +256,30,search,3225.46,1422.7,853.496,1.0155,90.8802,0.91789,0.103308,-1.0,-1.0,31.605,0.384109,1076,866391 +256,31,insert,535.243,-1.0,-1.0,-1.0,-1.0,338.05,0.554277,1118,1239816 +256,32,search,3496.15,1769.06,1097.16,0.937941,105.159,0.91086,0.10801,-1.0,-1.0,90.883,0.554304,1124,1239816 +256,33,insert,65.119,-1.0,-1.0,-1.0,-1.0,10.187,0.574957,1124,1291211 +256,34,search,3590.48,1796.63,1128.59,0.965828,94.8247,0.91146,0.107481,-1.0,-1.0,65.144,0.574972,1128,1291211 +256,35,insert,44.384,-1.0,-1.0,-1.0,-1.0,27.639,0.586001,1131,1324978 +256,36,search,3621.82,1836.28,1149.15,0.949324,101.944,0.91204,0.106854,-1.0,-1.0,32.279,0.586003,1133,1324978 +256,37,insert,73.402,-1.0,-1.0,-1.0,-1.0,25.869,0.605043,1135,1371885 +256,38,search,3680.43,1867.93,1186.04,0.982134,106.601,0.91197,0.107114,-1.0,-1.0,9.939,0.605043,1135,1371885 +256,39,insert,14.426,-1.0,-1.0,-1.0,-1.0,9.05,0.608616,1135,1379335 +256,40,search,3678.11,1877.3,1190.66,0.975344,96.5397,0.91244,0.107044,-1.0,-1.0,11.623,0.608616,1135,1379335 +256,41,insert,69.362,-1.0,-1.0,-1.0,-1.0,61.036,0.625497,1142,1413312 +256,42,search,3678.45,1889.6,1184.38,0.982888,97.1329,0.91241,0.107267,-1.0,-1.0,10.661,0.625497,1142,1413312 +256,43,insert,21.297,-1.0,-1.0,-1.0,-1.0,23.908,0.630431,1144,1423411 +256,44,search,3680.06,1896.38,1198.79,0.997243,99.01,0.91228,0.107439,-1.0,-1.0,17.045,0.63057,1145,1423411 +256,45,insert,224.293,-1.0,-1.0,-1.0,-1.0,48.184,0.64474,1151,1457559 +256,46,search,3700.12,1897.31,1192.76,0.971471,99.873,0.91199,0.106876,-1.0,-1.0,10.125,0.64474,1151,1457559 +256,47,insert,334.543,-1.0,-1.0,-1.0,-1.0,297.361,0.749693,1181,1688593 +256,48,search,3925.2,2031.93,1323.09,0.943001,115.165,0.91136,0.109144,-1.0,-1.0,101.757,0.749692,1188,1688593 +256,49,insert,49.098,-1.0,-1.0,-1.0,-1.0,49.923,0.764331,1193,1723505 +256,50,search,3931.86,2031.26,1315.67,0.95153,115.329,0.91121,0.109561,-1.0,-1.0,53.741,0.764331,1196,1723505 +256,51,insert,403.629,-1.0,-1.0,-1.0,-1.0,275.633,0.903658,1231,2035040 +256,52,search,4049.83,2227.73,1365.35,1.04454,109.593,0.91414,0.108013,-1.0,-1.0,65.174,0.903742,1239,2035040 +256,53,insert,48.623,-1.0,-1.0,-1.0,-1.0,23.635,0.920441,1241,2078341 +256,54,search,4026.55,2234.68,1315.46,1.03059,105.334,0.9147,0.10715,-1.0,-1.0,10.294,0.920441,1241,2078341 +256,55,insert,388.35,-1.0,-1.0,-1.0,-1.0,431.382,1.0691,1294,2399905 +256,56,search,4508.55,2386.52,1653.19,0.934143,138.976,0.9147,0.109905,-1.0,-1.0,79.398,1.0691,1302,2399905 +256,57,insert,23.073,-1.0,-1.0,-1.0,-1.0,16.346,1.07595,1303,2419836 +256,58,search,4482.37,2367.03,1641.36,0.958809,136.756,0.91404,0.110006,-1.0,-1.0,38.654,1.07596,1306,2419836 +256,59,insert,262.715,-1.0,-1.0,-1.0,-1.0,138.061,1.17566,1317,2643083 +256,60,search,4556.06,2445.47,1668.06,0.986127,135.888,0.90783,0.117466,-1.0,-1.0,32.403,1.17566,1320,2643083 +256,61,insert,12.504,-1.0,-1.0,-1.0,-1.0,9.865,1.17978,1320,2653064 +256,62,search,4657.78,2450.05,1728.37,0.933409,140.965,0.90795,0.117649,-1.0,-1.0,12.166,1.17978,1320,2653064 +256,63,insert,5.623,-1.0,-1.0,-1.0,-1.0,10.825,1.18088,1320,2656118 +256,64,search,4649.03,2453.17,1739.01,0.933658,149.92,0.90797,0.117517,-1.0,-1.0,12.316,1.18088,1320,2656118 +256,65,insert,106.05,-1.0,-1.0,-1.0,-1.0,94.81,1.20921,1332,2723424 +256,66,search,4634.52,2493.86,1714.58,0.947239,143.737,0.90884,0.116484,-1.0,-1.0,10.767,1.20921,1332,2723424 +256,67,insert,51.307,-1.0,-1.0,-1.0,-1.0,112.12,1.23034,1348,2769021 +256,68,search,4542.21,2497.1,1581.95,1.08508,132.845,0.90983,0.115543,-1.0,-1.0,16.93,1.23034,1349,2769021 +256,69,insert,61.417,-1.0,-1.0,-1.0,-1.0,86.023,1.2527,1359,2821777 +256,70,search,4780.07,2507.73,1801.43,0.966099,158.153,0.91055,0.11448,-1.0,-1.0,20.453,1.2527,1360,2821777 +256,71,insert,18.378,-1.0,-1.0,-1.0,-1.0,37.117,1.25796,1364,2834921 +256,72,search,4834.85,2509.13,1817.14,0.922589,167.184,0.91112,0.114098,-1.0,-1.0,11.275,1.25796,1364,2834921 +256,73,insert,12.435,-1.0,-1.0,-1.0,-1.0,10.345,1.26159,1364,2842044 +256,74,search,4778.43,2514.37,1796.74,0.960248,153.768,0.91113,0.114122,-1.0,-1.0,11.267,1.26159,1364,2842044 +256,75,insert,29.17,-1.0,-1.0,-1.0,-1.0,24.27,1.2705,1366,2858890 +256,76,search,4562.94,2524.17,1607.64,1.09059,132.605,0.91126,0.113863,-1.0,-1.0,11.524,1.2705,1366,2858890 +256,77,insert,39.64,-1.0,-1.0,-1.0,-1.0,10.593,1.28015,1366,2882818 +256,78,search,4782.35,2537.01,1817.36,0.958382,148.02,0.91092,0.114045,-1.0,-1.0,11.797,1.28015,1366,2882818 +256,79,insert,514.244,-1.0,-1.0,-1.0,-1.0,18.395,1.30633,1367,2947806 +256,80,search,4880.56,2575.58,1870.15,0.921732,161.661,0.91098,0.114338,-1.0,-1.0,18.337,1.30654,1368,2947806 +256,81,insert,38.66,-1.0,-1.0,-1.0,-1.0,55.447,1.31819,1374,2976895 +256,82,search,4784,2581.7,1753.4,0.995773,143.095,0.91195,0.113698,-1.0,-1.0,11.056,1.31819,1374,2976895 +256,83,insert,17.081,-1.0,-1.0,-1.0,-1.0,35.815,1.32434,1378,2990682 +256,84,search,4633.4,2577.11,1646.72,1.06359,129.527,0.91184,0.113599,-1.0,-1.0,11.679,1.32434,1378,2990682 +256,85,insert,39.969,-1.0,-1.0,-1.0,-1.0,61.15,1.3397,1385,3025613 +256,86,search,4928.29,2585.27,1887.93,0.92073,154.339,0.91249,0.113156,-1.0,-1.0,11.172,1.3397,1385,3025613 +256,87,insert,569.139,-1.0,-1.0,-1.0,-1.0,773.146,1.5473,1487,3468632 +256,88,search,5326.17,2689.83,2057.41,0.925801,163.066,0.91491,0.110715,-1.0,-1.0,116.108,1.54746,1499,3468632 +256,89,insert,414.199,-1.0,-1.0,-1.0,-1.0,504.674,1.70663,1574,3831692 +256,90,search,5748.78,2764.38,2270.67,0.903898,186.019,0.91887,0.106379,-1.0,-1.0,77.178,1.70668,1582,3831692 +256,91,insert,36.18,-1.0,-1.0,-1.0,-1.0,60.497,1.71714,1589,3864067 +256,92,search,5785.23,2750.71,2285.2,0.881235,184.123,0.91881,0.106611,-1.0,-1.0,12.829,1.71714,1589,3864067 +256,93,insert,307.138,-1.0,-1.0,-1.0,-1.0,259.868,1.82079,1619,4103827 +256,94,search,5876.67,2829.51,2316.68,0.946039,198.877,0.91912,0.105968,-1.0,-1.0,29.653,1.82079,1621,4103827 +256,95,insert,18.506,-1.0,-1.0,-1.0,-1.0,51.153,1.82827,1627,4119898 +256,96,search,5964.28,2825.48,2380.98,0.911465,187.887,0.91964,0.105314,-1.0,-1.0,13.181,1.82827,1627,4119898 +256,97,insert,12.408,-1.0,-1.0,-1.0,-1.0,20.91,1.8316,1628,4129367 +256,98,search,5828.58,2828.91,2275.18,0.97034,184.668,0.91976,0.105313,-1.0,-1.0,13.327,1.8316,1628,4129367 +256,99,insert,8.518,-1.0,-1.0,-1.0,-1.0,12.492,1.83383,1628,4134502 +256,100,search,5949.02,2832.12,2384.27,0.899291,193.251,0.91983,0.105226,-1.0,-1.0,13.375,1.83383,1628,4134502 +256,101,insert,38.787,-1.0,-1.0,-1.0,-1.0,76.393,1.84491,1636,4164482 +256,102,search,5794.24,2833.34,2234.06,1.04672,185.466,0.91992,0.105399,-1.0,-1.0,12.908,1.84491,1636,4164482 +256,103,insert,17.349,-1.0,-1.0,-1.0,-1.0,50.529,1.85182,1642,4178601 +256,104,search,6008.08,2826.94,2408.94,0.909554,195.455,0.9199,0.105363,-1.0,-1.0,12.987,1.85182,1642,4178601 +256,105,insert,8.85,-1.0,-1.0,-1.0,-1.0,12.267,1.85431,1642,4187774 +256,106,search,5813.17,2832.66,2229.58,1.02871,187.355,0.91983,0.105587,-1.0,-1.0,13.668,1.85431,1642,4187774 +256,107,insert,18.205,-1.0,-1.0,-1.0,-1.0,12.785,1.85962,1642,4200109 +256,108,search,5842.4,2837.99,2244.56,0.989398,186.496,0.91989,0.105406,-1.0,-1.0,23.361,1.85983,1643,4200109 +256,109,insert,500.814,-1.0,-1.0,-1.0,-1.0,624.11,2.0205,1720,4557867 +256,110,search,6396.58,2901.06,2596.91,0.929012,225.455,0.92197,0.103476,-1.0,-1.0,80.567,2.02122,1728,4557867 +256,111,insert,31.387,-1.0,-1.0,-1.0,-1.0,75.726,2.03278,1737,4587252 +256,112,search,6109.01,2884.77,2291.3,1.078,182.615,0.9221,0.103374,-1.0,-1.0,17.436,2.03278,1737,4587252 +256,113,insert,14.404,-1.0,-1.0,-1.0,-1.0,32.235,2.03735,1739,4600056 +256,114,search,6524.41,2888.06,2480.84,0.961726,199.095,0.92215,0.103368,-1.0,-1.0,17.999,2.03735,1739,4600056 +256,115,insert,43.309,-1.0,-1.0,-1.0,-1.0,47.142,2.04779,1743,4627422 +256,116,search,6492.41,2899.32,2630.39,0.920091,220.396,0.92267,0.102705,-1.0,-1.0,17.624,2.04779,1743,4627422 +256,117,insert,13.62,-1.0,-1.0,-1.0,-1.0,34.505,2.05203,1745,4639292 +256,118,search,6134.54,2899.47,2336.07,1.07785,183.971,0.92248,0.102926,-1.0,-1.0,18.03,2.05203,1745,4639292 +256,119,insert,6.298,-1.0,-1.0,-1.0,-1.0,24.207,2.05389,1746,4643423 +256,120,search,6400,2899.72,2550.14,0.950733,214.22,0.92252,0.102928,-1.0,-1.0,17.738,2.05389,1746,4643423 +256,121,insert,20.098,-1.0,-1.0,-1.0,-1.0,26.22,2.06078,1747,4657252 +256,122,search,6498.7,2903.95,2633.75,0.930259,224.178,0.92294,0.102599,-1.0,-1.0,17.682,2.06078,1747,4657252 +256,123,insert,379.336,-1.0,-1.0,-1.0,-1.0,485.271,2.18936,1795,4938477 +256,124,search,6715.31,2956.24,2744.62,0.919639,231.059,0.92372,0.103096,-1.0,-1.0,141.044,2.18969,1806,4938477 +256,125,insert,56.514,-1.0,-1.0,-1.0,-1.0,68.034,2.20571,1813,4980492 +256,126,search,6732.05,2942.33,2743.16,0.933147,215.235,0.92371,0.102968,-1.0,-1.0,40.245,2.20572,1815,4980492 +256,127,insert,57.406,-1.0,-1.0,-1.0,-1.0,94.105,2.22736,1827,5030515 +256,128,search,6795.51,2946,2758.05,0.922444,222.095,0.92462,0.102437,-1.0,-1.0,38.843,2.22736,1828,5030515 +256,129,delete,28.434,-1.0,-1.0,-1.0,-1.0,90.75,2.2254,1797,5009391 +256,130,search,6696.33,2939.33,2725.81,0.918491,222.668,0.92307,0.103577,-1.0,-1.0,22.207,2.2254,1797,5009391 +256,131,delete,7.269,-1.0,-1.0,-1.0,-1.0,20.997,2.2254,1797,5004093 +256,132,search,6810.16,2936.88,2733.65,0.910311,226.461,0.92292,0.103672,-1.0,-1.0,22.719,2.2254,1797,5004093 +256,133,delete,18.759,-1.0,-1.0,-1.0,-1.0,36.288,2.22391,1796,4991787 +256,134,search,6683.1,2925.37,2720.98,0.915476,218.388,0.92282,0.103783,-1.0,-1.0,21.952,2.22391,1796,4991787 +256,135,delete,23.352,-1.0,-1.0,-1.0,-1.0,44.279,2.22165,1794,4974857 +256,136,search,6310.71,2918.86,2377.65,1.02803,193.222,0.92308,0.103255,-1.0,-1.0,22.246,2.22165,1794,4974857 +256,137,delete,4.952,-1.0,-1.0,-1.0,-1.0,20.796,2.22165,1794,4971386 +256,138,search,6671.55,2914.84,2701.87,0.90167,229.279,0.92288,0.103539,-1.0,-1.0,21.552,2.22165,1794,4971386 +256,139,delete,9.822,-1.0,-1.0,-1.0,-1.0,21.417,2.22165,1794,4963487 +256,140,search,6640,2907.48,2685.61,0.910211,217.249,0.92285,0.103533,-1.0,-1.0,21.345,2.22165,1794,4963487 +256,141,delete,25.446,-1.0,-1.0,-1.0,-1.0,75.564,2.21747,1788,4941830 +256,142,search,6357.26,2898.23,2441.96,0.997016,193.82,0.9226,0.103556,-1.0,-1.0,22.321,2.21747,1788,4941830 +256,143,delete,24.262,-1.0,-1.0,-1.0,-1.0,84.533,2.21355,1785,4925657 +256,144,search,6583.42,2894.91,2664.5,0.931983,220.398,0.92211,0.104108,-1.0,-1.0,21.51,2.21355,1785,4925657 +256,145,delete,3.551,-1.0,-1.0,-1.0,-1.0,28.247,2.21299,1784,4922686 +256,146,search,6543.69,2895.28,2643.58,0.91346,209.633,0.92193,0.104468,-1.0,-1.0,21.319,2.21299,1784,4922686 +256,147,delete,10.092,-1.0,-1.0,-1.0,-1.0,20.777,2.21299,1784,4913636 +256,148,search,6308.84,2888.37,2396.65,1.03852,191.941,0.92179,0.104427,-1.0,-1.0,23.12,2.21299,1784,4913636 +256,149,delete,244.769,-1.0,-1.0,-1.0,-1.0,1058.02,2.13531,1709,4717250 +256,150,search,6101.67,2880.55,2358.59,1.01385,189.857,0.9211,0.10525,-1.0,-1.0,75.315,2.13636,1715,4717250 +256,151,delete,157.611,-1.0,-1.0,-1.0,-1.0,914.043,2.06733,1647,4560269 +512,2,search,3081.92,37.5312,516.295,1.35883,171.176,0.8551,0.130106,-1.0,-1.0,8.376,0.0184881,1000,38806 +512,3,insert,21.57,-1.0,-1.0,-1.0,-1.0,16.587,0.0226681,998,49334 +512,4,search,2588.01,63.6871,110.971,1.29105,164.717,0.86613,0.128797,-1.0,-1.0,14.859,0.0226611,997,49334 +512,5,insert,39.785,-1.0,-1.0,-1.0,-1.0,176.651,0.0329766,993,73557 +512,6,search,2628.96,173.704,192.378,1.40674,127.296,0.92433,0.104255,-1.0,-1.0,114.93,0.0330819,995,73557 +512,7,insert,47.141,-1.0,-1.0,-1.0,-1.0,104.677,0.0426107,995,95766 +512,8,search,2650.17,218.462,207.347,1.26948,124.911,0.92582,0.103612,-1.0,-1.0,94.22,0.0428012,997,95766 +512,9,insert,7.661,-1.0,-1.0,-1.0,-1.0,78.959,0.0445267,997,100025 +512,10,search,2660.56,224.701,201.025,1.21733,137.267,0.92703,0.100737,-1.0,-1.0,81.375,0.0445267,997,100025 +512,11,insert,18.595,-1.0,-1.0,-1.0,-1.0,81.095,0.0497772,997,111417 +512,12,search,2668.86,265.794,234.323,1.14347,134.971,0.92784,0.100473,-1.0,-1.0,82.498,0.0497772,997,111417 +512,13,insert,41.156,-1.0,-1.0,-1.0,-1.0,81.183,0.0610515,997,137139 +512,14,search,2694.24,309.176,256.388,1.09032,131.669,0.92449,0.102927,-1.0,-1.0,22.082,0.061098,998,137139 +512,15,insert,37.121,-1.0,-1.0,-1.0,-1.0,7.587,0.0698157,998,156710 +512,16,search,2757.94,340.453,280.127,0.995509,138.001,0.92241,0.104278,-1.0,-1.0,22.401,0.0698182,1000,156710 +512,17,insert,8.947,-1.0,-1.0,-1.0,-1.0,7.636,0.0717762,1000,161773 +512,18,search,2774.93,355.577,284.893,0.973773,145.56,0.92371,0.103298,-1.0,-1.0,14.669,0.0718484,1001,161773 +512,19,insert,43.717,-1.0,-1.0,-1.0,-1.0,7.452,0.0784384,1001,177161 +512,20,search,2701.02,391.993,306.172,0.932754,124.974,0.92743,0.0998076,-1.0,-1.0,8.73,0.0784384,1001,177161 +512,21,insert,463.774,-1.0,-1.0,-1.0,-1.0,131.862,0.199859,1022,448898 +512,22,search,2948.97,864.317,597.271,0.751287,103.408,0.91911,0.101388,-1.0,-1.0,17.117,0.199859,1023,448898 +512,23,insert,483.702,-1.0,-1.0,-1.0,-1.0,300.494,0.327972,1065,733493 +512,24,search,3264.54,1225.65,868.738,0.733657,115.833,0.91418,0.105459,-1.0,-1.0,79.017,0.32797,1075,733493 +512,25,insert,52.827,-1.0,-1.0,-1.0,-1.0,28.909,0.344801,1078,777443 +512,26,search,3362.24,1225.59,858.826,0.737085,120.645,0.91477,0.104863,-1.0,-1.0,20.832,0.344805,1079,777443 +512,27,insert,42.216,-1.0,-1.0,-1.0,-1.0,18.415,0.357304,1080,805394 +512,28,search,3317.34,1248.94,877.108,0.742225,99.2403,0.91561,0.104355,-1.0,-1.0,11.356,0.357304,1080,805394 +512,29,insert,85.074,-1.0,-1.0,-1.0,-1.0,59.617,0.385079,1086,866391 +512,30,search,3185.1,1317.81,823.326,0.747189,88.5482,0.91631,0.10471,-1.0,-1.0,32.504,0.385078,1088,866391 +512,31,insert,558.846,-1.0,-1.0,-1.0,-1.0,322.519,0.550965,1128,1239816 +512,32,search,3731.35,1675.45,1236.99,0.664256,118.219,0.91144,0.107881,-1.0,-1.0,124.58,0.55147,1139,1239816 +512,33,insert,79.799,-1.0,-1.0,-1.0,-1.0,23.179,0.573302,1141,1291211 +512,34,search,3793.96,1681.91,1256.77,0.630706,122.548,0.91095,0.107869,-1.0,-1.0,69.28,0.573315,1145,1291211 +512,35,insert,48.661,-1.0,-1.0,-1.0,-1.0,48.619,0.586868,1152,1324978 +512,36,search,3824.76,1703.33,1276.63,0.691769,120.023,0.91059,0.10817,-1.0,-1.0,35.953,0.58687,1154,1324978 +512,37,insert,68.793,-1.0,-1.0,-1.0,-1.0,27.225,0.605706,1156,1371885 +512,38,search,3800.46,1737.65,1256.95,0.752149,114.153,0.91053,0.108734,-1.0,-1.0,17.938,0.605705,1157,1371885 +512,39,insert,16.888,-1.0,-1.0,-1.0,-1.0,9.552,0.609685,1157,1379335 +512,40,search,3867.03,1743.21,1323.8,0.674302,109.203,0.91065,0.109062,-1.0,-1.0,10.452,0.609685,1157,1379335 +512,41,insert,60.531,-1.0,-1.0,-1.0,-1.0,51.73,0.625533,1163,1413312 +512,42,search,3881.89,1764.45,1338.56,0.659283,118.142,0.91106,0.109194,-1.0,-1.0,10.27,0.625533,1163,1413312 +512,43,insert,20.684,-1.0,-1.0,-1.0,-1.0,14.461,0.630493,1164,1423411 +512,44,search,3882.78,1772.39,1337.51,0.644442,119.439,0.9113,0.108826,-1.0,-1.0,10.48,0.630493,1164,1423411 +512,45,insert,234.58,-1.0,-1.0,-1.0,-1.0,56.999,0.64488,1172,1457559 +512,46,search,3918.2,1768.07,1346.81,0.660804,119.2,0.91107,0.108305,-1.0,-1.0,10.153,0.64488,1172,1457559 +512,47,insert,335.362,-1.0,-1.0,-1.0,-1.0,308.094,0.748794,1205,1688593 +512,48,search,3993.75,1901.18,1337.68,0.707576,122.709,0.9109,0.110261,-1.0,-1.0,104.577,0.748794,1212,1688593 +512,49,insert,49.072,-1.0,-1.0,-1.0,-1.0,47.586,0.763736,1217,1723505 +512,50,search,3921.81,1898.64,1271.33,0.780305,117.901,0.91129,0.110653,-1.0,-1.0,51.845,0.763766,1221,1723505 +512,51,insert,405.153,-1.0,-1.0,-1.0,-1.0,286.593,0.903308,1260,2035040 +512,52,search,4443.97,2085.42,1659.54,0.585809,156.198,0.91413,0.109514,-1.0,-1.0,68.825,0.903336,1268,2035040 +512,53,insert,50.925,-1.0,-1.0,-1.0,-1.0,18.409,0.92077,1269,2078341 +512,54,search,4482.87,2097.86,1678.09,0.636733,147.067,0.91366,0.10979,-1.0,-1.0,10.874,0.92077,1269,2078341 +512,55,insert,397.23,-1.0,-1.0,-1.0,-1.0,423.494,1.07045,1318,2399905 +512,56,search,4776.24,2267.77,1879.58,0.639122,150.332,0.91344,0.111654,-1.0,-1.0,84.038,1.07076,1325,2399905 +512,57,insert,25.015,-1.0,-1.0,-1.0,-1.0,28.281,1.07864,1328,2419836 +512,58,search,4808.94,2252.93,1877.33,0.587096,162.016,0.91372,0.111281,-1.0,-1.0,56.903,1.07867,1333,2419836 +512,59,insert,244.817,-1.0,-1.0,-1.0,-1.0,145.65,1.17433,1347,2643083 +512,60,search,4629.96,2325.58,1670.39,0.710197,133.376,0.90771,0.117562,-1.0,-1.0,60.565,1.17482,1354,2643083 +512,61,insert,11.55,-1.0,-1.0,-1.0,-1.0,10.481,1.17852,1354,2653064 +512,62,search,4987.84,2319.44,1964.48,0.563008,173.155,0.90754,0.117818,-1.0,-1.0,20.038,1.17852,1355,2653064 +512,63,insert,6.224,-1.0,-1.0,-1.0,-1.0,10.684,1.17993,1355,2656118 +512,64,search,4972.27,2318.93,1966.38,0.570011,169.715,0.90759,0.117622,-1.0,-1.0,12.398,1.17993,1355,2656118 +512,65,insert,119.725,-1.0,-1.0,-1.0,-1.0,99.531,1.20809,1368,2723424 +512,66,search,4807.67,2356.59,1798.47,0.644744,150.982,0.90837,0.117018,-1.0,-1.0,12.408,1.20809,1368,2723424 +512,67,insert,59.906,-1.0,-1.0,-1.0,-1.0,104.963,1.22948,1383,2769021 +512,68,search,5082.92,2359.33,2025.23,0.580215,165.056,0.90922,0.116233,-1.0,-1.0,18.318,1.22948,1384,2769021 +512,69,insert,61.881,-1.0,-1.0,-1.0,-1.0,63.782,1.25058,1391,2821777 +512,70,search,4936.27,2381.43,1877.1,0.726242,162.436,0.91037,0.114581,-1.0,-1.0,23.297,1.25058,1392,2821777 +512,71,insert,14.918,-1.0,-1.0,-1.0,-1.0,16.711,1.25397,1393,2834921 +512,72,search,4611.86,2396.06,1591.7,0.831008,131.348,0.91097,0.11416,-1.0,-1.0,20.597,1.25397,1394,2834921 +512,73,insert,8.671,-1.0,-1.0,-1.0,-1.0,24.391,1.25693,1396,2842044 +512,74,search,4651.18,2392.53,1598.17,0.792235,129.101,0.91099,0.114251,-1.0,-1.0,11.667,1.25693,1396,2842044 +512,75,insert,26.555,-1.0,-1.0,-1.0,-1.0,19.584,1.26347,1397,2858890 +512,76,search,5170.78,2403.75,2085.79,0.619548,178.94,0.91135,0.113769,-1.0,-1.0,11.319,1.26347,1397,2858890 +512,77,insert,44.222,-1.0,-1.0,-1.0,-1.0,10.823,1.27511,1397,2882818 +512,78,search,5166.66,2418.08,2091.55,0.671844,182.177,0.911,0.114026,-1.0,-1.0,11.415,1.27511,1397,2882818 +512,79,insert,476.658,-1.0,-1.0,-1.0,-1.0,10.197,1.30272,1397,2947806 +512,80,search,4822.15,2463.4,1775.46,0.675066,142.889,0.91097,0.114335,-1.0,-1.0,11.222,1.30272,1397,2947806 +512,81,insert,43.264,-1.0,-1.0,-1.0,-1.0,56.613,1.31738,1403,2976895 +512,82,search,5244.31,2471.72,2141.85,0.578193,189.208,0.91223,0.113488,-1.0,-1.0,10.882,1.31738,1403,2976895 +512,83,insert,19.189,-1.0,-1.0,-1.0,-1.0,43.692,1.32452,1408,2990682 +512,84,search,5233.7,2470.63,2112.28,0.671295,180.946,0.91241,0.113228,-1.0,-1.0,11.203,1.32452,1408,2990682 +512,85,insert,36.68,-1.0,-1.0,-1.0,-1.0,53.746,1.33787,1414,3025613 +512,86,search,4950.47,2481.92,1818.88,0.703009,158.136,0.91319,0.112903,-1.0,-1.0,11.061,1.33787,1414,3025613 +512,87,insert,567.909,-1.0,-1.0,-1.0,-1.0,770.438,1.54343,1513,3468632 +512,88,search,5501.3,2598.16,2163.89,0.676666,176.28,0.91582,0.110623,-1.0,-1.0,125.195,1.54374,1526,3468632 +512,89,insert,453.461,-1.0,-1.0,-1.0,-1.0,497.151,1.70287,1603,3831692 +512,90,search,5702.51,2668.34,2189.04,0.71032,178.989,0.91955,0.106091,-1.0,-1.0,50.412,1.70335,1608,3831692 +512,91,insert,32.159,-1.0,-1.0,-1.0,-1.0,44.157,1.71428,1613,3864067 +512,92,search,6107.53,2670.1,2542.61,0.627766,218.173,0.91937,0.106426,-1.0,-1.0,12.795,1.71428,1613,3864067 +512,93,insert,294.786,-1.0,-1.0,-1.0,-1.0,230.27,1.82032,1642,4103827 +512,94,search,6340.33,2753.31,2706.44,0.623243,229.026,0.91981,0.105658,-1.0,-1.0,43.453,1.8206,1646,4103827 +512,95,insert,14.277,-1.0,-1.0,-1.0,-1.0,24.167,1.82533,1648,4119898 +512,96,search,5888.4,2751.38,2288.82,0.728323,186.163,0.9201,0.105395,-1.0,-1.0,13.465,1.82533,1648,4119898 +512,97,insert,11.469,-1.0,-1.0,-1.0,-1.0,26.373,1.82872,1650,4129367 +512,98,search,6178.56,2752.44,2567.13,0.679557,215.853,0.92015,0.105233,-1.0,-1.0,13.573,1.82872,1650,4129367 +512,99,insert,7.803,-1.0,-1.0,-1.0,-1.0,24.642,1.83062,1652,4134502 +512,100,search,6391.94,2747.24,2742.74,0.57341,240.976,0.91998,0.105392,-1.0,-1.0,13.328,1.83062,1652,4134502 +512,101,insert,38.886,-1.0,-1.0,-1.0,-1.0,62.067,1.84203,1658,4164482 +512,102,search,6011.95,2752.25,2374.12,0.738195,204.204,0.92009,0.105303,-1.0,-1.0,12.924,1.84203,1658,4164482 +512,103,insert,17.819,-1.0,-1.0,-1.0,-1.0,31.192,1.84868,1660,4178601 +512,104,search,6214.56,2760.63,2567.53,0.648883,207.183,0.92053,0.104989,-1.0,-1.0,13.12,1.84868,1660,4178601 +512,105,insert,8.564,-1.0,-1.0,-1.0,-1.0,27.611,1.85143,1662,4187774 +512,106,search,5751.47,2759.93,2155.88,0.824231,173.431,0.92042,0.105291,-1.0,-1.0,13.33,1.85143,1662,4187774 +512,107,insert,18.726,-1.0,-1.0,-1.0,-1.0,18.893,1.85624,1663,4200109 +512,108,search,6402.99,2763.01,2750.9,0.615754,234.532,0.9202,0.105456,-1.0,-1.0,13.765,1.85624,1663,4200109 +512,109,insert,453.74,-1.0,-1.0,-1.0,-1.0,599.418,2.0185,1741,4557867 +512,110,search,6855.67,2823.18,2972.17,0.590868,237.469,0.92177,0.103787,-1.0,-1.0,79.842,2.01902,1746,4557867 +512,111,insert,32.532,-1.0,-1.0,-1.0,-1.0,95.567,2.03066,1757,4587252 +512,112,search,6908.58,2814.09,2983.51,0.592655,255.799,0.92215,0.103677,-1.0,-1.0,18.519,2.03066,1757,4587252 +512,113,insert,15.137,-1.0,-1.0,-1.0,-1.0,17.487,2.03591,1757,4600056 +512,114,search,6928.64,2819.81,3014.81,0.65285,251.051,0.92232,0.103657,-1.0,-1.0,18.944,2.03591,1757,4600056 +512,115,insert,42.619,-1.0,-1.0,-1.0,-1.0,87.358,2.04542,1766,4627422 +512,116,search,6593.19,2820.7,2712.59,0.634338,217.08,0.92254,0.103405,-1.0,-1.0,18.956,2.04542,1766,4627422 +512,117,insert,17.602,-1.0,-1.0,-1.0,-1.0,27.627,2.05006,1767,4639292 +512,118,search,6581,2825.46,2665.46,0.676412,221.738,0.92288,0.103317,-1.0,-1.0,18.943,2.05006,1767,4639292 +512,119,insert,5.172,-1.0,-1.0,-1.0,-1.0,17.954,2.05101,1767,4643423 +512,120,search,6865.76,2828.13,2926.09,0.678233,251.76,0.92295,0.103358,-1.0,-1.0,19.736,2.05101,1767,4643423 +512,121,insert,26.05,-1.0,-1.0,-1.0,-1.0,44.866,2.05868,1770,4657252 +512,122,search,6963.5,2831.94,3032.65,0.605625,264.525,0.92295,0.1033,-1.0,-1.0,19.192,2.05868,1770,4657252 +512,123,insert,405.919,-1.0,-1.0,-1.0,-1.0,431.684,2.18824,1812,4938477 +512,124,search,6759.88,2900.36,2778.96,0.672563,226.031,0.9235,0.103249,-1.0,-1.0,138.763,2.1889,1822,4938477 +512,125,insert,57.65,-1.0,-1.0,-1.0,-1.0,77.841,2.20656,1831,4980492 +512,126,search,7242.35,2884.61,3148.94,0.635022,258.651,0.92393,0.103134,-1.0,-1.0,100.548,2.20672,1836,4980492 +512,127,insert,58.397,-1.0,-1.0,-1.0,-1.0,88.907,2.22588,1846,5030515 +512,128,search,7357.2,2884.76,3209.58,0.626576,279.585,0.9246,0.102437,-1.0,-1.0,55.391,2.22589,1848,5030515 +512,129,delete,27.019,-1.0,-1.0,-1.0,-1.0,94.329,2.22438,1817,5009391 +512,130,search,7220.03,2884.87,3160.09,0.565198,272.008,0.92342,0.103273,-1.0,-1.0,23.694,2.22438,1817,5009391 +512,131,delete,7.264,-1.0,-1.0,-1.0,-1.0,22.432,2.22438,1817,5004093 +512,132,search,6961.85,2883.13,2912.6,0.643764,236.532,0.92346,0.10332,-1.0,-1.0,22.87,2.22438,1817,5004093 +512,133,delete,14.969,-1.0,-1.0,-1.0,-1.0,21.678,2.22438,1817,4991787 +512,134,search,6791.35,2869.03,2789.76,0.664741,227.585,0.92336,0.103664,-1.0,-1.0,23.223,2.22438,1817,4991787 +512,135,delete,21.123,-1.0,-1.0,-1.0,-1.0,49.407,2.222,1815,4974857 +512,136,search,7006.68,2861.27,2955,0.613294,255.432,0.92336,0.103461,-1.0,-1.0,23.104,2.222,1815,4974857 +512,137,delete,4.225,-1.0,-1.0,-1.0,-1.0,22.027,2.222,1815,4971386 +512,138,search,7181.92,2856.89,3108.05,0.567017,263.827,0.92334,0.103556,-1.0,-1.0,22.951,2.222,1815,4971386 +512,139,delete,9.612,-1.0,-1.0,-1.0,-1.0,22.991,2.222,1815,4963487 +512,140,search,6944.26,2851.74,2908.12,0.645543,234.228,0.92325,0.103631,-1.0,-1.0,22.751,2.222,1815,4963487 +512,141,delete,28.646,-1.0,-1.0,-1.0,-1.0,73.261,2.21797,1810,4941830 +512,142,search,7116,2842.22,3073.73,0.626838,248.974,0.92283,0.103728,-1.0,-1.0,26.163,2.21797,1810,4941830 +512,143,delete,19.414,-1.0,-1.0,-1.0,-1.0,59.73,2.2144,1807,4925657 +512,144,search,7089.58,2837.8,3060.71,0.589932,251.4,0.92311,0.103579,-1.0,-1.0,22.428,2.2144,1807,4925657 +512,145,delete,2.941,-1.0,-1.0,-1.0,-1.0,21.721,2.2144,1807,4922686 +512,146,search,7108.02,2834.44,3058.27,0.57106,262.15,0.92303,0.103629,-1.0,-1.0,22.461,2.2144,1807,4922686 +512,147,delete,9.492,-1.0,-1.0,-1.0,-1.0,22.069,2.2144,1807,4913636 +512,148,search,7104.68,2827.43,3062.03,0.570765,254.555,0.92295,0.10357,-1.0,-1.0,22.418,2.2144,1807,4913636 +512,149,delete,214.69,-1.0,-1.0,-1.0,-1.0,1177.88,2.12967,1720,4717250 +512,150,search,6799.12,2845.64,2927.37,0.58456,256.01,0.92191,0.105071,-1.0,-1.0,81.759,2.13043,1726,4717250 +512,151,delete,168.829,-1.0,-1.0,-1.0,-1.0,859.111,2.06375,1658,4560269 +1024,2,search,4156.4,37.7178,1683.56,1.08576,180.653,0.8551,0.130106,-1.0,-1.0,8.616,0.0184881,1000,38806 +1024,3,insert,22.422,-1.0,-1.0,-1.0,-1.0,7.153,0.0226644,1000,49334 +1024,4,search,2593.21,60.1351,115.112,1.13082,157.639,0.86604,0.12886,-1.0,-1.0,8.504,0.0226644,1000,49334 +1024,5,insert,44.055,-1.0,-1.0,-1.0,-1.0,172.688,0.0330953,997,73557 +1024,6,search,2566.38,152.235,189.893,1.3901,112.374,0.92386,0.105036,-1.0,-1.0,151.81,0.0330953,997,73557 +1024,7,insert,47.627,-1.0,-1.0,-1.0,-1.0,133.077,0.0428118,998,95766 +1024,8,search,2595.91,196.454,213.153,1.24889,122.614,0.92518,0.103938,-1.0,-1.0,97.464,0.043126,1001,95766 +1024,9,insert,8.509,-1.0,-1.0,-1.0,-1.0,77.332,0.0448678,1001,100025 +1024,10,search,2618.16,198.864,205.362,1.1218,126.702,0.92729,0.100268,-1.0,-1.0,79.583,0.0448678,1001,100025 +1024,11,insert,19.687,-1.0,-1.0,-1.0,-1.0,79.851,0.0495679,1001,111417 +1024,12,search,2604.84,231.702,227.512,1.13444,105.362,0.92776,0.10035,-1.0,-1.0,81.82,0.0495679,1001,111417 +1024,13,insert,47.896,-1.0,-1.0,-1.0,-1.0,79.416,0.0609091,1001,137139 +1024,14,search,2697.94,272.126,264.564,1.04029,118.966,0.925,0.102401,-1.0,-1.0,22.793,0.0612763,1002,137139 +1024,15,insert,38.358,-1.0,-1.0,-1.0,-1.0,7.971,0.0699749,1002,156710 +1024,16,search,2626.4,302.617,276.521,0.916084,119.266,0.92291,0.103342,-1.0,-1.0,16.589,0.0699785,1004,156710 +1024,17,insert,5.059,-1.0,-1.0,-1.0,-1.0,5.989,0.0718398,1004,161773 +1024,18,search,2520.66,315.478,261.15,1.08999,73.8584,0.92324,0.103581,-1.0,-1.0,13.758,0.0720618,1006,161773 +1024,19,insert,27.544,-1.0,-1.0,-1.0,-1.0,6.228,0.0785082,1006,177161 +1024,20,search,2514.58,345.081,274.751,1.05888,69.5007,0.92715,0.100366,-1.0,-1.0,8.848,0.0785082,1006,177161 +1024,21,insert,475.581,-1.0,-1.0,-1.0,-1.0,155.617,0.200219,1032,448898 +1024,22,search,2833.4,755.351,534.382,0.866427,84.0046,0.91772,0.102929,-1.0,-1.0,13.424,0.200219,1033,448898 +1024,23,insert,489.331,-1.0,-1.0,-1.0,-1.0,304.132,0.328777,1077,733493 +1024,24,search,3107.86,1085.92,757.957,0.741973,95.6478,0.91205,0.105883,-1.0,-1.0,74.838,0.328776,1086,733493 +1024,25,insert,54.695,-1.0,-1.0,-1.0,-1.0,38.112,0.346067,1090,777443 +1024,26,search,3139.71,1101.92,766.936,0.753231,84.8261,0.91281,0.105451,-1.0,-1.0,23.219,0.346096,1092,777443 +1024,27,insert,43.049,-1.0,-1.0,-1.0,-1.0,23.783,0.357867,1094,805394 +1024,28,search,3181.04,1113.12,806.053,0.7277,77.2537,0.91366,0.104823,-1.0,-1.0,12.014,0.357867,1094,805394 +1024,29,insert,85.375,-1.0,-1.0,-1.0,-1.0,58.664,0.385698,1101,866391 +1024,30,search,3233.21,1179.57,836.831,0.783533,90.6841,0.91502,0.105035,-1.0,-1.0,27.609,0.385698,1103,866391 +1024,31,insert,520.827,-1.0,-1.0,-1.0,-1.0,331.181,0.552656,1145,1239816 +1024,32,search,3564.76,1503.64,1045.57,0.717741,96.3234,0.9084,0.110647,-1.0,-1.0,110.586,0.553129,1154,1239816 +1024,33,insert,67.492,-1.0,-1.0,-1.0,-1.0,15.381,0.574155,1155,1291211 +1024,34,search,3592.48,1522.11,1061.16,0.706883,92.1074,0.90896,0.110692,-1.0,-1.0,78.639,0.574166,1159,1291211 +1024,35,insert,60.493,-1.0,-1.0,-1.0,-1.0,14.333,0.587188,1160,1324978 +1024,36,search,3648.26,1564.94,1101.79,0.673446,96.4307,0.91053,0.109293,-1.0,-1.0,41.522,0.587191,1163,1324978 +1024,37,insert,77.78,-1.0,-1.0,-1.0,-1.0,40.151,0.607736,1167,1371885 +1024,38,search,3670.18,1589.1,1114.98,0.700587,94.313,0.91012,0.1092,-1.0,-1.0,10.694,0.607736,1167,1371885 +1024,39,insert,11.776,-1.0,-1.0,-1.0,-1.0,9.12,0.610103,1167,1379335 +1024,40,search,3688.25,1593.49,1086.3,0.720698,93.588,0.91046,0.109269,-1.0,-1.0,10.014,0.610103,1167,1379335 +1024,41,insert,58.86,-1.0,-1.0,-1.0,-1.0,40.122,0.625259,1171,1413312 +1024,42,search,3822.89,1616.52,1204.23,0.649201,107.438,0.91075,0.109144,-1.0,-1.0,10.427,0.625259,1171,1413312 +1024,43,insert,15.894,-1.0,-1.0,-1.0,-1.0,9.172,0.628465,1171,1423411 +1024,44,search,3735.7,1632.61,1196.86,0.643744,104.404,0.91088,0.108697,-1.0,-1.0,10.535,0.628465,1171,1423411 +1024,45,insert,249.706,-1.0,-1.0,-1.0,-1.0,56.59,0.643933,1179,1457559 +1024,46,search,4054.73,1638.96,1463.71,0.561357,126.567,0.91109,0.107577,-1.0,-1.0,10.946,0.643933,1179,1457559 +1024,47,insert,366.881,-1.0,-1.0,-1.0,-1.0,273.832,0.74974,1210,1688593 +1024,48,search,4236.87,1770.27,1570.21,0.502076,141.142,0.91084,0.108671,-1.0,-1.0,94.613,0.749914,1218,1688593 +1024,49,insert,49.705,-1.0,-1.0,-1.0,-1.0,33.041,0.764466,1221,1723505 +1024,50,search,4277.12,1778.09,1584.78,0.530734,140.745,0.91194,0.108094,-1.0,-1.0,27.187,0.764466,1223,1723505 +1024,51,insert,421.391,-1.0,-1.0,-1.0,-1.0,242.778,0.903494,1254,2035040 +1024,52,search,4518.28,1991.14,1750.36,0.55433,138.836,0.9144,0.107707,-1.0,-1.0,40.605,0.903493,1258,2035040 +1024,53,insert,47.289,-1.0,-1.0,-1.0,-1.0,47.679,0.919506,1263,2078341 +1024,54,search,4609.23,2002.07,1802.07,0.537326,160.335,0.91418,0.107988,-1.0,-1.0,10.555,0.919506,1263,2078341 +1024,55,insert,393.18,-1.0,-1.0,-1.0,-1.0,416.97,1.06716,1310,2399905 +1024,56,search,4611.28,2190.59,1739.05,0.570061,132.609,0.91468,0.108578,-1.0,-1.0,83.312,1.06757,1320,2399905 +1024,57,insert,26.971,-1.0,-1.0,-1.0,-1.0,14.844,1.07568,1321,2419836 +1024,58,search,4982.06,2178.2,2041.18,0.529086,175.504,0.91488,0.108426,-1.0,-1.0,46.572,1.07577,1325,2419836 +1024,59,insert,261.219,-1.0,-1.0,-1.0,-1.0,187.767,1.17357,1344,2643083 +1024,60,search,4895.08,2259.06,1924.61,0.528271,157.962,0.90831,0.115342,-1.0,-1.0,52.311,1.17363,1347,2643083 +1024,61,insert,17.139,-1.0,-1.0,-1.0,-1.0,23.684,1.17866,1349,2653064 +1024,62,search,4743.62,2258.67,1775.5,0.638926,138.017,0.90837,0.115565,-1.0,-1.0,12.467,1.17866,1349,2653064 +1024,63,insert,6.244,-1.0,-1.0,-1.0,-1.0,10.66,1.18016,1349,2656118 +1024,64,search,5116.73,2260.15,2131.22,0.52302,175.461,0.90843,0.115564,-1.0,-1.0,11.893,1.18016,1349,2656118 +1024,65,insert,104.033,-1.0,-1.0,-1.0,-1.0,140.93,1.20831,1370,2723424 +1024,66,search,5092.93,2271.19,2047.74,0.493882,183.315,0.90874,0.115489,-1.0,-1.0,17.422,1.20831,1371,2723424 +1024,67,insert,50.39,-1.0,-1.0,-1.0,-1.0,88.455,1.22886,1382,2769021 +1024,68,search,4842.73,2286.9,1795.38,0.577678,151.773,0.90992,0.113859,-1.0,-1.0,29.179,1.22886,1385,2769021 +1024,69,insert,64.163,-1.0,-1.0,-1.0,-1.0,72.011,1.25155,1394,2821777 +1024,70,search,5290.77,2304.92,2178.33,0.528784,197.873,0.91151,0.112646,-1.0,-1.0,11.223,1.25155,1394,2821777 +1024,71,insert,21.267,-1.0,-1.0,-1.0,-1.0,42.659,1.25732,1399,2834921 +1024,72,search,5173.23,2304.07,2084.12,0.53491,181.783,0.91113,0.112915,-1.0,-1.0,11.75,1.25732,1399,2834921 +1024,73,insert,12.111,-1.0,-1.0,-1.0,-1.0,18.318,1.26035,1400,2842044 +1024,74,search,4969.72,2307.32,1877.41,0.602331,161.38,0.91157,0.112699,-1.0,-1.0,11.465,1.26035,1400,2842044 +1024,75,insert,28.075,-1.0,-1.0,-1.0,-1.0,18.1,1.26732,1401,2858890 +1024,76,search,5315.01,2317.4,2195.96,0.555956,202.132,0.91178,0.112339,-1.0,-1.0,21.143,1.2676,1402,2858890 +1024,77,insert,44.183,-1.0,-1.0,-1.0,-1.0,19.336,1.27827,1403,2882818 +1024,78,search,5238.5,2329.58,2120.09,0.544313,186.06,0.91141,0.112962,-1.0,-1.0,12.033,1.27827,1403,2882818 +1024,79,insert,476.444,-1.0,-1.0,-1.0,-1.0,22.699,1.30627,1405,2947806 +1024,80,search,5335.75,2366.81,2240.07,0.483499,189.294,0.91107,0.11342,-1.0,-1.0,11.588,1.30627,1405,2947806 +1024,81,insert,43.065,-1.0,-1.0,-1.0,-1.0,52.053,1.31924,1410,2976895 +1024,82,search,5374.53,2380.83,2231.17,0.535648,198.968,0.91196,0.112857,-1.0,-1.0,11.438,1.31924,1410,2976895 +1024,83,insert,21.132,-1.0,-1.0,-1.0,-1.0,28.954,1.3255,1413,2990682 +1024,84,search,5102.89,2382.48,2018.97,0.497242,160.719,0.9119,0.112762,-1.0,-1.0,11.413,1.3255,1413,2990682 +1024,85,insert,43.103,-1.0,-1.0,-1.0,-1.0,16.692,1.34017,1414,3025613 +1024,86,search,5482.32,2405.37,2313.82,0.487979,198.817,0.91309,0.111948,-1.0,-1.0,11.255,1.34017,1414,3025613 +1024,87,insert,570.854,-1.0,-1.0,-1.0,-1.0,762.307,1.54546,1508,3468632 +1024,88,search,5691.59,2525.04,2340.01,0.513802,193.902,0.91552,0.109972,-1.0,-1.0,134.244,1.54555,1522,3468632 +1024,89,insert,460.769,-1.0,-1.0,-1.0,-1.0,527.896,1.70471,1598,3831692 +1024,90,search,6345.62,2604.67,2785.81,0.429967,238.102,0.9198,0.104873,-1.0,-1.0,82.505,1.705,1606,3831692 +1024,91,insert,46.403,-1.0,-1.0,-1.0,-1.0,25,1.71846,1608,3864067 +1024,92,search,6386.95,2608.74,2799.86,0.406024,243.026,0.92019,0.1043,-1.0,-1.0,20.829,1.71846,1609,3864067 +1024,93,insert,299.929,-1.0,-1.0,-1.0,-1.0,248.579,1.82062,1637,4103827 +1024,94,search,6610.87,2698.59,2976.39,0.453617,234.24,0.92055,0.103021,-1.0,-1.0,66.195,1.82114,1643,4103827 +1024,95,insert,17.629,-1.0,-1.0,-1.0,-1.0,36.143,1.82687,1647,4119898 +1024,96,search,6088.04,2690.95,2474.01,0.570459,194.634,0.92122,0.102616,-1.0,-1.0,10.356,1.82687,1647,4119898 +1024,97,insert,7.11,-1.0,-1.0,-1.0,-1.0,18.275,1.83046,1649,4129367 +1024,98,search,6189.24,2692.6,2627.06,0.526888,207.319,0.92118,0.102716,-1.0,-1.0,14.332,1.83046,1649,4129367 +1024,99,insert,7.265,-1.0,-1.0,-1.0,-1.0,29.129,1.83226,1651,4134502 +1024,100,search,6025.99,2691.24,2434.69,0.53804,182.205,0.92111,0.102676,-1.0,-1.0,13.558,1.83226,1651,4134502 +1024,101,insert,33.673,-1.0,-1.0,-1.0,-1.0,34.253,1.84153,1654,4164482 +1024,102,search,6355.73,2702.54,2721.85,0.449682,214.393,0.92143,0.1029,-1.0,-1.0,13.044,1.84153,1654,4164482 +1024,103,insert,16.949,-1.0,-1.0,-1.0,-1.0,34.685,1.84755,1657,4178601 +1024,104,search,6652.48,2706.25,2973.83,0.43,243.91,0.92153,0.10295,-1.0,-1.0,13.879,1.84755,1657,4178601 +1024,105,insert,10.163,-1.0,-1.0,-1.0,-1.0,29.102,1.8512,1659,4187774 +1024,106,search,6513.58,2704.54,2857.85,0.447926,221.593,0.92135,0.103329,-1.0,-1.0,13.838,1.8512,1659,4187774 +1024,107,insert,27.049,-1.0,-1.0,-1.0,-1.0,33.904,1.85701,1660,4200109 +1024,108,search,6670.84,2710.27,2782.3,0.517149,222.806,0.92158,0.103009,-1.0,-1.0,14.829,1.85701,1660,4200109 +1024,109,insert,522.06,-1.0,-1.0,-1.0,-1.0,518.574,2.02064,1727,4557867 +1024,110,search,6711.63,2800.18,2895.61,0.532292,220.407,0.92276,0.101839,-1.0,-1.0,86.302,2.02146,1734,4557867 +1024,111,insert,34.372,-1.0,-1.0,-1.0,-1.0,36.52,2.03452,1737,4587252 +1024,112,search,6622.39,2805.76,2785.39,0.550835,214.625,0.92287,0.101961,-1.0,-1.0,25.707,2.03484,1738,4587252 +1024,113,insert,12.449,-1.0,-1.0,-1.0,-1.0,38.711,2.03916,1741,4600056 +1024,114,search,6988.4,2806.92,3097.77,0.507906,253.511,0.92329,0.101878,-1.0,-1.0,26.444,2.03943,1742,4600056 +1024,115,insert,24.896,-1.0,-1.0,-1.0,-1.0,49.344,2.05031,1749,4627422 +1024,116,search,6747.23,2804.77,2922.9,0.552354,231.931,0.92368,0.101535,-1.0,-1.0,15.284,2.05031,1749,4627422 +1024,117,insert,10.355,-1.0,-1.0,-1.0,-1.0,14.584,2.05359,1749,4639292 +1024,118,search,6257.71,2811.71,2493.2,0.636731,181.535,0.92384,0.101548,-1.0,-1.0,17.944,2.05359,1749,4639292 +1024,119,insert,5.23,-1.0,-1.0,-1.0,-1.0,16.974,2.05533,1749,4643423 +1024,120,search,6337.98,2814.64,2521.65,0.598795,197.447,0.92393,0.101443,-1.0,-1.0,18.08,2.05533,1749,4643423 +1024,121,insert,22.115,-1.0,-1.0,-1.0,-1.0,32.572,2.06132,1751,4657252 +1024,122,search,6320.37,2820.75,2517.17,0.627765,186.781,0.92403,0.101291,-1.0,-1.0,17.71,2.06132,1751,4657252 +1024,123,insert,385.938,-1.0,-1.0,-1.0,-1.0,512.513,2.19323,1803,4938477 +1024,124,search,6549.86,2862.33,2621.58,0.602142,199.883,0.92429,0.101724,-1.0,-1.0,121.496,2.19351,1810,4938477 +1024,125,insert,53.762,-1.0,-1.0,-1.0,-1.0,48.923,2.20886,1814,4980492 +1024,126,search,6591.26,2860.05,2636.09,0.626291,191.325,0.92476,0.101477,-1.0,-1.0,102.647,2.20916,1820,4980492 +1024,127,insert,53.362,-1.0,-1.0,-1.0,-1.0,123.37,2.22969,1836,5030515 +1024,128,search,6660.44,2850.55,2657.45,0.579917,191.531,0.92509,0.101655,-1.0,-1.0,82.714,2.22995,1840,5030515 +1024,129,delete,24.09,-1.0,-1.0,-1.0,-1.0,87.607,2.22829,1811,5009391 +1024,130,search,7087.24,2855.28,3075.81,0.482755,242.471,0.92381,0.102261,-1.0,-1.0,23.519,2.22829,1811,5009391 +1024,131,delete,8.208,-1.0,-1.0,-1.0,-1.0,22.402,2.22829,1811,5004093 +1024,132,search,7306.4,2854.41,3291.7,0.443693,262.82,0.92376,0.102455,-1.0,-1.0,22.78,2.22829,1811,5004093 +1024,133,delete,18.047,-1.0,-1.0,-1.0,-1.0,21.171,2.2281,1811,4991787 +1024,134,search,7397.86,2842.02,3380.93,0.472723,260.765,0.92355,0.102818,-1.0,-1.0,22.546,2.2281,1811,4991787 +1024,135,delete,23.857,-1.0,-1.0,-1.0,-1.0,48.985,2.22583,1809,4974857 +1024,136,search,7454.09,2836.22,3423.97,0.466644,263.887,0.92378,0.102658,-1.0,-1.0,23.168,2.22583,1809,4974857 +1024,137,delete,5.584,-1.0,-1.0,-1.0,-1.0,21.481,2.22583,1809,4971386 +1024,138,search,7112.33,2833.97,3120.98,0.517431,244.175,0.92373,0.102765,-1.0,-1.0,22.493,2.22583,1809,4971386 +1024,139,delete,13.198,-1.0,-1.0,-1.0,-1.0,22.138,2.22583,1809,4963487 +1024,140,search,7070.88,2827.44,3085.6,0.48844,233.113,0.92373,0.102785,-1.0,-1.0,22.942,2.22583,1809,4963487 +1024,141,delete,30.309,-1.0,-1.0,-1.0,-1.0,103.479,2.22028,1802,4941830 +1024,142,search,6953.21,2822.55,2985.95,0.543922,240.907,0.92327,0.103116,-1.0,-1.0,22.36,2.22028,1802,4941830 +1024,143,delete,20.909,-1.0,-1.0,-1.0,-1.0,69.646,2.21654,1799,4925657 +1024,144,search,6763.82,2818.91,2826.28,0.487967,219.737,0.92333,0.102825,-1.0,-1.0,21.816,2.21654,1799,4925657 +1024,145,delete,3.38,-1.0,-1.0,-1.0,-1.0,29.49,2.21596,1798,4922686 +1024,146,search,6924.72,2817.45,2979.24,0.496258,230.761,0.92319,0.103032,-1.0,-1.0,21.57,2.21596,1798,4922686 +1024,147,delete,9.394,-1.0,-1.0,-1.0,-1.0,20.955,2.21596,1798,4913636 +1024,148,search,7081.63,2809.68,3122.49,0.523638,242.846,0.923,0.103226,-1.0,-1.0,21.597,2.21596,1798,4913636 +1024,149,delete,214.356,-1.0,-1.0,-1.0,-1.0,959.296,2.13906,1723,4717250 +1024,150,search,6810.74,2808.36,2978.95,0.479757,234.225,0.92222,0.103874,-1.0,-1.0,44.357,2.13936,1725,4717250 +1024,151,delete,188.117,-1.0,-1.0,-1.0,-1.0,933.706,2.07349,1656,4560269 +2048,2,search,6742.84,38.1808,4094.74,0.625102,225.389,0.8551,0.130106,-1.0,-1.0,8.889,0.0184881,1000,38806 +2048,3,insert,22.519,-1.0,-1.0,-1.0,-1.0,13.022,0.0226428,999,49334 +2048,4,search,2740.93,56.9112,149.111,0.993296,153.74,0.86574,0.128987,-1.0,-1.0,15.323,0.0226534,998,49334 +2048,5,insert,44.239,-1.0,-1.0,-1.0,-1.0,171.948,0.0329117,995,73557 +2048,6,search,2723.23,129.722,201.192,1.1712,144.326,0.92412,0.104554,-1.0,-1.0,114.753,0.0330823,997,73557 +2048,7,insert,48.179,-1.0,-1.0,-1.0,-1.0,104.634,0.0427115,997,95766 +2048,8,search,2628.34,164.396,222.138,1.19541,128.463,0.92484,0.104181,-1.0,-1.0,93.325,0.0429608,999,95766 +2048,9,insert,7.252,-1.0,-1.0,-1.0,-1.0,79.074,0.0446793,999,100025 +2048,10,search,2705.43,164.114,227.821,1.10528,132.361,0.92622,0.1008,-1.0,-1.0,81.467,0.0446793,999,100025 +2048,11,insert,20.081,-1.0,-1.0,-1.0,-1.0,84.72,0.0499514,1000,111417 +2048,12,search,2755.25,190.563,257.286,1.07247,136.184,0.92709,0.101116,-1.0,-1.0,82.433,0.0499514,1000,111417 +2048,13,insert,43.435,-1.0,-1.0,-1.0,-1.0,80.953,0.0611204,1000,137139 +2048,14,search,2748.16,223.38,281.738,1.04415,116.204,0.92369,0.103096,-1.0,-1.0,23.415,0.0611731,1001,137139 +2048,15,insert,36.85,-1.0,-1.0,-1.0,-1.0,12.185,0.0698492,1002,156710 +2048,16,search,2727.03,252.366,302.489,0.919837,128.508,0.92188,0.104282,-1.0,-1.0,22.227,0.0698525,1004,156710 +2048,17,insert,8.338,-1.0,-1.0,-1.0,-1.0,12.027,0.0717117,1005,161773 +2048,18,search,2744.01,261.259,315.974,0.901413,127.232,0.92195,0.10416,-1.0,-1.0,25.607,0.0720242,1009,161773 +2048,19,insert,44.976,-1.0,-1.0,-1.0,-1.0,7.581,0.0788276,1009,177161 +2048,20,search,2872.16,286.768,335.37,0.869878,138.31,0.92592,0.10087,-1.0,-1.0,16.595,0.0788403,1011,177161 +2048,21,insert,468.412,-1.0,-1.0,-1.0,-1.0,184.684,0.201129,1048,448898 +2048,22,search,3135.37,614.833,653.794,0.71205,125.427,0.9153,0.104003,-1.0,-1.0,41.049,0.201127,1055,448898 +2048,23,insert,497.115,-1.0,-1.0,-1.0,-1.0,335.132,0.329899,1118,733493 +2048,24,search,3454.43,855.454,906.361,0.628783,122.254,0.9075,0.108471,-1.0,-1.0,94.077,0.330124,1131,733493 +2048,25,insert,52.842,-1.0,-1.0,-1.0,-1.0,26.818,0.347154,1134,777443 +2048,26,search,3504.74,864.581,922.437,0.605595,119.575,0.90754,0.109082,-1.0,-1.0,33.388,0.347255,1138,777443 +2048,27,insert,39.612,-1.0,-1.0,-1.0,-1.0,18.687,0.358619,1139,805394 +2048,28,search,3515.79,881.488,941.29,0.601415,116.161,0.90782,0.108748,-1.0,-1.0,24.031,0.358666,1141,805394 +2048,29,insert,104.047,-1.0,-1.0,-1.0,-1.0,76.768,0.385983,1153,866391 +2048,30,search,3389.89,926.83,857.141,0.597795,103.732,0.90867,0.10922,-1.0,-1.0,18.4,0.385983,1154,866391 +2048,31,insert,575.665,-1.0,-1.0,-1.0,-1.0,366.747,0.554142,1217,1239816 +2048,32,search,3965.82,1190.22,1292.23,0.534433,117.167,0.90248,0.114527,-1.0,-1.0,75.247,0.554141,1221,1239816 +2048,33,insert,70.154,-1.0,-1.0,-1.0,-1.0,33.874,0.574312,1226,1291211 +2048,34,search,3966.8,1221,1292.74,0.555395,126.132,0.90381,0.113789,-1.0,-1.0,79.728,0.574419,1231,1291211 +2048,35,insert,42.024,-1.0,-1.0,-1.0,-1.0,35.024,0.585956,1238,1324978 +2048,36,search,3980.07,1240.34,1247.54,0.548734,123.797,0.90463,0.112786,-1.0,-1.0,30.401,0.58596,1240,1324978 +2048,37,insert,65.165,-1.0,-1.0,-1.0,-1.0,41.666,0.605734,1246,1371885 +2048,38,search,4036.46,1269.6,1306.84,0.518811,134.771,0.90411,0.113705,-1.0,-1.0,10.906,0.605734,1246,1371885 +2048,39,insert,12.073,-1.0,-1.0,-1.0,-1.0,13.037,0.609226,1247,1379335 +2048,40,search,4083.56,1273.97,1332.48,0.52759,135.55,0.90497,0.113576,-1.0,-1.0,10.738,0.609226,1247,1379335 +2048,41,insert,56.732,-1.0,-1.0,-1.0,-1.0,46.351,0.624915,1253,1413312 +2048,42,search,4106.06,1293.05,1370.49,0.622234,126.676,0.90511,0.11338,-1.0,-1.0,10.861,0.624915,1253,1413312 +2048,43,insert,16.844,-1.0,-1.0,-1.0,-1.0,14.469,0.629132,1254,1423411 +2048,44,search,4117.58,1304.56,1367.39,0.55397,131.252,0.90559,0.112642,-1.0,-1.0,11.525,0.629132,1254,1423411 +2048,45,insert,249.536,-1.0,-1.0,-1.0,-1.0,45.909,0.64488,1263,1457559 +2048,46,search,4165.78,1321.11,1404.98,0.519079,131.042,0.90547,0.112419,-1.0,-1.0,10.64,0.64488,1263,1457559 +2048,47,insert,349.308,-1.0,-1.0,-1.0,-1.0,261.785,0.749534,1295,1688593 +2048,48,search,4438.71,1458.96,1583.71,0.500374,133.171,0.90606,0.112788,-1.0,-1.0,91.571,0.749653,1303,1688593 +2048,49,insert,47.098,-1.0,-1.0,-1.0,-1.0,36.469,0.763584,1306,1723505 +2048,50,search,4409.39,1476.08,1548.69,0.507646,134.448,0.90694,0.112199,-1.0,-1.0,49.604,0.763585,1309,1723505 +2048,51,insert,422.672,-1.0,-1.0,-1.0,-1.0,250.498,0.9014,1349,2035040 +2048,52,search,4520.03,1654.38,1544.48,0.522612,144.672,0.90984,0.111246,-1.0,-1.0,22.663,0.9014,1350,2035040 +2048,53,insert,60.141,-1.0,-1.0,-1.0,-1.0,31.006,0.920433,1352,2078341 +2048,54,search,4547.03,1685.71,1590.01,0.558232,128.025,0.90997,0.111087,-1.0,-1.0,18.675,0.920439,1353,2078341 +2048,55,insert,435.085,-1.0,-1.0,-1.0,-1.0,454.65,1.06876,1407,2399905 +2048,56,search,4969.16,1856.72,1857.27,0.500387,157.96,0.91043,0.112915,-1.0,-1.0,140.484,1.06922,1425,2399905 +2048,57,insert,31.869,-1.0,-1.0,-1.0,-1.0,10.703,1.07779,1425,2419836 +2048,58,search,4917.67,1827.12,1813.66,0.514087,143.291,0.90925,0.113682,-1.0,-1.0,39.174,1.0778,1427,2419836 +2048,59,insert,280.612,-1.0,-1.0,-1.0,-1.0,155.791,1.17593,1439,2643083 +2048,60,search,4907.59,1939.02,1765.11,0.50224,140.903,0.90268,0.121482,-1.0,-1.0,26.122,1.17622,1442,2643083 +2048,61,insert,15.047,-1.0,-1.0,-1.0,-1.0,19.272,1.18079,1444,2653064 +2048,62,search,4693.67,1935.9,1567.94,0.572034,127.276,0.90252,0.121329,-1.0,-1.0,11.725,1.18079,1444,2653064 +2048,63,insert,4.471,-1.0,-1.0,-1.0,-1.0,10.946,1.18198,1444,2656118 +2048,64,search,4719.93,1937.83,1602.29,0.552913,123.179,0.90257,0.121347,-1.0,-1.0,11.875,1.18198,1444,2656118 +2048,65,insert,82.519,-1.0,-1.0,-1.0,-1.0,154.654,1.20928,1469,2723424 +2048,66,search,5328.4,1953.4,2097.21,0.465194,167.973,0.90364,0.120202,-1.0,-1.0,15.347,1.20942,1470,2723424 +2048,67,insert,57.976,-1.0,-1.0,-1.0,-1.0,60.674,1.22984,1477,2769021 +2048,68,search,5143.77,1981.12,1882.15,0.438606,159.532,0.90529,0.11916,-1.0,-1.0,21.895,1.22984,1478,2769021 +2048,69,insert,74.258,-1.0,-1.0,-1.0,-1.0,69.73,1.2524,1486,2821777 +2048,70,search,5428.52,2001.89,2133.43,0.472196,184.831,0.90686,0.118102,-1.0,-1.0,23.774,1.2524,1487,2821777 +2048,71,insert,27.641,-1.0,-1.0,-1.0,-1.0,26.049,1.25813,1489,2834921 +2048,72,search,5352.68,2009.71,2074.71,0.3972,167.95,0.90739,0.117505,-1.0,-1.0,13.639,1.25813,1489,2834921 +2048,73,insert,12.847,-1.0,-1.0,-1.0,-1.0,54.782,1.26204,1494,2842044 +2048,74,search,5019.52,2008.45,1766.47,0.656906,141.656,0.90749,0.117568,-1.0,-1.0,12.456,1.26204,1494,2842044 +2048,75,insert,31.311,-1.0,-1.0,-1.0,-1.0,33.025,1.26869,1497,2858890 +2048,76,search,5399.3,2011.77,2139.59,0.50247,177.551,0.90744,0.117533,-1.0,-1.0,13.086,1.26869,1497,2858890 +2048,77,insert,47.355,-1.0,-1.0,-1.0,-1.0,18.416,1.27953,1498,2882818 +2048,78,search,5298.95,2024.41,2033.74,0.448168,165.197,0.90698,0.118,-1.0,-1.0,13.241,1.27953,1498,2882818 +2048,79,insert,537.525,-1.0,-1.0,-1.0,-1.0,12.226,1.30734,1498,2947806 +2048,80,search,5518.07,2064.93,2230.95,0.455563,184.548,0.90674,0.118262,-1.0,-1.0,12.701,1.30734,1498,2947806 +2048,81,insert,44.161,-1.0,-1.0,-1.0,-1.0,32.475,1.31975,1501,2976895 +2048,82,search,5354.69,2078.04,2111.04,0.432005,170.838,0.90828,0.116564,-1.0,-1.0,17.744,1.31992,1502,2976895 +2048,83,insert,16.296,-1.0,-1.0,-1.0,-1.0,30.149,1.32547,1505,2990682 +2048,84,search,5350.32,2080.57,2072.76,0.540098,159.506,0.90826,0.116674,-1.0,-1.0,12.538,1.32547,1505,2990682 +2048,85,insert,41.798,-1.0,-1.0,-1.0,-1.0,40.527,1.33918,1509,3025613 +2048,86,search,5628.2,2097.94,2278.99,0.443742,188.055,0.90921,0.116184,-1.0,-1.0,25.277,1.33931,1510,3025613 +2048,87,insert,592.922,-1.0,-1.0,-1.0,-1.0,614.934,1.5415,1591,3468632 +2048,88,search,5921.72,2269.6,2412.94,0.394859,183.692,0.91216,0.113617,-1.0,-1.0,116.447,1.54172,1602,3468632 +2048,89,insert,467.709,-1.0,-1.0,-1.0,-1.0,429.346,1.69991,1661,3831692 +2048,90,search,6229.87,2388.86,2585.22,0.392498,191.146,0.91705,0.108027,-1.0,-1.0,87.633,1.70015,1672,3831692 +2048,91,insert,39.709,-1.0,-1.0,-1.0,-1.0,46.311,1.71321,1677,3864067 +2048,92,search,6506.9,2386.25,2813.45,0.421943,214.461,0.91692,0.107888,-1.0,-1.0,13.854,1.71321,1677,3864067 +2048,93,insert,321.099,-1.0,-1.0,-1.0,-1.0,232.426,1.82048,1703,4103827 +2048,94,search,6468.13,2477.07,2740.28,0.461241,199.34,0.91734,0.106625,-1.0,-1.0,56.038,1.821,1710,4103827 +2048,95,insert,18.091,-1.0,-1.0,-1.0,-1.0,20.218,1.8269,1711,4119898 +2048,96,search,6828.39,2472.01,3061.25,0.441398,215.175,0.91745,0.106627,-1.0,-1.0,15.204,1.8269,1711,4119898 +2048,97,insert,12.364,-1.0,-1.0,-1.0,-1.0,28.418,1.831,1713,4129367 +2048,98,search,6468.58,2472.4,2729.24,0.411449,204.303,0.9175,0.106464,-1.0,-1.0,15.568,1.831,1713,4129367 +2048,99,insert,6.452,-1.0,-1.0,-1.0,-1.0,27.485,1.83268,1715,4134502 +2048,100,search,6786.15,2475.46,2989.64,0.418089,221.037,0.91779,0.106131,-1.0,-1.0,51.51,1.83329,1719,4134502 +2048,101,insert,38.952,-1.0,-1.0,-1.0,-1.0,50.831,1.84454,1724,4164482 +2048,102,search,6813.99,2474.08,2988.68,0.370013,228.209,0.91795,0.10623,-1.0,-1.0,17.647,1.84454,1724,4164482 +2048,103,insert,20.525,-1.0,-1.0,-1.0,-1.0,21.771,1.84924,1725,4178601 +2048,104,search,6606.28,2481.58,2841.28,0.382123,206.268,0.91804,0.106241,-1.0,-1.0,17.022,1.84924,1725,4178601 +2048,105,insert,11.356,-1.0,-1.0,-1.0,-1.0,15.685,1.85258,1725,4187774 +2048,106,search,6552.14,2485.92,2776.62,0.390638,209.825,0.91794,0.106552,-1.0,-1.0,17.834,1.85258,1725,4187774 +2048,107,insert,29.104,-1.0,-1.0,-1.0,-1.0,26.982,1.8574,1726,4200109 +2048,108,search,6736.98,2489.03,2956.82,0.385353,223.947,0.91786,0.106677,-1.0,-1.0,17.187,1.8574,1726,4200109 +2048,109,insert,522.425,-1.0,-1.0,-1.0,-1.0,579.021,2.01818,1799,4557867 +2048,110,search,6923.31,2576.33,2942.92,0.410608,217.347,0.91983,0.104607,-1.0,-1.0,71.862,2.01924,1805,4557867 +2048,111,insert,32.28,-1.0,-1.0,-1.0,-1.0,76.373,2.03042,1814,4587252 +2048,112,search,6884.34,2571.63,2901.69,0.468932,212.906,0.92042,0.104423,-1.0,-1.0,20.572,2.03042,1814,4587252 +2048,113,insert,15.221,-1.0,-1.0,-1.0,-1.0,19.938,2.03474,1814,4600056 +2048,114,search,7151.75,2581.25,3155.64,0.411619,240.399,0.92058,0.104346,-1.0,-1.0,20.618,2.03474,1814,4600056 +2048,115,insert,36.134,-1.0,-1.0,-1.0,-1.0,35.605,2.04452,1816,4627422 +2048,116,search,7196.25,2594.02,3192.28,0.411645,227.222,0.92095,0.104131,-1.0,-1.0,22.449,2.04452,1816,4627422 +2048,117,insert,14.037,-1.0,-1.0,-1.0,-1.0,43.694,2.04869,1819,4639292 +2048,118,search,7121.47,2593.25,3134.87,0.500174,218.009,0.92074,0.104242,-1.0,-1.0,24.287,2.04869,1819,4639292 +2048,119,insert,7.38,-1.0,-1.0,-1.0,-1.0,20.988,2.04994,1819,4643423 +2048,120,search,6920.43,2595.38,2952.9,0.427788,209.225,0.92081,0.104142,-1.0,-1.0,22.215,2.04994,1819,4643423 +2048,121,insert,23.277,-1.0,-1.0,-1.0,-1.0,40.747,2.05642,1821,4657252 +2048,122,search,7036.68,2600.16,3035.27,0.400545,218.883,0.92134,0.103846,-1.0,-1.0,22.389,2.05642,1821,4657252 +2048,123,insert,436.066,-1.0,-1.0,-1.0,-1.0,512.294,2.18478,1872,4938477 +2048,124,search,7341.63,2665.3,3213.85,0.436446,221.918,0.92153,0.104665,-1.0,-1.0,169.19,2.18551,1885,4938477 +2048,125,insert,64.48,-1.0,-1.0,-1.0,-1.0,74.46,2.20322,1893,4980492 +2048,126,search,7704.98,2654.29,3537.74,0.417725,245.543,0.92206,0.104337,-1.0,-1.0,66.207,2.2034,1896,4980492 +2048,127,insert,61.834,-1.0,-1.0,-1.0,-1.0,117.969,2.2231,1911,5030515 +2048,128,search,7116.91,2653.63,2942.42,0.509782,207.576,0.92256,0.104145,-1.0,-1.0,29.028,2.2231,1912,5030515 +2048,129,delete,18.517,-1.0,-1.0,-1.0,-1.0,95.691,2.22108,1880,5009391 +2048,130,search,7309.35,2677.13,3174.26,0.362236,224.065,0.92113,0.105126,-1.0,-1.0,27.897,2.22108,1880,5009391 +2048,131,delete,12.08,-1.0,-1.0,-1.0,-1.0,25.325,2.22108,1880,5004093 +2048,132,search,7625.39,2674.78,3479.55,0.3292,238.119,0.92118,0.105169,-1.0,-1.0,25.809,2.22108,1880,5004093 +2048,133,delete,20.43,-1.0,-1.0,-1.0,-1.0,33.932,2.22018,1879,4991787 +2048,134,search,7582.44,2667.67,3438.84,0.334598,249.479,0.92086,0.105415,-1.0,-1.0,26.374,2.22018,1879,4991787 +2048,135,delete,30.135,-1.0,-1.0,-1.0,-1.0,57.904,2.21743,1876,4974857 +2048,136,search,7361.16,2660.84,3238.58,0.369661,233.985,0.92106,0.105195,-1.0,-1.0,25.816,2.21743,1876,4974857 +2048,137,delete,7.7,-1.0,-1.0,-1.0,-1.0,24.36,2.21743,1876,4971386 +2048,138,search,7334.13,2660,3201.2,0.374534,232.622,0.92098,0.105254,-1.0,-1.0,25.297,2.21743,1876,4971386 +2048,139,delete,13.23,-1.0,-1.0,-1.0,-1.0,25.487,2.21743,1876,4963487 +2048,140,search,7465.1,2655.27,3322.74,0.348421,252.004,0.92083,0.105372,-1.0,-1.0,25.653,2.21743,1876,4963487 +2048,141,delete,32.069,-1.0,-1.0,-1.0,-1.0,78.888,2.21332,1870,4941830 +2048,142,search,7478.11,2650.41,3356.26,0.389279,249.511,0.92058,0.105462,-1.0,-1.0,24.636,2.21332,1870,4941830 +2048,143,delete,19.449,-1.0,-1.0,-1.0,-1.0,70.413,2.20912,1867,4925657 +2048,144,search,7470.87,2646.64,3432.68,0.36775,237.806,0.92083,0.105229,-1.0,-1.0,24.636,2.20912,1867,4925657 +2048,145,delete,3.359,-1.0,-1.0,-1.0,-1.0,36.81,2.20836,1865,4922686 +2048,146,search,7131.68,2645.3,3036.48,0.386342,216.476,0.92031,0.105418,-1.0,-1.0,28.188,2.20836,1865,4922686 +2048,147,delete,20.35,-1.0,-1.0,-1.0,-1.0,24.718,2.20836,1865,4913636 +2048,148,search,7141.47,2640.91,3045.77,0.377016,219.715,0.92033,0.105436,-1.0,-1.0,25.784,2.20836,1865,4913636 +2048,149,delete,240.523,-1.0,-1.0,-1.0,-1.0,1031.16,2.13667,1789,4717250 +2048,150,search,6839.69,2637.2,2910.69,0.483281,212.888,0.91992,0.106344,-1.0,-1.0,59.668,2.13755,1791,4717250 +2048,151,delete,183.961,-1.0,-1.0,-1.0,-1.0,1015.17,2.06786,1713,4560269 +4096,2,search,11537.4,38.5561,9121.22,0.649295,167.197,0.8551,0.130106,-1.0,-1.0,8.612,0.0184881,1000,38806 +4096,3,insert,22.857,-1.0,-1.0,-1.0,-1.0,7.288,0.0226644,1000,49334 +4096,4,search,2799.94,52.5464,270.13,0.683308,254.698,0.86604,0.12886,-1.0,-1.0,8.689,0.0226644,1000,49334 +4096,5,insert,44.31,-1.0,-1.0,-1.0,-1.0,172.752,0.0330953,997,73557 +4096,6,search,2754.42,107.344,205.689,1.13465,158.168,0.92386,0.105036,-1.0,-1.0,151.793,0.0330953,997,73557 +4096,7,insert,49.145,-1.0,-1.0,-1.0,-1.0,133.301,0.0428118,998,95766 +4096,8,search,2768.75,132.008,239.421,1.13271,152.771,0.92518,0.103938,-1.0,-1.0,95.591,0.043126,1001,95766 +4096,9,insert,7.458,-1.0,-1.0,-1.0,-1.0,76.451,0.0448678,1001,100025 +4096,10,search,2745.72,130.256,231.426,1.08274,151.019,0.92729,0.100268,-1.0,-1.0,78.206,0.0448678,1001,100025 +4096,11,insert,17.922,-1.0,-1.0,-1.0,-1.0,78.23,0.0495679,1001,111417 +4096,12,search,2755.88,148.883,281.088,1.04146,148.926,0.92776,0.10035,-1.0,-1.0,79.991,0.0495679,1001,111417 +4096,13,insert,40.671,-1.0,-1.0,-1.0,-1.0,78.801,0.0609091,1001,137139 +4096,14,search,2840.34,175.671,296.333,0.985786,156.798,0.925,0.102401,-1.0,-1.0,23.024,0.0612763,1002,137139 +4096,15,insert,35.517,-1.0,-1.0,-1.0,-1.0,7.929,0.0699749,1002,156710 +4096,16,search,2855.65,198.98,317.811,0.900885,161.626,0.92291,0.103342,-1.0,-1.0,24.559,0.0699785,1004,156710 +4096,17,insert,9.365,-1.0,-1.0,-1.0,-1.0,7.949,0.0718398,1004,161773 +4096,18,search,2861.04,206.396,333.949,0.908278,145.187,0.92324,0.103581,-1.0,-1.0,18.224,0.0720618,1006,161773 +4096,19,insert,47.924,-1.0,-1.0,-1.0,-1.0,7.903,0.0785082,1006,177161 +4096,20,search,2804.17,227.616,369.988,0.966953,123.585,0.92715,0.100366,-1.0,-1.0,9.062,0.0785082,1006,177161 +4096,21,insert,491.853,-1.0,-1.0,-1.0,-1.0,168.208,0.200555,1036,448898 +4096,22,search,3095.19,528.188,721.056,0.769119,113.52,0.91727,0.103129,-1.0,-1.0,35.698,0.200553,1041,448898 +4096,23,insert,523.075,-1.0,-1.0,-1.0,-1.0,302.251,0.328603,1083,733493 +4096,24,search,3489.73,784.676,997.183,0.694697,126.244,0.91257,0.105198,-1.0,-1.0,88.678,0.328664,1094,733493 +4096,25,insert,58.378,-1.0,-1.0,-1.0,-1.0,41.8,0.345761,1099,777443 +4096,26,search,3463.02,804.589,1005.93,0.662109,123.056,0.91175,0.106454,-1.0,-1.0,20.478,0.345766,1100,777443 +4096,27,insert,48.337,-1.0,-1.0,-1.0,-1.0,22.061,0.357943,1101,805394 +4096,28,search,3573.48,821.934,1044.3,0.678013,121.137,0.91364,0.105069,-1.0,-1.0,12.981,0.357943,1101,805394 +4096,29,insert,93.307,-1.0,-1.0,-1.0,-1.0,61.904,0.385594,1108,866391 +4096,30,search,3672.01,869.688,1078.15,0.634186,126.102,0.91483,0.105673,-1.0,-1.0,29.306,0.385593,1110,866391 +4096,31,insert,581.097,-1.0,-1.0,-1.0,-1.0,389.249,0.553916,1161,1239816 +4096,32,search,3960.5,1143.99,1370.29,0.597561,132.723,0.9091,0.110302,-1.0,-1.0,109.853,0.554335,1169,1239816 +4096,33,insert,66.458,-1.0,-1.0,-1.0,-1.0,22.409,0.574261,1171,1291211 +4096,34,search,4017.04,1174.43,1374.24,0.542666,137.238,0.90811,0.111001,-1.0,-1.0,57.884,0.57434,1175,1291211 +4096,35,insert,44.444,-1.0,-1.0,-1.0,-1.0,24.812,0.586385,1178,1324978 +4096,36,search,3939.25,1195.55,1322.23,0.55502,139.24,0.90887,0.110037,-1.0,-1.0,48.177,0.586387,1181,1324978 +4096,37,insert,75.677,-1.0,-1.0,-1.0,-1.0,29.571,0.607092,1183,1371885 +4096,38,search,3853.88,1229.65,1230.77,0.607154,121.091,0.90916,0.109973,-1.0,-1.0,18.994,0.607091,1184,1371885 +4096,39,insert,9.905,-1.0,-1.0,-1.0,-1.0,10.018,0.608986,1184,1379335 +4096,40,search,3898.2,1234.71,1294.14,0.570971,126.337,0.90899,0.110654,-1.0,-1.0,11.122,0.608986,1184,1379335 +4096,41,insert,66.038,-1.0,-1.0,-1.0,-1.0,44.678,0.623349,1189,1413312 +4096,42,search,3870.12,1255.56,1240.32,0.580447,124.789,0.90972,0.110182,-1.0,-1.0,11.791,0.623349,1189,1413312 +4096,43,insert,23.114,-1.0,-1.0,-1.0,-1.0,15.579,0.628273,1190,1423411 +4096,44,search,3910.16,1264.86,1298.99,0.559865,122.918,0.90991,0.109793,-1.0,-1.0,11.04,0.628273,1190,1423411 +4096,45,insert,263.118,-1.0,-1.0,-1.0,-1.0,54.475,0.644578,1198,1457559 +4096,46,search,3894.73,1281.24,1261.15,0.680918,114.928,0.90971,0.109171,-1.0,-1.0,10.355,0.644578,1198,1457559 +4096,47,insert,376.972,-1.0,-1.0,-1.0,-1.0,254.505,0.749623,1225,1688593 +4096,48,search,4343.41,1429.73,1630.46,0.50611,149.003,0.90995,0.109918,-1.0,-1.0,100.956,0.749754,1234,1688593 +4096,49,insert,53.041,-1.0,-1.0,-1.0,-1.0,32.993,0.764617,1237,1723505 +4096,50,search,4405.9,1444.44,1647.52,0.508175,149.463,0.91121,0.109342,-1.0,-1.0,29.689,0.764617,1239,1723505 +4096,51,insert,429.229,-1.0,-1.0,-1.0,-1.0,229.188,0.901089,1269,2035040 +4096,52,search,4590.49,1638.63,1778.94,0.499362,131.895,0.91364,0.107785,-1.0,-1.0,34.504,0.901089,1272,2035040 +4096,53,insert,50.689,-1.0,-1.0,-1.0,-1.0,44.777,0.918886,1276,2078341 +4096,54,search,4508.4,1663.8,1671.16,0.483307,138.783,0.91337,0.108109,-1.0,-1.0,20.977,0.918887,1277,2078341 +4096,55,insert,437.42,-1.0,-1.0,-1.0,-1.0,452.122,1.06679,1322,2399905 +4096,56,search,4672.69,1856.9,1748.64,0.484284,130.126,0.91426,0.109757,-1.0,-1.0,107.807,1.06718,1335,2399905 +4096,57,insert,33.547,-1.0,-1.0,-1.0,-1.0,23.068,1.07592,1337,2419836 +4096,58,search,5098.63,1850.79,2111.06,0.523936,175.457,0.91347,0.109913,-1.0,-1.0,53.512,1.07595,1342,2419836 +4096,59,insert,311.752,-1.0,-1.0,-1.0,-1.0,137.822,1.17537,1354,2643083 +4096,60,search,4899.23,1982.2,1917.81,0.46155,151.064,0.90847,0.115012,-1.0,-1.0,88.181,1.17571,1362,2643083 +4096,61,insert,16.138,-1.0,-1.0,-1.0,-1.0,30.006,1.18049,1365,2653064 +4096,62,search,5309.94,1970.91,2294.68,0.436011,161.393,0.90728,0.116262,-1.0,-1.0,11.674,1.18049,1365,2653064 +4096,63,insert,5.649,-1.0,-1.0,-1.0,-1.0,10.376,1.1818,1365,2656118 +4096,64,search,5210.06,1973.07,2185.5,0.4237,172.645,0.90743,0.116167,-1.0,-1.0,11.874,1.1818,1365,2656118 +4096,65,insert,100.981,-1.0,-1.0,-1.0,-1.0,136.555,1.20999,1385,2723424 +4096,66,search,4948.19,1993.67,1896.19,0.437753,149.379,0.90789,0.115619,-1.0,-1.0,11.422,1.20999,1385,2723424 +4096,67,insert,60.125,-1.0,-1.0,-1.0,-1.0,93.799,1.23121,1397,2769021 +4096,68,search,5444.03,2007.79,2273.09,0.459303,187.885,0.90935,0.114185,-1.0,-1.0,18.531,1.23121,1398,2769021 +4096,69,insert,67.153,-1.0,-1.0,-1.0,-1.0,76.888,1.2544,1407,2821777 +4096,70,search,5537.21,2027.73,2365.68,0.39594,188.286,0.91111,0.112628,-1.0,-1.0,12.941,1.2544,1407,2821777 +4096,71,insert,23.002,-1.0,-1.0,-1.0,-1.0,36.406,1.25961,1410,2834921 +4096,72,search,5547.46,2031.97,2431.2,0.390223,168.778,0.91144,0.112492,-1.0,-1.0,13.946,1.25961,1410,2834921 +4096,73,insert,10.863,-1.0,-1.0,-1.0,-1.0,30.17,1.2625,1412,2842044 +4096,74,search,5441.66,2033.48,2320.63,0.378742,183.498,0.91132,0.112676,-1.0,-1.0,12.394,1.2625,1412,2842044 +4096,75,insert,31.21,-1.0,-1.0,-1.0,-1.0,27.19,1.26976,1414,2858890 +4096,76,search,5165.45,2046.38,2048.37,0.465175,146.866,0.91155,0.112409,-1.0,-1.0,12.075,1.26976,1414,2858890 +4096,77,insert,53.452,-1.0,-1.0,-1.0,-1.0,24.016,1.28098,1416,2882818 +4096,78,search,5482.48,2054.93,2357.63,0.374653,185.122,0.91097,0.11329,-1.0,-1.0,12.482,1.28098,1416,2882818 +4096,79,insert,513.061,-1.0,-1.0,-1.0,-1.0,24.024,1.308,1418,2947806 +4096,80,search,5180.5,2099.72,2083.37,0.399101,158.031,0.91061,0.113692,-1.0,-1.0,12.138,1.308,1418,2947806 +4096,81,insert,46.767,-1.0,-1.0,-1.0,-1.0,36.942,1.32035,1421,2976895 +4096,82,search,5610.66,2114.16,2441.24,0.373434,183.814,0.91171,0.11318,-1.0,-1.0,11.635,1.32035,1421,2976895 +4096,83,insert,19.421,-1.0,-1.0,-1.0,-1.0,19.759,1.32527,1422,2990682 +4096,84,search,5402.42,2122.71,2274.53,0.392965,173.631,0.91202,0.112614,-1.0,-1.0,23.585,1.32539,1423,2990682 +4096,85,insert,50.858,-1.0,-1.0,-1.0,-1.0,26.646,1.34123,1425,3025613 +4096,86,search,5286.02,2142.15,2159.19,0.456139,151.799,0.91262,0.112591,-1.0,-1.0,11.733,1.34123,1425,3025613 +4096,87,insert,587.825,-1.0,-1.0,-1.0,-1.0,752.543,1.54568,1520,3468632 +4096,88,search,6099.7,2290.88,2708.21,0.339739,207.368,0.91535,0.110627,-1.0,-1.0,133.677,1.54574,1533,3468632 +4096,89,insert,446.131,-1.0,-1.0,-1.0,-1.0,515.886,1.70627,1607,3831692 +4096,90,search,6490.04,2390.55,2936.66,0.313765,218.65,0.91944,0.105925,-1.0,-1.0,90.857,1.70639,1617,3831692 +4096,91,insert,33.781,-1.0,-1.0,-1.0,-1.0,31.547,1.71499,1620,3864067 +4096,92,search,6205.87,2391.83,2636.04,0.397321,187.294,0.91939,0.105661,-1.0,-1.0,13.996,1.71499,1620,3864067 +4096,93,insert,307.688,-1.0,-1.0,-1.0,-1.0,298.73,1.8202,1655,4103827 +4096,94,search,6464.47,2482.82,2825.25,0.404702,186.64,0.92019,0.104663,-1.0,-1.0,44.223,1.82042,1659,4103827 +4096,95,insert,23.736,-1.0,-1.0,-1.0,-1.0,12.922,1.82791,1659,4119898 +4096,96,search,6188.74,2487.29,2577.01,0.383051,177.416,0.92061,0.104447,-1.0,-1.0,13.583,1.82791,1659,4119898 +4096,97,insert,11.805,-1.0,-1.0,-1.0,-1.0,24.876,1.83119,1661,4129367 +4096,98,search,6771.21,2490.06,3073.32,0.332482,237.509,0.92081,0.104334,-1.0,-1.0,21.59,1.83131,1662,4129367 +4096,99,insert,7.959,-1.0,-1.0,-1.0,-1.0,19.973,1.83383,1663,4134502 +4096,100,search,6496.61,2490.21,2844.78,0.394863,192.706,0.9209,0.10425,-1.0,-1.0,16.766,1.83383,1663,4134502 +4096,101,insert,52.454,-1.0,-1.0,-1.0,-1.0,62.471,1.84586,1669,4164482 +4096,102,search,6569.1,2498.76,2711.24,0.346932,197.719,0.92102,0.103942,-1.0,-1.0,14.331,1.84586,1669,4164482 +4096,103,insert,22.453,-1.0,-1.0,-1.0,-1.0,31.045,1.8521,1671,4178601 +4096,104,search,6903.95,2506.03,3031.24,0.366423,226.193,0.92097,0.104024,-1.0,-1.0,28.344,1.85231,1672,4178601 +4096,105,insert,13.386,-1.0,-1.0,-1.0,-1.0,39.262,1.85552,1675,4187774 +4096,106,search,7030.28,2503.9,3140.33,0.326805,234.901,0.92107,0.104309,-1.0,-1.0,15.084,1.85552,1675,4187774 +4096,107,insert,30.63,-1.0,-1.0,-1.0,-1.0,41.566,1.86065,1677,4200109 +4096,108,search,7175.58,2508.2,3277.4,0.328379,224.37,0.92093,0.10439,-1.0,-1.0,14.399,1.86065,1677,4200109 +4096,109,insert,521.929,-1.0,-1.0,-1.0,-1.0,466.844,2.02066,1735,4557867 +4096,110,search,6815.46,2630.61,2822.69,0.36691,189.815,0.92255,0.102885,-1.0,-1.0,60.346,2.02108,1741,4557867 +4096,111,insert,29.804,-1.0,-1.0,-1.0,-1.0,58.894,2.03292,1748,4587252 +4096,112,search,7458.85,2625.53,3388.75,0.358886,216.216,0.92282,0.102756,-1.0,-1.0,28.748,2.03316,1749,4587252 +4096,113,insert,14.724,-1.0,-1.0,-1.0,-1.0,35.564,2.03889,1751,4600056 +4096,114,search,7633.12,2628.06,3557.25,0.335104,229.629,0.92277,0.10292,-1.0,-1.0,19.745,2.03889,1751,4600056 +4096,115,insert,51.483,-1.0,-1.0,-1.0,-1.0,75.368,2.05096,1757,4627422 +4096,116,search,7693.53,2634.63,3595.67,0.302892,226.727,0.92333,0.102289,-1.0,-1.0,28.141,2.05119,1758,4627422 +4096,117,insert,15.346,-1.0,-1.0,-1.0,-1.0,27.896,2.05471,1759,4639292 +4096,118,search,7220.39,2638.34,3159.84,0.3979,198.438,0.92334,0.102322,-1.0,-1.0,19.363,2.05471,1759,4639292 +4096,119,insert,6.402,-1.0,-1.0,-1.0,-1.0,17.877,2.05602,1759,4643423 +4096,120,search,7199.61,2640.66,3155.08,0.325026,210.354,0.9234,0.102292,-1.0,-1.0,18.852,2.05602,1759,4643423 +4096,121,insert,23.268,-1.0,-1.0,-1.0,-1.0,61.097,2.06126,1763,4657252 +4096,122,search,7130.64,2642.62,3070.67,0.340196,209.09,0.9236,0.101959,-1.0,-1.0,33.995,2.06168,1765,4657252 +4096,123,insert,391.016,-1.0,-1.0,-1.0,-1.0,504.237,2.1885,1815,4938477 +4096,124,search,7965.69,2719.32,3750.84,0.303044,242.027,0.92404,0.102309,-1.0,-1.0,180.113,2.18895,1828,4938477 +4096,125,insert,62.575,-1.0,-1.0,-1.0,-1.0,58.021,2.20754,1834,4980492 +4096,126,search,7833.4,2713.13,3547.79,0.31354,253.025,0.92454,0.102277,-1.0,-1.0,97.297,2.20758,1838,4980492 +4096,127,insert,71.531,-1.0,-1.0,-1.0,-1.0,111.287,2.22997,1852,5030515 +4096,128,search,7811.23,2711.34,3455.76,0.320415,243.675,0.92452,0.102294,-1.0,-1.0,40.519,2.22997,1853,5030515 +4096,129,delete,28.343,-1.0,-1.0,-1.0,-1.0,95.209,2.22825,1822,5009391 +4096,130,search,7661.59,2746.96,3406.54,0.305481,245.874,0.92342,0.102894,-1.0,-1.0,24.101,2.22825,1822,5009391 +4096,131,delete,8.353,-1.0,-1.0,-1.0,-1.0,22.636,2.22825,1822,5004093 +4096,132,search,7683.01,2743.88,3433.27,0.313807,247.491,0.92333,0.103,-1.0,-1.0,23.053,2.22825,1822,5004093 +4096,133,delete,15.1,-1.0,-1.0,-1.0,-1.0,34.605,2.22726,1821,4991787 +4096,134,search,7752.9,2738.21,3504.2,0.283415,251.929,0.9232,0.103135,-1.0,-1.0,23.23,2.22726,1821,4991787 +4096,135,delete,20.635,-1.0,-1.0,-1.0,-1.0,49.06,2.22456,1819,4974857 +4096,136,search,7773.97,2732.2,3515.24,0.28179,248.904,0.92303,0.103067,-1.0,-1.0,26.667,2.22456,1819,4974857 +4096,137,delete,9.494,-1.0,-1.0,-1.0,-1.0,23.031,2.22456,1819,4971386 +4096,138,search,7790.28,2730.25,3516.06,0.273334,254.58,0.92301,0.103095,-1.0,-1.0,24.909,2.22456,1819,4971386 +4096,139,delete,18.595,-1.0,-1.0,-1.0,-1.0,23.739,2.22456,1819,4963487 +4096,140,search,7382.75,2725.63,3177.17,0.29731,210.383,0.92291,0.103314,-1.0,-1.0,23.773,2.22456,1819,4963487 +4096,141,delete,39.383,-1.0,-1.0,-1.0,-1.0,97.7,2.21991,1813,4941830 +4096,142,search,7433.65,2720.77,3240.03,0.399913,209.053,0.92276,0.10336,-1.0,-1.0,23.754,2.21991,1813,4941830 +4096,143,delete,25.403,-1.0,-1.0,-1.0,-1.0,93.184,2.21495,1809,4925657 +4096,144,search,7314.47,2719.58,3149.11,0.289173,216.74,0.92282,0.103406,-1.0,-1.0,34.209,2.21498,1810,4925657 +4096,145,delete,3.372,-1.0,-1.0,-1.0,-1.0,30.403,2.2144,1809,4922686 +4096,146,search,7388.57,2717.94,3180.17,0.282511,228.958,0.9227,0.103805,-1.0,-1.0,26.416,2.2144,1809,4922686 +4096,147,delete,20.965,-1.0,-1.0,-1.0,-1.0,22.915,2.2144,1809,4913636 +4096,148,search,7195.7,2711.48,3019.84,0.331553,215.287,0.92268,0.103803,-1.0,-1.0,24.401,2.2144,1809,4913636 +4096,149,delete,276.274,-1.0,-1.0,-1.0,-1.0,1051.77,2.13983,1730,4717250 +4096,150,search,7088.36,2722.99,3071.76,0.281484,211.682,0.92226,0.104236,-1.0,-1.0,55.043,2.14033,1733,4717250 +4096,151,delete,183.103,-1.0,-1.0,-1.0,-1.0,858.619,2.07207,1666,4560269 +8192,2,search,27565.3,38.7532,24395.7,1.30728,115.15,0.8551,0.130106,-1.0,-1.0,8.553,0.0184881,1000,38806 +8192,3,insert,22.577,-1.0,-1.0,-1.0,-1.0,16.498,0.0226681,998,49334 +8192,4,search,12399.3,50.3434,7828.17,0.825033,165.07,0.86613,0.128797,-1.0,-1.0,14.952,0.0226611,997,49334 +8192,5,insert,40.973,-1.0,-1.0,-1.0,-1.0,176.879,0.0329766,993,73557 +8192,6,search,4042.9,91.1717,774.971,0.683991,253.458,0.92433,0.104255,-1.0,-1.0,114.347,0.0330819,995,73557 +8192,7,insert,48.308,-1.0,-1.0,-1.0,-1.0,104.903,0.0426107,995,95766 +8192,8,search,2799.43,109.607,232.022,1.14273,147.147,0.92582,0.103612,-1.0,-1.0,94.258,0.0428012,997,95766 +8192,9,insert,7.508,-1.0,-1.0,-1.0,-1.0,79.078,0.0445267,997,100025 +8192,10,search,2765.81,105.062,249.059,1.08397,148.386,0.92703,0.100737,-1.0,-1.0,80.411,0.0445267,997,100025 +8192,11,insert,19.307,-1.0,-1.0,-1.0,-1.0,81.366,0.0497772,997,111417 +8192,12,search,2805.15,119.099,284.257,1.11498,131.466,0.92784,0.100473,-1.0,-1.0,82.539,0.0497772,997,111417 +8192,13,insert,40.199,-1.0,-1.0,-1.0,-1.0,86.656,0.0610515,997,137139 +8192,14,search,2845.37,142.935,302.696,1.00175,142.35,0.92449,0.102927,-1.0,-1.0,24.298,0.061098,998,137139 +8192,15,insert,39.328,-1.0,-1.0,-1.0,-1.0,8.292,0.0698157,998,156710 +8192,16,search,2865.05,161.365,339.939,1.0034,146.305,0.92241,0.104278,-1.0,-1.0,23.602,0.0698182,1000,156710 +8192,17,insert,10.93,-1.0,-1.0,-1.0,-1.0,11.833,0.0717892,1001,161773 +8192,18,search,2880.69,166.103,348.815,0.931538,154.133,0.92357,0.10331,-1.0,-1.0,20.208,0.07197,1003,161773 +8192,19,insert,48.928,-1.0,-1.0,-1.0,-1.0,7.77,0.07856,1003,177161 +8192,20,search,2909,182.593,367.665,0.94727,147.493,0.92703,0.10003,-1.0,-1.0,8.824,0.07856,1003,177161 +8192,21,insert,499.527,-1.0,-1.0,-1.0,-1.0,199.508,0.201212,1039,448898 +8192,22,search,3172.07,431.198,709.276,0.849123,117.527,0.91539,0.10383,-1.0,-1.0,28.701,0.201211,1042,448898 +8192,23,insert,518.481,-1.0,-1.0,-1.0,-1.0,317.188,0.328128,1088,733493 +8192,24,search,3241.73,672.368,822.975,0.896863,98.9298,0.91169,0.106081,-1.0,-1.0,90.829,0.328126,1100,733493 +8192,25,insert,59.504,-1.0,-1.0,-1.0,-1.0,45.874,0.345871,1106,777443 +8192,26,search,3347.11,701.086,893.971,0.656742,119.87,0.91074,0.107309,-1.0,-1.0,20.215,0.345873,1107,777443 +8192,27,insert,46.304,-1.0,-1.0,-1.0,-1.0,13.065,0.357604,1107,805394 +8192,28,search,3540.13,725.635,1034.44,0.698074,119.065,0.91184,0.106671,-1.0,-1.0,24.35,0.357714,1108,805394 +8192,29,insert,94.716,-1.0,-1.0,-1.0,-1.0,61.834,0.38539,1115,866391 +8192,30,search,3562.24,775.002,1065.18,0.6865,129.469,0.91289,0.10726,-1.0,-1.0,31.502,0.38539,1117,866391 +8192,31,insert,582.361,-1.0,-1.0,-1.0,-1.0,364.37,0.552645,1171,1239816 +8192,32,search,3998.64,1057.88,1365.6,0.554018,133.373,0.9057,0.112014,-1.0,-1.0,113.838,0.552897,1179,1239816 +8192,33,insert,82.806,-1.0,-1.0,-1.0,-1.0,13.304,0.573687,1180,1291211 +8192,34,search,4061.73,1093.34,1396.13,0.539061,138.055,0.90713,0.110999,-1.0,-1.0,72.964,0.573978,1186,1291211 +8192,35,insert,49.616,-1.0,-1.0,-1.0,-1.0,26.451,0.585004,1190,1324978 +8192,36,search,3853.65,1112.51,1210.89,0.553019,132.283,0.90718,0.110544,-1.0,-1.0,43.757,0.58501,1193,1324978 +8192,37,insert,73.951,-1.0,-1.0,-1.0,-1.0,27.081,0.604548,1195,1371885 +8192,38,search,4105.31,1147.08,1435.28,0.532165,135.079,0.90718,0.111175,-1.0,-1.0,10.591,0.604548,1195,1371885 +8192,39,insert,17.421,-1.0,-1.0,-1.0,-1.0,15.747,0.6081,1196,1379335 +8192,40,search,4100.15,1151.39,1465.26,0.533851,130.947,0.90763,0.111373,-1.0,-1.0,18.14,0.608219,1197,1379335 +8192,41,insert,63.876,-1.0,-1.0,-1.0,-1.0,41.647,0.623929,1202,1413312 +8192,42,search,4160.05,1173.88,1492.12,0.558511,131.177,0.90792,0.11131,-1.0,-1.0,11.251,0.623929,1202,1413312 +8192,43,insert,19.625,-1.0,-1.0,-1.0,-1.0,15.026,0.627703,1203,1423411 +8192,44,search,4106.37,1181.28,1471.83,0.551127,135.371,0.90817,0.111101,-1.0,-1.0,10.587,0.627703,1203,1423411 +8192,45,insert,248.527,-1.0,-1.0,-1.0,-1.0,32.825,0.642549,1207,1457559 +8192,46,search,4035.01,1205.62,1361.81,0.533036,124.136,0.90912,0.109805,-1.0,-1.0,10.387,0.642549,1207,1457559 +8192,47,insert,370.411,-1.0,-1.0,-1.0,-1.0,275.658,0.748957,1237,1688593 +8192,48,search,4430.78,1361.81,1674.91,0.497981,148.391,0.90942,0.110545,-1.0,-1.0,117.808,0.748956,1245,1688593 +8192,49,insert,56.928,-1.0,-1.0,-1.0,-1.0,50.983,0.763786,1251,1723505 +8192,50,search,4121.75,1374.44,1374.35,0.522257,130.457,0.90992,0.109936,-1.0,-1.0,47.022,0.763785,1254,1723505 +8192,51,insert,470.613,-1.0,-1.0,-1.0,-1.0,268.323,0.902564,1291,2035040 +8192,52,search,4425.91,1573.92,1576.98,0.470998,138.53,0.91274,0.108916,-1.0,-1.0,64.459,0.902624,1298,2035040 +8192,53,insert,61.764,-1.0,-1.0,-1.0,-1.0,23.878,0.920093,1299,2078341 +8192,54,search,4790.64,1597.52,1908.38,0.428668,155.489,0.91205,0.109496,-1.0,-1.0,21.213,0.920303,1300,2078341 +8192,55,insert,457.122,-1.0,-1.0,-1.0,-1.0,480.599,1.07101,1357,2399905 +8192,56,search,4937.2,1765.96,1942.93,0.425863,151.744,0.91214,0.110282,-1.0,-1.0,120.538,1.07107,1369,2399905 +8192,57,insert,28.545,-1.0,-1.0,-1.0,-1.0,27.7,1.07944,1372,2419836 +8192,58,search,5124.88,1761.19,2072.63,0.406386,169.861,0.91137,0.111192,-1.0,-1.0,54.552,1.07946,1377,2419836 +8192,59,insert,304.754,-1.0,-1.0,-1.0,-1.0,211.281,1.18038,1394,2643083 +8192,60,search,4930.78,1893.35,1846.41,0.424805,143.685,0.90594,0.118054,-1.0,-1.0,86.225,1.18088,1401,2643083 +8192,61,insert,16.023,-1.0,-1.0,-1.0,-1.0,18.753,1.18529,1402,2653064 +8192,62,search,5339.38,1889.67,2205.53,0.3813,172.066,0.90631,0.117868,-1.0,-1.0,13.144,1.18529,1402,2653064 +8192,63,insert,4.547,-1.0,-1.0,-1.0,-1.0,11.297,1.18588,1402,2656118 +8192,64,search,5116.23,1891.85,2019.09,0.407683,156.989,0.90633,0.11773,-1.0,-1.0,13.184,1.18588,1402,2656118 +8192,65,insert,117.909,-1.0,-1.0,-1.0,-1.0,125.498,1.21169,1420,2723424 +8192,66,search,4852.88,1917.9,1743.11,0.466117,137.498,0.90733,0.116659,-1.0,-1.0,11.668,1.21169,1420,2723424 +8192,67,insert,65.925,-1.0,-1.0,-1.0,-1.0,118.319,1.23214,1437,2769021 +8192,68,search,5436.8,1926.95,2241.65,0.372674,181.425,0.90769,0.116154,-1.0,-1.0,27.977,1.23214,1439,2769021 +8192,69,insert,76.671,-1.0,-1.0,-1.0,-1.0,75.56,1.25331,1447,2821777 +8192,70,search,5327.4,1950.09,2147.68,0.384288,160.508,0.90948,0.114699,-1.0,-1.0,17.875,1.25345,1448,2821777 +8192,71,insert,19.312,-1.0,-1.0,-1.0,-1.0,31.252,1.25843,1451,2834921 +8192,72,search,5134.81,1953.77,1951.81,0.463296,148.114,0.9097,0.114723,-1.0,-1.0,14.841,1.25843,1451,2834921 +8192,73,insert,11.596,-1.0,-1.0,-1.0,-1.0,12.747,1.26121,1451,2842044 +8192,74,search,5215.42,1958.68,2029.49,0.370849,159.953,0.90969,0.114613,-1.0,-1.0,14.86,1.26121,1451,2842044 +8192,75,insert,38.825,-1.0,-1.0,-1.0,-1.0,34.746,1.26706,1453,2858890 +8192,76,search,5303.64,1967.58,2119.63,0.369611,166.499,0.91007,0.114344,-1.0,-1.0,13.078,1.26706,1453,2858890 +8192,77,insert,58.39,-1.0,-1.0,-1.0,-1.0,20.514,1.27736,1454,2882818 +8192,78,search,5670.94,1982.68,2435,0.364955,173.682,0.91025,0.11435,-1.0,-1.0,23.729,1.27744,1455,2882818 +8192,79,insert,533.076,-1.0,-1.0,-1.0,-1.0,18.132,1.30368,1456,2947806 +8192,80,search,5341.12,2024.59,2153.74,0.370877,162.05,0.90978,0.11485,-1.0,-1.0,12.629,1.30368,1456,2947806 +8192,81,insert,47.764,-1.0,-1.0,-1.0,-1.0,26.131,1.31749,1458,2976895 +8192,82,search,5648.94,2041.77,2428.61,0.361755,174.395,0.9108,0.113888,-1.0,-1.0,14.892,1.31749,1458,2976895 +8192,83,insert,24.806,-1.0,-1.0,-1.0,-1.0,22.184,1.32363,1459,2990682 +8192,84,search,5553.11,2049.82,2335.34,0.344939,171.562,0.91098,0.113601,-1.0,-1.0,14.935,1.32363,1459,2990682 +8192,85,insert,60.989,-1.0,-1.0,-1.0,-1.0,19.867,1.33978,1460,3025613 +8192,86,search,5286.85,2072.34,2062.76,0.414272,156.159,0.91203,0.113107,-1.0,-1.0,13.248,1.33978,1460,3025613 +8192,87,insert,631.356,-1.0,-1.0,-1.0,-1.0,715.326,1.54402,1550,3468632 +8192,88,search,6254.67,2237.83,2810.73,0.324781,200.353,0.91507,0.110386,-1.0,-1.0,112.843,1.54402,1561,3468632 +8192,89,insert,453.651,-1.0,-1.0,-1.0,-1.0,468.107,1.70228,1625,3831692 +8192,90,search,6621.61,2357.96,3032.51,0.314883,205.523,0.91908,0.105571,-1.0,-1.0,65.102,1.70242,1633,3831692 +8192,91,insert,32.477,-1.0,-1.0,-1.0,-1.0,49.861,1.71305,1638,3864067 +8192,92,search,6735.16,2359.02,3074.76,0.303641,211.278,0.91921,0.105488,-1.0,-1.0,16.226,1.71305,1638,3864067 +8192,93,insert,350.506,-1.0,-1.0,-1.0,-1.0,222.082,1.81842,1666,4103827 +8192,94,search,6930.91,2460.35,3236.88,0.308049,213.079,0.91986,0.104438,-1.0,-1.0,40.547,1.8185,1669,4103827 +8192,95,insert,24.082,-1.0,-1.0,-1.0,-1.0,51.934,1.8257,1674,4119898 +8192,96,search,6606.96,2458.2,2912.07,0.342406,200.054,0.92011,0.104232,-1.0,-1.0,14.123,1.8257,1674,4119898 +8192,97,insert,15.511,-1.0,-1.0,-1.0,-1.0,24.735,1.82997,1675,4129367 +8192,98,search,6389.08,2462.38,2720.02,0.343961,199.724,0.92035,0.10404,-1.0,-1.0,14.265,1.82997,1675,4129367 +8192,99,insert,8.655,-1.0,-1.0,-1.0,-1.0,13.287,1.83221,1675,4134502 +8192,100,search,6834.22,2465.44,3130.51,0.319901,224.746,0.92046,0.103952,-1.0,-1.0,16.75,1.83221,1675,4134502 +8192,101,insert,50.478,-1.0,-1.0,-1.0,-1.0,88.292,1.84226,1683,4164482 +8192,102,search,7005.34,2471.53,3268.29,0.308466,215.502,0.92086,0.103481,-1.0,-1.0,16.07,1.84226,1683,4164482 +8192,103,insert,31.092,-1.0,-1.0,-1.0,-1.0,34.872,1.849,1685,4178601 +8192,104,search,6960.83,2476.97,3246.07,0.313931,221.263,0.92098,0.103269,-1.0,-1.0,15.822,1.849,1685,4178601 +8192,105,insert,17.25,-1.0,-1.0,-1.0,-1.0,44.796,1.85329,1688,4187774 +8192,106,search,6962.69,2478,3241.28,0.31205,220.841,0.92116,0.103197,-1.0,-1.0,24.817,1.85333,1689,4187774 +8192,107,insert,27.477,-1.0,-1.0,-1.0,-1.0,14.05,1.85883,1689,4200109 +8192,108,search,6769.1,2483.83,3048.23,0.33667,214.721,0.92091,0.103435,-1.0,-1.0,15.124,1.85883,1689,4200109 +8192,109,insert,494.398,-1.0,-1.0,-1.0,-1.0,557.68,2.0177,1760,4557867 +8192,110,search,7146.39,2586.78,3221.1,0.321154,218.577,0.92209,0.102209,-1.0,-1.0,69.707,2.01814,1763,4557867 +8192,111,insert,45.849,-1.0,-1.0,-1.0,-1.0,89.292,2.0318,1772,4587252 +8192,112,search,7070.46,2585.85,3136.72,0.291171,209.268,0.92212,0.102458,-1.0,-1.0,19.975,2.0318,1772,4587252 +8192,113,insert,18.157,-1.0,-1.0,-1.0,-1.0,44.694,2.03766,1775,4600056 +8192,114,search,7006.56,2588.69,3085.4,0.318095,222.53,0.92258,0.102156,-1.0,-1.0,19.393,2.03766,1775,4600056 +8192,115,insert,46.474,-1.0,-1.0,-1.0,-1.0,66.893,2.04733,1780,4627422 +8192,116,search,7170.25,2596.78,3233.81,0.304241,224.977,0.92328,0.101834,-1.0,-1.0,19.577,2.04733,1780,4627422 +8192,117,insert,12.698,-1.0,-1.0,-1.0,-1.0,18.161,2.05008,1780,4639292 +8192,118,search,7392.91,2603.44,3434.97,0.312541,246.193,0.9234,0.101497,-1.0,-1.0,34.449,2.05016,1781,4639292 +8192,119,insert,7.315,-1.0,-1.0,-1.0,-1.0,41.273,2.05214,1783,4643423 +8192,120,search,7036.11,2601.38,3104.92,0.357798,205.222,0.92337,0.10157,-1.0,-1.0,21.596,2.05214,1783,4643423 +8192,121,insert,25.754,-1.0,-1.0,-1.0,-1.0,32.389,2.05929,1784,4657252 +8192,122,search,6857.13,2607.67,2938.57,0.329138,210.235,0.92342,0.101367,-1.0,-1.0,34.779,2.05932,1785,4657252 +8192,123,insert,473.435,-1.0,-1.0,-1.0,-1.0,517.542,2.19035,1837,4938477 +8192,124,search,7642.46,2685.44,3570.99,0.280006,254.132,0.92357,0.102073,-1.0,-1.0,173.211,2.19174,1848,4938477 +8192,125,insert,57.507,-1.0,-1.0,-1.0,-1.0,76.128,2.2081,1855,4980492 +8192,126,search,7713.69,2682.04,3588.83,0.306597,248.986,0.92413,0.101583,-1.0,-1.0,175.556,2.20867,1864,4980492 +8192,127,insert,70.373,-1.0,-1.0,-1.0,-1.0,100.452,2.23104,1876,5030515 +8192,128,search,7693.47,2678.68,3513.74,0.29093,248.194,0.9249,0.101125,-1.0,-1.0,39.427,2.23104,1877,5030515 +8192,129,delete,28.464,-1.0,-1.0,-1.0,-1.0,95.492,2.22913,1845,5009391 +8192,130,search,7589.27,2715.12,3509.87,0.259519,245.114,0.92323,0.1024,-1.0,-1.0,24.819,2.22913,1845,5009391 +8192,131,delete,9.509,-1.0,-1.0,-1.0,-1.0,23.661,2.22913,1845,5004093 +8192,132,search,7390.59,2712.25,3311.2,0.265846,241.806,0.92319,0.102526,-1.0,-1.0,23.834,2.22913,1845,5004093 +8192,133,delete,15.927,-1.0,-1.0,-1.0,-1.0,23.198,2.2289,1845,4991787 +8192,134,search,6680.22,2705.58,2653.1,0.335831,183.033,0.92303,0.102513,-1.0,-1.0,26.692,2.2289,1845,4991787 +8192,135,delete,37.285,-1.0,-1.0,-1.0,-1.0,75.236,2.2259,1842,4974857 +8192,136,search,7639.75,2700.79,3543,0.249413,247.792,0.92328,0.102246,-1.0,-1.0,25.977,2.2259,1842,4974857 +8192,137,delete,11.887,-1.0,-1.0,-1.0,-1.0,25.114,2.2259,1842,4971386 +8192,138,search,7113,2698.91,3126.47,0.267756,219.484,0.9232,0.102395,-1.0,-1.0,25.437,2.2259,1842,4971386 +8192,139,delete,15.956,-1.0,-1.0,-1.0,-1.0,23.211,2.2259,1842,4963487 +8192,140,search,7273.39,2694.62,3212.27,0.267613,225.756,0.92306,0.102488,-1.0,-1.0,24.497,2.2259,1842,4963487 +8192,141,delete,36.837,-1.0,-1.0,-1.0,-1.0,82.322,2.22131,1836,4941830 +8192,142,search,7347.15,2691.63,3294.47,0.265634,242.144,0.92275,0.102675,-1.0,-1.0,27.342,2.22131,1836,4941830 +8192,143,delete,31.469,-1.0,-1.0,-1.0,-1.0,79.263,2.21694,1832,4925657 +8192,144,search,7444.06,2688.68,3379.64,0.256733,251.063,0.92256,0.102539,-1.0,-1.0,26.662,2.21694,1832,4925657 +8192,145,delete,8.052,-1.0,-1.0,-1.0,-1.0,36.026,2.21602,1830,4922686 +8192,146,search,7585.19,2689.99,3515.08,0.251207,245.758,0.9223,0.103048,-1.0,-1.0,25.48,2.21602,1830,4922686 +8192,147,delete,22.881,-1.0,-1.0,-1.0,-1.0,23.918,2.21602,1830,4913636 +8192,148,search,7354.09,2685.05,3323.94,0.288858,219.784,0.92238,0.103047,-1.0,-1.0,24.987,2.21602,1830,4913636 +8192,149,delete,263.512,-1.0,-1.0,-1.0,-1.0,1064.58,2.13653,1750,4717250 +8192,150,search,7194.72,2695.57,3295,0.253421,246.721,0.92125,0.104502,-1.0,-1.0,49.497,2.13706,1753,4717250 +8192,151,delete,162.583,-1.0,-1.0,-1.0,-1.0,1026.41,2.06946,1675,4560269 diff --git a/scripts/big_ann_perf_numbers/perf_debug_scan_0.15_vary_num_search_workers.csv b/scripts/big_ann_perf_numbers/perf_debug_scan_0.15_vary_num_search_workers.csv new file mode 100644 index 00000000..b3e274fc --- /dev/null +++ b/scripts/big_ann_perf_numbers/perf_debug_scan_0.15_vary_num_search_workers.csv @@ -0,0 +1,901 @@ +num_search_workers,step_num,step_type,search_latency_ms,worker_partition_size,worker_scan_throughput,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +1,2,search,3041.02,37.5515,2.35049,0.8551,0.130106,-1.0,-1.0,6.569,0.0184881,1000,38806 +1,3,insert,12.216,-1.0,-1.0,-1.0,-1.0,13.346,0.022655,996,49334 +1,4,search,3219.15,63.4255,1.99793,0.86686,0.128427,-1.0,-1.0,17.589,0.0226615,993,49334 +1,5,insert,18.915,-1.0,-1.0,-1.0,-1.0,161.53,0.033017,984,73557 +1,6,search,4124.28,173.596,1.72628,0.92515,0.103785,-1.0,-1.0,121.338,0.0331618,986,73557 +1,7,insert,22.396,-1.0,-1.0,-1.0,-1.0,121.051,0.0426658,987,95766 +1,8,search,4544.66,220.032,1.52979,0.92462,0.103689,-1.0,-1.0,89.651,0.0428151,991,95766 +1,9,insert,4.503,-1.0,-1.0,-1.0,-1.0,74.201,0.0445618,991,100025 +1,10,search,4570.37,226.619,1.59938,0.92565,0.101549,-1.0,-1.0,80.666,0.0446017,994,100025 +1,11,insert,8.797,-1.0,-1.0,-1.0,-1.0,78.762,0.0498511,995,111417 +1,12,search,4870.43,261.637,1.48005,0.92608,0.101825,-1.0,-1.0,22.111,0.0498866,996,111417 +1,13,insert,25.643,-1.0,-1.0,-1.0,-1.0,7.968,0.0611449,997,137139 +1,14,search,5269.66,299.297,1.29259,0.92196,0.104759,-1.0,-1.0,15.705,0.0611514,999,137139 +1,15,insert,15.116,-1.0,-1.0,-1.0,-1.0,6.531,0.0699051,999,156710 +1,16,search,5533.3,330.607,1.22835,0.92003,0.105131,-1.0,-1.0,21.036,0.06991,1003,156710 +1,17,insert,5.158,-1.0,-1.0,-1.0,-1.0,8.72,0.0717831,1004,161773 +1,18,search,5637.13,338.418,1.16719,0.92029,0.105277,-1.0,-1.0,6.641,0.0717831,1004,161773 +1,19,insert,30.59,-1.0,-1.0,-1.0,-1.0,10.387,0.0787456,1006,177161 +1,20,search,5925.42,373.957,1.17113,0.92341,0.102841,-1.0,-1.0,9.065,0.0787456,1006,177161 +1,21,insert,332.691,-1.0,-1.0,-1.0,-1.0,162.922,0.200847,1051,448898 +1,22,search,9672.62,768.462,0.974077,0.91039,0.107984,-1.0,-1.0,39.3,0.200845,1061,448898 +1,23,insert,385.399,-1.0,-1.0,-1.0,-1.0,251.556,0.328422,1116,733493 +1,24,search,12577.6,1064.04,0.900795,0.90613,0.110085,-1.0,-1.0,72.28,0.328846,1129,733493 +1,25,insert,39.546,-1.0,-1.0,-1.0,-1.0,22.684,0.345771,1133,777443 +1,26,search,9218.54,1068.79,1.36891,0.9063,0.110352,-1.0,-1.0,24.831,0.345966,1137,777443 +1,27,insert,26.449,-1.0,-1.0,-1.0,-1.0,20.158,0.357754,1140,805394 +1,28,search,9415.66,1079.21,1.27462,0.90756,0.109157,-1.0,-1.0,9.438,0.357754,1140,805394 +1,29,insert,53.934,-1.0,-1.0,-1.0,-1.0,59.476,0.385616,1153,866391 +1,30,search,9687.31,1130.84,1.26147,0.90884,0.109052,-1.0,-1.0,37.249,0.385713,1157,866391 +1,31,insert,499.712,-1.0,-1.0,-1.0,-1.0,316.025,0.554314,1222,1239816 +1,32,search,11999.8,1403.7,1.23337,0.9035,0.113402,-1.0,-1.0,40.458,0.554578,1229,1239816 +1,33,insert,45.495,-1.0,-1.0,-1.0,-1.0,26.435,0.575056,1233,1291211 +1,34,search,12195.2,1430.14,1.33286,0.9036,0.113131,-1.0,-1.0,21.768,0.575285,1235,1291211 +1,35,insert,42.954,-1.0,-1.0,-1.0,-1.0,31.873,0.586476,1241,1324978 +1,36,search,12528.4,1457.45,1.20186,0.90569,0.111542,-1.0,-1.0,9.11,0.586476,1241,1324978 +1,37,insert,42.945,-1.0,-1.0,-1.0,-1.0,39.799,0.606427,1248,1371885 +1,38,search,12661.1,1480.18,1.19221,0.90461,0.112684,-1.0,-1.0,17.839,0.606427,1249,1371885 +1,39,insert,9.151,-1.0,-1.0,-1.0,-1.0,12.12,0.608926,1250,1379335 +1,40,search,12711.8,1482.08,1.23777,0.90496,0.112945,-1.0,-1.0,18.692,0.608925,1251,1379335 +1,41,insert,53.392,-1.0,-1.0,-1.0,-1.0,31.297,0.624188,1256,1413312 +1,42,search,12893,1502.65,1.33865,0.90518,0.112837,-1.0,-1.0,12.926,0.624188,1257,1413312 +1,43,insert,10.728,-1.0,-1.0,-1.0,-1.0,9.036,0.627895,1257,1423411 +1,44,search,13007.2,1513.1,1.30939,0.90522,0.112906,-1.0,-1.0,10.922,0.627895,1257,1423411 +1,45,insert,212.004,-1.0,-1.0,-1.0,-1.0,34.05,0.644498,1265,1457559 +1,46,search,13126.9,1526.57,1.20765,0.90558,0.111887,-1.0,-1.0,9.265,0.644498,1265,1457559 +1,47,insert,270.324,-1.0,-1.0,-1.0,-1.0,257.348,0.749159,1302,1688593 +1,48,search,14267,1647.21,1.16508,0.90715,0.111944,-1.0,-1.0,87.88,0.74928,1312,1688593 +1,49,insert,41.69,-1.0,-1.0,-1.0,-1.0,32.611,0.764772,1316,1723505 +1,50,search,14421.7,1652.59,1.25374,0.90783,0.111678,-1.0,-1.0,36.094,0.764799,1320,1723505 +1,51,insert,340.488,-1.0,-1.0,-1.0,-1.0,267.538,0.904316,1368,2035040 +1,52,search,16192,1816.7,1.1727,0.91084,0.110713,-1.0,-1.0,37.51,0.904316,1373,2035040 +1,53,insert,40.587,-1.0,-1.0,-1.0,-1.0,27.702,0.92232,1377,2078341 +1,54,search,16424.8,1835.53,1.05095,0.91104,0.110612,-1.0,-1.0,21.08,0.92246,1378,2078341 +1,55,insert,376.04,-1.0,-1.0,-1.0,-1.0,343.893,1.06819,1428,2399905 +1,56,search,18190.4,2005.34,1.13022,0.91146,0.113147,-1.0,-1.0,110.494,1.06853,1446,2399905 +1,57,insert,24.763,-1.0,-1.0,-1.0,-1.0,10.311,1.0757,1446,2419836 +1,58,search,18239.1,1975.18,1.18929,0.91027,0.114083,-1.0,-1.0,38.572,1.07571,1449,2419836 +1,59,insert,223.012,-1.0,-1.0,-1.0,-1.0,153.288,1.17443,1467,2643083 +1,60,search,18895.6,2046.33,1.1691,0.90396,0.121349,-1.0,-1.0,48.185,1.17467,1473,2643083 +1,61,insert,11.389,-1.0,-1.0,-1.0,-1.0,18.057,1.17897,1475,2653064 +1,62,search,18868.6,2034.29,1.09455,0.90343,0.122239,-1.0,-1.0,33.74,1.17897,1477,2653064 +1,63,insert,3.327,-1.0,-1.0,-1.0,-1.0,10.78,1.18007,1477,2656118 +1,64,search,18872.5,2035.01,1.11813,0.90337,0.122128,-1.0,-1.0,10.967,1.18007,1477,2656118 +1,65,insert,74.924,-1.0,-1.0,-1.0,-1.0,104.973,1.2077,1498,2723424 +1,66,search,19248.3,2046.74,1.09577,0.90431,0.120908,-1.0,-1.0,13.283,1.2077,1498,2723424 +1,67,insert,47.134,-1.0,-1.0,-1.0,-1.0,85.911,1.22905,1510,2769021 +1,68,search,19692.9,2069.1,1.04242,0.90574,0.119845,-1.0,-1.0,29.403,1.22908,1513,2769021 +1,69,insert,74.313,-1.0,-1.0,-1.0,-1.0,86.767,1.25251,1526,2821777 +1,70,search,19884.3,2076.75,1.16115,0.90762,0.118339,-1.0,-1.0,24.1,1.25252,1528,2821777 +1,71,insert,16.359,-1.0,-1.0,-1.0,-1.0,11.324,1.25746,1528,2834921 +1,72,search,20108.4,2086.32,1.16793,0.90859,0.11757,-1.0,-1.0,11.825,1.25746,1528,2834921 +1,73,insert,11.288,-1.0,-1.0,-1.0,-1.0,18.508,1.26103,1529,2842044 +1,74,search,20038.5,2089.51,1.22707,0.90865,0.117458,-1.0,-1.0,11.755,1.26103,1529,2842044 +1,75,insert,21.897,-1.0,-1.0,-1.0,-1.0,22.823,1.26745,1531,2858890 +1,76,search,20081.6,2099.23,1.16853,0.90883,0.117172,-1.0,-1.0,11.751,1.26745,1531,2858890 +1,77,insert,32.323,-1.0,-1.0,-1.0,-1.0,17.737,1.27654,1532,2882818 +1,78,search,20165.7,2109.99,1.15245,0.90837,0.117274,-1.0,-1.0,11.78,1.27654,1532,2882818 +1,79,insert,496.959,-1.0,-1.0,-1.0,-1.0,15.973,1.306,1533,2947806 +1,80,search,20489,2147.08,1.14943,0.90817,0.117139,-1.0,-1.0,13.549,1.306,1533,2947806 +1,81,insert,45.492,-1.0,-1.0,-1.0,-1.0,19.005,1.31888,1534,2976895 +1,82,search,20712.6,2161.9,1.08715,0.90956,0.115938,-1.0,-1.0,20.492,1.31888,1535,2976895 +1,83,insert,23.294,-1.0,-1.0,-1.0,-1.0,42.008,1.32482,1539,2990682 +1,84,search,20862.9,2161.48,1.06221,0.90964,0.115826,-1.0,-1.0,11.871,1.32482,1539,2990682 +1,85,insert,40.832,-1.0,-1.0,-1.0,-1.0,59.913,1.33944,1548,3025613 +1,86,search,20941.7,2163.01,1.11224,0.9107,0.114971,-1.0,-1.0,26.67,1.3396,1550,3025613 +1,87,insert,589.157,-1.0,-1.0,-1.0,-1.0,694.865,1.54209,1653,3468632 +1,88,search,23208.7,2295.57,1.16528,0.91363,0.11126,-1.0,-1.0,105.548,1.54266,1668,3468632 +1,89,insert,384.214,-1.0,-1.0,-1.0,-1.0,467.021,1.7034,1743,3831692 +1,90,search,25344.1,2383.57,1.05045,0.9182,0.106233,-1.0,-1.0,113.544,1.70406,1754,3831692 +1,91,insert,33.999,-1.0,-1.0,-1.0,-1.0,45.218,1.71379,1759,3864067 +1,92,search,25686.5,2375.4,1.0641,0.91838,0.10623,-1.0,-1.0,18.872,1.71379,1759,3864067 +1,93,insert,301.532,-1.0,-1.0,-1.0,-1.0,281.691,1.81925,1800,4103827 +1,94,search,27210.3,2434.02,1.06267,0.9187,0.105629,-1.0,-1.0,55.701,1.81954,1806,4103827 +1,95,insert,18.538,-1.0,-1.0,-1.0,-1.0,67.53,1.82616,1816,4119898 +1,96,search,26839.5,2411.5,1.13688,0.91844,0.10598,-1.0,-1.0,20.941,1.82616,1816,4119898 +1,97,insert,13.85,-1.0,-1.0,-1.0,-1.0,25.361,1.82956,1817,4129367 +1,98,search,26775.9,2414.53,1.12283,0.91849,0.105796,-1.0,-1.0,24.067,1.82956,1817,4129367 +1,99,insert,13.378,-1.0,-1.0,-1.0,-1.0,31.307,1.83169,1818,4134502 +1,100,search,26830.4,2416.9,1.12109,0.91857,0.105814,-1.0,-1.0,23.206,1.83169,1818,4134502 +1,101,insert,58.333,-1.0,-1.0,-1.0,-1.0,85.61,1.84323,1826,4164482 +1,102,search,27088.1,2419.71,1.09843,0.91865,0.105877,-1.0,-1.0,22.016,1.84323,1826,4164482 +1,103,insert,23.192,-1.0,-1.0,-1.0,-1.0,54.56,1.84952,1831,4178601 +1,104,search,27049,2421.94,1.17848,0.91888,0.105524,-1.0,-1.0,21.449,1.84952,1831,4178601 +1,105,insert,12.052,-1.0,-1.0,-1.0,-1.0,37.175,1.85301,1833,4187774 +1,106,search,27022.3,2423.68,1.13577,0.91875,0.105755,-1.0,-1.0,25.738,1.85301,1833,4187774 +1,107,insert,36.091,-1.0,-1.0,-1.0,-1.0,32.634,1.85861,1834,4200109 +1,108,search,27399,2427.01,1.16731,0.91902,0.105516,-1.0,-1.0,41.12,1.85888,1836,4200109 +1,109,insert,532.018,-1.0,-1.0,-1.0,-1.0,567.056,2.01715,1916,4557867 +1,110,search,29073.2,2500.08,1.02343,0.92089,0.104333,-1.0,-1.0,114.965,2.01799,1929,4557867 +1,111,insert,32.011,-1.0,-1.0,-1.0,-1.0,55.554,2.02985,1935,4587252 +1,112,search,29103.3,2488.06,1.16266,0.92124,0.104354,-1.0,-1.0,38.985,2.03016,1936,4587252 +1,113,insert,16.645,-1.0,-1.0,-1.0,-1.0,33.021,2.03619,1937,4600056 +1,114,search,29182.7,2493.22,1.12484,0.92124,0.104306,-1.0,-1.0,30.348,2.03619,1937,4600056 +1,115,insert,55.663,-1.0,-1.0,-1.0,-1.0,91.788,2.04608,1945,4627422 +1,116,search,29414.3,2497.54,1.09819,0.92178,0.103821,-1.0,-1.0,38.11,2.04616,1946,4627422 +1,117,insert,16.611,-1.0,-1.0,-1.0,-1.0,61.986,2.0508,1951,4639292 +1,118,search,29601.1,2493.74,1.11525,0.92188,0.103609,-1.0,-1.0,27.882,2.0508,1951,4639292 +1,119,insert,6.609,-1.0,-1.0,-1.0,-1.0,26.508,2.05212,1951,4643423 +1,120,search,29514.8,2495.48,1.08879,0.92191,0.103644,-1.0,-1.0,29.764,2.05212,1951,4643423 +1,121,insert,25.536,-1.0,-1.0,-1.0,-1.0,47.392,2.05838,1953,4657252 +1,122,search,29667.7,2499.6,1.12896,0.92212,0.103439,-1.0,-1.0,41.421,2.05851,1954,4657252 +1,123,insert,441.318,-1.0,-1.0,-1.0,-1.0,394.679,2.1853,2001,4938477 +1,124,search,31092.7,2552.94,1.09875,0.9227,0.104324,-1.0,-1.0,150.875,2.18609,2017,4938477 +1,125,insert,66.458,-1.0,-1.0,-1.0,-1.0,88.746,2.20322,2027,4980492 +1,126,search,30867.6,2529.58,1.15167,0.92297,0.104232,-1.0,-1.0,79.895,2.20375,2033,4980492 +1,127,insert,61.024,-1.0,-1.0,-1.0,-1.0,127.246,2.22506,2050,5030515 +1,128,search,31279.9,2527,1.02765,0.92343,0.103974,-1.0,-1.0,34.822,2.22506,2050,5030515 +1,129,delete,47.274,-1.0,-1.0,-1.0,-1.0,56.363,2.22344,2021,5009391 +1,130,search,30939.4,2527.96,1.08557,0.9225,0.104636,-1.0,-1.0,33.142,2.22344,2021,5009391 +1,131,delete,10.986,-1.0,-1.0,-1.0,-1.0,31.055,2.22344,2021,5004093 +1,132,search,30961.6,2525.17,1.06026,0.92251,0.104598,-1.0,-1.0,32.226,2.22344,2021,5004093 +1,133,delete,17.161,-1.0,-1.0,-1.0,-1.0,30.524,2.22344,2021,4991787 +1,134,search,30542.5,2515.05,1.10369,0.92255,0.104611,-1.0,-1.0,34.636,2.22344,2021,4991787 +1,135,delete,40.929,-1.0,-1.0,-1.0,-1.0,57.954,2.22061,2018,4974857 +1,136,search,30610.8,2507.9,1.12306,0.92213,0.104712,-1.0,-1.0,33.452,2.22061,2018,4974857 +1,137,delete,8.038,-1.0,-1.0,-1.0,-1.0,31.016,2.22061,2018,4971386 +1,138,search,30676.7,2506.68,1.08796,0.9221,0.104825,-1.0,-1.0,32.099,2.22061,2018,4971386 +1,139,delete,12.664,-1.0,-1.0,-1.0,-1.0,30.713,2.22061,2018,4963487 +1,140,search,30628.7,2500.68,1.09972,0.92203,0.104831,-1.0,-1.0,34.227,2.22061,2018,4963487 +1,141,delete,49.162,-1.0,-1.0,-1.0,-1.0,104.43,2.21454,2010,4941830 +1,142,search,30669.6,2496.64,1.09216,0.92204,0.104832,-1.0,-1.0,33.196,2.21454,2010,4941830 +1,143,delete,27.193,-1.0,-1.0,-1.0,-1.0,57.255,2.2117,2007,4925657 +1,144,search,30398.4,2490.26,1.12346,0.92212,0.104698,-1.0,-1.0,32.262,2.2117,2007,4925657 +1,145,delete,3.933,-1.0,-1.0,-1.0,-1.0,41.687,2.21131,2006,4922686 +1,146,search,30044.9,2490.88,1.12654,0.92143,0.105387,-1.0,-1.0,33.459,2.21131,2006,4922686 +1,147,delete,23.872,-1.0,-1.0,-1.0,-1.0,30.86,2.21131,2006,4913636 +1,148,search,29974.5,2485.2,1.12118,0.9213,0.105497,-1.0,-1.0,33.342,2.21131,2006,4913636 +1,149,delete,271.29,-1.0,-1.0,-1.0,-1.0,1172.09,2.13327,1914,4717250 +1,150,search,28869.3,2493.96,1.0694,0.92059,0.106552,-1.0,-1.0,129.75,2.13493,1927,4717250 +1,151,delete,163.281,-1.0,-1.0,-1.0,-1.0,954.39,2.06358,1846,4560269 +2,2,search,2622.87,37.5515,2.68484,0.8551,0.130106,-1.0,-1.0,6.887,0.0184881,1000,38806 +2,3,insert,9.276,-1.0,-1.0,-1.0,-1.0,13.925,0.022655,996,49334 +2,4,search,2735.03,63.4255,1.7385,0.86686,0.128427,-1.0,-1.0,12.3,0.0226615,993,49334 +2,5,insert,20.579,-1.0,-1.0,-1.0,-1.0,162.297,0.0329603,983,73557 +2,6,search,2854.5,173.821,1.70125,0.92559,0.103361,-1.0,-1.0,124.129,0.0331112,985,73557 +2,7,insert,25.151,-1.0,-1.0,-1.0,-1.0,123.007,0.0427474,986,95766 +2,8,search,2971.72,219.876,1.4926,0.92461,0.104001,-1.0,-1.0,88.291,0.0428988,990,95766 +2,9,insert,3.502,-1.0,-1.0,-1.0,-1.0,74.374,0.0446421,990,100025 +2,10,search,3008.89,226.697,1.44461,0.92644,0.101101,-1.0,-1.0,82.823,0.0446707,993,100025 +2,11,insert,10.034,-1.0,-1.0,-1.0,-1.0,81.162,0.0497634,995,111417 +2,12,search,3132.37,262.143,1.3789,0.92604,0.10176,-1.0,-1.0,16.724,0.0498139,996,111417 +2,13,insert,28.447,-1.0,-1.0,-1.0,-1.0,9.872,0.0610221,996,137139 +2,14,search,3239.37,298.935,1.17255,0.92257,0.104007,-1.0,-1.0,21.857,0.0611274,999,137139 +2,15,insert,21.913,-1.0,-1.0,-1.0,-1.0,9.669,0.0699557,1000,156710 +2,16,search,3204.3,330.019,1.27262,0.92096,0.104291,-1.0,-1.0,19.105,0.069963,1004,156710 +2,17,insert,6.122,-1.0,-1.0,-1.0,-1.0,8.861,0.0721458,1005,161773 +2,18,search,3179.88,339.902,1.41786,0.92005,0.105323,-1.0,-1.0,13.175,0.0722572,1007,161773 +2,19,insert,34.962,-1.0,-1.0,-1.0,-1.0,12.462,0.0789908,1009,177161 +2,20,search,3473.31,372.582,1.12162,0.92394,0.102681,-1.0,-1.0,9.612,0.0789912,1010,177161 +2,21,insert,261.815,-1.0,-1.0,-1.0,-1.0,116.736,0.202448,1069,448898 +2,22,search,4376.49,737.114,1.09947,0.90854,0.108821,-1.0,-1.0,57.076,0.202448,1082,448898 +2,23,insert,235.101,-1.0,-1.0,-1.0,-1.0,208.2,0.329637,1147,733493 +2,24,search,5781.84,1002.24,0.824772,0.90335,0.112134,-1.0,-1.0,105.092,0.330475,1175,733493 +2,25,insert,34.492,-1.0,-1.0,-1.0,-1.0,24.483,0.346577,1180,777443 +2,26,search,5566.65,969.016,0.956156,0.90152,0.113747,-1.0,-1.0,19.396,0.34669,1185,777443 +2,27,insert,20.974,-1.0,-1.0,-1.0,-1.0,16.754,0.357375,1189,805394 +2,28,search,5593.76,972.996,0.911176,0.90191,0.113691,-1.0,-1.0,16.712,0.357599,1191,805394 +2,29,insert,61.353,-1.0,-1.0,-1.0,-1.0,67.285,0.384967,1206,866391 +2,30,search,5399.15,1018.22,0.93607,0.9037,0.113636,-1.0,-1.0,17.583,0.384967,1211,866391 +2,31,insert,226.388,-1.0,-1.0,-1.0,-1.0,201.031,0.555095,1293,1239816 +2,32,search,5873.16,1251.44,0.927909,0.89675,0.117471,-1.0,-1.0,38.566,0.555304,1303,1239816 +2,33,insert,32.247,-1.0,-1.0,-1.0,-1.0,21.848,0.575302,1311,1291211 +2,34,search,7228.25,1258.37,0.8108,0.89778,0.118167,-1.0,-1.0,13.817,0.575325,1312,1291211 +2,35,insert,42.78,-1.0,-1.0,-1.0,-1.0,20.86,0.586996,1317,1324978 +2,36,search,7466.19,1286.04,0.769098,0.89898,0.117229,-1.0,-1.0,10.704,0.586996,1317,1324978 +2,37,insert,55.749,-1.0,-1.0,-1.0,-1.0,44.84,0.607123,1325,1371885 +2,38,search,7469.77,1308.58,0.814424,0.89885,0.117051,-1.0,-1.0,16.289,0.607123,1326,1371885 +2,39,insert,10.446,-1.0,-1.0,-1.0,-1.0,18.827,0.610253,1329,1379335 +2,40,search,7855.46,1306.97,0.740569,0.89908,0.117339,-1.0,-1.0,13.571,0.610254,1330,1379335 +2,41,insert,43.425,-1.0,-1.0,-1.0,-1.0,24.914,0.625029,1333,1413312 +2,42,search,7655.77,1331.6,0.859088,0.89972,0.117315,-1.0,-1.0,15.759,0.625034,1335,1413312 +2,43,insert,11.887,-1.0,-1.0,-1.0,-1.0,15.179,0.629252,1337,1423411 +2,44,search,7746.13,1336.27,0.801166,0.89988,0.116567,-1.0,-1.0,12.605,0.629252,1337,1423411 +2,45,insert,251.92,-1.0,-1.0,-1.0,-1.0,59.529,0.644735,1353,1457559 +2,46,search,7877.17,1334.49,0.851159,0.89993,0.115641,-1.0,-1.0,10.259,0.644735,1353,1457559 +2,47,insert,291.085,-1.0,-1.0,-1.0,-1.0,239.235,0.747711,1393,1688593 +2,48,search,8469.82,1451.52,0.756066,0.90206,0.115209,-1.0,-1.0,73.103,0.747883,1401,1688593 +2,49,insert,40.099,-1.0,-1.0,-1.0,-1.0,35.081,0.764194,1408,1723505 +2,50,search,8473.38,1454.77,0.797663,0.90358,0.11361,-1.0,-1.0,34.384,0.764268,1412,1723505 +2,51,insert,322.679,-1.0,-1.0,-1.0,-1.0,291.354,0.904976,1477,2035040 +2,52,search,9258.04,1591.15,0.77644,0.90669,0.112468,-1.0,-1.0,39.385,0.905048,1483,2035040 +2,53,insert,39.477,-1.0,-1.0,-1.0,-1.0,33.376,0.922384,1488,2078341 +2,54,search,9511.5,1600.53,0.755075,0.90694,0.112608,-1.0,-1.0,13.996,0.922384,1488,2078341 +2,55,insert,348.494,-1.0,-1.0,-1.0,-1.0,326.501,1.06856,1559,2399905 +2,56,search,8307.67,1747.65,1.07564,0.90864,0.11261,-1.0,-1.0,129.971,1.0695,1581,2399905 +2,57,insert,24.296,-1.0,-1.0,-1.0,-1.0,11.264,1.0766,1581,2419836 +2,58,search,10458.1,1713.75,0.776625,0.90815,0.11283,-1.0,-1.0,61.064,1.07665,1587,2419836 +2,59,insert,245.827,-1.0,-1.0,-1.0,-1.0,192.683,1.1748,1612,2643083 +2,60,search,11234.5,1758.44,0.697863,0.90112,0.121304,-1.0,-1.0,34.306,1.17485,1617,2643083 +2,61,insert,10.514,-1.0,-1.0,-1.0,-1.0,26.086,1.17871,1621,2653064 +2,62,search,11273,1749.71,0.690127,0.90109,0.121204,-1.0,-1.0,15.051,1.17871,1621,2653064 +2,63,insert,5.668,-1.0,-1.0,-1.0,-1.0,12.601,1.1797,1621,2656118 +2,64,search,11229.1,1750.71,0.759291,0.90126,0.121037,-1.0,-1.0,14.49,1.1797,1621,2656118 +2,65,insert,106.115,-1.0,-1.0,-1.0,-1.0,137.157,1.20972,1653,2723424 +2,66,search,10775.7,1754.85,0.813801,0.90142,0.120116,-1.0,-1.0,16.838,1.20979,1654,2723424 +2,67,insert,42.991,-1.0,-1.0,-1.0,-1.0,75.173,1.22933,1672,2769021 +2,68,search,10816.2,1762.53,0.828548,0.90262,0.119441,-1.0,-1.0,25.062,1.22951,1675,2769021 +2,69,insert,49.907,-1.0,-1.0,-1.0,-1.0,75.983,1.25182,1687,2821777 +2,70,search,10841.5,1776.03,0.771849,0.90441,0.117906,-1.0,-1.0,13.237,1.25182,1688,2821777 +2,71,insert,8.558,-1.0,-1.0,-1.0,-1.0,17.199,1.25778,1692,2834921 +2,72,search,10758.3,1781.19,0.836105,0.90467,0.117858,-1.0,-1.0,16.076,1.25778,1692,2834921 +2,73,insert,6.875,-1.0,-1.0,-1.0,-1.0,14.347,1.25976,1692,2842044 +2,74,search,11592.8,1784.12,0.733229,0.90467,0.117884,-1.0,-1.0,16.53,1.25976,1692,2842044 +2,75,insert,34.574,-1.0,-1.0,-1.0,-1.0,19.116,1.26646,1693,2858890 +2,76,search,10717.5,1793.88,0.809648,0.90492,0.117903,-1.0,-1.0,15.598,1.26646,1693,2858890 +2,77,insert,47.949,-1.0,-1.0,-1.0,-1.0,19.568,1.2767,1694,2882818 +2,78,search,9197.41,1804.2,1.04464,0.905,0.117738,-1.0,-1.0,14.783,1.2767,1694,2882818 +2,79,insert,410.481,-1.0,-1.0,-1.0,-1.0,18.173,1.30315,1698,2947806 +2,80,search,9127.76,1832.23,1.10464,0.90462,0.117982,-1.0,-1.0,14.542,1.30315,1698,2947806 +2,81,insert,38.046,-1.0,-1.0,-1.0,-1.0,34.851,1.31631,1702,2976895 +2,82,search,9304.43,1842.15,1.03268,0.90598,0.117141,-1.0,-1.0,14.477,1.31631,1702,2976895 +2,83,insert,15.115,-1.0,-1.0,-1.0,-1.0,31.38,1.3224,1706,2990682 +2,84,search,9304.42,1844.16,1.04165,0.90608,0.117089,-1.0,-1.0,14.92,1.3224,1706,2990682 +2,85,insert,34.561,-1.0,-1.0,-1.0,-1.0,36.295,1.33694,1709,3025613 +2,86,search,9435.46,1861.59,1.06848,0.90692,0.116369,-1.0,-1.0,16.285,1.33694,1709,3025613 +2,87,insert,471.289,-1.0,-1.0,-1.0,-1.0,559.258,1.54077,1809,3468632 +2,88,search,10435,2004.02,0.985132,0.91044,0.113806,-1.0,-1.0,108.46,1.54079,1822,3468632 +2,89,insert,311.964,-1.0,-1.0,-1.0,-1.0,491.747,1.70225,1925,3831692 +2,90,search,10998.7,2060.14,1.01067,0.91457,0.108833,-1.0,-1.0,87.647,1.7027,1935,3831692 +2,91,insert,33.889,-1.0,-1.0,-1.0,-1.0,63.145,1.71386,1942,3864067 +2,92,search,10836.4,2059.49,1.1055,0.91551,0.107688,-1.0,-1.0,45.413,1.7144,1945,3864067 +2,93,insert,282.536,-1.0,-1.0,-1.0,-1.0,268.715,1.81848,1988,4103827 +2,94,search,11139.2,2110.25,1.13448,0.91595,0.107422,-1.0,-1.0,56.783,1.81879,1993,4103827 +2,95,insert,16.346,-1.0,-1.0,-1.0,-1.0,51.138,1.82422,1998,4119898 +2,96,search,10910.1,2106.72,1.0892,0.91607,0.107545,-1.0,-1.0,24.122,1.82423,1999,4119898 +2,97,insert,7.635,-1.0,-1.0,-1.0,-1.0,26.692,1.82934,2001,4129367 +2,98,search,11027,2105.71,1.09818,0.91617,0.107333,-1.0,-1.0,27.279,1.82954,2002,4129367 +2,99,insert,5.776,-1.0,-1.0,-1.0,-1.0,20.894,1.83117,2002,4134502 +2,100,search,11152.5,2108.08,1.18036,0.91631,0.107257,-1.0,-1.0,33.603,1.83117,2002,4134502 +2,101,insert,50.088,-1.0,-1.0,-1.0,-1.0,44.94,1.84336,2004,4164482 +2,102,search,11338.2,2121.6,1.10594,0.91628,0.107175,-1.0,-1.0,40.111,1.84346,2005,4164482 +2,103,insert,21.997,-1.0,-1.0,-1.0,-1.0,44.11,1.85056,2007,4178601 +2,104,search,11637.7,2126.2,1.08574,0.91682,0.106858,-1.0,-1.0,36.743,1.85063,2008,4178601 +2,105,insert,9.666,-1.0,-1.0,-1.0,-1.0,45.321,1.85401,2010,4187774 +2,106,search,11586.7,2127.37,1.09732,0.91654,0.106957,-1.0,-1.0,31.608,1.85401,2010,4187774 +2,107,insert,18.797,-1.0,-1.0,-1.0,-1.0,37.759,1.85839,2011,4200109 +2,108,search,11602.4,2129.81,1.06016,0.91637,0.107267,-1.0,-1.0,32.963,1.85839,2011,4200109 +2,109,insert,495.046,-1.0,-1.0,-1.0,-1.0,493.929,2.01985,2094,4557867 +2,110,search,11718.8,2205.76,1.11053,0.91902,0.104918,-1.0,-1.0,86.253,2.0212,2111,4557867 +2,111,insert,16.295,-1.0,-1.0,-1.0,-1.0,54.711,2.03354,2123,4587252 +2,112,search,12240.9,2186.1,1.09333,0.91947,0.104973,-1.0,-1.0,37.383,2.03354,2123,4587252 +2,113,insert,12.797,-1.0,-1.0,-1.0,-1.0,57.449,2.03787,2126,4600056 +2,114,search,12417.5,2189.23,1.10715,0.91945,0.105078,-1.0,-1.0,38.195,2.03787,2126,4600056 +2,115,insert,48.612,-1.0,-1.0,-1.0,-1.0,57.932,2.04858,2130,4627422 +2,116,search,12261.6,2197.43,1.13185,0.92001,0.104622,-1.0,-1.0,30.143,2.04884,2131,4627422 +2,117,insert,9.377,-1.0,-1.0,-1.0,-1.0,24.399,2.05297,2131,4639292 +2,118,search,12350,2200.12,1.04967,0.91996,0.104482,-1.0,-1.0,37.497,2.05297,2131,4639292 +2,119,insert,5.715,-1.0,-1.0,-1.0,-1.0,35.253,2.05445,2131,4643423 +2,120,search,12531.7,2202.67,1.07882,0.91998,0.104392,-1.0,-1.0,37.51,2.05445,2131,4643423 +2,121,insert,22.101,-1.0,-1.0,-1.0,-1.0,58.241,2.06029,2135,4657252 +2,122,search,12535.8,2201.42,1.11668,0.92024,0.104231,-1.0,-1.0,36.267,2.06029,2135,4657252 +2,123,insert,379.367,-1.0,-1.0,-1.0,-1.0,501.033,2.18966,2192,4938477 +2,124,search,12939.9,2251.48,1.14946,0.92104,0.104985,-1.0,-1.0,112.442,2.19082,2213,4938477 +2,125,insert,35.285,-1.0,-1.0,-1.0,-1.0,76.917,2.20853,2223,4980492 +2,126,search,13019.6,2230.79,1.09592,0.92136,0.104736,-1.0,-1.0,53.808,2.20864,2225,4980492 +2,127,insert,61.168,-1.0,-1.0,-1.0,-1.0,99.831,2.22879,2237,5030515 +2,128,search,13266.6,2238.79,1.05305,0.92201,0.10452,-1.0,-1.0,38.213,2.22879,2237,5030515 +2,129,delete,29.933,-1.0,-1.0,-1.0,-1.0,69.706,2.22675,2203,5009391 +2,130,search,12714.2,2240.61,1.12574,0.92103,0.105021,-1.0,-1.0,26.686,2.22675,2203,5009391 +2,131,delete,5.729,-1.0,-1.0,-1.0,-1.0,24.447,2.22675,2203,5004093 +2,132,search,12777.4,2237.64,1.09228,0.92097,0.105096,-1.0,-1.0,37.637,2.22675,2203,5004093 +2,133,delete,17.745,-1.0,-1.0,-1.0,-1.0,35.395,2.22675,2203,4991787 +2,134,search,12178.9,2228.22,1.1266,0.92076,0.105241,-1.0,-1.0,25.304,2.22675,2203,4991787 +2,135,delete,12.956,-1.0,-1.0,-1.0,-1.0,43.194,2.22455,2200,4974857 +2,136,search,12682.4,2222.2,1.07994,0.9208,0.105187,-1.0,-1.0,36.379,2.22455,2200,4974857 +2,137,delete,5.963,-1.0,-1.0,-1.0,-1.0,35.415,2.22455,2200,4971386 +2,138,search,12937.6,2220.74,1.08516,0.92077,0.105184,-1.0,-1.0,37.908,2.22455,2200,4971386 +2,139,delete,16.301,-1.0,-1.0,-1.0,-1.0,36.547,2.22455,2200,4963487 +2,140,search,12938.4,2216.32,1.18035,0.92065,0.105145,-1.0,-1.0,37.983,2.22455,2200,4963487 +2,141,delete,36.952,-1.0,-1.0,-1.0,-1.0,98.434,2.21998,2194,4941830 +2,142,search,12897.5,2209.62,1.08392,0.92046,0.105366,-1.0,-1.0,39.036,2.21998,2194,4941830 +2,143,delete,26.972,-1.0,-1.0,-1.0,-1.0,108.522,2.21458,2190,4925657 +2,144,search,12755.3,2206.7,1.11067,0.92037,0.105149,-1.0,-1.0,45.762,2.21467,2191,4925657 +2,145,delete,4.286,-1.0,-1.0,-1.0,-1.0,50.429,2.21359,2188,4922686 +2,146,search,12763.1,2206.43,1.09121,0.92014,0.105734,-1.0,-1.0,36.458,2.21359,2188,4922686 +2,147,delete,11.503,-1.0,-1.0,-1.0,-1.0,35.077,2.21359,2188,4913636 +2,148,search,12339.9,2200.89,1.09756,0.92013,0.105596,-1.0,-1.0,24.743,2.21359,2188,4913636 +2,149,delete,132.089,-1.0,-1.0,-1.0,-1.0,881.557,2.13101,2079,4717250 +2,150,search,12052.9,2225.63,1.09906,0.91965,0.106045,-1.0,-1.0,92.825,2.13244,2089,4717250 +2,151,delete,154.315,-1.0,-1.0,-1.0,-1.0,1144.17,2.06134,1991,4560269 +4,2,search,2648.89,37.5515,1.88773,0.8551,0.130106,-1.0,-1.0,7.64,0.0184881,1000,38806 +4,3,insert,12.009,-1.0,-1.0,-1.0,-1.0,14.808,0.022655,996,49334 +4,4,search,2539.51,63.4255,1.99265,0.86686,0.128427,-1.0,-1.0,17.91,0.0226819,989,49334 +4,5,insert,26.97,-1.0,-1.0,-1.0,-1.0,166.437,0.0328921,977,73557 +4,6,search,2658.6,177.353,1.62788,0.92606,0.103259,-1.0,-1.0,125.644,0.0330517,979,73557 +4,7,insert,30.268,-1.0,-1.0,-1.0,-1.0,125.916,0.0426943,980,95766 +4,8,search,2713.67,222.757,1.4305,0.92504,0.103629,-1.0,-1.0,32.131,0.0432982,985,95766 +4,9,insert,4.869,-1.0,-1.0,-1.0,-1.0,6.717,0.0450828,985,100025 +4,10,search,2759.96,227.581,1.28173,0.92592,0.101876,-1.0,-1.0,22.944,0.0451207,989,100025 +4,11,insert,13.801,-1.0,-1.0,-1.0,-1.0,13.078,0.0499266,992,111417 +4,12,search,2799.06,258.957,1.16719,0.92491,0.102748,-1.0,-1.0,23.034,0.0501546,997,111417 +4,13,insert,26.965,-1.0,-1.0,-1.0,-1.0,6.758,0.0612286,997,137139 +4,14,search,2638.15,293.564,1.37405,0.92153,0.10454,-1.0,-1.0,13.427,0.0612375,999,137139 +4,15,insert,25.853,-1.0,-1.0,-1.0,-1.0,13.327,0.069562,1001,156710 +4,16,search,2851.49,324.727,1.10201,0.91974,0.105152,-1.0,-1.0,7.265,0.069562,1001,156710 +4,17,insert,6.305,-1.0,-1.0,-1.0,-1.0,12.1,0.0715575,1003,161773 +4,18,search,2853.58,339.476,1.05827,0.92081,0.105107,-1.0,-1.0,7.396,0.0715575,1003,161773 +4,19,insert,37.852,-1.0,-1.0,-1.0,-1.0,12.907,0.0785114,1005,177161 +4,20,search,2826.04,375.64,1.0772,0.92445,0.101871,-1.0,-1.0,10.978,0.0785114,1006,177161 +4,21,insert,361.589,-1.0,-1.0,-1.0,-1.0,201.885,0.201554,1063,448898 +4,22,search,3374.26,744.238,0.854057,0.90874,0.108744,-1.0,-1.0,53.438,0.201553,1074,448898 +4,23,insert,394.692,-1.0,-1.0,-1.0,-1.0,259.614,0.329691,1134,733493 +4,24,search,3973.07,1027.45,0.777998,0.90456,0.110585,-1.0,-1.0,119.009,0.330318,1161,733493 +4,25,insert,41.615,-1.0,-1.0,-1.0,-1.0,25.088,0.347655,1165,777443 +4,26,search,3980.72,989.61,0.808181,0.90156,0.113667,-1.0,-1.0,21.389,0.347813,1168,777443 +4,27,insert,31.006,-1.0,-1.0,-1.0,-1.0,13.945,0.359663,1169,805394 +4,28,search,4042.56,1012.24,0.791303,0.90312,0.112023,-1.0,-1.0,14.428,0.359665,1170,805394 +4,29,insert,66.039,-1.0,-1.0,-1.0,-1.0,66.568,0.38664,1184,866391 +4,30,search,4162.44,1063.32,0.782776,0.90536,0.110946,-1.0,-1.0,28.54,0.386639,1187,866391 +4,31,insert,428.089,-1.0,-1.0,-1.0,-1.0,310.7,0.551452,1251,1239816 +4,32,search,4801.03,1334.93,0.683276,0.9004,0.115641,-1.0,-1.0,119.586,0.552172,1263,1239816 +4,33,insert,69.549,-1.0,-1.0,-1.0,-1.0,23.87,0.57367,1268,1291211 +4,34,search,4792.31,1335.96,0.758786,0.90063,0.115592,-1.0,-1.0,70.503,0.573836,1274,1291211 +4,35,insert,42.976,-1.0,-1.0,-1.0,-1.0,28.067,0.585118,1279,1324978 +4,36,search,4572.43,1359.92,0.850717,0.90129,0.114334,-1.0,-1.0,42.211,0.58512,1282,1324978 +4,37,insert,65.822,-1.0,-1.0,-1.0,-1.0,34.349,0.606691,1288,1371885 +4,38,search,4978,1380.02,0.769329,0.9023,0.113605,-1.0,-1.0,10.303,0.606691,1288,1371885 +4,39,insert,9.559,-1.0,-1.0,-1.0,-1.0,14.027,0.609658,1290,1379335 +4,40,search,4785.97,1381.63,0.847907,0.90231,0.113761,-1.0,-1.0,10.332,0.609658,1290,1379335 +4,41,insert,49.91,-1.0,-1.0,-1.0,-1.0,36.364,0.625428,1296,1413312 +4,42,search,4731.48,1400.39,0.778653,0.90314,0.113453,-1.0,-1.0,10.085,0.625428,1296,1413312 +4,43,insert,11.275,-1.0,-1.0,-1.0,-1.0,13.184,0.628646,1297,1423411 +4,44,search,4790.58,1410.94,0.823537,0.90341,0.113374,-1.0,-1.0,10.323,0.628646,1297,1423411 +4,45,insert,227.598,-1.0,-1.0,-1.0,-1.0,35.812,0.643613,1305,1457559 +4,46,search,5068.68,1425.96,0.731091,0.90349,0.11292,-1.0,-1.0,9.942,0.643613,1305,1457559 +4,47,insert,299.283,-1.0,-1.0,-1.0,-1.0,247.048,0.749626,1343,1688593 +4,48,search,5039.08,1552.73,0.957053,0.90646,0.111837,-1.0,-1.0,83.071,0.749902,1352,1688593 +4,49,insert,36.858,-1.0,-1.0,-1.0,-1.0,28.453,0.765001,1356,1723505 +4,50,search,5489.19,1556.37,0.707924,0.907,0.111429,-1.0,-1.0,31.424,0.765,1359,1723505 +4,51,insert,330.217,-1.0,-1.0,-1.0,-1.0,283.815,0.905145,1417,2035040 +4,52,search,5719.34,1704.72,0.852746,0.90888,0.112071,-1.0,-1.0,39.682,0.905144,1422,2035040 +4,53,insert,41.901,-1.0,-1.0,-1.0,-1.0,17.864,0.921853,1423,2078341 +4,54,search,5861.86,1723.97,0.774239,0.90922,0.11148,-1.0,-1.0,11.964,0.921853,1423,2078341 +4,55,insert,352.446,-1.0,-1.0,-1.0,-1.0,431.969,1.06922,1489,2399905 +4,56,search,6301.59,1874.54,0.740148,0.91068,0.11224,-1.0,-1.0,101.913,1.06982,1505,2399905 +4,57,insert,23.727,-1.0,-1.0,-1.0,-1.0,21.8,1.07862,1508,2419836 +4,58,search,6559.39,1843.56,0.701931,0.90999,0.113061,-1.0,-1.0,40.52,1.07863,1511,2419836 +4,59,insert,248.512,-1.0,-1.0,-1.0,-1.0,182.297,1.17688,1531,2643083 +4,60,search,6699.78,1901.36,0.70948,0.90319,0.120837,-1.0,-1.0,67.913,1.17725,1539,2643083 +4,61,insert,10.687,-1.0,-1.0,-1.0,-1.0,14.857,1.18091,1540,2653064 +4,62,search,6700.39,1897.07,0.745074,0.90331,0.12042,-1.0,-1.0,26.509,1.18092,1543,2653064 +4,63,insert,3.528,-1.0,-1.0,-1.0,-1.0,11.123,1.18201,1543,2656118 +4,64,search,6685.34,1888.97,0.746461,0.90299,0.120674,-1.0,-1.0,11.788,1.18201,1543,2656118 +4,65,insert,74.34,-1.0,-1.0,-1.0,-1.0,131.029,1.20991,1572,2723424 +4,66,search,6888.25,1895.61,0.683151,0.90394,0.119328,-1.0,-1.0,22.013,1.21014,1573,2723424 +4,67,insert,46.322,-1.0,-1.0,-1.0,-1.0,78.547,1.23016,1587,2769021 +4,68,search,6995.43,1909.97,0.685908,0.90523,0.118301,-1.0,-1.0,27.766,1.23016,1590,2769021 +4,69,insert,67.441,-1.0,-1.0,-1.0,-1.0,69.323,1.25091,1599,2821777 +4,70,search,7002.65,1927.43,0.651686,0.90672,0.116866,-1.0,-1.0,29.874,1.25091,1601,2821777 +4,71,insert,23.412,-1.0,-1.0,-1.0,-1.0,29.66,1.257,1605,2834921 +4,72,search,6870.86,1928.3,0.732529,0.90711,0.116825,-1.0,-1.0,22.036,1.257,1606,2834921 +4,73,insert,7.286,-1.0,-1.0,-1.0,-1.0,11.805,1.25884,1606,2842044 +4,74,search,6762.91,1932.73,0.784468,0.90711,0.116885,-1.0,-1.0,13.017,1.25884,1606,2842044 +4,75,insert,26.468,-1.0,-1.0,-1.0,-1.0,36.577,1.26508,1610,2858890 +4,76,search,6968.68,1938.27,0.769314,0.90753,0.116645,-1.0,-1.0,17.094,1.26528,1611,2858890 +4,77,insert,33.335,-1.0,-1.0,-1.0,-1.0,11.373,1.27555,1611,2882818 +4,78,search,7140.85,1948.65,0.685717,0.90725,0.116846,-1.0,-1.0,12.119,1.27555,1611,2882818 +4,79,insert,473.154,-1.0,-1.0,-1.0,-1.0,21.667,1.30353,1614,2947806 +4,80,search,7226.32,1976.62,0.70684,0.90699,0.116916,-1.0,-1.0,17.937,1.30359,1615,2947806 +4,81,insert,29.663,-1.0,-1.0,-1.0,-1.0,45.95,1.3149,1621,2976895 +4,82,search,6810.25,1982.52,0.782588,0.90821,0.116149,-1.0,-1.0,12.236,1.3149,1621,2976895 +4,83,insert,17.009,-1.0,-1.0,-1.0,-1.0,18.907,1.32219,1622,2990682 +4,84,search,6926.49,1991.13,0.723782,0.90831,0.115947,-1.0,-1.0,12.076,1.32219,1622,2990682 +4,85,insert,36.181,-1.0,-1.0,-1.0,-1.0,41.221,1.33756,1627,3025613 +4,86,search,7097.74,2005.05,0.729937,0.90951,0.115039,-1.0,-1.0,14.288,1.33756,1627,3025613 +4,87,insert,577.638,-1.0,-1.0,-1.0,-1.0,661.733,1.54487,1740,3468632 +4,88,search,7767.6,2121.4,0.729338,0.91298,0.111325,-1.0,-1.0,101.632,1.54554,1752,3468632 +4,89,insert,411.545,-1.0,-1.0,-1.0,-1.0,467.189,1.70655,1839,3831692 +4,90,search,8261.35,2195.93,0.719684,0.917,0.106903,-1.0,-1.0,85.11,1.70673,1849,3831692 +4,91,insert,29.693,-1.0,-1.0,-1.0,-1.0,79.226,1.71688,1861,3864067 +4,92,search,8359.11,2182.33,0.730181,0.9171,0.106494,-1.0,-1.0,28.629,1.71693,1862,3864067 +4,93,insert,284.488,-1.0,-1.0,-1.0,-1.0,219.341,1.82177,1892,4103827 +4,94,search,8287.5,2256.56,0.873714,0.91791,0.106212,-1.0,-1.0,83.117,1.82215,1901,4103827 +4,95,insert,16.547,-1.0,-1.0,-1.0,-1.0,34.717,1.82807,1904,4119898 +4,96,search,7791.13,2248.21,0.968586,0.91818,0.105858,-1.0,-1.0,29.828,1.82823,1905,4119898 +4,97,insert,10.473,-1.0,-1.0,-1.0,-1.0,37.435,1.83185,1908,4129367 +4,98,search,9015.74,2245.99,0.655722,0.9181,0.105883,-1.0,-1.0,27.074,1.83185,1908,4129367 +4,99,insert,10.006,-1.0,-1.0,-1.0,-1.0,32.305,1.83379,1909,4134502 +4,100,search,8988.86,2247.48,0.714673,0.91818,0.105801,-1.0,-1.0,28.105,1.83379,1909,4134502 +4,101,insert,51.863,-1.0,-1.0,-1.0,-1.0,57.339,1.84591,1913,4164482 +4,102,search,8850.84,2258.9,0.742483,0.91828,0.106099,-1.0,-1.0,27.98,1.84591,1913,4164482 +4,103,insert,20.444,-1.0,-1.0,-1.0,-1.0,46.398,1.85193,1916,4178601 +4,104,search,8920.03,2264.44,0.73229,0.91851,0.106034,-1.0,-1.0,34.209,1.85205,1917,4178601 +4,105,insert,12.976,-1.0,-1.0,-1.0,-1.0,33.917,1.85626,1918,4187774 +4,106,search,8853.27,2266.53,0.672879,0.91855,0.105945,-1.0,-1.0,26.994,1.85626,1918,4187774 +4,107,insert,19.602,-1.0,-1.0,-1.0,-1.0,25.712,1.86002,1918,4200109 +4,108,search,9084.87,2270.15,0.666449,0.91843,0.10604,-1.0,-1.0,27.101,1.86002,1918,4200109 +4,109,insert,468.415,-1.0,-1.0,-1.0,-1.0,636.48,2.02206,2014,4557867 +4,110,search,9628.79,2327.85,0.697749,0.92049,0.10383,-1.0,-1.0,92.488,2.02302,2026,4557867 +4,111,insert,29.913,-1.0,-1.0,-1.0,-1.0,72.646,2.03557,2035,4587252 +4,112,search,9742.57,2310.57,0.690824,0.92065,0.103806,-1.0,-1.0,48.949,2.03598,2036,4587252 +4,113,insert,14.657,-1.0,-1.0,-1.0,-1.0,57.675,2.04124,2040,4600056 +4,114,search,9540.73,2310.61,0.733133,0.92096,0.103656,-1.0,-1.0,39.694,2.04133,2041,4600056 +4,115,insert,35.234,-1.0,-1.0,-1.0,-1.0,61.425,2.04849,2046,4627422 +4,116,search,9788.63,2315.14,0.650192,0.92128,0.103612,-1.0,-1.0,34.838,2.04849,2046,4627422 +4,117,insert,17.306,-1.0,-1.0,-1.0,-1.0,36.272,2.05248,2047,4639292 +4,118,search,8741.98,2318.28,0.877847,0.92155,0.103384,-1.0,-1.0,33.183,2.05248,2047,4639292 +4,119,insert,5.926,-1.0,-1.0,-1.0,-1.0,46.34,2.05441,2049,4643423 +4,120,search,9853.57,2316.8,0.670908,0.92149,0.103387,-1.0,-1.0,33.062,2.05441,2049,4643423 +4,121,insert,23.242,-1.0,-1.0,-1.0,-1.0,40.227,2.05936,2050,4657252 +4,122,search,9828.76,2321.93,0.657878,0.92151,0.103251,-1.0,-1.0,33.081,2.05936,2050,4657252 +4,123,insert,366.663,-1.0,-1.0,-1.0,-1.0,436.498,2.18515,2092,4938477 +4,124,search,10015.9,2387.89,0.701238,0.92241,0.103768,-1.0,-1.0,223.511,2.18602,2110,4938477 +4,125,insert,60.021,-1.0,-1.0,-1.0,-1.0,87.829,2.2052,2119,4980492 +4,126,search,10274.8,2373.28,0.687235,0.92244,0.103677,-1.0,-1.0,129.009,2.20565,2127,4980492 +4,127,insert,65.731,-1.0,-1.0,-1.0,-1.0,134.898,2.2263,2149,5030515 +4,128,search,10429.2,2358.04,0.682777,0.92328,0.103084,-1.0,-1.0,77.967,2.22646,2154,5030515 +4,129,delete,18.805,-1.0,-1.0,-1.0,-1.0,58.088,2.22483,2126,5009391 +4,130,search,9838.25,2349.94,0.777618,0.92231,0.103546,-1.0,-1.0,36.301,2.22483,2126,5009391 +4,131,delete,6.265,-1.0,-1.0,-1.0,-1.0,40.916,2.22483,2126,5004093 +4,132,search,9770.16,2348.02,0.752127,0.92222,0.103691,-1.0,-1.0,37.315,2.22483,2126,5004093 +4,133,delete,15.734,-1.0,-1.0,-1.0,-1.0,35.202,2.2248,2126,4991787 +4,134,search,10013,2336.42,0.712997,0.92204,0.103845,-1.0,-1.0,35.808,2.2248,2126,4991787 +4,135,delete,22.078,-1.0,-1.0,-1.0,-1.0,80.696,2.22171,2122,4974857 +4,136,search,10018.2,2333.46,0.684522,0.92231,0.103575,-1.0,-1.0,36.159,2.22171,2122,4974857 +4,137,delete,6.066,-1.0,-1.0,-1.0,-1.0,35.121,2.22171,2122,4971386 +4,138,search,9706.09,2331.39,0.761608,0.92225,0.103658,-1.0,-1.0,35.748,2.22171,2122,4971386 +4,139,delete,8.958,-1.0,-1.0,-1.0,-1.0,35.77,2.22171,2122,4963487 +4,140,search,9921.51,2326.33,0.741001,0.92224,0.103564,-1.0,-1.0,37.479,2.22171,2122,4963487 +4,141,delete,36.536,-1.0,-1.0,-1.0,-1.0,111.073,2.21759,2115,4941830 +4,142,search,9625.52,2321.07,0.78902,0.92212,0.103729,-1.0,-1.0,38.118,2.21759,2115,4941830 +4,143,delete,26.653,-1.0,-1.0,-1.0,-1.0,106.51,2.2123,2111,4925657 +4,144,search,10129.5,2317.06,0.687404,0.92216,0.103679,-1.0,-1.0,38.092,2.2123,2111,4925657 +4,145,delete,5.385,-1.0,-1.0,-1.0,-1.0,45.165,2.21141,2109,4922686 +4,146,search,10075.1,2317.57,0.66731,0.92202,0.103733,-1.0,-1.0,36.875,2.21141,2109,4922686 +4,147,delete,12.72,-1.0,-1.0,-1.0,-1.0,35.394,2.21141,2109,4913636 +4,148,search,10100.2,2311.94,0.680439,0.92196,0.103714,-1.0,-1.0,36.754,2.21141,2109,4913636 +4,149,delete,227.333,-1.0,-1.0,-1.0,-1.0,1262.45,2.12783,2007,4717250 +4,150,search,9254.72,2333.41,0.774102,0.92147,0.10475,-1.0,-1.0,86.86,2.1292,2014,4717250 +4,151,delete,166.137,-1.0,-1.0,-1.0,-1.0,1087.57,2.05883,1920,4560269 +8,2,search,3176.13,37.5515,0.685469,0.8551,0.130106,-1.0,-1.0,7.519,0.0184881,1000,38806 +8,3,insert,14.061,-1.0,-1.0,-1.0,-1.0,15.915,0.022655,996,49334 +8,4,search,2689.35,63.4255,1.05853,0.86686,0.128427,-1.0,-1.0,14.033,0.0226615,993,49334 +8,5,insert,29.243,-1.0,-1.0,-1.0,-1.0,167.398,0.032935,982,73557 +8,6,search,2602.73,175.313,1.44387,0.92586,0.10314,-1.0,-1.0,144.996,0.0329729,983,73557 +8,7,insert,33.212,-1.0,-1.0,-1.0,-1.0,149.245,0.0426748,985,95766 +8,8,search,2602.98,220.999,1.27385,0.92475,0.103899,-1.0,-1.0,91.19,0.0429489,989,95766 +8,9,insert,5.46,-1.0,-1.0,-1.0,-1.0,74.862,0.0447658,989,100025 +8,10,search,2603,227.069,1.22283,0.92685,0.101224,-1.0,-1.0,86.632,0.0448143,993,100025 +8,11,insert,14.031,-1.0,-1.0,-1.0,-1.0,84.775,0.0499416,996,111417 +8,12,search,2618.52,260.537,1.16238,0.92563,0.102114,-1.0,-1.0,19.706,0.0499841,997,111417 +8,13,insert,29.638,-1.0,-1.0,-1.0,-1.0,11.314,0.0612257,999,137139 +8,14,search,2671.04,294.111,0.993493,0.92128,0.10523,-1.0,-1.0,28.055,0.0613236,1002,137139 +8,15,insert,32.741,-1.0,-1.0,-1.0,-1.0,15.749,0.0700361,1005,156710 +8,16,search,2697.61,322.814,0.995554,0.91926,0.105592,-1.0,-1.0,20.677,0.0700409,1009,156710 +8,17,insert,6.873,-1.0,-1.0,-1.0,-1.0,11.988,0.0720871,1011,161773 +8,18,search,2718.78,331.447,0.949806,0.91922,0.106363,-1.0,-1.0,21.651,0.0722678,1016,161773 +8,19,insert,43.185,-1.0,-1.0,-1.0,-1.0,17.725,0.0792022,1020,177161 +8,20,search,2779.57,358.088,0.966152,0.92312,0.102872,-1.0,-1.0,14.555,0.0792022,1022,177161 +8,21,insert,385.316,-1.0,-1.0,-1.0,-1.0,228.807,0.202194,1094,448898 +8,22,search,3056.74,691.199,0.815705,0.90671,0.108756,-1.0,-1.0,81.923,0.20226,1112,448898 +8,23,insert,429.022,-1.0,-1.0,-1.0,-1.0,269.899,0.328827,1180,733493 +8,24,search,3343.87,945.68,0.731784,0.90352,0.111145,-1.0,-1.0,119.369,0.32956,1210,733493 +8,25,insert,48.946,-1.0,-1.0,-1.0,-1.0,34.083,0.347375,1218,777443 +8,26,search,3481.64,909.867,0.771286,0.90169,0.112692,-1.0,-1.0,29.199,0.347516,1225,777443 +8,27,insert,35.257,-1.0,-1.0,-1.0,-1.0,28.761,0.359307,1230,805394 +8,28,search,3471.19,918.894,0.767757,0.90131,0.113193,-1.0,-1.0,16.102,0.359438,1232,805394 +8,29,insert,73.173,-1.0,-1.0,-1.0,-1.0,70.278,0.386063,1248,866391 +8,30,search,3599.43,959.047,0.828177,0.90373,0.112592,-1.0,-1.0,33.904,0.386066,1253,866391 +8,31,insert,485.523,-1.0,-1.0,-1.0,-1.0,311.352,0.554971,1336,1239816 +8,32,search,3936.92,1190.76,0.763297,0.89809,0.117141,-1.0,-1.0,52.913,0.555102,1345,1239816 +8,33,insert,52.131,-1.0,-1.0,-1.0,-1.0,48.984,0.574791,1356,1291211 +8,34,search,3982.56,1197.43,0.684143,0.89932,0.116194,-1.0,-1.0,17.949,0.574833,1358,1291211 +8,35,insert,36.475,-1.0,-1.0,-1.0,-1.0,28.721,0.586575,1365,1324978 +8,36,search,3986.52,1221.58,0.686129,0.90058,0.115202,-1.0,-1.0,10.591,0.586575,1365,1324978 +8,37,insert,58.661,-1.0,-1.0,-1.0,-1.0,43.29,0.607156,1371,1371885 +8,38,search,4067.71,1248.28,0.685673,0.89998,0.115727,-1.0,-1.0,23.157,0.607161,1374,1371885 +8,39,insert,10.475,-1.0,-1.0,-1.0,-1.0,12.095,0.610528,1375,1379335 +8,40,search,4065.6,1245.38,0.68669,0.89994,0.116434,-1.0,-1.0,15.046,0.610528,1376,1379335 +8,41,insert,40.328,-1.0,-1.0,-1.0,-1.0,32.987,0.62445,1382,1413312 +8,42,search,4077.28,1264.69,0.702071,0.90119,0.115424,-1.0,-1.0,18.169,0.624455,1384,1413312 +8,43,insert,12.139,-1.0,-1.0,-1.0,-1.0,18.337,0.628464,1387,1423411 +8,44,search,4109.98,1266.03,0.701159,0.90151,0.115193,-1.0,-1.0,10.705,0.628464,1387,1423411 +8,45,insert,236.562,-1.0,-1.0,-1.0,-1.0,33.403,0.642904,1396,1457559 +8,46,search,4121.1,1268.71,0.682604,0.90136,0.115086,-1.0,-1.0,11.027,0.642904,1396,1457559 +8,47,insert,292.157,-1.0,-1.0,-1.0,-1.0,241.667,0.747845,1434,1688593 +8,48,search,4306.7,1397.23,0.678486,0.90367,0.114957,-1.0,-1.0,89.633,0.748038,1443,1688593 +8,49,insert,38.484,-1.0,-1.0,-1.0,-1.0,27.452,0.761785,1449,1723505 +8,50,search,4343.53,1398.53,0.682565,0.90418,0.114696,-1.0,-1.0,34.902,0.761785,1452,1723505 +8,51,insert,406.498,-1.0,-1.0,-1.0,-1.0,281.605,0.905733,1516,2035040 +8,52,search,4448.59,1533.08,0.734685,0.90781,0.112832,-1.0,-1.0,52.812,0.906125,1522,2035040 +8,53,insert,51.825,-1.0,-1.0,-1.0,-1.0,22.037,0.923049,1525,2078341 +8,54,search,4669.03,1548.07,0.645888,0.90762,0.113182,-1.0,-1.0,12.546,0.923049,1525,2078341 +8,55,insert,375.867,-1.0,-1.0,-1.0,-1.0,394.89,1.07034,1595,2399905 +8,56,search,4783.98,1694.62,0.710547,0.90918,0.113835,-1.0,-1.0,170.218,1.07162,1628,2399905 +8,57,insert,23.926,-1.0,-1.0,-1.0,-1.0,20.019,1.07807,1629,2419836 +8,58,search,5051.66,1640.23,0.653922,0.90808,0.114567,-1.0,-1.0,64.517,1.07811,1637,2419836 +8,59,insert,158.018,-1.0,-1.0,-1.0,-1.0,90.116,1.17634,1659,2643083 +8,60,search,4351.15,1682.8,0.808787,0.90131,0.121765,-1.0,-1.0,35.709,1.17634,1662,2643083 +8,61,insert,13.848,-1.0,-1.0,-1.0,-1.0,17.253,1.1806,1664,2653064 +8,62,search,5041.17,1683.09,0.73793,0.90154,0.121725,-1.0,-1.0,13.262,1.1806,1664,2653064 +8,63,insert,4.825,-1.0,-1.0,-1.0,-1.0,12.61,1.18193,1664,2656118 +8,64,search,5110.88,1683.41,0.727436,0.90152,0.121617,-1.0,-1.0,17.684,1.18196,1665,2656118 +8,65,insert,83.676,-1.0,-1.0,-1.0,-1.0,107.23,1.20876,1690,2723424 +8,66,search,5373.68,1700.4,0.705238,0.90262,0.120814,-1.0,-1.0,11.716,1.20876,1690,2723424 +8,67,insert,26.402,-1.0,-1.0,-1.0,-1.0,34.132,1.22762,1700,2769021 +8,68,search,4886.46,1726.67,0.851401,0.90453,0.119315,-1.0,-1.0,35.85,1.22772,1705,2769021 +8,69,insert,51.785,-1.0,-1.0,-1.0,-1.0,46.064,1.25083,1715,2821777 +8,70,search,5127.14,1740.81,0.723305,0.90625,0.117871,-1.0,-1.0,23.995,1.25083,1716,2821777 +8,71,insert,17.388,-1.0,-1.0,-1.0,-1.0,15.564,1.25673,1716,2834921 +8,72,search,5109.72,1752.4,0.6421,0.90696,0.117325,-1.0,-1.0,10.691,1.25673,1716,2834921 +8,73,insert,4.319,-1.0,-1.0,-1.0,-1.0,10.353,1.2596,1716,2842044 +8,74,search,5033.51,1756.55,0.794706,0.90711,0.117346,-1.0,-1.0,16.356,1.2596,1716,2842044 +8,75,insert,20.184,-1.0,-1.0,-1.0,-1.0,27.704,1.26635,1719,2858890 +8,76,search,4303.88,1764.55,0.774132,0.90728,0.117017,-1.0,-1.0,10.955,1.26635,1719,2858890 +8,77,insert,18.86,-1.0,-1.0,-1.0,-1.0,13.697,1.27844,1720,2882818 +8,78,search,5233.81,1774.78,0.698777,0.90726,0.117219,-1.0,-1.0,11.338,1.27844,1720,2882818 +8,79,insert,302.014,-1.0,-1.0,-1.0,-1.0,33.94,1.30644,1727,2947806 +8,80,search,5285.79,1795.69,0.653776,0.9067,0.117696,-1.0,-1.0,11.464,1.30644,1727,2947806 +8,81,insert,22.483,-1.0,-1.0,-1.0,-1.0,32.321,1.31943,1734,2976895 +8,82,search,4963.58,1802.23,0.777655,0.90783,0.117099,-1.0,-1.0,15.303,1.31943,1734,2976895 +8,83,insert,10.363,-1.0,-1.0,-1.0,-1.0,35.623,1.32481,1740,2990682 +8,84,search,4899.55,1800.34,0.69544,0.90828,0.116521,-1.0,-1.0,12.173,1.32481,1740,2990682 +8,85,insert,22.006,-1.0,-1.0,-1.0,-1.0,39.106,1.33931,1749,3025613 +8,86,search,4241.48,1806.75,0.876135,0.90876,0.116276,-1.0,-1.0,16.928,1.33947,1750,3025613 +8,87,insert,289.598,-1.0,-1.0,-1.0,-1.0,367.117,1.54024,1849,3468632 +8,88,search,5512.9,1951.34,0.805257,0.91199,0.113166,-1.0,-1.0,112.214,1.54077,1863,3468632 +8,89,insert,416.423,-1.0,-1.0,-1.0,-1.0,314.865,1.69997,1957,3831692 +8,90,search,6012.27,2023.44,0.653866,0.91651,0.108674,-1.0,-1.0,106.121,1.70023,1970,3831692 +8,91,insert,37.445,-1.0,-1.0,-1.0,-1.0,61.029,1.71205,1978,3864067 +8,92,search,5959.5,2014.52,0.817775,0.91645,0.108514,-1.0,-1.0,28.546,1.71213,1979,3864067 +8,93,insert,278.89,-1.0,-1.0,-1.0,-1.0,290.921,1.81872,2020,4103827 +8,94,search,6054.24,2075.54,0.708834,0.9175,0.10711,-1.0,-1.0,56.813,1.81956,2033,4103827 +8,95,insert,9.402,-1.0,-1.0,-1.0,-1.0,47.104,1.82588,2039,4119898 +8,96,search,6627.87,2059.2,0.68994,0.91742,0.1071,-1.0,-1.0,32.618,1.82588,2039,4119898 +8,97,insert,8.503,-1.0,-1.0,-1.0,-1.0,30.094,1.83006,2041,4129367 +8,98,search,6222.6,2062.4,0.714383,0.91766,0.106837,-1.0,-1.0,32.944,1.83006,2041,4129367 +8,99,insert,7.136,-1.0,-1.0,-1.0,-1.0,35.909,1.8322,2042,4134502 +8,100,search,6221.55,2063.59,0.759507,0.91764,0.106875,-1.0,-1.0,30.65,1.8322,2042,4134502 +8,101,insert,21.42,-1.0,-1.0,-1.0,-1.0,45.521,1.84339,2046,4164482 +8,102,search,6565.39,2072.3,0.647449,0.91775,0.106861,-1.0,-1.0,24.993,1.84339,2046,4164482 +8,103,insert,9.864,-1.0,-1.0,-1.0,-1.0,53.827,1.84903,2055,4178601 +8,104,search,6093.72,2068.72,0.735142,0.91805,0.106529,-1.0,-1.0,46.444,1.8493,2057,4178601 +8,105,insert,13.583,-1.0,-1.0,-1.0,-1.0,40.515,1.8543,2058,4187774 +8,106,search,6585.51,2068.95,0.690668,0.9179,0.106911,-1.0,-1.0,30.959,1.8543,2058,4187774 +8,107,insert,15.151,-1.0,-1.0,-1.0,-1.0,31.821,1.85872,2058,4200109 +8,108,search,6404.43,2071.97,0.604212,0.91794,0.106955,-1.0,-1.0,35.873,1.85872,2058,4200109 +8,109,insert,320.646,-1.0,-1.0,-1.0,-1.0,602.967,2.02124,2150,4557867 +8,110,search,7153.59,2140.69,0.693012,0.92017,0.105304,-1.0,-1.0,104.239,2.02202,2163,4557867 +8,111,insert,32.898,-1.0,-1.0,-1.0,-1.0,53.763,2.0339,2171,4587252 +8,112,search,7158.54,2127.7,0.657832,0.92017,0.105428,-1.0,-1.0,34.968,2.03393,2172,4587252 +8,113,insert,11.741,-1.0,-1.0,-1.0,-1.0,46.723,2.03913,2175,4600056 +8,114,search,6383.12,2128.96,0.71362,0.92038,0.105364,-1.0,-1.0,30.166,2.03913,2175,4600056 +8,115,insert,17.14,-1.0,-1.0,-1.0,-1.0,42.07,2.04729,2181,4627422 +8,116,search,7327.13,2133.52,0.647492,0.92103,0.104697,-1.0,-1.0,38.295,2.04729,2181,4627422 +8,117,insert,11.883,-1.0,-1.0,-1.0,-1.0,46.386,2.04997,2182,4639292 +8,118,search,6738.96,2137.51,0.659331,0.92111,0.104185,-1.0,-1.0,40.459,2.05014,2183,4639292 +8,119,insert,4.012,-1.0,-1.0,-1.0,-1.0,35.517,2.05197,2183,4643423 +8,120,search,7528.54,2137.22,0.661673,0.92109,0.104264,-1.0,-1.0,38.142,2.05197,2183,4643423 +8,121,insert,17.641,-1.0,-1.0,-1.0,-1.0,50.454,2.05669,2185,4657252 +8,122,search,7099.18,2143.5,0.672221,0.92114,0.104374,-1.0,-1.0,35.404,2.05669,2185,4657252 +8,123,insert,286.358,-1.0,-1.0,-1.0,-1.0,320.932,2.18474,2248,4938477 +8,124,search,7608.35,2181.7,0.693711,0.92161,0.105792,-1.0,-1.0,119.299,2.18546,2258,4938477 +8,125,insert,61.145,-1.0,-1.0,-1.0,-1.0,88.448,2.20134,2267,4980492 +8,126,search,7301.51,2175.8,0.679197,0.92208,0.105404,-1.0,-1.0,29.687,2.20139,2268,4980492 +8,127,insert,41.357,-1.0,-1.0,-1.0,-1.0,105.157,2.22438,2285,5030515 +8,128,search,7995.05,2178.81,0.637915,0.92257,0.105088,-1.0,-1.0,38.616,2.22438,2285,5030515 +8,129,delete,25.744,-1.0,-1.0,-1.0,-1.0,69.265,2.22253,2253,5009391 +8,130,search,7814.56,2180.7,0.633245,0.92157,0.105796,-1.0,-1.0,38.277,2.22253,2253,5009391 +8,131,delete,7.197,-1.0,-1.0,-1.0,-1.0,37.148,2.22253,2253,5004093 +8,132,search,7768.08,2177.63,0.65513,0.9215,0.105847,-1.0,-1.0,38.684,2.22253,2253,5004093 +8,133,delete,15.394,-1.0,-1.0,-1.0,-1.0,53.829,2.22124,2251,4991787 +8,134,search,6701.67,2171.67,0.804316,0.92154,0.105829,-1.0,-1.0,34.674,2.22124,2251,4991787 +8,135,delete,12.995,-1.0,-1.0,-1.0,-1.0,74.343,2.21864,2248,4974857 +8,136,search,7085.72,2165.1,0.859149,0.92153,0.105502,-1.0,-1.0,26.391,2.21864,2248,4974857 +8,137,delete,3.77,-1.0,-1.0,-1.0,-1.0,25.479,2.21864,2248,4971386 +8,138,search,7208.63,2163.21,0.785935,0.92144,0.105715,-1.0,-1.0,38.403,2.21864,2248,4971386 +8,139,delete,9.67,-1.0,-1.0,-1.0,-1.0,37.649,2.21864,2248,4963487 +8,140,search,7705.17,2158.85,0.683466,0.92137,0.105704,-1.0,-1.0,37.723,2.21864,2248,4963487 +8,141,delete,23.182,-1.0,-1.0,-1.0,-1.0,80.88,2.21502,2242,4941830 +8,142,search,7775.59,2152.13,0.669927,0.92127,0.105854,-1.0,-1.0,37.55,2.21502,2242,4941830 +8,143,delete,17.744,-1.0,-1.0,-1.0,-1.0,88.389,2.21103,2238,4925657 +8,144,search,7750.08,2148.97,0.653724,0.92122,0.106047,-1.0,-1.0,37.699,2.21103,2238,4925657 +8,145,delete,3.158,-1.0,-1.0,-1.0,-1.0,54.308,2.20993,2234,4922686 +8,146,search,7712.17,2150.62,0.660421,0.92127,0.106081,-1.0,-1.0,39,2.20993,2234,4922686 +8,147,delete,14.882,-1.0,-1.0,-1.0,-1.0,36.937,2.20993,2234,4913636 +8,148,search,7730.15,2145.36,0.66315,0.92106,0.106189,-1.0,-1.0,39.756,2.20993,2234,4913636 +8,149,delete,238.758,-1.0,-1.0,-1.0,-1.0,1269.57,2.13175,2130,4717250 +8,150,search,7429.46,2156.56,0.630906,0.92091,0.106747,-1.0,-1.0,100.956,2.13338,2138,4717250 +8,151,delete,161.725,-1.0,-1.0,-1.0,-1.0,1053.46,2.06482,2046,4560269 +16,2,search,2817.57,37.5515,1.01605,0.8551,0.130106,-1.0,-1.0,9.901,0.0184881,1000,38806 +16,3,insert,20.093,-1.0,-1.0,-1.0,-1.0,18.702,0.022655,996,49334 +16,4,search,2732.33,63.4255,1.05145,0.86686,0.128427,-1.0,-1.0,19.63,0.0226615,993,49334 +16,5,insert,42.354,-1.0,-1.0,-1.0,-1.0,208.22,0.033017,984,73557 +16,6,search,2709.92,173.596,1.14988,0.92515,0.103785,-1.0,-1.0,165.305,0.0330521,985,73557 +16,7,insert,84.96,-1.0,-1.0,-1.0,-1.0,159.301,0.0425699,987,95766 +16,8,search,2733.96,221.115,1.01108,0.92467,0.103645,-1.0,-1.0,118.73,0.0428123,991,95766 +16,9,insert,16.124,-1.0,-1.0,-1.0,-1.0,76.406,0.0446389,991,100025 +16,10,search,2633.24,226.599,0.941417,0.925,0.102401,-1.0,-1.0,98.236,0.0447131,996,100025 +16,11,insert,33.207,-1.0,-1.0,-1.0,-1.0,83.788,0.0498507,997,111417 +16,12,search,2746.25,260.601,0.888242,0.92407,0.103321,-1.0,-1.0,36.232,0.0498804,998,111417 +16,13,insert,69.913,-1.0,-1.0,-1.0,-1.0,14.412,0.0612045,1000,137139 +16,14,search,2794.07,295.052,0.772953,0.92074,0.105501,-1.0,-1.0,39.099,0.06132,1003,137139 +16,15,insert,67.323,-1.0,-1.0,-1.0,-1.0,21.626,0.0698961,1005,156710 +16,16,search,2786.13,322.174,0.747958,0.91771,0.106381,-1.0,-1.0,32.804,0.0698982,1009,156710 +16,17,insert,16.874,-1.0,-1.0,-1.0,-1.0,18.425,0.0718627,1011,161773 +16,18,search,2776.61,328.292,0.79122,0.91744,0.107279,-1.0,-1.0,24.74,0.0720365,1015,161773 +16,19,insert,63.682,-1.0,-1.0,-1.0,-1.0,14.765,0.0789864,1016,177161 +16,20,search,2861.3,361.6,0.81055,0.92123,0.103851,-1.0,-1.0,8.506,0.0789864,1016,177161 +16,21,insert,516.636,-1.0,-1.0,-1.0,-1.0,297.165,0.202475,1087,448898 +16,22,search,3036.86,703.72,0.583757,0.90764,0.109503,-1.0,-1.0,137.383,0.202472,1107,448898 +16,23,insert,737.783,-1.0,-1.0,-1.0,-1.0,352.994,0.329412,1173,733493 +16,24,search,3282.01,955.037,0.61648,0.90117,0.11289,-1.0,-1.0,150.251,0.330087,1200,733493 +16,25,insert,72.504,-1.0,-1.0,-1.0,-1.0,40.715,0.346546,1208,777443 +16,26,search,3367.46,919.947,0.617036,0.89993,0.114372,-1.0,-1.0,49.409,0.34667,1216,777443 +16,27,insert,62.555,-1.0,-1.0,-1.0,-1.0,19.055,0.358247,1217,805394 +16,28,search,3338.27,927.827,0.585935,0.89987,0.11462,-1.0,-1.0,15.156,0.358252,1218,805394 +16,29,insert,137.986,-1.0,-1.0,-1.0,-1.0,89.811,0.385584,1232,866391 +16,30,search,3388.86,971.62,0.664151,0.90176,0.114279,-1.0,-1.0,53.422,0.385866,1238,866391 +16,31,insert,794.46,-1.0,-1.0,-1.0,-1.0,428.749,0.554962,1313,1239816 +16,32,search,3672.23,1205.18,0.62597,0.89529,0.119038,-1.0,-1.0,156.606,0.555114,1325,1239816 +16,33,insert,134.751,-1.0,-1.0,-1.0,-1.0,40.655,0.574731,1332,1291211 +16,34,search,3754.63,1209.75,0.58402,0.89595,0.118634,-1.0,-1.0,96.799,0.574907,1339,1291211 +16,35,insert,66.629,-1.0,-1.0,-1.0,-1.0,45.457,0.586082,1348,1324978 +16,36,search,3736.79,1223.14,0.623036,0.89787,0.116258,-1.0,-1.0,52.92,0.586082,1352,1324978 +16,37,insert,92.917,-1.0,-1.0,-1.0,-1.0,35.545,0.606319,1355,1371885 +16,38,search,3778.56,1250.97,0.607034,0.89722,0.117355,-1.0,-1.0,25.115,0.606319,1357,1371885 +16,39,insert,18.239,-1.0,-1.0,-1.0,-1.0,15.538,0.610146,1359,1379335 +16,40,search,3809.96,1247.57,0.574779,0.89679,0.118082,-1.0,-1.0,11.53,0.610146,1359,1379335 +16,41,insert,71.735,-1.0,-1.0,-1.0,-1.0,40.759,0.624828,1364,1413312 +16,42,search,3785.31,1269.57,0.644573,0.89786,0.117603,-1.0,-1.0,11.067,0.624828,1364,1413312 +16,43,insert,18.626,-1.0,-1.0,-1.0,-1.0,16.044,0.628473,1366,1423411 +16,44,search,3844.81,1274.95,0.602784,0.89774,0.117697,-1.0,-1.0,18.054,0.628473,1366,1423411 +16,45,insert,341.749,-1.0,-1.0,-1.0,-1.0,46.519,0.644515,1374,1457559 +16,46,search,3974.18,1292.36,0.458881,0.89947,0.116045,-1.0,-1.0,11.558,0.644515,1374,1457559 +16,47,insert,506.119,-1.0,-1.0,-1.0,-1.0,308.225,0.748912,1415,1688593 +16,48,search,3937.23,1415.03,0.643038,0.9027,0.115195,-1.0,-1.0,110.563,0.749405,1425,1688593 +16,49,insert,62.741,-1.0,-1.0,-1.0,-1.0,31.081,0.763859,1428,1723505 +16,50,search,4010.81,1419.92,0.610572,0.90368,0.114848,-1.0,-1.0,46.65,0.76386,1431,1723505 +16,51,insert,503.026,-1.0,-1.0,-1.0,-1.0,266.592,0.904667,1483,2035040 +16,52,search,4255.14,1585.05,0.5986,0.90786,0.111588,-1.0,-1.0,58.248,0.904667,1489,2035040 +16,53,insert,79.282,-1.0,-1.0,-1.0,-1.0,56.206,0.921761,1495,2078341 +16,54,search,4413.42,1593.68,0.523337,0.9077,0.112333,-1.0,-1.0,22.732,0.921763,1496,2078341 +16,55,insert,726.496,-1.0,-1.0,-1.0,-1.0,520.786,1.06983,1569,2399905 +16,56,search,4685.83,1727.96,0.526217,0.91008,0.112374,-1.0,-1.0,184.365,1.07079,1593,2399905 +16,57,insert,43.863,-1.0,-1.0,-1.0,-1.0,35.983,1.07804,1597,2419836 +16,58,search,4695.34,1687.35,0.560566,0.90881,0.113087,-1.0,-1.0,61.224,1.07806,1602,2419836 +16,59,insert,317.248,-1.0,-1.0,-1.0,-1.0,239.614,1.17733,1620,2643083 +16,60,search,4634.47,1730.57,0.613968,0.90048,0.123551,-1.0,-1.0,74.559,1.17747,1625,2643083 +16,61,insert,14.966,-1.0,-1.0,-1.0,-1.0,23.928,1.18108,1628,2653064 +16,62,search,4654.67,1721.19,0.613519,0.90039,0.123363,-1.0,-1.0,46.007,1.18108,1629,2653064 +16,63,insert,6.906,-1.0,-1.0,-1.0,-1.0,13.393,1.18216,1629,2656118 +16,64,search,4793.88,1723.34,0.546362,0.90053,0.123248,-1.0,-1.0,14.845,1.18216,1629,2656118 +16,65,insert,155.212,-1.0,-1.0,-1.0,-1.0,138.071,1.21245,1655,2723424 +16,66,search,4569.15,1743.68,0.659975,0.90144,0.122511,-1.0,-1.0,21.439,1.21245,1656,2723424 +16,67,insert,74.376,-1.0,-1.0,-1.0,-1.0,73.182,1.23228,1667,2769021 +16,68,search,4814.98,1763.99,0.582264,0.90307,0.121137,-1.0,-1.0,23.424,1.23238,1669,2769021 +16,69,insert,83.006,-1.0,-1.0,-1.0,-1.0,77.04,1.2549,1680,2821777 +16,70,search,4823.18,1777.17,0.596263,0.90506,0.119497,-1.0,-1.0,32.98,1.2549,1682,2821777 +16,71,insert,18.471,-1.0,-1.0,-1.0,-1.0,28.471,1.25961,1685,2834921 +16,72,search,4834.46,1783.22,0.594418,0.90494,0.119449,-1.0,-1.0,15.635,1.25961,1685,2834921 +16,73,insert,12.026,-1.0,-1.0,-1.0,-1.0,20.782,1.26303,1686,2842044 +16,74,search,4834.28,1785.39,0.583287,0.905,0.119417,-1.0,-1.0,15.543,1.26303,1686,2842044 +16,75,insert,26.46,-1.0,-1.0,-1.0,-1.0,27.639,1.26892,1688,2858890 +16,76,search,4887.8,1792.61,0.591188,0.90584,0.118653,-1.0,-1.0,20.843,1.26908,1689,2858890 +16,77,insert,43.241,-1.0,-1.0,-1.0,-1.0,14.605,1.27934,1689,2882818 +16,78,search,4886.24,1803.27,0.609436,0.90539,0.119204,-1.0,-1.0,26.663,1.27934,1689,2882818 +16,79,insert,548.593,-1.0,-1.0,-1.0,-1.0,24.469,1.30775,1691,2947806 +16,80,search,4940.3,1833.42,0.556805,0.90537,0.119374,-1.0,-1.0,16.126,1.30775,1691,2947806 +16,81,insert,58.835,-1.0,-1.0,-1.0,-1.0,38.402,1.31962,1694,2976895 +16,82,search,4972.19,1845.27,0.548699,0.90738,0.117193,-1.0,-1.0,39.754,1.31985,1696,2976895 +16,83,insert,32.489,-1.0,-1.0,-1.0,-1.0,53.262,1.32589,1700,2990682 +16,84,search,5002.44,1842.46,0.553189,0.90769,0.116807,-1.0,-1.0,22.656,1.32593,1701,2990682 +16,85,insert,59.175,-1.0,-1.0,-1.0,-1.0,42.931,1.33998,1705,3025613 +16,86,search,4898.37,1855.52,0.60492,0.90778,0.116864,-1.0,-1.0,16.583,1.33998,1705,3025613 +16,87,insert,767.31,-1.0,-1.0,-1.0,-1.0,684.167,1.54372,1807,3468632 +16,88,search,5398.51,1996.52,0.576777,0.91163,0.112764,-1.0,-1.0,133.81,1.54408,1823,3468632 +16,89,insert,512.088,-1.0,-1.0,-1.0,-1.0,533.546,1.70224,1914,3831692 +16,90,search,5632.56,2058.69,0.576022,0.91581,0.108058,-1.0,-1.0,114.917,1.70308,1927,3831692 +16,91,insert,45.967,-1.0,-1.0,-1.0,-1.0,53.022,1.71469,1932,3864067 +16,92,search,5655.79,2053.73,0.592965,0.91599,0.108021,-1.0,-1.0,28.66,1.71469,1932,3864067 +16,93,insert,320.686,-1.0,-1.0,-1.0,-1.0,230.135,1.81799,1965,4103827 +16,94,search,5849.83,2117.22,0.577777,0.91615,0.108323,-1.0,-1.0,118.236,1.81868,1972,4103827 +16,95,insert,35.635,-1.0,-1.0,-1.0,-1.0,55.287,1.82467,1975,4119898 +16,96,search,5938.31,2115.66,0.581457,0.91673,0.107949,-1.0,-1.0,44.931,1.82495,1977,4119898 +16,97,insert,20.705,-1.0,-1.0,-1.0,-1.0,37.086,1.82902,1978,4129367 +16,98,search,5872.13,2114.62,0.6034,0.91663,0.10783,-1.0,-1.0,31.66,1.82902,1978,4129367 +16,99,insert,12.325,-1.0,-1.0,-1.0,-1.0,30.61,1.83103,1978,4134502 +16,100,search,5820.14,2116.75,0.625382,0.91668,0.107799,-1.0,-1.0,31.27,1.83103,1978,4134502 +16,101,insert,57.475,-1.0,-1.0,-1.0,-1.0,47.831,1.84334,1981,4164482 +16,102,search,5738.46,2129.3,0.62206,0.91707,0.107525,-1.0,-1.0,46.131,1.8436,1983,4164482 +16,103,insert,25.27,-1.0,-1.0,-1.0,-1.0,48.071,1.85038,1986,4178601 +16,104,search,5793.29,2132.74,0.600731,0.91713,0.107535,-1.0,-1.0,37.711,1.85056,1987,4178601 +16,105,insert,11.873,-1.0,-1.0,-1.0,-1.0,40.594,1.85378,1989,4187774 +16,106,search,5753.62,2131.1,0.569714,0.91732,0.107382,-1.0,-1.0,36.792,1.85381,1990,4187774 +16,107,insert,26.384,-1.0,-1.0,-1.0,-1.0,30.471,1.85976,1990,4200109 +16,108,search,5682.18,2135.19,0.623803,0.91701,0.107663,-1.0,-1.0,34.446,1.85976,1990,4200109 +16,109,insert,698.569,-1.0,-1.0,-1.0,-1.0,651.196,2.02263,2074,4557867 +16,110,search,6199.29,2207.29,0.574436,0.91943,0.10517,-1.0,-1.0,158.412,2.02343,2086,4557867 +16,111,insert,55.811,-1.0,-1.0,-1.0,-1.0,83.162,2.03652,2095,4587252 +16,112,search,6339.84,2194.57,0.550224,0.91987,0.10465,-1.0,-1.0,36.396,2.03652,2095,4587252 +16,113,insert,20.809,-1.0,-1.0,-1.0,-1.0,44.662,2.04163,2096,4600056 +16,114,search,6309.39,2199.62,0.575326,0.91998,0.104612,-1.0,-1.0,35.985,2.04163,2096,4600056 +16,115,insert,44.993,-1.0,-1.0,-1.0,-1.0,68.986,2.05195,2101,4627422 +16,116,search,6311.47,2207.42,0.571418,0.92046,0.104585,-1.0,-1.0,43.334,2.05201,2102,4627422 +16,117,insert,16.663,-1.0,-1.0,-1.0,-1.0,42.2,2.05581,2103,4639292 +16,118,search,6202.23,2212.04,0.559177,0.92044,0.104345,-1.0,-1.0,36.269,2.05581,2103,4639292 +16,119,insert,5.425,-1.0,-1.0,-1.0,-1.0,42.443,2.05725,2104,4643423 +16,120,search,6225.6,2213.47,0.586058,0.92051,0.104244,-1.0,-1.0,40.686,2.05725,2104,4643423 +16,121,insert,32.965,-1.0,-1.0,-1.0,-1.0,53.254,2.06358,2106,4657252 +16,122,search,6425.24,2217.93,0.5072,0.9205,0.104241,-1.0,-1.0,39.496,2.06358,2106,4657252 +16,123,insert,561.921,-1.0,-1.0,-1.0,-1.0,518.741,2.1905,2161,4938477 +16,124,search,6533.45,2265.37,0.551593,0.92145,0.104814,-1.0,-1.0,257.183,2.19091,2180,4938477 +16,125,insert,104.858,-1.0,-1.0,-1.0,-1.0,86.276,2.20755,2187,4980492 +16,126,search,6526.68,2250.4,0.542127,0.92186,0.104236,-1.0,-1.0,101.913,2.2078,2194,4980492 +16,127,insert,82.796,-1.0,-1.0,-1.0,-1.0,128.381,2.22715,2212,5030515 +16,128,search,6541.78,2244.59,0.576165,0.92202,0.104444,-1.0,-1.0,44.99,2.22733,2213,5030515 +16,129,delete,21.272,-1.0,-1.0,-1.0,-1.0,71.383,2.22527,2179,5009391 +16,130,search,6402.65,2244.97,0.573326,0.92079,0.105098,-1.0,-1.0,37.724,2.22527,2179,5009391 +16,131,delete,6.096,-1.0,-1.0,-1.0,-1.0,37.006,2.22527,2179,5004093 +16,132,search,6416.31,2244.82,0.578844,0.92087,0.105065,-1.0,-1.0,37.375,2.22527,2179,5004093 +16,133,delete,13.569,-1.0,-1.0,-1.0,-1.0,44.018,2.22464,2178,4991787 +16,134,search,6425.04,2236.06,0.550619,0.92082,0.105179,-1.0,-1.0,37.412,2.22464,2178,4991787 +16,135,delete,18.85,-1.0,-1.0,-1.0,-1.0,67.414,2.22224,2175,4974857 +16,136,search,6478.56,2229.62,0.596052,0.92079,0.105335,-1.0,-1.0,42.323,2.22224,2175,4974857 +16,137,delete,10.507,-1.0,-1.0,-1.0,-1.0,37.488,2.22224,2175,4971386 +16,138,search,6488.41,2227.44,0.549725,0.92081,0.105383,-1.0,-1.0,38.477,2.22224,2175,4971386 +16,139,delete,12.554,-1.0,-1.0,-1.0,-1.0,37.799,2.22224,2175,4963487 +16,140,search,6559.85,2222.19,0.558919,0.9207,0.105432,-1.0,-1.0,37.391,2.22224,2175,4963487 +16,141,delete,28.797,-1.0,-1.0,-1.0,-1.0,107.622,2.21712,2168,4941830 +16,142,search,6393.39,2216.68,0.612253,0.92065,0.105705,-1.0,-1.0,37.301,2.21712,2168,4941830 +16,143,delete,19.598,-1.0,-1.0,-1.0,-1.0,112.971,2.21238,2164,4925657 +16,144,search,6385.35,2214.98,0.550079,0.92038,0.105724,-1.0,-1.0,37.345,2.21238,2164,4925657 +16,145,delete,3.585,-1.0,-1.0,-1.0,-1.0,48.467,2.21133,2162,4922686 +16,146,search,6366.52,2215.54,0.598512,0.92003,0.105917,-1.0,-1.0,37.584,2.21133,2162,4922686 +16,147,delete,10.237,-1.0,-1.0,-1.0,-1.0,36.012,2.21133,2162,4913636 +16,148,search,6396.97,2210.63,0.613517,0.92003,0.105861,-1.0,-1.0,37.44,2.21133,2162,4913636 +16,149,delete,229.895,-1.0,-1.0,-1.0,-1.0,1408.22,2.1276,2050,4717250 +16,150,search,6129.75,2240.28,0.598705,0.91978,0.106231,-1.0,-1.0,70.144,2.12806,2055,4717250 +16,151,delete,179.278,-1.0,-1.0,-1.0,-1.0,1164.6,2.0563,1956,4560269 +32,2,search,3152.99,37.5515,1.09161,0.8551,0.130106,-1.0,-1.0,9.845,0.0184881,1000,38806 +32,3,insert,20.979,-1.0,-1.0,-1.0,-1.0,18.146,0.022655,996,49334 +32,4,search,2809.98,63.4255,0.677353,0.86686,0.128427,-1.0,-1.0,19.314,0.0226615,993,49334 +32,5,insert,37.142,-1.0,-1.0,-1.0,-1.0,173.187,0.032935,982,73557 +32,6,search,2751.87,175.313,0.777635,0.92586,0.10314,-1.0,-1.0,123.81,0.032935,982,73557 +32,7,insert,49.092,-1.0,-1.0,-1.0,-1.0,129.267,0.0426217,983,95766 +32,8,search,2761.65,222.921,0.667287,0.92535,0.103423,-1.0,-1.0,86.385,0.0426805,984,95766 +32,9,insert,8.69,-1.0,-1.0,-1.0,-1.0,77.525,0.0446446,984,100025 +32,10,search,2773.43,232.551,0.729023,0.92768,0.100456,-1.0,-1.0,25.301,0.0450518,985,100025 +32,11,insert,18.085,-1.0,-1.0,-1.0,-1.0,12.179,0.0496924,986,111417 +32,12,search,2791.95,268.048,0.665201,0.92761,0.101391,-1.0,-1.0,24.448,0.0498906,988,111417 +32,13,insert,41.374,-1.0,-1.0,-1.0,-1.0,12.136,0.0611151,989,137139 +32,14,search,2807.11,307.155,0.609106,0.9233,0.103527,-1.0,-1.0,10.343,0.0611151,989,137139 +32,15,insert,36.061,-1.0,-1.0,-1.0,-1.0,9.942,0.0697011,989,156710 +32,16,search,2841.7,343.221,0.560725,0.92311,0.103114,-1.0,-1.0,10.494,0.0697011,989,156710 +32,17,insert,9.865,-1.0,-1.0,-1.0,-1.0,9.443,0.0717366,989,161773 +32,18,search,2886.36,363.437,0.547855,0.92423,0.103307,-1.0,-1.0,10.559,0.0717366,989,161773 +32,19,insert,65.892,-1.0,-1.0,-1.0,-1.0,20.761,0.0787309,992,177161 +32,20,search,2889.06,396.114,0.56205,0.92696,0.100685,-1.0,-1.0,12.274,0.0787309,992,177161 +32,21,insert,613.371,-1.0,-1.0,-1.0,-1.0,156.316,0.198936,1009,448898 +32,22,search,2955.49,898.062,0.504195,0.92019,0.101747,-1.0,-1.0,61.885,0.199224,1013,448898 +32,23,insert,783.995,-1.0,-1.0,-1.0,-1.0,366.396,0.328246,1049,733493 +32,24,search,3063.41,1264.86,0.457447,0.91566,0.104648,-1.0,-1.0,68.485,0.328244,1056,733493 +32,25,insert,87.178,-1.0,-1.0,-1.0,-1.0,18.538,0.346121,1057,777443 +32,26,search,3066.36,1290.03,0.480837,0.91614,0.104417,-1.0,-1.0,17.395,0.346121,1057,777443 +32,27,insert,94.104,-1.0,-1.0,-1.0,-1.0,29.762,0.3581,1059,805394 +32,28,search,3068.55,1315.35,0.466007,0.91624,0.1045,-1.0,-1.0,10.929,0.3581,1059,805394 +32,29,insert,119.32,-1.0,-1.0,-1.0,-1.0,60.289,0.384547,1064,866391 +32,30,search,3082.21,1381.23,0.469378,0.91804,0.103832,-1.0,-1.0,32.417,0.384547,1066,866391 +32,31,insert,690.271,-1.0,-1.0,-1.0,-1.0,551.883,0.555044,1126,1239816 +32,32,search,3345.37,1680.42,0.478118,0.90916,0.109836,-1.0,-1.0,183.371,0.555051,1135,1239816 +32,33,insert,145.198,-1.0,-1.0,-1.0,-1.0,11.551,0.575612,1135,1291211 +32,34,search,3333.53,1687.35,0.433124,0.90883,0.110004,-1.0,-1.0,82.311,0.57563,1139,1291211 +32,35,insert,64.531,-1.0,-1.0,-1.0,-1.0,44.355,0.586911,1144,1324978 +32,36,search,3261.72,1714.62,0.471049,0.90924,0.109507,-1.0,-1.0,67.516,0.587002,1150,1324978 +32,37,insert,79.106,-1.0,-1.0,-1.0,-1.0,56.05,0.606628,1156,1371885 +32,38,search,3335.41,1731.06,0.451322,0.90906,0.109021,-1.0,-1.0,17.922,0.606627,1157,1371885 +32,39,insert,15.853,-1.0,-1.0,-1.0,-1.0,9.873,0.610073,1157,1379335 +32,40,search,3366.05,1736.66,0.535322,0.90916,0.109617,-1.0,-1.0,10.584,0.610073,1157,1379335 +32,41,insert,62.141,-1.0,-1.0,-1.0,-1.0,31.523,0.62424,1160,1413312 +32,42,search,3374.14,1764.83,0.504377,0.91017,0.109498,-1.0,-1.0,17.55,0.62424,1160,1413312 +32,43,insert,38.567,-1.0,-1.0,-1.0,-1.0,20.237,0.628433,1161,1423411 +32,44,search,3397.31,1773.5,0.439306,0.91018,0.109458,-1.0,-1.0,10.994,0.628433,1161,1423411 +32,45,insert,260.44,-1.0,-1.0,-1.0,-1.0,42.571,0.643658,1167,1457559 +32,46,search,3373.22,1785.09,0.463988,0.91056,0.108557,-1.0,-1.0,11.086,0.643658,1167,1457559 +32,47,insert,423.869,-1.0,-1.0,-1.0,-1.0,273.435,0.749948,1194,1688593 +32,48,search,3471.22,1937.53,0.447244,0.91201,0.10815,-1.0,-1.0,105.543,0.750168,1203,1688593 +32,49,insert,49.99,-1.0,-1.0,-1.0,-1.0,38.076,0.763668,1206,1723505 +32,50,search,3454.73,1934.4,0.527042,0.91174,0.108089,-1.0,-1.0,38.551,0.763668,1208,1723505 +32,51,insert,429.602,-1.0,-1.0,-1.0,-1.0,352.086,0.904785,1256,2035040 +32,52,search,3628.58,2095.1,0.483117,0.91372,0.107114,-1.0,-1.0,67.437,0.904813,1264,2035040 +32,53,insert,58.234,-1.0,-1.0,-1.0,-1.0,24.359,0.921852,1266,2078341 +32,54,search,3615.88,2107.39,0.533986,0.91343,0.10822,-1.0,-1.0,18.975,0.921959,1267,2078341 +32,55,insert,375.731,-1.0,-1.0,-1.0,-1.0,455.464,1.07141,1320,2399905 +32,56,search,3779.43,2280.94,0.495227,0.91403,0.108017,-1.0,-1.0,134.108,1.07164,1336,2399905 +32,57,insert,25.53,-1.0,-1.0,-1.0,-1.0,15.967,1.07834,1337,2419836 +32,58,search,3829.07,2236.79,0.55455,0.91346,0.10874,-1.0,-1.0,88.426,1.07835,1344,2419836 +32,59,insert,407.146,-1.0,-1.0,-1.0,-1.0,214.552,1.17965,1359,2643083 +32,60,search,3961.96,2306.28,0.41845,0.90639,0.11691,-1.0,-1.0,60.34,1.18002,1362,2643083 +32,61,insert,15.277,-1.0,-1.0,-1.0,-1.0,16.61,1.18332,1363,2653064 +32,62,search,3877.72,2308.89,0.487889,0.90677,0.116483,-1.0,-1.0,28.438,1.18332,1364,2653064 +32,63,insert,6.116,-1.0,-1.0,-1.0,-1.0,17.528,1.18481,1365,2656118 +32,64,search,3921.93,2306.46,0.506081,0.90661,0.116879,-1.0,-1.0,24.976,1.18481,1366,2656118 +32,65,insert,97.344,-1.0,-1.0,-1.0,-1.0,112.638,1.21099,1381,2723424 +32,66,search,4021.02,2335.2,0.482632,0.90789,0.116034,-1.0,-1.0,11.671,1.21099,1381,2723424 +32,67,insert,56.494,-1.0,-1.0,-1.0,-1.0,77.504,1.23026,1390,2769021 +32,68,search,3992.36,2361.96,0.528996,0.90906,0.115757,-1.0,-1.0,19.219,1.23026,1391,2769021 +32,69,insert,70.921,-1.0,-1.0,-1.0,-1.0,102.063,1.25386,1404,2821777 +32,70,search,3990.65,2375.91,0.517398,0.91096,0.113715,-1.0,-1.0,19.771,1.25386,1405,2821777 +32,71,insert,22.685,-1.0,-1.0,-1.0,-1.0,23.472,1.26051,1407,2834921 +32,72,search,4031.11,2385.71,0.547386,0.91158,0.113305,-1.0,-1.0,12.195,1.26051,1407,2834921 +32,73,insert,12.922,-1.0,-1.0,-1.0,-1.0,17.428,1.26464,1408,2842044 +32,74,search,4043.87,2386.59,0.519457,0.91154,0.113341,-1.0,-1.0,11.952,1.26464,1408,2842044 +32,75,insert,26.402,-1.0,-1.0,-1.0,-1.0,36.801,1.27259,1412,2858890 +32,76,search,3320.32,2390.01,0.488242,0.91143,0.113519,-1.0,-1.0,7.408,1.27259,1412,2858890 +32,77,insert,19.017,-1.0,-1.0,-1.0,-1.0,14.661,1.28196,1414,2882818 +32,78,search,2979.92,2401,0.631615,0.91117,0.113574,-1.0,-1.0,7.223,1.28196,1414,2882818 +32,79,insert,319.671,-1.0,-1.0,-1.0,-1.0,21.851,1.3102,1418,2947806 +32,80,search,2981.07,2428.15,0.600424,0.91101,0.114224,-1.0,-1.0,7.528,1.3102,1418,2947806 +32,81,insert,21.277,-1.0,-1.0,-1.0,-1.0,39.747,1.32343,1426,2976895 +32,82,search,3006.92,2431.06,0.642677,0.91189,0.11401,-1.0,-1.0,7.46,1.32343,1426,2976895 +32,83,insert,10.895,-1.0,-1.0,-1.0,-1.0,15.845,1.32988,1428,2990682 +32,84,search,3010.46,2438.48,0.60147,0.91265,0.113296,-1.0,-1.0,7.278,1.32988,1428,2990682 +32,85,insert,22.524,-1.0,-1.0,-1.0,-1.0,48.272,1.34419,1438,3025613 +32,86,search,3073.21,2438.24,0.554873,0.91303,0.113119,-1.0,-1.0,9.622,1.34419,1438,3025613 +32,87,insert,449.234,-1.0,-1.0,-1.0,-1.0,564.093,1.54861,1539,3468632 +32,88,search,3326.41,2555.12,0.578607,0.91498,0.110661,-1.0,-1.0,109.324,1.54901,1552,3468632 +32,89,insert,412.742,-1.0,-1.0,-1.0,-1.0,430.639,1.70968,1636,3831692 +32,90,search,3549.71,2612.39,0.521716,0.91902,0.105289,-1.0,-1.0,49.236,1.70996,1646,3831692 +32,91,insert,20.308,-1.0,-1.0,-1.0,-1.0,34.443,1.72081,1653,3864067 +32,92,search,3441.92,2601.47,0.653902,0.9191,0.105075,-1.0,-1.0,18.533,1.72109,1655,3864067 +32,93,insert,160.597,-1.0,-1.0,-1.0,-1.0,162.336,1.82303,1691,4103827 +32,94,search,3578.61,2669.61,0.631089,0.91971,0.103989,-1.0,-1.0,26.195,1.82325,1695,4103827 +32,95,insert,10.488,-1.0,-1.0,-1.0,-1.0,29.344,1.82952,1701,4119898 +32,96,search,3598.81,2662.1,0.623401,0.9204,0.103501,-1.0,-1.0,9.745,1.82952,1701,4119898 +32,97,insert,6.098,-1.0,-1.0,-1.0,-1.0,12.916,1.83247,1702,4129367 +32,98,search,3583.04,2667.15,0.582576,0.92056,0.103148,-1.0,-1.0,14.945,1.83254,1703,4129367 +32,99,insert,4.394,-1.0,-1.0,-1.0,-1.0,13.348,1.83469,1704,4134502 +32,100,search,3592.21,2667.44,0.600535,0.92049,0.103347,-1.0,-1.0,10.132,1.83469,1704,4134502 +32,101,insert,21.561,-1.0,-1.0,-1.0,-1.0,39.122,1.84626,1711,4164482 +32,102,search,3557.15,2673.6,0.609146,0.92077,0.103167,-1.0,-1.0,15.253,1.84628,1712,4164482 +32,103,insert,8.822,-1.0,-1.0,-1.0,-1.0,29.006,1.8516,1717,4178601 +32,104,search,3632.88,2676.3,0.611925,0.92106,0.103053,-1.0,-1.0,10.378,1.8516,1717,4178601 +32,105,insert,5.905,-1.0,-1.0,-1.0,-1.0,17.712,1.8554,1719,4187774 +32,106,search,3622.74,2679,0.634476,0.92095,0.10334,-1.0,-1.0,10.613,1.8554,1719,4187774 +32,107,insert,10.507,-1.0,-1.0,-1.0,-1.0,10.284,1.8605,1719,4200109 +32,108,search,3496.85,2680.74,0.631482,0.9208,0.103615,-1.0,-1.0,10.286,1.8605,1719,4200109 +32,109,insert,236.43,-1.0,-1.0,-1.0,-1.0,414.991,2.0182,1797,4557867 +32,110,search,3824.08,2751.86,0.624412,0.92327,0.101187,-1.0,-1.0,44.082,2.01856,1804,4557867 +32,111,insert,18.026,-1.0,-1.0,-1.0,-1.0,48.553,2.03147,1813,4587252 +32,112,search,3824.64,2751.52,0.596598,0.92369,0.100851,-1.0,-1.0,17.616,2.03154,1814,4587252 +32,113,insert,7.503,-1.0,-1.0,-1.0,-1.0,28.71,2.03675,1818,4600056 +32,114,search,3826.58,2750.43,0.631511,0.92397,0.100526,-1.0,-1.0,18.993,2.03675,1818,4600056 +32,115,insert,32.211,-1.0,-1.0,-1.0,-1.0,54.411,2.04892,1827,4627422 +32,116,search,3887.49,2752.19,0.589801,0.9246,0.100227,-1.0,-1.0,17.399,2.04916,1828,4627422 +32,117,insert,9.807,-1.0,-1.0,-1.0,-1.0,25.725,2.05452,1830,4639292 +32,118,search,3888.31,2755.85,0.612499,0.92452,0.10023,-1.0,-1.0,17.042,2.05452,1830,4639292 +32,119,insert,4.54,-1.0,-1.0,-1.0,-1.0,23.543,2.05697,1832,4643423 +32,120,search,3890.01,2752.58,0.571857,0.92452,0.10017,-1.0,-1.0,14.071,2.05697,1832,4643423 +32,121,insert,10.759,-1.0,-1.0,-1.0,-1.0,28.499,2.06233,1835,4657252 +32,122,search,3879.81,2756.35,0.610568,0.92474,0.10008,-1.0,-1.0,14.884,2.06233,1835,4657252 +32,123,insert,265.918,-1.0,-1.0,-1.0,-1.0,302.129,2.19105,1883,4938477 +32,124,search,4018.88,2812.42,0.624501,0.92523,0.101005,-1.0,-1.0,127.475,2.19167,1896,4938477 +32,125,insert,49.845,-1.0,-1.0,-1.0,-1.0,43.173,2.20899,1902,4980492 +32,126,search,4048.5,2810.72,0.593577,0.92576,0.100583,-1.0,-1.0,57.557,2.20909,1909,4980492 +32,127,insert,40.787,-1.0,-1.0,-1.0,-1.0,59.317,2.22996,1920,5030515 +32,128,search,4076.3,2815.91,0.645079,0.92675,0.0998028,-1.0,-1.0,25.699,2.23018,1922,5030515 +32,129,delete,11.287,-1.0,-1.0,-1.0,-1.0,30.707,2.22864,1893,5009391 +32,130,search,4016.26,2813.58,0.615433,0.92526,0.100871,-1.0,-1.0,16.425,2.22864,1893,5009391 +32,131,delete,3.149,-1.0,-1.0,-1.0,-1.0,15.971,2.22864,1893,5004093 +32,132,search,3984.39,2809.86,0.61735,0.92521,0.100976,-1.0,-1.0,15.833,2.22864,1893,5004093 +32,133,delete,7.07,-1.0,-1.0,-1.0,-1.0,15.629,2.22828,1893,4991787 +32,134,search,4045.95,2797.66,0.616977,0.92532,0.101013,-1.0,-1.0,18.564,2.22828,1893,4991787 +32,135,delete,22.934,-1.0,-1.0,-1.0,-1.0,48.688,2.22478,1889,4974857 +32,136,search,4050.92,2791.07,0.603725,0.92538,0.100812,-1.0,-1.0,20.261,2.2249,1890,4974857 +32,137,delete,2.717,-1.0,-1.0,-1.0,-1.0,16.051,2.2249,1890,4971386 +32,138,search,4037.52,2785.44,0.647033,0.92526,0.100989,-1.0,-1.0,16.75,2.2249,1890,4971386 +32,139,delete,8.05,-1.0,-1.0,-1.0,-1.0,16.303,2.2249,1890,4963487 +32,140,search,4037.92,2779.6,0.634301,0.92523,0.100936,-1.0,-1.0,16.247,2.2249,1890,4963487 +32,141,delete,15.493,-1.0,-1.0,-1.0,-1.0,62.528,2.22014,1885,4941830 +32,142,search,4041.46,2771.32,0.677359,0.92509,0.100993,-1.0,-1.0,26.282,2.22022,1886,4941830 +32,143,delete,16.51,-1.0,-1.0,-1.0,-1.0,45.137,2.21621,1883,4925657 +32,144,search,4060.78,2764.64,0.623418,0.9251,0.101059,-1.0,-1.0,16.361,2.21621,1883,4925657 +32,145,delete,2.121,-1.0,-1.0,-1.0,-1.0,18.728,2.21579,1882,4922686 +32,146,search,4025.03,2763.33,0.612749,0.92487,0.10134,-1.0,-1.0,16.643,2.21579,1882,4922686 +32,147,delete,6.556,-1.0,-1.0,-1.0,-1.0,15.995,2.21579,1882,4913636 +32,148,search,4073.35,2756.05,0.625526,0.92488,0.101322,-1.0,-1.0,19.815,2.21579,1882,4913636 +32,149,delete,184.94,-1.0,-1.0,-1.0,-1.0,733.934,2.13643,1798,4717250 +32,150,search,3881.7,2759.03,0.547399,0.92341,0.102958,-1.0,-1.0,43.829,2.13711,1804,4717250 +32,151,delete,107.072,-1.0,-1.0,-1.0,-1.0,609.644,2.06459,1728,4560269 diff --git a/scripts/big_ann_perf_numbers/scan_0.12_no_aps_refinment_wma_delete.csv b/scripts/big_ann_perf_numbers/scan_0.12_no_aps_refinment_wma_delete.csv new file mode 100644 index 00000000..2ec7394c --- /dev/null +++ b/scripts/big_ann_perf_numbers/scan_0.12_no_aps_refinment_wma_delete.csv @@ -0,0 +1,1280 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,2819.48,0.82013,0.14515,27.0527,17.7427,6.012,0.0184873,1024,38806 +3,insert,13.18,-1.0,-1.0,-1.0,-1.0,12.711,0.0226399,1020,49334 +4,search,2296.29,0.83661,0.14314,25.2135,17.2457,11.828,0.0226509,1017,49334 +5,insert,18.38,-1.0,-1.0,-1.0,-1.0,173.097,0.0331879,1008,73557 +6,search,2154.01,0.90249,0.119256,18.2191,16.1598,112.693,0.033274,1011,73557 +7,insert,26.243,-1.0,-1.0,-1.0,-1.0,116.238,0.0430081,1011,95766 +8,search,2172.58,0.89956,0.120187,18.1714,15.5465,84.763,0.0431558,1016,95766 +9,insert,4.183,-1.0,-1.0,-1.0,-1.0,72.427,0.0448967,1016,100025 +10,search,2162.64,0.89898,0.11857,18.1642,15.222,80.648,0.0448992,1018,100025 +11,insert,17.777,-1.0,-1.0,-1.0,-1.0,78.882,0.0497682,1020,111417 +12,search,2135.12,0.89918,0.118288,18.097,15.0779,75.841,0.0497682,1020,111417 +13,insert,22.916,-1.0,-1.0,-1.0,-1.0,78.721,0.0609858,1022,137139 +14,search,2158.7,0.89359,0.122992,18.2578,14.9262,17.882,0.0613763,1024,137139 +15,insert,17.269,-1.0,-1.0,-1.0,-1.0,9.732,0.0695815,1026,156710 +16,search,2231.91,0.88962,0.124808,18.4363,14.666,14.858,0.0695844,1028,156710 +17,insert,4.479,-1.0,-1.0,-1.0,-1.0,13.183,0.0718655,1033,161773 +18,search,2233.91,0.88844,0.125566,18.4228,14.6267,17.441,0.0721103,1037,161773 +19,insert,40.501,-1.0,-1.0,-1.0,-1.0,12.538,0.0790973,1041,177161 +20,search,2242.42,0.89172,0.12179,18.3386,14.5539,12.475,0.0791022,1044,177161 +21,insert,358.483,-1.0,-1.0,-1.0,-1.0,206.46,0.202279,1110,448898 +22,search,2510.4,0.87696,0.127124,20.5626,15.6941,61.57,0.202339,1128,448898 +23,insert,377.899,-1.0,-1.0,-1.0,-1.0,268.954,0.329047,1197,733493 +24,search,2712.51,0.871,0.129739,21.4788,16.392,127.525,0.330218,1234,733493 +25,insert,54.435,-1.0,-1.0,-1.0,-1.0,22.158,0.34713,1239,777443 +26,search,2814.86,0.8675,0.132112,21.9584,16.6427,21.56,0.347176,1244,777443 +27,insert,33.089,-1.0,-1.0,-1.0,-1.0,26.448,0.358537,1249,805394 +28,search,2856.33,0.86719,0.132299,21.9418,16.629,7.825,0.358623,1251,805394 +29,insert,31.837,-1.0,-1.0,-1.0,-1.0,43.19,0.38611,1271,866391 +30,search,2888.24,0.87082,0.130932,21.634,16.5702,20.502,0.386117,1276,866391 +31,insert,470.18,-1.0,-1.0,-1.0,-1.0,221.934,0.553713,1361,1239816 +32,search,3008.12,0.86487,0.136168,22.4565,17.1789,50.395,0.553906,1374,1239816 +33,insert,51.52,-1.0,-1.0,-1.0,-1.0,28.172,0.57357,1380,1291211 +34,search,3130.07,0.86611,0.134976,22.4059,17.2331,8.166,0.573626,1381,1291211 +35,insert,23.043,-1.0,-1.0,-1.0,-1.0,23.585,0.585766,1391,1324978 +36,search,3275.37,0.86711,0.133861,22.3074,17.1842,10.215,0.585766,1391,1324978 +37,insert,58.822,-1.0,-1.0,-1.0,-1.0,34.93,0.605394,1399,1371885 +38,search,3265.79,0.86746,0.133755,22.418,17.4658,22.055,0.605443,1402,1371885 +39,insert,10.044,-1.0,-1.0,-1.0,-1.0,11.865,0.608672,1403,1379335 +40,search,3193.58,0.86832,0.133531,22.4442,17.5762,9.879,0.608672,1403,1379335 +41,insert,28.917,-1.0,-1.0,-1.0,-1.0,14.688,0.623574,1407,1413312 +42,search,3092.64,0.86917,0.133389,22.3312,17.6052,9.675,0.623574,1407,1413312 +43,insert,12.135,-1.0,-1.0,-1.0,-1.0,13.73,0.62731,1409,1423411 +44,search,3023.98,0.86981,0.132778,22.3791,17.6557,6.64,0.62731,1409,1423411 +45,insert,150.707,-1.0,-1.0,-1.0,-1.0,26.163,0.641533,1420,1457559 +46,search,3326.98,0.87022,0.132487,22.3876,17.7134,9.838,0.641533,1420,1457559 +47,insert,223.585,-1.0,-1.0,-1.0,-1.0,164.715,0.746675,1465,1688593 +48,search,3386.7,0.87338,0.13363,22.0334,17.8572,94.833,0.746992,1479,1688593 +49,insert,44.358,-1.0,-1.0,-1.0,-1.0,27.304,0.761896,1484,1723505 +50,search,3555.84,0.87546,0.132107,21.943,17.8973,38.566,0.76199,1489,1723505 +51,insert,367.719,-1.0,-1.0,-1.0,-1.0,215.888,0.903029,1553,2035040 +52,search,3701.77,0.88182,0.129171,21.4981,18.166,30.076,0.903121,1563,2035040 +53,insert,30.203,-1.0,-1.0,-1.0,-1.0,21.571,0.920373,1568,2078341 +54,search,3777.61,0.88236,0.12875,21.4871,18.1465,7.054,0.920373,1568,2078341 +55,insert,212.543,-1.0,-1.0,-1.0,-1.0,284.102,1.06805,1642,2399905 +56,search,3917.53,0.88589,0.128152,21.1166,18.2308,164.743,1.06906,1676,2399905 +57,insert,25.215,-1.0,-1.0,-1.0,-1.0,15.761,1.07702,1677,2419836 +58,search,4202.76,0.88436,0.129192,21.3257,18.3327,46.309,1.07702,1681,2419836 +59,insert,228.33,-1.0,-1.0,-1.0,-1.0,139.254,1.17595,1696,2643083 +60,search,4257.76,0.87855,0.135346,22.4206,19.4819,50.639,1.17611,1702,2643083 +61,insert,11.488,-1.0,-1.0,-1.0,-1.0,19.971,1.17992,1704,2653064 +62,search,4274.71,0.87821,0.135799,22.4815,19.562,21.268,1.18006,1706,2653064 +63,insert,4.243,-1.0,-1.0,-1.0,-1.0,12.258,1.18158,1706,2656118 +64,search,4094.16,0.87832,0.13561,22.4734,19.5735,13.148,1.18158,1706,2656118 +65,insert,77.325,-1.0,-1.0,-1.0,-1.0,110.681,1.20883,1738,2723424 +66,search,4376.59,0.87891,0.134528,22.4755,19.5986,16.213,1.20883,1738,2723424 +67,insert,60.422,-1.0,-1.0,-1.0,-1.0,79.027,1.22962,1757,2769021 +68,search,4266.67,0.88005,0.13366,22.3045,19.4955,19.765,1.2297,1758,2769021 +69,insert,61.575,-1.0,-1.0,-1.0,-1.0,66.774,1.24981,1768,2821777 +70,search,4496.96,0.88252,0.132208,22.0513,19.3808,24.376,1.24981,1769,2821777 +71,insert,19.46,-1.0,-1.0,-1.0,-1.0,28.468,1.25513,1772,2834921 +72,search,4189.68,0.88321,0.131664,21.9397,19.3137,17.138,1.25513,1772,2834921 +73,insert,10.807,-1.0,-1.0,-1.0,-1.0,20.768,1.25859,1773,2842044 +74,search,4477.73,0.88311,0.131709,21.929,19.315,16.717,1.25859,1773,2842044 +75,insert,25.405,-1.0,-1.0,-1.0,-1.0,33.817,1.26723,1777,2858890 +76,search,4215.89,0.88334,0.13162,21.8792,19.2534,16.564,1.26723,1777,2858890 +77,insert,34.616,-1.0,-1.0,-1.0,-1.0,18.521,1.27747,1778,2882818 +78,search,4500.46,0.88334,0.131658,21.9509,19.3011,16.559,1.27747,1778,2882818 +79,insert,530.73,-1.0,-1.0,-1.0,-1.0,48.219,1.30615,1787,2947806 +80,search,4556.32,0.88298,0.131908,22.0778,19.4783,18.191,1.30615,1787,2947806 +81,insert,48.941,-1.0,-1.0,-1.0,-1.0,24.089,1.31915,1789,2976895 +82,search,4333.07,0.8841,0.130863,21.8776,19.3785,20.582,1.31917,1790,2976895 +83,insert,14.381,-1.0,-1.0,-1.0,-1.0,24.704,1.32435,1792,2990682 +84,search,4457.43,0.88504,0.130283,21.8483,19.3678,25.76,1.3245,1794,2990682 +85,insert,51.476,-1.0,-1.0,-1.0,-1.0,44.603,1.33889,1800,3025613 +86,search,4438.05,0.88595,0.130118,21.7465,19.3817,12.008,1.33889,1800,3025613 +87,insert,497.437,-1.0,-1.0,-1.0,-1.0,701.709,1.54462,1933,3468632 +88,search,4889.31,0.88965,0.127362,21.2515,19.1936,107.801,1.54488,1949,3468632 +89,insert,391.713,-1.0,-1.0,-1.0,-1.0,392.827,1.70077,2035,3831692 +90,search,5252.61,0.89506,0.121518,20.6746,18.7578,69.155,1.70146,2049,3831692 +91,insert,22.394,-1.0,-1.0,-1.0,-1.0,42.981,1.71228,2056,3864067 +92,search,4938.63,0.89495,0.121311,20.6944,18.7848,31.649,1.71228,2057,3864067 +93,insert,215.562,-1.0,-1.0,-1.0,-1.0,326.612,1.82009,2106,4103827 +94,search,5370.82,0.89548,0.121323,20.5744,18.7796,48.045,1.82029,2112,4103827 +95,insert,15.42,-1.0,-1.0,-1.0,-1.0,47.88,1.82701,2118,4119898 +96,search,5114.47,0.89602,0.121042,20.5233,18.7499,31.491,1.82729,2120,4119898 +97,insert,6.268,-1.0,-1.0,-1.0,-1.0,34.852,1.83058,2123,4129367 +98,search,5175.03,0.89571,0.120922,20.5378,18.748,24.91,1.83058,2123,4129367 +99,insert,4.421,-1.0,-1.0,-1.0,-1.0,26.125,1.83218,2123,4134502 +100,search,5729.91,0.89572,0.121015,20.514,18.7408,29.912,1.83218,2123,4134502 +101,insert,45.061,-1.0,-1.0,-1.0,-1.0,64.225,1.84414,2130,4164482 +102,search,5752.75,0.89616,0.121057,20.4725,18.7317,30.308,1.84414,2130,4164482 +103,insert,25.77,-1.0,-1.0,-1.0,-1.0,54.016,1.85165,2136,4178601 +104,search,5605.08,0.89655,0.120745,20.4561,18.7379,33.101,1.85175,2137,4178601 +105,insert,14.593,-1.0,-1.0,-1.0,-1.0,45.172,1.85609,2140,4187774 +106,search,5645.7,0.89622,0.121203,20.458,18.7306,30.578,1.85609,2140,4187774 +107,insert,17.451,-1.0,-1.0,-1.0,-1.0,35.381,1.8606,2141,4200109 +108,search,5581.3,0.89597,0.121323,20.4694,18.7457,30.218,1.8606,2141,4200109 +109,insert,437.681,-1.0,-1.0,-1.0,-1.0,520.513,2.02021,2225,4557867 +110,search,4713.71,0.89929,0.119237,20.1499,18.689,57.936,2.02104,2237,4557867 +111,insert,18.159,-1.0,-1.0,-1.0,-1.0,39.396,2.03279,2246,4587252 +112,search,4660.93,0.89944,0.119275,20.0991,18.684,22.528,2.03279,2246,4587252 +113,insert,8.706,-1.0,-1.0,-1.0,-1.0,24.725,2.03725,2247,4600056 +114,search,4720.91,0.89953,0.119331,20.0614,18.6614,22.553,2.03725,2247,4600056 +115,insert,22.244,-1.0,-1.0,-1.0,-1.0,27.979,2.04577,2249,4627422 +116,search,4691.98,0.90004,0.119231,19.982,18.6513,22.529,2.04577,2249,4627422 +117,insert,8.554,-1.0,-1.0,-1.0,-1.0,23.569,2.04938,2250,4639292 +118,search,4760.08,0.9006,0.11878,19.9803,18.6632,33.454,2.04969,2253,4639292 +119,insert,4.719,-1.0,-1.0,-1.0,-1.0,21.473,2.05197,2253,4643423 +120,search,4763.12,0.9006,0.118805,19.9626,18.6546,22.498,2.05197,2253,4643423 +121,insert,10.998,-1.0,-1.0,-1.0,-1.0,33.532,2.05722,2257,4657252 +122,search,4730.68,0.9008,0.118614,19.9484,18.6672,22.379,2.05722,2257,4657252 +123,insert,178.663,-1.0,-1.0,-1.0,-1.0,285.88,2.18607,2313,4938477 +124,search,4848.87,0.90183,0.119048,19.662,18.4961,88.466,2.18677,2332,4938477 +125,insert,30.379,-1.0,-1.0,-1.0,-1.0,64.452,2.20566,2346,4980492 +126,search,4938.48,0.90217,0.118781,19.6551,18.5636,23.903,2.20566,2346,4980492 +127,insert,32.023,-1.0,-1.0,-1.0,-1.0,69.858,2.22426,2364,5030515 +128,search,4950.72,0.90283,0.118098,19.5708,18.5454,14.426,2.22426,2364,5030515 +129,delete,11.967,-1.0,-1.0,-1.0,-1.0,15.817,2.22355,2358,5009391 +130,search,4902.7,0.90244,0.118391,19.6048,18.5991,15.755,2.22355,2358,5009391 +131,delete,3.887,-1.0,-1.0,-1.0,-1.0,14.104,2.22355,2358,5004093 +132,search,4943.46,0.9026,0.118315,19.5881,18.5922,15.588,2.22355,2358,5004093 +133,delete,6.931,-1.0,-1.0,-1.0,-1.0,14.008,2.22355,2358,4991787 +134,search,4847.19,0.9025,0.118344,19.6048,18.5937,15.748,2.22355,2358,4991787 +135,delete,10.638,-1.0,-1.0,-1.0,-1.0,22.151,2.22185,2357,4974857 +136,search,4944.82,0.90264,0.118379,19.6113,18.6162,24.214,2.22185,2357,4974857 +137,delete,2.046,-1.0,-1.0,-1.0,-1.0,22.582,2.22185,2357,4971386 +138,search,4923.29,0.90253,0.118506,19.6256,18.6276,23.777,2.22185,2357,4971386 +139,delete,4.214,-1.0,-1.0,-1.0,-1.0,22.395,2.22185,2357,4963487 +140,search,4895.77,0.90231,0.118352,19.6339,18.6318,23.706,2.22185,2357,4963487 +141,delete,11.353,-1.0,-1.0,-1.0,-1.0,22.51,2.2199,2357,4941830 +142,search,4926.02,0.90252,0.118553,19.6331,18.6513,24.427,2.2199,2357,4941830 +143,delete,8.871,-1.0,-1.0,-1.0,-1.0,34.134,2.21751,2355,4925657 +144,search,4872.89,0.9026,0.118484,19.6404,18.6479,25.076,2.21751,2355,4925657 +145,delete,2.015,-1.0,-1.0,-1.0,-1.0,22.489,2.21722,2355,4922686 +146,search,4913.85,0.90236,0.118653,19.6571,18.64,24.832,2.21722,2355,4922686 +147,delete,6.5,-1.0,-1.0,-1.0,-1.0,22.361,2.21722,2355,4913636 +148,search,4903.9,0.90236,0.118518,19.6823,18.6604,24.63,2.21722,2355,4913636 +149,delete,128.055,-1.0,-1.0,-1.0,-1.0,83.697,2.1865,2343,4717250 +150,search,4826.65,0.90153,0.119573,19.7834,18.7481,29.14,2.1866,2344,4717250 +151,delete,108.357,-1.0,-1.0,-1.0,-1.0,106.191,2.14494,2328,4560269 +152,search,4635.3,0.90037,0.120542,19.9054,18.7933,24.168,2.14494,2328,4560269 +153,delete,18.871,-1.0,-1.0,-1.0,-1.0,50.224,2.13495,2322,4521936 +154,search,4698.99,0.89931,0.121292,20.0455,18.8794,23.448,2.13495,2322,4521936 +155,delete,11.207,-1.0,-1.0,-1.0,-1.0,30.19,2.13131,2321,4502222 +156,search,4670.8,0.89919,0.121341,20.0752,18.9093,23.337,2.13131,2321,4502222 +157,delete,23.244,-1.0,-1.0,-1.0,-1.0,35.858,2.12403,2319,4456039 +158,search,4667.83,0.89859,0.121772,20.1359,18.9751,23.349,2.12403,2319,4456039 +159,delete,193.098,-1.0,-1.0,-1.0,-1.0,304.645,1.98503,2264,4143922 +160,search,4421.26,0.89674,0.123659,20.3788,19.1921,23.323,1.98503,2264,4143922 +161,delete,31.146,-1.0,-1.0,-1.0,-1.0,21.357,1.97923,2264,4101445 +162,search,4459.94,0.89699,0.123295,20.3778,19.1848,23.152,1.97923,2264,4101445 +163,delete,13.655,-1.0,-1.0,-1.0,-1.0,26.354,1.97421,2263,4080175 +164,search,4445.82,0.89664,0.123391,20.4191,19.1965,23.159,1.97421,2263,4080175 +165,delete,22.685,-1.0,-1.0,-1.0,-1.0,21.183,1.96716,2263,4047313 +166,search,4434.14,0.8967,0.123494,20.3819,19.165,23.269,1.96716,2263,4047313 +167,delete,6.114,-1.0,-1.0,-1.0,-1.0,21.242,1.96479,2263,4040738 +168,search,4424.85,0.89662,0.123369,20.4128,19.1746,22.855,1.96479,2263,4040738 +169,delete,13.683,-1.0,-1.0,-1.0,-1.0,35.281,1.95767,2261,4022674 +170,search,4436.82,0.89635,0.123576,20.4644,19.2106,22.854,1.95767,2261,4022674 +171,delete,5.216,-1.0,-1.0,-1.0,-1.0,21.128,1.95566,2261,4015835 +172,search,4425.38,0.89617,0.123823,20.4661,19.2149,22.617,1.95566,2261,4015835 +173,delete,23.436,-1.0,-1.0,-1.0,-1.0,21.331,1.94754,2261,3986424 +174,search,4370.75,0.89595,0.124146,20.5282,19.2546,22.848,1.94754,2261,3986424 +175,delete,115.968,-1.0,-1.0,-1.0,-1.0,306.028,1.86541,2200,3804693 +176,search,4264.88,0.89452,0.124553,20.8343,19.4217,27.521,1.86541,2201,3804693 +177,delete,13.017,-1.0,-1.0,-1.0,-1.0,20.4,1.86073,2201,3786352 +178,search,4284.2,0.89415,0.124764,20.8761,19.4317,22.392,1.86073,2201,3786352 +179,delete,179.046,-1.0,-1.0,-1.0,-1.0,242.681,1.74394,2131,3556994 +180,search,4064.63,0.88846,0.12721,21.6756,19.6591,28.95,1.74416,2133,3556994 +181,delete,22.23,-1.0,-1.0,-1.0,-1.0,21.615,1.73511,2132,3526824 +182,search,4059.59,0.88824,0.127371,21.696,19.701,22.024,1.73511,2132,3526824 +183,delete,121.409,-1.0,-1.0,-1.0,-1.0,258.727,1.65509,2069,3350245 +184,search,3895.89,0.87925,0.134225,22.8945,20.2389,21.317,1.65509,2069,3350245 +185,delete,9.303,-1.0,-1.0,-1.0,-1.0,20.277,1.6511,2067,3340234 +186,search,3914.18,0.88097,0.132111,22.7058,20.1374,20.819,1.6511,2067,3340234 +187,delete,133.301,-1.0,-1.0,-1.0,-1.0,183.5,1.55699,2030,3150857 +188,search,3811.2,0.88318,0.129679,22.1821,19.5923,22.833,1.55705,2031,3150857 +189,delete,5.023,-1.0,-1.0,-1.0,-1.0,18.663,1.5555,2031,3143609 +190,search,3815.84,0.88275,0.130293,22.2171,19.6008,20.258,1.5555,2031,3143609 +191,delete,1.825,-1.0,-1.0,-1.0,-1.0,18.796,1.55495,2031,3141038 +192,search,3769.44,0.88274,0.13033,22.2083,19.5878,20.059,1.55495,2031,3141038 +193,delete,42.987,-1.0,-1.0,-1.0,-1.0,19.325,1.53067,2031,3081734 +194,search,3774.83,0.88114,0.131431,22.411,19.7055,20.131,1.53067,2031,3081734 +195,delete,25.929,-1.0,-1.0,-1.0,-1.0,30.756,1.51323,2029,3043966 +196,search,3744.61,0.8801,0.13236,22.502,19.7694,20.978,1.51323,2029,3043966 +197,delete,29.364,-1.0,-1.0,-1.0,-1.0,40.625,1.4988,2027,3008145 +198,search,3717.92,0.87957,0.1325,22.5923,19.8148,20.707,1.4988,2027,3008145 +199,delete,9.255,-1.0,-1.0,-1.0,-1.0,18.851,1.49494,2027,2999384 +200,search,3754.83,0.87934,0.132473,22.6517,19.8478,20.662,1.49494,2027,2999384 +201,delete,5.092,-1.0,-1.0,-1.0,-1.0,19.077,1.49249,2027,2993528 +202,search,3747.34,0.8791,0.132723,22.6819,19.8518,20.495,1.49249,2027,2993528 +203,delete,9.951,-1.0,-1.0,-1.0,-1.0,18.612,1.48903,2027,2980973 +204,search,3737.37,0.87914,0.13257,22.6908,19.851,20.777,1.48903,2027,2980973 +205,delete,18.476,-1.0,-1.0,-1.0,-1.0,18.915,1.48045,2027,2960291 +206,search,3741.85,0.87901,0.132325,22.7402,19.8635,20.599,1.48045,2027,2960291 +207,delete,33.148,-1.0,-1.0,-1.0,-1.0,30.9,1.45967,2018,2920620 +208,search,3706.92,0.87831,0.132892,22.7719,19.8303,20.415,1.45967,2018,2920620 +209,delete,15.331,-1.0,-1.0,-1.0,-1.0,26.808,1.45225,2017,2900501 +210,search,3699.98,0.8776,0.13303,22.9155,19.8649,20.137,1.45225,2017,2900501 +211,delete,6.094,-1.0,-1.0,-1.0,-1.0,18.518,1.44952,2017,2891660 +212,search,3680.45,0.87727,0.133236,22.9495,19.881,20.019,1.44952,2017,2891660 +213,delete,12.097,-1.0,-1.0,-1.0,-1.0,24.91,1.44233,2015,2873257 +214,search,3646.65,0.87625,0.133945,23.0074,19.8618,19.835,1.44233,2015,2873257 +215,delete,245.835,-1.0,-1.0,-1.0,-1.0,369.569,1.25737,1917,2537571 +216,search,3448.23,0.87204,0.136502,23.6432,20.101,26.946,1.25748,1918,2537571 +217,delete,157.588,-1.0,-1.0,-1.0,-1.0,202.672,1.14403,1870,2308194 +218,search,3300.66,0.86822,0.139188,24.0721,20.319,18.971,1.14403,1870,2308194 +219,delete,20.527,-1.0,-1.0,-1.0,-1.0,16.564,1.13825,1870,2285820 +220,search,3278.25,0.86703,0.140095,24.1672,20.3359,18.396,1.13825,1870,2285820 +221,delete,157.829,-1.0,-1.0,-1.0,-1.0,207.851,1.02589,1796,2082858 +222,search,3130.7,0.86358,0.14281,24.5167,20.4545,17.733,1.02589,1796,2082858 +223,delete,9.87,-1.0,-1.0,-1.0,-1.0,19.581,1.02192,1795,2071207 +224,search,3136.21,0.8617,0.144283,24.7043,20.5445,17.318,1.02192,1795,2071207 +225,delete,4.402,-1.0,-1.0,-1.0,-1.0,15.189,1.02081,1795,2065400 +226,search,3146.03,0.86158,0.144343,24.7423,20.5608,16.879,1.02081,1795,2065400 +227,delete,2.693,-1.0,-1.0,-1.0,-1.0,15.322,1.01992,1795,2062105 +228,search,3084.84,0.8615,0.144419,24.7481,20.5643,17.225,1.01992,1795,2062105 +229,delete,15.748,-1.0,-1.0,-1.0,-1.0,21.324,1.01234,1793,2039610 +230,search,3104.05,0.86037,0.144683,24.9508,20.6028,17.42,1.01234,1793,2039610 +231,delete,9.929,-1.0,-1.0,-1.0,-1.0,22.408,1.00468,1790,2027795 +232,search,3080.79,0.85944,0.14515,25.0359,20.6046,17.028,1.00468,1790,2027795 +233,delete,3.634,-1.0,-1.0,-1.0,-1.0,15.206,1.00306,1790,2022859 +234,search,3088.1,0.85916,0.145319,25.0434,20.605,16.878,1.00306,1790,2022859 +235,delete,5.658,-1.0,-1.0,-1.0,-1.0,16.987,1.00002,1787,2016264 +236,search,3084.33,0.85902,0.145337,25.1024,20.6355,16.72,1.00002,1787,2016264 +237,delete,172.193,-1.0,-1.0,-1.0,-1.0,214.49,0.88084,1718,1782866 +238,search,2991.91,0.85515,0.147735,25.6267,20.94,16.267,0.88084,1718,1782866 +239,delete,14.972,-1.0,-1.0,-1.0,-1.0,35.638,0.869252,1711,1758931 +240,search,2950.99,0.85281,0.148894,25.9156,21.0402,16.376,0.869252,1711,1758931 +241,delete,5.591,-1.0,-1.0,-1.0,-1.0,19.934,0.86711,1710,1751275 +242,search,2921.09,0.85238,0.148785,25.9883,21.0705,16.2,0.86711,1710,1751275 +243,delete,12.757,-1.0,-1.0,-1.0,-1.0,14.508,0.862142,1710,1733264 +244,search,2954.12,0.8509,0.149299,26.2022,21.0935,17.984,0.862142,1710,1733264 +245,delete,11.347,-1.0,-1.0,-1.0,-1.0,19.458,0.858286,1709,1722658 +246,search,2970.42,0.85048,0.149365,26.3065,21.1166,16.587,0.858286,1709,1722658 +247,delete,3.644,-1.0,-1.0,-1.0,-1.0,14.437,0.857302,1709,1719601 +248,search,2960.62,0.85038,0.149441,26.3419,21.1347,16.295,0.857302,1709,1719601 +249,delete,7.603,-1.0,-1.0,-1.0,-1.0,14.651,0.853426,1709,1711648 +250,search,2965.1,0.84991,0.149603,26.399,21.1728,16.759,0.853426,1709,1711648 +251,delete,191.205,-1.0,-1.0,-1.0,-1.0,240.409,0.726617,1615,1470624 +252,search,2745.07,0.84819,0.150632,26.4705,21.1024,31.914,0.726617,1617,1470624 +253,delete,21.313,-1.0,-1.0,-1.0,-1.0,19.239,0.715733,1615,1442070 +254,search,2785.41,0.84723,0.150899,26.5916,21.1194,15.598,0.715733,1615,1442070 +255,delete,29.45,-1.0,-1.0,-1.0,-1.0,16.174,0.700021,1613,1405549 +256,search,2775.65,0.8435,0.153008,27.0613,21.2757,21.912,0.700022,1614,1405549 +257,insert,6.455,-1.0,-1.0,-1.0,-1.0,13.897,0.702302,1614,1416288 +258,search,2730,0.84379,0.152949,27.0388,21.25,15.228,0.702302,1614,1416288 +259,insert,16.849,-1.0,-1.0,-1.0,-1.0,18.465,0.708545,1615,1443010 +260,search,2771.65,0.84407,0.15316,27.0032,21.3093,15.336,0.708545,1615,1443010 +261,insert,1.332,-1.0,-1.0,-1.0,-1.0,13.901,0.708683,1615,1447529 +262,search,2780.88,0.84467,0.152665,26.923,21.297,15.17,0.708683,1615,1447529 +263,insert,82.053,-1.0,-1.0,-1.0,-1.0,42.04,0.747569,1620,1569110 +264,search,2753.05,0.84634,0.153442,26.517,21.3223,20.036,0.747569,1621,1569110 +265,insert,45.073,-1.0,-1.0,-1.0,-1.0,14.2,0.767608,1621,1636119 +266,search,2834.41,0.85003,0.151125,26.1002,21.22,20.436,0.767614,1622,1636119 +267,insert,23.653,-1.0,-1.0,-1.0,-1.0,24.479,0.778431,1624,1665152 +268,search,2843.15,0.8522,0.150407,25.7405,21.1295,15.339,0.778431,1624,1665152 +269,insert,179.541,-1.0,-1.0,-1.0,-1.0,86.152,0.872611,1636,1916952 +270,search,2906.08,0.8546,0.148651,25.4714,21.0758,57.422,0.872739,1643,1916952 +271,insert,4.016,-1.0,-1.0,-1.0,-1.0,14.439,0.874215,1643,1920898 +272,search,2917.88,0.85488,0.148663,25.4171,21.042,24.846,0.874214,1645,1920898 +273,insert,19.591,-1.0,-1.0,-1.0,-1.0,18.006,0.882635,1646,1942757 +274,search,2916.2,0.85682,0.147718,25.1912,20.9736,15.934,0.882635,1646,1942757 +275,insert,324.501,-1.0,-1.0,-1.0,-1.0,168.27,1.04517,1678,2328372 +276,search,3029.32,0.86068,0.144136,24.8361,20.7816,70.954,1.0453,1689,2328372 +277,insert,29.94,-1.0,-1.0,-1.0,-1.0,14.321,1.0578,1689,2363257 +278,search,3085.88,0.86179,0.143461,24.7037,20.7305,25.649,1.0578,1691,2363257 +279,insert,20.055,-1.0,-1.0,-1.0,-1.0,14.081,1.06657,1691,2388933 +280,search,3073.99,0.86278,0.142819,24.6311,20.6921,15.355,1.06657,1691,2388933 +281,insert,220.393,-1.0,-1.0,-1.0,-1.0,105.255,1.20017,1706,2684315 +282,search,3141.13,0.86403,0.142598,24.5614,20.9276,41.283,1.20017,1710,2684315 +283,insert,342.882,-1.0,-1.0,-1.0,-1.0,174.833,1.38747,1745,3109157 +284,search,3344.7,0.87019,0.138965,23.8041,20.4843,79.716,1.38747,1757,3109157 +285,insert,10.529,-1.0,-1.0,-1.0,-1.0,14.924,1.39245,1757,3122751 +286,search,3395.91,0.87077,0.138835,23.7482,20.4923,29.338,1.39245,1760,3122751 +287,insert,27.482,-1.0,-1.0,-1.0,-1.0,14.86,1.40753,1760,3163679 +288,search,3387.1,0.87172,0.13846,23.6567,20.4411,18.372,1.40753,1761,3163679 +289,insert,221.028,-1.0,-1.0,-1.0,-1.0,158.647,1.54547,1790,3472031 +290,search,3567.86,0.87504,0.136598,23.2109,20.4557,80.302,1.5461,1803,3472031 +291,insert,18.089,-1.0,-1.0,-1.0,-1.0,15.331,1.55739,1803,3495577 +292,search,3608.18,0.87742,0.134492,23.0026,20.3295,26.45,1.55739,1805,3495577 +293,insert,11.772,-1.0,-1.0,-1.0,-1.0,26.963,1.56647,1808,3515193 +294,search,3594.81,0.87735,0.134767,22.9777,20.363,16.365,1.56647,1808,3515193 +295,insert,146.339,-1.0,-1.0,-1.0,-1.0,109.324,1.65865,1826,3723063 +296,search,3720.62,0.87962,0.133864,22.6467,20.2314,48.564,1.65889,1833,3723063 +297,insert,3.528,-1.0,-1.0,-1.0,-1.0,15.685,1.66076,1833,3727399 +298,search,3702.38,0.87946,0.133991,22.6362,20.2483,21.989,1.66076,1834,3727399 +299,insert,56.42,-1.0,-1.0,-1.0,-1.0,24.948,1.68585,1838,3788002 +300,search,3748.67,0.88188,0.131786,22.463,20.1491,21.038,1.68601,1840,3788002 +301,insert,450.33,-1.0,-1.0,-1.0,-1.0,404.458,1.94921,1953,4370866 +302,search,4155.34,0.90387,0.118005,19.6158,18.9384,58.062,1.94949,1969,4370866 +303,insert,29.288,-1.0,-1.0,-1.0,-1.0,42.342,1.96941,1974,4420306 +304,search,4163.12,0.90345,0.118311,19.6595,18.9396,16.471,1.96941,1974,4420306 +305,insert,231.341,-1.0,-1.0,-1.0,-1.0,260.382,2.13414,2020,4789669 +306,search,4330.44,0.90573,0.116557,19.359,18.8201,97.64,2.13465,2040,4789669 +307,insert,17.287,-1.0,-1.0,-1.0,-1.0,15.495,2.14404,2040,4815550 +308,search,4350.13,0.90587,0.116318,19.3133,18.7665,21.474,2.14404,2041,4815550 +309,insert,4.274,-1.0,-1.0,-1.0,-1.0,19.111,2.14616,2042,4823345 +310,search,4368.32,0.90615,0.116205,19.3108,18.7724,27.172,2.14661,2046,4823345 +311,insert,9.427,-1.0,-1.0,-1.0,-1.0,15.669,2.15296,2046,4843493 +312,search,4322.01,0.90582,0.116514,19.3463,18.8024,16.578,2.15296,2046,4843493 +313,insert,28.979,-1.0,-1.0,-1.0,-1.0,42.769,2.17409,2053,4885489 +314,search,4378.38,0.90599,0.116292,19.2962,18.7571,39.588,2.17437,2058,4885489 +315,insert,4.493,-1.0,-1.0,-1.0,-1.0,16.041,2.17686,2058,4892335 +316,search,4386.26,0.90582,0.116402,19.3257,18.7959,16.383,2.17686,2058,4892335 +317,insert,32.431,-1.0,-1.0,-1.0,-1.0,43.58,2.19544,2066,4932996 +318,search,4425.59,0.90595,0.116691,19.2491,18.7427,20.671,2.1955,2067,4932996 +319,insert,399.197,-1.0,-1.0,-1.0,-1.0,429.918,2.41217,2156,5430684 +320,search,4713.33,0.90959,0.113436,18.6932,18.4726,115.317,2.41337,2182,5430684 +321,insert,31.001,-1.0,-1.0,-1.0,-1.0,50.625,2.42984,2195,5471239 +322,search,4684.16,0.9102,0.113118,18.6372,18.4358,36.371,2.42991,2198,5471239 +323,insert,76.398,-1.0,-1.0,-1.0,-1.0,121.529,2.47705,2233,5575560 +324,search,4848.07,0.91027,0.113123,18.5625,18.431,30.676,2.47731,2237,5575560 +325,insert,948.436,-1.0,-1.0,-1.0,-1.0,574.684,2.70384,2328,6069499 +326,search,5110.68,0.91354,0.111268,18.0326,18.193,148.689,2.70507,2358,6069499 +327,insert,42.552,-1.0,-1.0,-1.0,-1.0,60.403,2.73491,2369,6139647 +328,search,5183.73,0.91392,0.111331,17.9149,18.1294,45.508,2.73508,2377,6139647 +329,insert,5.042,-1.0,-1.0,-1.0,-1.0,12.083,2.73776,2377,6146884 +330,search,5160.34,0.91398,0.111124,17.8869,18.0966,13.944,2.73776,2377,6146884 +331,insert,27.059,-1.0,-1.0,-1.0,-1.0,47.104,2.75537,2386,6188816 +332,search,5284.84,0.91401,0.111048,17.887,18.1108,20.871,2.75549,2388,6188816 +333,insert,1.743,-1.0,-1.0,-1.0,-1.0,11.949,2.7559,2388,6190810 +334,search,5227.51,0.91401,0.11112,17.8851,18.1072,13.715,2.7559,2388,6190810 +335,insert,35.515,-1.0,-1.0,-1.0,-1.0,47.165,2.77262,2401,6230108 +336,search,5283.66,0.91432,0.110481,17.8633,18.0816,13.337,2.77262,2401,6230108 +337,insert,15.593,-1.0,-1.0,-1.0,-1.0,17.643,2.7802,2402,6249699 +338,search,5317.58,0.91465,0.110236,17.8187,18.0696,13.673,2.7802,2402,6249699 +339,insert,32.396,-1.0,-1.0,-1.0,-1.0,54.677,2.79596,2414,6290226 +340,search,5296.5,0.91533,0.109644,17.7376,17.9928,15.801,2.79602,2415,6290226 +341,insert,7.6,-1.0,-1.0,-1.0,-1.0,28.822,2.80017,2420,6297556 +342,search,5274.29,0.9155,0.109586,17.7026,17.9711,17.671,2.80018,2421,6297556 +343,insert,23.79,-1.0,-1.0,-1.0,-1.0,62.141,2.81502,2435,6330843 +344,search,5317.94,0.91562,0.109584,17.681,17.9766,20.477,2.81512,2437,6330843 +345,insert,28.843,-1.0,-1.0,-1.0,-1.0,45.01,2.83073,2448,6370265 +346,search,5374.02,0.91566,0.109531,17.6431,17.9511,18.69,2.83092,2449,6370265 +347,insert,392.628,-1.0,-1.0,-1.0,-1.0,576.514,3.03809,2588,6831086 +348,search,5753.14,0.91878,0.107222,17.2214,17.7598,107.14,3.04053,2616,6831086 +349,insert,30.735,-1.0,-1.0,-1.0,-1.0,53.212,3.05502,2629,6876154 +350,search,5739.4,0.91902,0.107097,17.2034,17.7477,21.113,3.05519,2630,6876154 +351,insert,259.337,-1.0,-1.0,-1.0,-1.0,413.16,3.23634,2718,7288266 +352,search,5993.3,0.918,0.108436,17.2102,17.6943,134.83,3.23832,2747,7288266 +353,insert,34.408,-1.0,-1.0,-1.0,-1.0,64.88,3.25846,2762,7333132 +354,search,6096.04,0.91806,0.108716,17.1544,17.6919,29.676,3.25846,2764,7333132 +355,insert,248.269,-1.0,-1.0,-1.0,-1.0,424.316,3.42183,2857,7696838 +356,search,6328.89,0.91954,0.107359,16.8386,17.4891,128.651,3.42376,2884,7696838 +357,insert,13.956,-1.0,-1.0,-1.0,-1.0,25.225,3.42962,2885,7714498 +358,search,6340.62,0.92014,0.107181,16.7756,17.439,37.746,3.42995,2891,7714498 +359,insert,9.669,-1.0,-1.0,-1.0,-1.0,26.288,3.43463,2893,7725755 +360,search,6387.01,0.92023,0.107122,16.7623,17.435,25.245,3.4348,2894,7725755 +361,insert,49.946,-1.0,-1.0,-1.0,-1.0,81.173,3.46246,2911,7789312 +362,search,6446.64,0.9211,0.106309,16.6174,17.3231,34.044,3.46292,2914,7789312 +363,insert,13.065,-1.0,-1.0,-1.0,-1.0,28.441,3.46777,2916,7803163 +364,search,6409.49,0.92126,0.106147,16.6027,17.3237,25.953,3.4678,2917,7803163 +365,insert,43.161,-1.0,-1.0,-1.0,-1.0,95.711,3.49232,2936,7863722 +366,search,6501.1,0.92189,0.10578,16.5382,17.2557,31.687,3.49248,2939,7863722 +367,insert,20.848,-1.0,-1.0,-1.0,-1.0,47.739,3.5051,2947,7895271 +368,search,6464.21,0.92231,0.105384,16.4794,17.2476,43.167,3.50559,2953,7895271 +369,insert,45.486,-1.0,-1.0,-1.0,-1.0,75.163,3.53114,2969,7961194 +370,search,6606.33,0.92276,0.104887,16.4305,17.2181,36.086,3.53153,2973,7961194 +371,insert,8.405,-1.0,-1.0,-1.0,-1.0,29.211,3.53516,2975,7970239 +372,search,6563.45,0.92302,0.104819,16.4187,17.2194,28.076,3.53537,2976,7970239 +373,insert,33.875,-1.0,-1.0,-1.0,-1.0,76.463,3.55161,2993,8010868 +374,search,6611.28,0.92338,0.104613,16.3686,17.1938,43.118,3.55243,2999,8010868 +375,insert,23.42,-1.0,-1.0,-1.0,-1.0,57.409,3.56424,3008,8040698 +376,search,6589.53,0.92343,0.104626,16.3219,17.1802,26.891,3.56447,3009,8040698 +377,insert,48.118,-1.0,-1.0,-1.0,-1.0,70.227,3.59258,3024,8104748 +378,search,6661.02,0.92402,0.10433,16.2207,17.1264,45.839,3.59342,3031,8104748 +379,insert,4.61,-1.0,-1.0,-1.0,-1.0,24.208,3.59637,3032,8109755 +380,search,6507.83,0.92399,0.104537,16.2237,17.1369,23.226,3.59637,3032,8109755 +381,insert,17.13,-1.0,-1.0,-1.0,-1.0,30.115,3.6053,3034,8130382 +382,search,6613.44,0.92429,0.104421,16.1941,17.136,33.58,3.60551,3037,8130382 +383,insert,18.332,-1.0,-1.0,-1.0,-1.0,34.139,3.61265,3041,8153290 +384,search,6583.82,0.92417,0.104416,16.1712,17.1088,23.402,3.61265,3041,8153290 +385,delete,9.626,-1.0,-1.0,-1.0,-1.0,23.899,3.61274,3041,8138873 +386,search,6662.56,0.92429,0.104363,16.1585,17.1077,23.218,3.61274,3041,8138873 +387,delete,13.844,-1.0,-1.0,-1.0,-1.0,21.758,3.61093,3041,8115069 +388,search,6621.21,0.92436,0.104261,16.1369,17.0894,22.527,3.61093,3041,8115069 +389,delete,9.509,-1.0,-1.0,-1.0,-1.0,21.637,3.61076,3041,8101157 +390,search,6663.5,0.92427,0.104427,16.1291,17.0901,23.027,3.61076,3041,8101157 +391,delete,55.533,-1.0,-1.0,-1.0,-1.0,97.083,3.5896,3034,8014644 +392,search,6573.87,0.92393,0.104185,16.2281,17.0745,38.48,3.58979,3037,8014644 +393,delete,21.682,-1.0,-1.0,-1.0,-1.0,33.913,3.5881,3036,7978358 +394,search,6616.64,0.92357,0.104264,16.2751,17.0997,28.252,3.5881,3037,7978358 +395,delete,14.187,-1.0,-1.0,-1.0,-1.0,47.181,3.58104,3034,7957094 +396,search,6555.08,0.92351,0.104265,16.2972,17.1093,23.817,3.58104,3034,7957094 +397,delete,78.959,-1.0,-1.0,-1.0,-1.0,125.744,3.547,3026,7823916 +398,search,6587.48,0.92355,0.104334,16.2962,17.1308,35.386,3.54721,3028,7823916 +399,delete,3.5,-1.0,-1.0,-1.0,-1.0,21.887,3.54721,3028,7818430 +400,search,6471.4,0.92349,0.104547,16.3008,17.1457,23.386,3.54721,3028,7818430 +401,delete,13.963,-1.0,-1.0,-1.0,-1.0,28.775,3.54402,3026,7800702 +402,search,6511.05,0.92329,0.104705,16.2919,17.1072,27.165,3.54412,3027,7800702 +403,delete,211.735,-1.0,-1.0,-1.0,-1.0,354.656,3.46246,2984,7517858 +404,search,6369.86,0.92226,0.105562,16.4731,17.2222,45.303,3.46302,2988,7517858 +405,delete,43.205,-1.0,-1.0,-1.0,-1.0,22.379,3.45942,2988,7451637 +406,search,6393.05,0.92236,0.105882,16.4494,17.2125,37.529,3.45974,2990,7451637 +407,delete,67.804,-1.0,-1.0,-1.0,-1.0,91.244,3.43441,2981,7360417 +408,search,6276.86,0.92208,0.106151,16.5097,17.2482,24.179,3.43441,2981,7360417 +409,delete,151.569,-1.0,-1.0,-1.0,-1.0,370.266,3.33999,2941,7131620 +410,search,6188.32,0.92191,0.105934,16.5616,17.2924,40.32,3.34004,2944,7131620 +411,delete,170.885,-1.0,-1.0,-1.0,-1.0,246.517,3.26409,2911,6867617 +412,search,5989.61,0.92112,0.106403,16.71,17.4123,27.642,3.26425,2912,6867617 +413,delete,12.987,-1.0,-1.0,-1.0,-1.0,21.12,3.26068,2912,6850098 +414,search,6057.78,0.92102,0.106415,16.7236,17.415,22.908,3.26068,2912,6850098 +415,delete,51.212,-1.0,-1.0,-1.0,-1.0,36.342,3.24227,2910,6782921 +416,search,6074.6,0.92088,0.106441,16.7495,17.4063,23.222,3.24227,2910,6782921 +417,delete,163.508,-1.0,-1.0,-1.0,-1.0,216.486,3.17649,2885,6562039 +418,search,5899.48,0.92057,0.106596,16.8039,17.4025,27.774,3.17667,2886,6562039 +419,delete,19.038,-1.0,-1.0,-1.0,-1.0,20.742,3.17271,2886,6536285 +420,search,5940.68,0.92055,0.10687,16.7943,17.4164,22.773,3.17271,2886,6536285 +421,delete,14.742,-1.0,-1.0,-1.0,-1.0,67.033,3.16346,2880,6510820 +422,search,5886.74,0.9206,0.106942,16.7772,17.4234,22.163,3.16346,2880,6510820 +423,delete,123.252,-1.0,-1.0,-1.0,-1.0,219.771,3.09202,2841,6334168 +424,search,5844.47,0.91968,0.107306,16.932,17.5263,22.323,3.09202,2841,6334168 +425,delete,16.823,-1.0,-1.0,-1.0,-1.0,20.065,3.08629,2841,6316443 +426,search,5750.99,0.91975,0.107274,16.9362,17.5664,22.66,3.08629,2841,6316443 +427,delete,45.275,-1.0,-1.0,-1.0,-1.0,20.307,3.07287,2841,6266345 +428,search,5815.04,0.91938,0.107645,17.0024,17.6113,22.647,3.07287,2841,6266345 +429,delete,371.157,-1.0,-1.0,-1.0,-1.0,308.726,2.89414,2781,5883847 +430,search,5511.1,0.9118,0.112421,18.0883,18.0513,33.338,2.8942,2783,5883847 +431,delete,64.132,-1.0,-1.0,-1.0,-1.0,125.418,2.85541,2765,5801380 +432,search,5446,0.9114,0.112888,18.1802,18.1542,22.066,2.85541,2765,5801380 +433,delete,145.931,-1.0,-1.0,-1.0,-1.0,337.764,2.76279,2727,5607567 +434,search,5375.56,0.90997,0.113676,18.3842,18.2312,38.084,2.76279,2729,5607567 +435,delete,82.531,-1.0,-1.0,-1.0,-1.0,35.338,2.72037,2726,5512176 +436,search,5331.14,0.9103,0.113314,18.3825,18.1838,36.526,2.72037,2728,5512176 +437,delete,10.128,-1.0,-1.0,-1.0,-1.0,32.441,2.71378,2727,5501141 +438,search,5307.26,0.9104,0.113317,18.357,18.1713,31.746,2.71378,2729,5501141 +439,delete,81.769,-1.0,-1.0,-1.0,-1.0,235.392,2.65255,2701,5369822 +440,search,5273.92,0.9099,0.113918,18.3904,18.1751,21.454,2.65255,2701,5369822 +441,delete,26.566,-1.0,-1.0,-1.0,-1.0,28.292,2.64012,2700,5338985 +442,search,5281.28,0.91066,0.112931,18.3401,18.1587,21.556,2.64012,2700,5338985 +443,delete,26.233,-1.0,-1.0,-1.0,-1.0,71.928,2.62514,2691,5307279 +444,search,5241.34,0.91018,0.113416,18.351,18.1364,21.281,2.62514,2691,5307279 +445,delete,32.484,-1.0,-1.0,-1.0,-1.0,22.359,2.61099,2690,5268382 +446,search,5218.38,0.91007,0.113422,18.3545,18.1094,20.919,2.61099,2690,5268382 +447,delete,283.568,-1.0,-1.0,-1.0,-1.0,311.703,2.44073,2617,4922787 +448,search,5016.43,0.90661,0.115778,18.8917,18.4635,19.99,2.44073,2617,4922787 +449,delete,30.299,-1.0,-1.0,-1.0,-1.0,18.274,2.42687,2617,4887227 +450,search,5037.85,0.90653,0.115758,18.8956,18.4511,20.94,2.42687,2617,4887227 +451,delete,88.039,-1.0,-1.0,-1.0,-1.0,96.046,2.38218,2600,4795082 +452,search,4951.65,0.90584,0.116415,19.0182,18.545,25.102,2.38235,2601,4795082 +453,delete,241.883,-1.0,-1.0,-1.0,-1.0,568.526,2.23352,2535,4500175 +454,search,4802.01,0.90362,0.118217,19.3241,18.744,85.502,2.23453,2543,4500175 +455,delete,31.898,-1.0,-1.0,-1.0,-1.0,25.816,2.2211,2542,4462648 +456,search,4772.79,0.90313,0.118316,19.4233,18.765,46.6,2.2211,2546,4462648 +457,delete,5.921,-1.0,-1.0,-1.0,-1.0,18.368,2.218,2546,4456761 +458,search,4803.16,0.90319,0.118192,19.4133,18.7674,26.117,2.218,2547,4456761 +459,delete,29.77,-1.0,-1.0,-1.0,-1.0,46.07,2.19848,2541,4419992 +460,search,4740.36,0.90301,0.118142,19.4714,18.8161,19.647,2.19848,2541,4419992 +461,delete,3.905,-1.0,-1.0,-1.0,-1.0,17.434,2.19698,2541,4415725 +462,search,4749.81,0.90301,0.118151,19.4547,18.7936,20.125,2.19698,2541,4415725 +463,delete,40.827,-1.0,-1.0,-1.0,-1.0,33.113,2.17563,2533,4370609 +464,search,4745.63,0.90217,0.118865,19.5389,18.8217,20.05,2.17563,2533,4370609 +465,delete,19.516,-1.0,-1.0,-1.0,-1.0,48.43,2.16387,2526,4345244 +466,search,4768.58,0.90165,0.119196,19.6592,18.8869,19.896,2.16387,2526,4345244 +467,delete,24.059,-1.0,-1.0,-1.0,-1.0,26.543,2.15404,2524,4319734 +468,search,4680.2,0.90095,0.119631,19.747,18.9275,19.757,2.15404,2524,4319734 +469,delete,16.12,-1.0,-1.0,-1.0,-1.0,36.118,2.14379,2519,4298361 +470,search,4704.45,0.90038,0.120023,19.8688,18.963,19.711,2.14379,2519,4298361 +471,delete,93.577,-1.0,-1.0,-1.0,-1.0,137.657,2.08388,2493,4188315 +472,search,4614.57,0.90016,0.120049,19.9511,19.0217,19.607,2.08388,2493,4188315 +473,delete,100.8,-1.0,-1.0,-1.0,-1.0,77.109,2.03248,2485,4074567 +474,search,4560.41,0.89965,0.120228,19.9979,19.0247,19.014,2.03248,2485,4074567 +475,delete,298.575,-1.0,-1.0,-1.0,-1.0,498.839,1.81887,2382,3691698 +476,search,4278.05,0.89501,0.124531,20.4571,19.2513,24.311,1.81907,2383,3691698 +477,delete,38.226,-1.0,-1.0,-1.0,-1.0,47.747,1.79992,2378,3642619 +478,search,4241.19,0.89459,0.124366,20.4724,19.2384,18.83,1.79992,2378,3642619 +479,delete,177.284,-1.0,-1.0,-1.0,-1.0,209.453,1.70522,2343,3421368 +480,search,4219.18,0.89438,0.124762,20.7086,19.4544,40.627,1.70522,2345,3421368 +481,delete,44.302,-1.0,-1.0,-1.0,-1.0,57.744,1.6844,2339,3378034 +482,search,4183.44,0.89415,0.124403,20.7203,19.4082,30.774,1.6844,2340,3378034 +483,delete,220.117,-1.0,-1.0,-1.0,-1.0,417.85,1.53445,2251,3115682 +484,search,3987.97,0.89109,0.125805,21.2638,19.7639,22.819,1.53445,2251,3115682 +485,delete,20.17,-1.0,-1.0,-1.0,-1.0,20.409,1.52863,2251,3094629 +486,search,3978.01,0.89107,0.125804,21.2578,19.7474,22.452,1.52863,2251,3094629 +487,delete,7.958,-1.0,-1.0,-1.0,-1.0,20.326,1.52635,2251,3086025 +488,search,3985.35,0.89081,0.125651,21.3205,19.757,22.542,1.52635,2251,3086025 +489,delete,27.968,-1.0,-1.0,-1.0,-1.0,20.545,1.51272,2251,3052105 +490,search,3931.46,0.89082,0.125455,21.3158,19.7487,22.455,1.51272,2251,3052105 +491,delete,10.966,-1.0,-1.0,-1.0,-1.0,26.058,1.50697,2249,3038974 +492,search,3910.51,0.89033,0.125482,21.4023,19.7797,23.532,1.50697,2249,3038974 +493,delete,107.845,-1.0,-1.0,-1.0,-1.0,126.473,1.44188,2231,2911063 +494,search,3857.27,0.88928,0.126265,21.5477,19.8369,21.727,1.44188,2231,2911063 +495,delete,17.588,-1.0,-1.0,-1.0,-1.0,28.953,1.43277,2228,2888785 +496,search,3878.22,0.88904,0.126532,21.5763,19.8066,23.131,1.43277,2228,2888785 +497,delete,55.098,-1.0,-1.0,-1.0,-1.0,52.812,1.40605,2218,2829718 +498,search,3860.99,0.88796,0.127378,21.7236,19.9286,23.274,1.40605,2218,2829718 +499,delete,18.283,-1.0,-1.0,-1.0,-1.0,25.672,1.40021,2217,2816082 +500,search,3869.31,0.88817,0.127224,21.7198,19.9164,22.361,1.40021,2217,2816082 +501,delete,25.638,-1.0,-1.0,-1.0,-1.0,39.187,1.38658,2213,2790880 +502,search,3855.82,0.88782,0.127605,21.7274,19.9472,22.819,1.38658,2213,2790880 +503,delete,15.861,-1.0,-1.0,-1.0,-1.0,19.787,1.38226,2213,2774459 +504,search,3840.28,0.88743,0.127828,21.7727,19.9551,22.28,1.38226,2213,2774459 +505,delete,47.021,-1.0,-1.0,-1.0,-1.0,79.519,1.34889,2199,2714683 +506,search,3839.39,0.88651,0.127664,21.9351,20.0096,22.157,1.34889,2199,2714683 +507,delete,29.974,-1.0,-1.0,-1.0,-1.0,46.549,1.33398,2191,2680986 +508,search,3755.68,0.88649,0.127324,21.9707,19.9876,21.827,1.33398,2191,2680986 +509,delete,15.196,-1.0,-1.0,-1.0,-1.0,22.862,1.32667,2190,2662966 +510,search,3789.24,0.88633,0.127474,22.0082,20.014,21.335,1.32667,2190,2662966 +511,delete,17.794,-1.0,-1.0,-1.0,-1.0,25.071,1.31822,2187,2642855 +512,search,3770.99,0.88587,0.128033,22.0562,20.0217,21.238,1.31822,2187,2642855 +513,insert,216.618,-1.0,-1.0,-1.0,-1.0,84.719,1.40184,2197,2906838 +514,search,3831.24,0.88583,0.128548,22.1189,20.1942,30.931,1.40197,2198,2906838 +515,insert,4.568,-1.0,-1.0,-1.0,-1.0,19.893,1.40303,2198,2913746 +516,search,3842.25,0.88589,0.12851,22.0965,20.1848,22.801,1.40303,2198,2913746 +517,insert,13.374,-1.0,-1.0,-1.0,-1.0,19.915,1.40599,2198,2936862 +518,search,3853.7,0.88644,0.128309,22.0379,20.1478,22.327,1.40599,2198,2936862 +519,insert,64.845,-1.0,-1.0,-1.0,-1.0,70.39,1.43109,2206,3009732 +520,search,3868.04,0.8877,0.128545,21.736,20.0812,43.392,1.43109,2210,3009732 +521,insert,10.486,-1.0,-1.0,-1.0,-1.0,20.569,1.43398,2210,3025906 +522,search,3890.39,0.88838,0.128311,21.6857,20.0501,22.663,1.43398,2210,3025906 +523,insert,28.977,-1.0,-1.0,-1.0,-1.0,34.614,1.44497,2212,3061899 +524,search,3877.83,0.88868,0.128471,21.5428,19.9892,28.86,1.44497,2213,3061899 +525,insert,28.344,-1.0,-1.0,-1.0,-1.0,32.159,1.45396,2215,3101209 +526,search,3888.43,0.88903,0.128127,21.4934,19.9562,22.532,1.45396,2215,3101209 +527,insert,380.829,-1.0,-1.0,-1.0,-1.0,144.479,1.6099,2239,3522334 +528,search,4069.67,0.89323,0.125636,20.9131,19.6622,41.523,1.6099,2242,3522334 +529,insert,213.606,-1.0,-1.0,-1.0,-1.0,112.67,1.71417,2254,3797294 +530,search,4234.4,0.89954,0.122822,19.9691,19.4243,102.32,1.71438,2263,3797294 +531,insert,50.109,-1.0,-1.0,-1.0,-1.0,21.041,1.73313,2263,3853781 +532,search,4238.7,0.89982,0.122781,19.9106,19.39,47.67,1.73313,2265,3853781 +533,insert,20.1,-1.0,-1.0,-1.0,-1.0,21.105,1.7397,2265,3873352 +534,search,4280.65,0.89994,0.122536,19.8717,19.3805,45.836,1.7397,2267,3873352 +535,insert,21.354,-1.0,-1.0,-1.0,-1.0,21.183,1.74695,2267,3893308 +536,search,4275.22,0.90055,0.122122,19.8322,19.3467,48.125,1.74695,2271,3893308 +537,insert,4.631,-1.0,-1.0,-1.0,-1.0,21.289,1.74874,2271,3901271 +538,search,4295.21,0.90035,0.122144,19.8387,19.3397,28.844,1.74874,2272,3901271 +539,insert,22.96,-1.0,-1.0,-1.0,-1.0,25.574,1.75654,2273,3925164 +540,search,4293.07,0.90047,0.122097,19.8243,19.3296,23.161,1.75654,2273,3925164 +541,insert,384.242,-1.0,-1.0,-1.0,-1.0,198.598,1.92758,2310,4320915 +542,search,4499.9,0.9038,0.119133,19.367,19.0611,42.613,1.92777,2314,4320915 +543,insert,33.782,-1.0,-1.0,-1.0,-1.0,21.396,1.94385,2314,4360814 +544,search,4480.11,0.90422,0.118577,19.3291,19.037,22.539,1.94385,2314,4360814 +545,insert,7.044,-1.0,-1.0,-1.0,-1.0,20.868,1.94689,2314,4370220 +546,search,4479.82,0.90431,0.118595,19.3191,19.028,23.428,1.94689,2314,4370220 +547,insert,129.649,-1.0,-1.0,-1.0,-1.0,44.573,2.00618,2323,4510675 +548,search,4557.1,0.90652,0.116442,19.0128,18.8588,23.019,2.00618,2323,4510675 +549,insert,215.424,-1.0,-1.0,-1.0,-1.0,71.1,2.1066,2334,4744845 +550,search,4671.34,0.90236,0.121089,19.8253,19.7451,35.14,2.1066,2336,4744845 +551,insert,18.923,-1.0,-1.0,-1.0,-1.0,44.352,2.11815,2339,4771433 +552,search,4659.4,0.90216,0.12128,19.8288,19.7386,23.616,2.11815,2339,4771433 +553,insert,12.069,-1.0,-1.0,-1.0,-1.0,21.351,2.12274,2339,4782909 +554,search,4669.05,0.90243,0.121182,19.7827,19.7131,23.493,2.12274,2339,4782909 +555,insert,365.151,-1.0,-1.0,-1.0,-1.0,132.842,2.29217,2369,5180322 +556,search,4884.25,0.90891,0.114775,19.1523,19.3571,31.407,2.29242,2373,5180322 +557,insert,23.265,-1.0,-1.0,-1.0,-1.0,15.551,2.30357,2373,5203220 +558,search,4846.38,0.90929,0.114494,19.1117,19.3478,16.52,2.30357,2373,5203220 +559,insert,5.903,-1.0,-1.0,-1.0,-1.0,14.952,2.3064,2373,5210154 +560,search,4880.57,0.90951,0.114367,19.11,19.3595,16.997,2.3064,2373,5210154 +561,insert,5.952,-1.0,-1.0,-1.0,-1.0,15.006,2.3087,2373,5215426 +562,search,4893.18,0.90959,0.114279,19.1028,19.3568,16.976,2.3087,2373,5215426 +563,insert,41.468,-1.0,-1.0,-1.0,-1.0,22.02,2.32189,2374,5252726 +564,search,4896.46,0.90971,0.114186,19.0857,19.3503,16.894,2.32189,2374,5252726 +565,insert,196.409,-1.0,-1.0,-1.0,-1.0,82.645,2.40987,2386,5447274 +566,search,4968.44,0.9101,0.11344,19.0621,19.3445,32.146,2.40991,2388,5447274 +567,insert,11.764,-1.0,-1.0,-1.0,-1.0,15.626,2.41591,2388,5463135 +568,search,5015.08,0.90994,0.113561,19.0865,19.3717,22.62,2.41591,2389,5463135 +569,insert,12.888,-1.0,-1.0,-1.0,-1.0,15.238,2.42097,2389,5478168 +570,search,5015.61,0.90989,0.113536,19.0676,19.3416,17.023,2.42097,2389,5478168 +571,insert,13.664,-1.0,-1.0,-1.0,-1.0,23.146,2.42736,2390,5492309 +572,search,4959.01,0.90948,0.113815,19.1322,19.4145,16.591,2.42736,2390,5492309 +573,insert,53.979,-1.0,-1.0,-1.0,-1.0,47.167,2.45387,2397,5554599 +574,search,5005.37,0.91006,0.113295,19.048,19.3634,16.594,2.45387,2397,5554599 +575,insert,42.931,-1.0,-1.0,-1.0,-1.0,35.424,2.47493,2401,5604500 +576,search,5060.18,0.91059,0.113141,18.9662,19.3076,17.441,2.47493,2401,5604500 +577,insert,35.974,-1.0,-1.0,-1.0,-1.0,41.395,2.48897,2410,5638834 +578,search,5106.52,0.91101,0.112814,18.9403,19.3168,25.76,2.48926,2412,5638834 +579,insert,568.125,-1.0,-1.0,-1.0,-1.0,623.967,2.76616,2555,6229778 +580,search,5471.26,0.91415,0.109214,18.5775,19.1598,55.993,2.76635,2563,6229778 +581,insert,25.363,-1.0,-1.0,-1.0,-1.0,22.009,2.77694,2564,6261240 +582,search,5475.41,0.91407,0.109449,18.5712,19.1734,27.141,2.77703,2565,6261240 +583,insert,19.461,-1.0,-1.0,-1.0,-1.0,22.48,2.78533,2566,6283315 +584,search,5481.88,0.91405,0.109621,18.5519,19.1683,18.267,2.78533,2566,6283315 +585,insert,210.653,-1.0,-1.0,-1.0,-1.0,213.292,2.91052,2603,6556101 +586,search,5611,0.91535,0.108988,18.268,18.9414,84.965,2.91143,2620,6556101 +587,insert,25.757,-1.0,-1.0,-1.0,-1.0,41.196,2.92609,2626,6587187 +588,search,5658.93,0.91582,0.10845,18.2069,18.9133,27.821,2.92649,2628,6587187 +589,insert,47.739,-1.0,-1.0,-1.0,-1.0,39.051,2.94943,2633,6631526 +590,search,5707.69,0.91604,0.108224,18.1832,18.9107,24.102,2.94943,2634,6631526 +591,insert,401.419,-1.0,-1.0,-1.0,-1.0,406.047,3.1542,2726,7108942 +592,search,6049.9,0.91912,0.106213,17.7528,18.7515,84.503,3.15512,2740,7108942 +593,insert,143.626,-1.0,-1.0,-1.0,-1.0,178.711,3.24043,2773,7303976 +594,search,6160.48,0.92026,0.105245,17.512,18.5989,77.824,3.24093,2786,7303976 +595,insert,4.68,-1.0,-1.0,-1.0,-1.0,26.905,3.24271,2788,7309244 +596,search,6139.19,0.92031,0.105209,17.505,18.6021,24.794,3.24271,2789,7309244 +597,insert,159.787,-1.0,-1.0,-1.0,-1.0,214.74,3.34756,2831,7542477 +598,search,6269.41,0.92211,0.104185,17.2125,18.4905,83.117,3.34799,2847,7542477 +599,insert,93.319,-1.0,-1.0,-1.0,-1.0,154.705,3.39511,2884,7654438 +600,search,6454.26,0.9227,0.103902,17.1277,18.4557,42.568,3.39574,2890,7654438 +601,insert,52.34,-1.0,-1.0,-1.0,-1.0,60.675,3.42099,2901,7717936 +602,search,6445.1,0.92327,0.103223,17.0446,18.3802,24.408,3.42106,2902,7717936 +603,insert,6.521,-1.0,-1.0,-1.0,-1.0,18.778,3.4237,2902,7723521 +604,search,6436.21,0.92321,0.103306,17.0501,18.3823,24.987,3.42373,2903,7723521 +605,insert,43.141,-1.0,-1.0,-1.0,-1.0,67.409,3.44283,2916,7767404 +606,search,6473.86,0.92325,0.103147,17.0378,18.3863,23.632,3.44289,2917,7767404 +607,insert,42.044,-1.0,-1.0,-1.0,-1.0,44.809,3.46557,2924,7826115 +608,search,6404.35,0.92286,0.103586,17.0766,18.4001,29.926,3.46591,2926,7826115 +609,insert,145.003,-1.0,-1.0,-1.0,-1.0,214.242,3.55411,2964,8036351 +610,search,6589.82,0.92377,0.103196,16.8412,18.2472,71.459,3.55522,2977,8036351 +611,insert,17.316,-1.0,-1.0,-1.0,-1.0,51.176,3.56409,2985,8060364 +612,search,6698,0.92411,0.102971,16.8145,18.2273,31.359,3.56429,2988,8060364 +613,insert,158.522,-1.0,-1.0,-1.0,-1.0,242.084,3.65251,3035,8256083 +614,search,6821.4,0.92479,0.102962,16.6784,18.1367,70.959,3.65379,3050,8256083 +615,insert,284.625,-1.0,-1.0,-1.0,-1.0,482.9,3.8257,3137,8625707 +616,search,7052.77,0.92679,0.101109,16.325,17.9032,167.788,3.82824,3176,8625707 +617,insert,244.245,-1.0,-1.0,-1.0,-1.0,413.471,3.97388,3253,8949227 +618,search,7236.15,0.92827,0.100513,16.023,17.777,167.016,3.97571,3286,8949227 +619,insert,164.41,-1.0,-1.0,-1.0,-1.0,206.491,4.07759,3326,9195551 +620,search,7414.81,0.92908,0.100124,15.9455,17.7272,111.4,4.07959,3350,9195551 +621,insert,28.922,-1.0,-1.0,-1.0,-1.0,62.65,4.09275,3360,9232372 +622,search,7499.19,0.92951,0.0998432,15.8898,17.7093,33.405,4.09307,3363,9232372 +623,insert,20.06,-1.0,-1.0,-1.0,-1.0,35.626,4.10154,3367,9253561 +624,search,7487.6,0.9298,0.099624,15.8472,17.6899,36.533,4.1018,3370,9253561 +625,insert,58.792,-1.0,-1.0,-1.0,-1.0,102.762,4.13549,3387,9331230 +626,search,7472.93,0.9303,0.0992441,15.7396,17.637,61.726,4.13592,3396,9331230 +627,insert,38.4,-1.0,-1.0,-1.0,-1.0,93.687,4.15357,3415,9374063 +628,search,7566.73,0.93044,0.0992346,15.7047,17.6331,30.966,4.15366,3417,9374063 +629,insert,11.209,-1.0,-1.0,-1.0,-1.0,42.914,4.16235,3421,9387598 +630,search,7563.45,0.93059,0.0990341,15.676,17.602,24.018,4.16235,3421,9387598 +631,insert,29.034,-1.0,-1.0,-1.0,-1.0,65.991,4.17584,3430,9421724 +632,search,7624.4,0.93073,0.0988847,15.6555,17.5843,32.226,4.17603,3433,9421724 +633,insert,37.846,-1.0,-1.0,-1.0,-1.0,77.568,4.1924,3451,9461430 +634,search,7455.1,0.93087,0.0986654,15.604,17.5313,30.964,4.19244,3453,9461430 +635,insert,20.955,-1.0,-1.0,-1.0,-1.0,43.175,4.20168,3458,9479469 +636,search,7618.31,0.93082,0.0988289,15.5745,17.5238,22.974,4.20168,3458,9479469 +637,insert,354.854,-1.0,-1.0,-1.0,-1.0,602.255,4.3818,3585,9885997 +638,search,7889.22,0.93277,0.0976526,15.1912,17.3042,125.037,4.38387,3614,9885997 +639,insert,352.908,-1.0,-1.0,-1.0,-1.0,588.143,4.56701,3747,10292043 +640,search,8241.55,0.9349,0.0958349,14.8576,17.1239,120.571,4.56941,3771,10292043 +641,delete,142.933,-1.0,-1.0,-1.0,-1.0,235.89,4.52861,3751,10096584 +642,search,8222.28,0.93484,0.0958622,14.8691,17.1068,27.634,4.52861,3751,10096584 +643,delete,7.853,-1.0,-1.0,-1.0,-1.0,25.147,4.52861,3751,10084840 +644,search,8200.7,0.93481,0.0959495,14.8736,17.1182,27.394,4.52861,3751,10084840 +645,delete,11.082,-1.0,-1.0,-1.0,-1.0,25.018,4.52861,3751,10069802 +646,search,8193.05,0.93474,0.0960169,14.8784,17.121,28.711,4.52861,3751,10069802 +647,delete,56.577,-1.0,-1.0,-1.0,-1.0,191.543,4.50509,3737,9990953 +648,search,8236.14,0.93459,0.0962715,14.8839,17.1118,29.59,4.50509,3737,9990953 +649,delete,28.536,-1.0,-1.0,-1.0,-1.0,37.769,4.50331,3736,9951625 +650,search,8177.89,0.93455,0.0965273,14.8786,17.1195,28.177,4.50331,3736,9951625 +651,delete,15.628,-1.0,-1.0,-1.0,-1.0,24.947,4.50153,3736,9927065 +652,search,8208.91,0.93437,0.09662,14.9451,17.1622,28.257,4.50153,3736,9927065 +653,delete,70.201,-1.0,-1.0,-1.0,-1.0,171.759,4.47255,3719,9831098 +654,search,8035.3,0.93415,0.0967345,14.9914,17.1863,28.773,4.47255,3719,9831098 +655,delete,296.053,-1.0,-1.0,-1.0,-1.0,461.123,4.37901,3665,9462172 +656,search,7874.09,0.93252,0.0981123,15.2711,17.3412,32.483,4.37909,3666,9462172 +657,delete,178.075,-1.0,-1.0,-1.0,-1.0,168.042,4.31189,3613,9218902 +658,search,7754.15,0.92942,0.100426,15.7002,17.5837,26.847,4.31189,3613,9218902 +659,delete,86.669,-1.0,-1.0,-1.0,-1.0,196.409,4.27709,3594,9106546 +660,search,7569.22,0.92922,0.100667,15.7652,17.6157,27.088,4.27709,3594,9106546 +661,delete,35.586,-1.0,-1.0,-1.0,-1.0,45.12,4.27035,3592,9059776 +662,search,7647.81,0.92906,0.100921,15.753,17.5882,28.303,4.27035,3592,9059776 +663,delete,68.719,-1.0,-1.0,-1.0,-1.0,73.66,4.25159,3585,8990616 +664,search,7622.84,0.9287,0.101593,15.7675,17.5999,45.303,4.25183,3586,8990616 +665,delete,47.062,-1.0,-1.0,-1.0,-1.0,100.588,4.23535,3580,8937437 +666,search,7589.97,0.92862,0.101667,15.766,17.5968,27.767,4.23535,3580,8937437 +667,delete,155.598,-1.0,-1.0,-1.0,-1.0,247.156,4.16386,3554,8766390 +668,search,7446.49,0.92828,0.10174,15.8049,17.6173,26.342,4.16386,3554,8766390 +669,delete,175.725,-1.0,-1.0,-1.0,-1.0,398.06,4.081,3516,8532008 +670,search,7362.52,0.92793,0.102049,15.8911,17.67,26.88,4.081,3516,8532008 +671,delete,36.453,-1.0,-1.0,-1.0,-1.0,57.894,4.06979,3513,8489178 +672,search,7383.71,0.92769,0.102237,15.9375,17.7247,27.484,4.06979,3513,8489178 +673,delete,56.805,-1.0,-1.0,-1.0,-1.0,89.114,4.04865,3505,8421866 +674,search,7330.79,0.92777,0.102564,15.9489,17.7549,26.643,4.04865,3505,8421866 +675,delete,109.909,-1.0,-1.0,-1.0,-1.0,127.548,4.01288,3491,8288319 +676,search,7281.69,0.92717,0.103284,16.0209,17.826,25.753,4.01288,3491,8288319 +677,delete,144.042,-1.0,-1.0,-1.0,-1.0,137.403,3.93361,3479,8072197 +678,search,7209.72,0.9297,0.100816,15.5078,17.2749,26.297,3.93361,3479,8072197 +679,delete,27.589,-1.0,-1.0,-1.0,-1.0,62.206,3.92247,3477,8036947 +680,search,7195.37,0.92963,0.100846,15.519,17.2631,26.874,3.92247,3477,8036947 +681,delete,8.78,-1.0,-1.0,-1.0,-1.0,22.945,3.92138,3477,8026197 +682,search,7174.4,0.92975,0.100675,15.5158,17.2512,26.695,3.92138,3477,8026197 +683,delete,263.284,-1.0,-1.0,-1.0,-1.0,119.961,3.82631,3460,7744229 +684,search,7053.95,0.92851,0.101597,15.6704,17.29,31.031,3.82637,3461,7744229 +685,delete,207.559,-1.0,-1.0,-1.0,-1.0,92.149,3.72764,3447,7541066 +686,search,6891.36,0.92545,0.104202,16.0531,17.5897,30.643,3.72778,3448,7541066 +687,delete,18.352,-1.0,-1.0,-1.0,-1.0,30.312,3.7215,3447,7522190 +688,search,6806.83,0.92524,0.104505,16.0733,17.6026,25.989,3.7215,3447,7522190 +689,delete,101.58,-1.0,-1.0,-1.0,-1.0,257.473,3.66252,3412,7409400 +690,search,6771.3,0.92423,0.105035,16.2684,17.735,26.114,3.66252,3412,7409400 +691,delete,39.46,-1.0,-1.0,-1.0,-1.0,22.478,3.64817,3412,7375648 +692,search,6791.02,0.92404,0.105172,16.2967,17.7585,25.468,3.64817,3412,7375648 +693,delete,154.595,-1.0,-1.0,-1.0,-1.0,143.729,3.58232,3401,7222068 +694,search,6759.56,0.92421,0.104959,16.3403,17.7852,25.007,3.58232,3401,7222068 +695,delete,22.268,-1.0,-1.0,-1.0,-1.0,68.514,3.5705,3398,7196331 +696,search,6723.8,0.92455,0.104752,16.2801,17.7698,24.677,3.5705,3398,7196331 +697,delete,28.183,-1.0,-1.0,-1.0,-1.0,42.716,3.55968,3396,7170716 +698,search,6631.32,0.92453,0.104697,16.2973,17.7699,25.571,3.55968,3396,7170716 +699,delete,11.868,-1.0,-1.0,-1.0,-1.0,71.918,3.55033,3393,7155255 +700,search,6686.68,0.92496,0.104135,16.2667,17.7487,28.604,3.55033,3393,7155255 +701,delete,57.228,-1.0,-1.0,-1.0,-1.0,22.78,3.53166,3393,7097733 +702,search,6721.73,0.92479,0.104303,16.2694,17.7391,25.404,3.53166,3393,7097733 +703,delete,170.891,-1.0,-1.0,-1.0,-1.0,264.899,3.44243,3353,6924172 +704,search,6555.66,0.92476,0.103989,16.2973,17.6906,29.983,3.44253,3354,6924172 +705,delete,27.13,-1.0,-1.0,-1.0,-1.0,21.651,3.4295,3354,6896276 +706,search,6546.15,0.92462,0.103843,16.3117,17.7024,24.136,3.4295,3354,6896276 +707,delete,497.346,-1.0,-1.0,-1.0,-1.0,850.655,3.13885,3211,6385507 +708,search,6181.27,0.92174,0.106402,16.616,17.7932,52.009,3.13929,3215,6385507 +709,delete,163.685,-1.0,-1.0,-1.0,-1.0,520.309,3.04719,3155,6207658 +710,search,6072.08,0.92018,0.107048,16.918,17.971,47.78,3.04737,3158,6207658 +711,delete,44.896,-1.0,-1.0,-1.0,-1.0,40.848,3.0294,3156,6162720 +712,search,5983.77,0.9202,0.106823,16.9149,17.9647,31.274,3.0294,3157,6162720 +713,delete,231.028,-1.0,-1.0,-1.0,-1.0,464.259,2.90718,3103,5927803 +714,search,5915.22,0.9185,0.108691,17.1383,18.1629,28.775,2.90731,3104,5927803 +715,delete,26.325,-1.0,-1.0,-1.0,-1.0,48.004,2.8933,3100,5895666 +716,search,5780.71,0.91828,0.108669,17.1771,18.1635,22.344,2.8933,3100,5895666 +717,delete,26.131,-1.0,-1.0,-1.0,-1.0,28.92,2.88731,3099,5868774 +718,search,5750.69,0.91784,0.109159,17.2167,18.1937,23.101,2.88731,3099,5868774 +719,delete,334.482,-1.0,-1.0,-1.0,-1.0,541.185,2.71899,3004,5533540 +720,search,5646.44,0.91553,0.111663,17.4607,18.3258,33.953,2.71935,3006,5533540 +721,delete,98.82,-1.0,-1.0,-1.0,-1.0,138.326,2.67155,2985,5418387 +722,search,5599.37,0.91516,0.112038,17.5501,18.3596,22.42,2.67155,2985,5418387 +723,delete,18.884,-1.0,-1.0,-1.0,-1.0,25.454,2.66433,2984,5398291 +724,search,5554.33,0.91505,0.112011,17.561,18.3488,22.244,2.66433,2984,5398291 +725,delete,104.715,-1.0,-1.0,-1.0,-1.0,211.499,2.60872,2965,5273546 +726,search,5507.66,0.91425,0.112385,17.669,18.3741,44.306,2.60872,2967,5273546 +727,delete,78.715,-1.0,-1.0,-1.0,-1.0,49.009,2.56985,2962,5180937 +728,search,5424.87,0.91391,0.112596,17.7196,18.4468,38.145,2.56985,2964,5180937 +729,delete,94.683,-1.0,-1.0,-1.0,-1.0,168.179,2.51763,2939,5079524 +730,search,5358.44,0.91344,0.113069,17.7963,18.4727,21.825,2.51763,2939,5079524 +731,delete,76.447,-1.0,-1.0,-1.0,-1.0,19.505,2.49582,2939,5019265 +732,search,5351.1,0.91234,0.114484,17.8794,18.5083,21.816,2.49582,2939,5019265 +733,delete,80.774,-1.0,-1.0,-1.0,-1.0,81.81,2.46434,2932,4955565 +734,search,5362.33,0.91201,0.114501,17.9772,18.6216,22.209,2.46434,2932,4955565 +735,delete,146.477,-1.0,-1.0,-1.0,-1.0,542.721,2.37452,2878,4776948 +736,search,5248.76,0.91397,0.112648,18.0921,18.9202,39.16,2.37508,2880,4776948 +737,delete,107.017,-1.0,-1.0,-1.0,-1.0,112.156,2.32613,2870,4654026 +738,search,5204.25,0.91525,0.110854,17.812,18.6044,28.817,2.32613,2871,4654026 +739,delete,53.935,-1.0,-1.0,-1.0,-1.0,84.853,2.29511,2859,4589888 +740,search,5165.73,0.91548,0.110385,17.831,18.5932,28.743,2.29511,2860,4589888 +741,delete,144.109,-1.0,-1.0,-1.0,-1.0,206.776,2.21219,2834,4430862 +742,search,5075.53,0.9147,0.110522,17.8879,18.5051,27.773,2.21224,2835,4430862 +743,delete,181.883,-1.0,-1.0,-1.0,-1.0,361.701,2.11645,2787,4237049 +744,search,4944.35,0.91312,0.111923,18.0071,18.5605,38.518,2.1165,2789,4237049 +745,delete,231.196,-1.0,-1.0,-1.0,-1.0,529.938,1.98718,2718,4000195 +746,search,4862.79,0.91113,0.113251,18.398,18.7438,19.837,1.98718,2718,4000195 +747,delete,168.086,-1.0,-1.0,-1.0,-1.0,262.252,1.89345,2673,3803954 +748,search,4744.29,0.91022,0.113322,18.5713,18.7849,33.994,1.89362,2675,3803954 +749,delete,68.386,-1.0,-1.0,-1.0,-1.0,74.075,1.85842,2665,3733214 +750,search,4725.48,0.90995,0.113552,18.6939,18.8838,19.07,1.85842,2665,3733214 +751,delete,19.699,-1.0,-1.0,-1.0,-1.0,32.319,1.84728,2662,3709324 +752,search,4684.89,0.9098,0.113511,18.7632,18.907,18.509,1.84728,2662,3709324 +753,delete,51.525,-1.0,-1.0,-1.0,-1.0,133.733,1.81412,2647,3649123 +754,search,4669.61,0.90854,0.114874,18.9421,19.0797,28.183,1.81412,2648,3649123 +755,delete,26.644,-1.0,-1.0,-1.0,-1.0,26.477,1.80274,2647,3621475 +756,search,4665.78,0.90844,0.114648,18.9399,19.0191,20.158,1.80274,2647,3621475 +757,delete,34.524,-1.0,-1.0,-1.0,-1.0,38.622,1.79004,2643,3594479 +758,search,4598.75,0.9083,0.114524,18.998,19.0471,19.503,1.79004,2643,3594479 +759,delete,24.848,-1.0,-1.0,-1.0,-1.0,27.947,1.78077,2641,3569055 +760,search,4650.59,0.90788,0.114867,19.0391,19.0473,19.596,1.78077,2641,3569055 +761,delete,42.028,-1.0,-1.0,-1.0,-1.0,32.075,1.76395,2639,3527896 +762,search,4618.87,0.90728,0.115165,19.1011,19.0604,19.695,1.76395,2639,3527896 +763,delete,20.899,-1.0,-1.0,-1.0,-1.0,31.174,1.75444,2637,3509970 +764,search,4538.07,0.90714,0.115309,19.138,19.0931,18.948,1.75444,2637,3509970 +765,delete,305.617,-1.0,-1.0,-1.0,-1.0,440.663,1.58012,2515,3187549 +766,search,4380.39,0.90402,0.117696,19.5296,19.2017,17.498,1.58012,2515,3187549 +767,delete,337.969,-1.0,-1.0,-1.0,-1.0,561.444,1.37103,2361,2822906 +768,search,4032.46,0.89765,0.123147,20.4353,19.7379,26.292,1.37108,2362,2822906 +769,insert,5.77,-1.0,-1.0,-1.0,-1.0,14.927,1.37227,2362,2832899 +770,search,4015.63,0.89807,0.12282,20.3743,19.7123,17.225,1.37227,2362,2832899 +771,insert,32.301,-1.0,-1.0,-1.0,-1.0,14.4,1.38097,2362,2863523 +772,search,4071.09,0.89845,0.122622,20.341,19.743,17.245,1.38097,2362,2863523 +773,insert,50.326,-1.0,-1.0,-1.0,-1.0,15.302,1.39522,2362,2925169 +774,search,4151.4,0.89971,0.121515,20.1415,19.628,17.289,1.39522,2362,2925169 +775,insert,37.1,-1.0,-1.0,-1.0,-1.0,36.768,1.40695,2365,2961356 +776,search,4146.04,0.90001,0.121711,20.0759,19.6566,17.153,1.40695,2365,2961356 +777,insert,30.564,-1.0,-1.0,-1.0,-1.0,14.547,1.41501,2365,2998678 +778,search,4127.45,0.90106,0.12055,19.9366,19.5541,17.293,1.41501,2365,2998678 +779,insert,11.856,-1.0,-1.0,-1.0,-1.0,14.224,1.41767,2365,3011024 +780,search,4151.64,0.90163,0.120271,19.8475,19.5228,16.983,1.41767,2365,3011024 +781,insert,4.802,-1.0,-1.0,-1.0,-1.0,14.164,1.41842,2365,3019515 +782,search,4145.24,0.90177,0.120171,19.8358,19.5165,16.785,1.41842,2365,3019515 +783,insert,51.63,-1.0,-1.0,-1.0,-1.0,14.658,1.43242,2365,3082298 +784,search,4093.16,0.9025,0.119622,19.6864,19.4547,16.288,1.43242,2365,3082298 +785,insert,34.324,-1.0,-1.0,-1.0,-1.0,39.865,1.44674,2367,3122090 +786,search,4156.03,0.9038,0.118822,19.513,19.4034,40.098,1.44674,2370,3122090 +787,insert,19.991,-1.0,-1.0,-1.0,-1.0,14.754,1.45222,2370,3140294 +788,search,4197.54,0.90387,0.11911,19.478,19.3902,35.417,1.45222,2373,3140294 +789,insert,6.632,-1.0,-1.0,-1.0,-1.0,14.817,1.45353,2373,3147819 +790,search,4203.83,0.904,0.118994,19.4362,19.3607,17.159,1.45353,2373,3147819 +791,insert,31.773,-1.0,-1.0,-1.0,-1.0,14.445,1.46095,2373,3180668 +792,search,4172.82,0.90406,0.118894,19.4088,19.3678,17.396,1.46095,2373,3180668 +793,insert,2.211,-1.0,-1.0,-1.0,-1.0,14.298,1.46135,2373,3183235 +794,search,4174.52,0.90407,0.118933,19.4056,19.367,17.24,1.46135,2373,3183235 +795,insert,59.508,-1.0,-1.0,-1.0,-1.0,21.118,1.47987,2374,3242089 +796,search,4206.67,0.90483,0.118308,19.3224,19.3406,17.196,1.47987,2374,3242089 +797,insert,4.242,-1.0,-1.0,-1.0,-1.0,14.387,1.48068,2374,3247192 +798,search,4225.86,0.90487,0.118229,19.3215,19.3404,16.717,1.48068,2374,3247192 +799,insert,6.142,-1.0,-1.0,-1.0,-1.0,14.28,1.48209,2374,3254626 +800,search,4222.47,0.90503,0.118248,19.2935,19.3418,16.41,1.48209,2374,3254626 +801,insert,18.09,-1.0,-1.0,-1.0,-1.0,14.309,1.48772,2374,3274953 +802,search,4215.12,0.90535,0.118337,19.2112,19.3056,16.831,1.48772,2374,3274953 +803,insert,858.198,-1.0,-1.0,-1.0,-1.0,474.775,1.80987,2456,4032688 +804,search,4661.42,0.9117,0.113307,18.4461,18.9365,124.336,1.81001,2477,4032688 +805,insert,19.606,-1.0,-1.0,-1.0,-1.0,15.399,1.81516,2477,4050220 +806,search,4653.6,0.91179,0.113178,18.454,18.9645,31.74,1.81516,2480,4050220 +807,insert,19.143,-1.0,-1.0,-1.0,-1.0,15.387,1.8222,2480,4067940 +808,search,4673.31,0.91169,0.113337,18.4514,18.9663,18.693,1.8222,2480,4067940 +809,insert,33.89,-1.0,-1.0,-1.0,-1.0,23.148,1.83399,2482,4099369 +810,search,4682.23,0.91215,0.113036,18.398,18.94,17.821,1.83399,2482,4099369 +811,insert,30.122,-1.0,-1.0,-1.0,-1.0,15.157,1.84486,2482,4129509 +812,search,4839.7,0.91265,0.112623,18.386,18.9626,17.686,1.84486,2482,4129509 +813,insert,48.64,-1.0,-1.0,-1.0,-1.0,19.853,1.86459,2484,4178669 +814,search,4688.91,0.91385,0.11152,18.266,18.8741,17.238,1.86459,2484,4178669 +815,insert,2,-1.0,-1.0,-1.0,-1.0,14.997,1.86549,2484,4180960 +816,search,4680.62,0.91387,0.111388,18.268,18.8732,18.217,1.86549,2484,4180960 +817,insert,96.773,-1.0,-1.0,-1.0,-1.0,92.701,1.89685,2498,4255359 +818,search,4758.83,0.91386,0.111631,18.2122,18.8355,30.532,1.89696,2500,4255359 +819,insert,23.657,-1.0,-1.0,-1.0,-1.0,15.448,1.90449,2500,4272706 +820,search,4738.63,0.91431,0.111261,18.1731,18.8073,18.665,1.90449,2500,4272706 +821,insert,18.751,-1.0,-1.0,-1.0,-1.0,22.676,1.9109,2501,4287851 +822,search,4786.77,0.91423,0.111298,18.1875,18.8322,18.792,1.9109,2501,4287851 +823,insert,39.543,-1.0,-1.0,-1.0,-1.0,15.691,1.9251,2501,4322788 +824,search,4773.06,0.91435,0.111468,18.1271,18.8409,18.278,1.9251,2501,4322788 +825,insert,79.886,-1.0,-1.0,-1.0,-1.0,35.168,1.95584,2504,4396408 +826,search,4790.89,0.91411,0.111693,18.1493,18.8883,17.775,1.95584,2504,4396408 +827,insert,21.744,-1.0,-1.0,-1.0,-1.0,22.348,1.96382,2505,4415721 +828,search,4762.18,0.91389,0.112095,18.1568,18.8748,17.943,1.96382,2505,4415721 +829,insert,477.33,-1.0,-1.0,-1.0,-1.0,190.783,2.14384,2538,4832079 +830,search,5005.03,0.91649,0.11002,17.7732,18.7324,41.275,2.14432,2544,4832079 +831,insert,45.526,-1.0,-1.0,-1.0,-1.0,22.44,2.15861,2545,4864525 +832,search,5012.93,0.91665,0.109879,17.7505,18.7421,21.307,2.15863,2546,4864525 +833,insert,10.195,-1.0,-1.0,-1.0,-1.0,19.804,2.16133,2547,4874443 +834,search,4963.79,0.91661,0.109869,17.7472,18.739,18.811,2.16133,2547,4874443 +835,insert,73.609,-1.0,-1.0,-1.0,-1.0,33.959,2.18886,2551,4938267 +836,search,4967.3,0.91733,0.109362,17.6679,18.6926,18.339,2.18886,2551,4938267 +837,insert,26.853,-1.0,-1.0,-1.0,-1.0,32.691,2.19807,2554,4961147 +838,search,5003.34,0.91724,0.109492,17.6587,18.6898,18.003,2.19807,2554,4961147 +839,insert,75.917,-1.0,-1.0,-1.0,-1.0,50.236,2.22907,2561,5033740 +840,search,5107.62,0.91872,0.108374,17.5235,18.6423,18.063,2.22907,2561,5033740 +841,insert,52.284,-1.0,-1.0,-1.0,-1.0,35.702,2.24836,2564,5078736 +842,search,5123.7,0.91902,0.10824,17.4628,18.5911,22.214,2.2484,2565,5078736 +843,insert,458.377,-1.0,-1.0,-1.0,-1.0,214.459,2.41988,2601,5467376 +844,search,5343.05,0.92104,0.106554,17.1631,18.3889,76.018,2.42027,2614,5467376 +845,insert,384.143,-1.0,-1.0,-1.0,-1.0,252.852,2.57785,2663,5815547 +846,search,5495.24,0.92302,0.105324,16.8484,18.2583,78.75,2.57801,2676,5815547 +847,insert,7.162,-1.0,-1.0,-1.0,-1.0,18.211,2.58062,2676,5821782 +848,search,5420.75,0.92329,0.105248,16.8382,18.2571,27.881,2.58066,2678,5821782 +849,insert,2.423,-1.0,-1.0,-1.0,-1.0,21.81,2.58145,2679,5825594 +850,search,5452.44,0.92317,0.105286,16.8356,18.2627,19.281,2.58145,2679,5825594 +851,insert,10.562,-1.0,-1.0,-1.0,-1.0,17.465,2.58573,2679,5834880 +852,search,5445.77,0.92327,0.105229,16.8151,18.2508,19.646,2.58573,2679,5834880 +853,insert,27.076,-1.0,-1.0,-1.0,-1.0,17.598,2.59607,2679,5860274 +854,search,5456.97,0.9235,0.104978,16.7952,18.2678,20.035,2.59607,2679,5860274 +855,insert,10.598,-1.0,-1.0,-1.0,-1.0,19.674,2.59899,2680,5866064 +856,search,5483.03,0.92346,0.104933,16.797,18.2714,20.159,2.59899,2680,5866064 +857,insert,19.954,-1.0,-1.0,-1.0,-1.0,17.703,2.60415,2680,5884456 +858,search,5499.83,0.92373,0.10477,16.7585,18.2398,20.224,2.60415,2680,5884456 +859,insert,45.136,-1.0,-1.0,-1.0,-1.0,41.726,2.61837,2685,5916786 +860,search,5468.18,0.92388,0.104375,16.7233,18.2034,23.769,2.61843,2686,5916786 +861,insert,8.089,-1.0,-1.0,-1.0,-1.0,17.787,2.62143,2686,5923467 +862,search,5527.84,0.92402,0.104301,16.6949,18.1731,19.554,2.62143,2686,5923467 +863,insert,70.302,-1.0,-1.0,-1.0,-1.0,79.324,2.65223,2702,5997844 +864,search,5518.16,0.92341,0.104721,16.746,18.1673,32.059,2.65235,2704,5997844 +865,insert,10.637,-1.0,-1.0,-1.0,-1.0,18.304,2.6563,2704,6009561 +866,search,5563.32,0.92334,0.104766,16.7382,18.1589,20.195,2.6563,2704,6009561 +867,insert,52.199,-1.0,-1.0,-1.0,-1.0,36.165,2.67342,2707,6052876 +868,search,5619.16,0.92344,0.105183,16.7245,18.1906,27.985,2.67348,2708,6052876 +869,insert,8.145,-1.0,-1.0,-1.0,-1.0,18.185,2.67634,2708,6059234 +870,search,5647.94,0.92351,0.105135,16.7287,18.2101,20.804,2.67634,2708,6059234 +871,insert,45.016,-1.0,-1.0,-1.0,-1.0,40.118,2.69207,2712,6098225 +872,search,5680.72,0.9236,0.105175,16.7311,18.221,27.029,2.6922,2713,6098225 +873,insert,46.687,-1.0,-1.0,-1.0,-1.0,52.197,2.71051,2719,6137308 +874,search,5692.87,0.92371,0.105026,16.7405,18.24,22.321,2.71057,2720,6137308 +875,insert,16.459,-1.0,-1.0,-1.0,-1.0,23.74,2.71703,2721,6154089 +876,search,5603.77,0.92354,0.105133,16.7582,18.2631,20.158,2.71703,2721,6154089 +877,insert,1.386,-1.0,-1.0,-1.0,-1.0,21.871,2.71742,2722,6156061 +878,search,5605.61,0.9235,0.105082,16.7667,18.2741,20.571,2.71742,2722,6156061 +879,insert,306.492,-1.0,-1.0,-1.0,-1.0,308.444,2.84711,2779,6446004 +880,search,5874.35,0.92463,0.104127,16.6206,18.2213,62.794,2.84754,2787,6446004 +881,insert,522.406,-1.0,-1.0,-1.0,-1.0,358.288,3.08474,2860,6969764 +882,search,6155.42,0.92638,0.102714,16.299,17.9806,72.556,3.0854,2870,6969764 +883,insert,446.415,-1.0,-1.0,-1.0,-1.0,359.043,3.28807,2947,7422501 +884,search,6359.38,0.92851,0.100965,15.9096,17.8162,87.808,3.28853,2963,7422501 +885,insert,8.69,-1.0,-1.0,-1.0,-1.0,32.882,3.29354,2966,7432053 +886,search,6407.71,0.92865,0.100724,15.892,17.8122,28.587,3.29353,2967,7432053 +887,insert,304.472,-1.0,-1.0,-1.0,-1.0,447.129,3.44856,3048,7789886 +888,search,6516.33,0.93074,0.0998006,15.5523,17.6356,110.119,3.45053,3074,7789886 +889,insert,356.502,-1.0,-1.0,-1.0,-1.0,529.158,3.64263,3183,8210883 +890,search,6906.07,0.93189,0.0990997,15.2997,17.4957,117.534,3.64361,3208,8210883 +891,insert,33.393,-1.0,-1.0,-1.0,-1.0,59.591,3.65695,3217,8240504 +892,search,6988.11,0.93208,0.0990883,15.2803,17.5001,39.189,3.65711,3220,8240504 +893,insert,51.696,-1.0,-1.0,-1.0,-1.0,68.554,3.67943,3230,8299693 +894,search,6979.58,0.93235,0.0988719,15.2139,17.4548,28.238,3.67953,3231,8299693 +895,insert,38.542,-1.0,-1.0,-1.0,-1.0,55.38,3.69728,3240,8349126 +896,search,7002.44,0.93271,0.0986758,15.1839,17.423,39.633,3.69743,3243,8349126 +897,delete,45.865,-1.0,-1.0,-1.0,-1.0,145.685,3.67681,3229,8268842 +898,search,6930,0.93249,0.0987212,15.2393,17.4642,27.053,3.67681,3229,8268842 +899,delete,12.013,-1.0,-1.0,-1.0,-1.0,30.87,3.676,3228,8250527 +900,search,6933.16,0.93244,0.0987455,15.2552,17.4785,26.769,3.676,3228,8250527 +901,delete,42.283,-1.0,-1.0,-1.0,-1.0,39.532,3.67112,3226,8185565 +902,search,6929.54,0.9321,0.0990267,15.319,17.5097,27.007,3.67112,3226,8185565 +903,delete,40.234,-1.0,-1.0,-1.0,-1.0,126.869,3.64973,3209,8126915 +904,search,6965.41,0.93095,0.0994818,15.5192,17.5674,26.225,3.64973,3209,8126915 +905,delete,16.651,-1.0,-1.0,-1.0,-1.0,24.096,3.64882,3209,8098533 +906,search,6829.13,0.9309,0.0996417,15.5121,17.5809,25.908,3.64882,3209,8098533 +907,delete,16.598,-1.0,-1.0,-1.0,-1.0,45.579,3.64427,3208,8074833 +908,search,6736.31,0.93066,0.0997912,15.5179,17.5789,26.858,3.64427,3208,8074833 +909,delete,29.131,-1.0,-1.0,-1.0,-1.0,28.593,3.64202,3207,8035274 +910,search,6831.04,0.93053,0.0997406,15.5233,17.5576,27.189,3.64202,3207,8035274 +911,delete,60.1,-1.0,-1.0,-1.0,-1.0,31.074,3.63862,3206,7957223 +912,search,6806.75,0.93008,0.0999056,15.6143,17.5928,31.476,3.6388,3207,7957223 +913,delete,29.921,-1.0,-1.0,-1.0,-1.0,69.943,3.62771,3200,7917554 +914,search,6718.21,0.92995,0.0999436,15.6149,17.5746,26.44,3.62771,3200,7917554 +915,delete,41.465,-1.0,-1.0,-1.0,-1.0,93.637,3.61352,3191,7861503 +916,search,6733.68,0.9296,0.100317,15.667,17.6302,32.051,3.61357,3192,7861503 +917,delete,12.34,-1.0,-1.0,-1.0,-1.0,23.868,3.6122,3192,7844487 +918,search,6634.83,0.93003,0.0999808,15.6558,17.6143,26.695,3.6122,3192,7844487 +919,delete,21.773,-1.0,-1.0,-1.0,-1.0,24.18,3.6106,3192,7814993 +920,search,6760.88,0.93004,0.100035,15.6822,17.627,27.128,3.6106,3192,7814993 +921,delete,20.014,-1.0,-1.0,-1.0,-1.0,33.616,3.60603,3191,7793855 +922,search,6814.65,0.92972,0.100199,15.6843,17.6216,26.73,3.60603,3191,7793855 +923,delete,43.915,-1.0,-1.0,-1.0,-1.0,35.591,3.59832,3189,7736054 +924,search,6664.92,0.92929,0.100605,15.7488,17.6544,26.301,3.59832,3189,7736054 +925,delete,107.643,-1.0,-1.0,-1.0,-1.0,147.976,3.55068,3161,7587942 +926,search,6622.33,0.92819,0.10126,15.9163,17.7767,38.13,3.55078,3163,7587942 +927,delete,18.009,-1.0,-1.0,-1.0,-1.0,38.951,3.54789,3162,7565670 +928,search,6638.21,0.92829,0.101073,15.9387,17.7787,26.406,3.54789,3162,7565670 +929,delete,31.472,-1.0,-1.0,-1.0,-1.0,23.607,3.54349,3162,7529948 +930,search,6600.16,0.92818,0.101423,15.9238,17.8174,26.398,3.54349,3162,7529948 +931,delete,404.539,-1.0,-1.0,-1.0,-1.0,837.814,3.36443,3045,7016700 +932,search,6206.2,0.92686,0.102398,16.0404,17.8097,42.196,3.36474,3049,7016700 +933,delete,22.063,-1.0,-1.0,-1.0,-1.0,44.104,3.35501,3045,6980979 +934,search,6178.13,0.92671,0.102493,16.0654,17.8062,25.209,3.35501,3045,6980979 +935,delete,16.973,-1.0,-1.0,-1.0,-1.0,53.382,3.34631,3042,6954250 +936,search,6257.53,0.92669,0.102416,16.0955,17.8175,24.805,3.34631,3042,6954250 +937,delete,21.784,-1.0,-1.0,-1.0,-1.0,29.493,3.33636,3041,6924596 +938,search,6133.21,0.9263,0.102712,16.126,17.8202,25.143,3.33636,3041,6924596 +939,delete,120.802,-1.0,-1.0,-1.0,-1.0,38.484,3.30842,3039,6784831 +940,search,6209.65,0.92578,0.103867,16.1509,17.8809,31.264,3.30848,3040,6784831 +941,delete,38.363,-1.0,-1.0,-1.0,-1.0,22.874,3.30168,3040,6745819 +942,search,6089.49,0.92586,0.103874,16.1671,17.9229,33.243,3.30186,3041,6745819 +943,delete,3.676,-1.0,-1.0,-1.0,-1.0,22.361,3.30148,3041,6740944 +944,search,6142.24,0.92613,0.10349,16.1548,17.9201,25.281,3.30148,3041,6740944 +945,delete,106.64,-1.0,-1.0,-1.0,-1.0,227.688,3.24737,3017,6627202 +946,search,6120.78,0.92571,0.103975,16.2326,17.979,44.229,3.24795,3020,6627202 +947,delete,19.363,-1.0,-1.0,-1.0,-1.0,22.884,3.24163,3020,6607194 +948,search,5923.01,0.9253,0.104215,16.274,17.9943,24.17,3.24163,3020,6607194 +949,delete,48.07,-1.0,-1.0,-1.0,-1.0,57.542,3.22405,3015,6551065 +950,search,6066.47,0.92492,0.104758,16.3155,18.0527,25.046,3.22405,3015,6551065 +951,delete,33.352,-1.0,-1.0,-1.0,-1.0,87.928,3.19946,3000,6506798 +952,search,6021.98,0.92517,0.103879,16.3286,18.0371,25.078,3.19946,3000,6506798 +953,delete,44.651,-1.0,-1.0,-1.0,-1.0,35.079,3.18549,2998,6459301 +954,search,5988.19,0.92484,0.10422,16.3338,18.0296,25.154,3.18549,2998,6459301 +955,delete,17.493,-1.0,-1.0,-1.0,-1.0,41.848,3.1762,2994,6440069 +956,search,6041,0.92486,0.104042,16.3743,18.0611,25.029,3.1762,2994,6440069 +957,delete,301.736,-1.0,-1.0,-1.0,-1.0,307.974,3.02982,2946,6132470 +958,search,5783.41,0.92346,0.105352,16.6271,18.2566,23.641,3.02982,2946,6132470 +959,delete,39.638,-1.0,-1.0,-1.0,-1.0,47.313,3.01225,2942,6086241 +960,search,5721.18,0.92253,0.105913,16.7546,18.292,23.823,3.01225,2942,6086241 +961,delete,22.398,-1.0,-1.0,-1.0,-1.0,21.327,3.00629,2942,6064734 +962,search,5745.51,0.92245,0.105952,16.7445,18.2451,30.281,3.00631,2943,6064734 +963,delete,121.505,-1.0,-1.0,-1.0,-1.0,72.131,2.96025,2936,5967543 +964,search,5789.66,0.9222,0.105924,16.7865,18.3255,30.583,2.96046,2937,5967543 +965,delete,60.359,-1.0,-1.0,-1.0,-1.0,84.305,2.93253,2924,5906381 +966,search,5665.72,0.92173,0.106217,16.8793,18.4155,29.572,2.93262,2925,5906381 +967,delete,47.794,-1.0,-1.0,-1.0,-1.0,48.105,2.91159,2921,5851849 +968,search,5612.67,0.92154,0.106206,16.9706,18.4689,24.07,2.91159,2921,5851849 +969,delete,55.549,-1.0,-1.0,-1.0,-1.0,57.935,2.88108,2913,5783023 +970,search,5676.74,0.9208,0.106903,17.0556,18.5058,23.469,2.88108,2913,5783023 +971,delete,249.602,-1.0,-1.0,-1.0,-1.0,295.988,2.74224,2871,5496517 +972,search,5521.72,0.91904,0.108058,17.2906,18.6846,24.251,2.74224,2871,5496517 +973,delete,248.33,-1.0,-1.0,-1.0,-1.0,248.905,2.61386,2827,5237981 +974,search,5360.87,0.91716,0.109554,17.6557,18.9215,23.355,2.61386,2827,5237981 +975,delete,141.986,-1.0,-1.0,-1.0,-1.0,276.079,2.53443,2761,5099794 +976,search,5198.63,0.91596,0.110473,17.7418,18.867,22.8,2.53443,2761,5099794 +977,delete,58.444,-1.0,-1.0,-1.0,-1.0,108.052,2.50345,2747,5032210 +978,search,5164.34,0.91496,0.111434,17.8814,18.956,23.271,2.50345,2747,5032210 +979,delete,11.9,-1.0,-1.0,-1.0,-1.0,25.573,2.49933,2746,5020211 +980,search,5155.97,0.91479,0.11184,17.9032,18.9576,22.387,2.49933,2746,5020211 +981,delete,102.964,-1.0,-1.0,-1.0,-1.0,153.026,2.4434,2724,4902291 +982,search,5113.07,0.91323,0.112869,18.1022,19.0422,28.914,2.44356,2725,4902291 +983,delete,31.589,-1.0,-1.0,-1.0,-1.0,19.615,2.43232,2725,4872577 +984,search,5111.24,0.91325,0.112633,18.1414,19.0546,21.372,2.43232,2725,4872577 +985,delete,38.077,-1.0,-1.0,-1.0,-1.0,19.448,2.41931,2725,4836730 +986,search,5576.76,0.91288,0.112614,18.2149,19.088,22.507,2.41931,2725,4836730 +987,delete,53.858,-1.0,-1.0,-1.0,-1.0,28.592,2.40334,2723,4797164 +988,search,5055.1,0.91291,0.112648,18.2138,19.077,22.821,2.40334,2723,4797164 +989,delete,15.699,-1.0,-1.0,-1.0,-1.0,19.116,2.3993,2723,4784789 +990,search,5947.5,0.91286,0.112766,18.1989,19.0627,22.637,2.3993,2723,4784789 +991,delete,59.951,-1.0,-1.0,-1.0,-1.0,140.893,2.36585,2712,4705420 +992,search,5005.07,0.91425,0.111939,18.1536,19.1053,44.008,2.36621,2715,4705420 +993,delete,91.625,-1.0,-1.0,-1.0,-1.0,74.701,2.32109,2707,4621807 +994,search,4980.17,0.91379,0.111847,18.2424,19.1514,30.175,2.32109,2708,4621807 +995,delete,58.756,-1.0,-1.0,-1.0,-1.0,84.202,2.28975,2695,4555849 +996,search,4947.71,0.91374,0.111832,18.2588,19.1257,24.955,2.28982,2696,4555849 +997,delete,27.668,-1.0,-1.0,-1.0,-1.0,42.762,2.27801,2692,4526975 +998,search,4936.25,0.91379,0.111794,18.2498,19.1183,22.203,2.27801,2692,4526975 +999,delete,123.523,-1.0,-1.0,-1.0,-1.0,154.382,2.21526,2669,4408448 +1000,search,4953.88,0.91308,0.112274,18.4112,19.2198,22.233,2.21526,2669,4408448 +1001,delete,119.455,-1.0,-1.0,-1.0,-1.0,159.54,2.15704,2650,4303257 +1002,search,4870.07,0.91264,0.11309,18.4788,19.2427,23.048,2.15704,2650,4303257 +1003,delete,42.446,-1.0,-1.0,-1.0,-1.0,38.561,2.13794,2646,4259551 +1004,search,4811,0.91193,0.113792,18.5313,19.2611,22.108,2.13794,2646,4259551 +1005,delete,20.97,-1.0,-1.0,-1.0,-1.0,28.165,2.12893,2644,4241434 +1006,search,4846.7,0.91163,0.113788,18.6018,19.3298,21.47,2.12893,2644,4241434 +1007,delete,168.676,-1.0,-1.0,-1.0,-1.0,341.525,2.02637,2591,4052722 +1008,search,4683.35,0.90984,0.115151,18.9,19.4718,27.121,2.02656,2592,4052722 +1009,delete,276.898,-1.0,-1.0,-1.0,-1.0,344.514,1.86344,2523,3738432 +1010,search,4494.91,0.90794,0.115664,19.1069,19.4984,20.181,1.86344,2523,3738432 +1011,delete,347.14,-1.0,-1.0,-1.0,-1.0,446.825,1.67158,2425,3372539 +1012,search,4310.91,0.90541,0.117513,19.6973,19.9227,26.831,1.67178,2426,3372539 +1013,delete,13.389,-1.0,-1.0,-1.0,-1.0,22.11,1.66778,2424,3361432 +1014,search,4272.59,0.90465,0.118167,19.7757,19.9782,20.172,1.66778,2424,3361432 +1015,delete,254.48,-1.0,-1.0,-1.0,-1.0,350.469,1.53475,2351,3091692 +1016,search,4166.07,0.90172,0.119774,20.1979,20.0796,27.488,1.53475,2351,3091692 +1017,delete,311.206,-1.0,-1.0,-1.0,-1.0,420.726,1.37195,2272,2775418 +1018,search,3934.55,0.90041,0.12107,20.3901,20.2231,26.996,1.37195,2272,2775418 +1019,delete,22.156,-1.0,-1.0,-1.0,-1.0,39.601,1.36425,2269,2754073 +1020,search,3981.06,0.89999,0.121514,20.4792,20.3216,25.361,1.36425,2269,2754073 +1021,delete,112.634,-1.0,-1.0,-1.0,-1.0,66.938,1.31617,2259,2638917 +1022,search,3925.41,0.89799,0.122348,20.7489,20.4117,25.387,1.31617,2259,2638917 +1023,delete,71.866,-1.0,-1.0,-1.0,-1.0,102.395,1.27737,2234,2556494 +1024,search,3858.92,0.8959,0.123465,20.9969,20.5391,37.411,1.27749,2236,2556494 +1025,insert,21.013,-1.0,-1.0,-1.0,-1.0,22.936,1.28365,2236,2594934 +1026,search,3909.5,0.8965,0.12292,20.896,20.4739,25.268,1.28365,2236,2594934 +1027,insert,177.873,-1.0,-1.0,-1.0,-1.0,109.6,1.34352,2246,2802105 +1028,search,3959.81,0.89633,0.123924,20.8157,20.5095,63.724,1.34352,2251,2802105 +1029,insert,403.011,-1.0,-1.0,-1.0,-1.0,168.533,1.48498,2274,3203975 +1030,search,4173.05,0.90146,0.12138,20.1155,20.1714,112.014,1.48506,2291,3203975 +1031,insert,602.39,-1.0,-1.0,-1.0,-1.0,252.571,1.73736,2322,3830566 +1032,search,4410.6,0.90296,0.120723,19.8352,20.2113,110.694,1.73736,2333,3830566 +1033,insert,445.702,-1.0,-1.0,-1.0,-1.0,271.015,1.91499,2383,4252669 +1034,search,4585.76,0.9075,0.117738,18.924,19.5864,85.54,1.91502,2395,4252669 +1035,insert,248.997,-1.0,-1.0,-1.0,-1.0,205.961,2.02817,2425,4512107 +1036,search,4660.82,0.90923,0.117173,18.6963,19.5535,68.729,2.02858,2436,4512107 +1037,insert,10.027,-1.0,-1.0,-1.0,-1.0,16.078,2.03237,2436,4525815 +1038,search,4738.16,0.90941,0.117033,18.6536,19.5272,17.755,2.03237,2436,4525815 +1039,insert,73.591,-1.0,-1.0,-1.0,-1.0,19.454,2.06137,2437,4599215 +1040,search,4763.34,0.90956,0.11702,18.5499,19.4156,17.962,2.06137,2437,4599215 +1041,insert,12.068,-1.0,-1.0,-1.0,-1.0,30.091,2.06566,2440,4609851 +1042,search,4772.38,0.90955,0.117111,18.5255,19.4094,18.099,2.06566,2440,4609851 +1043,insert,23.384,-1.0,-1.0,-1.0,-1.0,20.687,2.07238,2441,4627411 +1044,search,4703.43,0.9096,0.116905,18.5146,19.3751,18.18,2.07238,2441,4627411 +1045,insert,47.006,-1.0,-1.0,-1.0,-1.0,15.876,2.08779,2441,4670607 +1046,search,4743.95,0.90999,0.116726,18.4698,19.3781,18.121,2.08779,2441,4670607 +1047,insert,5.649,-1.0,-1.0,-1.0,-1.0,15.592,2.08921,2441,4675386 +1048,search,4795.63,0.91007,0.116597,18.4525,19.369,18.074,2.08921,2441,4675386 +1049,insert,18.344,-1.0,-1.0,-1.0,-1.0,15.662,2.095,2441,4689998 +1050,search,4739,0.9102,0.116523,18.4217,19.3451,17.482,2.095,2441,4689998 +1051,insert,104.338,-1.0,-1.0,-1.0,-1.0,34.569,2.13582,2445,4783971 +1052,search,4821.43,0.91115,0.115646,18.3033,19.2953,27.44,2.13589,2447,4783971 +1053,insert,53.088,-1.0,-1.0,-1.0,-1.0,19.305,2.15355,2448,4826219 +1054,search,4840.51,0.91117,0.115331,18.2327,19.2426,18.221,2.15355,2448,4826219 +1055,insert,48.748,-1.0,-1.0,-1.0,-1.0,32.523,2.1687,2452,4862355 +1056,search,4858.63,0.91155,0.115169,18.1879,19.2343,23.648,2.1687,2453,4862355 +1057,insert,6.866,-1.0,-1.0,-1.0,-1.0,16.053,2.17035,2453,4869564 +1058,search,4859.41,0.9116,0.115042,18.1889,19.2321,18.518,2.17035,2453,4869564 +1059,insert,29.556,-1.0,-1.0,-1.0,-1.0,15.904,2.17907,2453,4898570 +1060,search,4873.17,0.91196,0.114972,18.1542,19.2146,18.035,2.17907,2453,4898570 +1061,insert,10.465,-1.0,-1.0,-1.0,-1.0,15.739,2.18251,2453,4907501 +1062,search,4874.85,0.9122,0.114666,18.1158,19.1959,17.794,2.18251,2453,4907501 +1063,insert,13.202,-1.0,-1.0,-1.0,-1.0,16.005,2.18631,2453,4916386 +1064,search,4850.95,0.91212,0.114719,18.1436,19.2283,17.338,2.18631,2453,4916386 +1065,insert,306.059,-1.0,-1.0,-1.0,-1.0,239.502,2.30929,2491,5194839 +1066,search,5001.32,0.91344,0.113836,17.9165,19.0745,63.297,2.30973,2500,5194839 +1067,insert,8.901,-1.0,-1.0,-1.0,-1.0,19.458,2.31195,2501,5200182 +1068,search,5027.78,0.91396,0.113447,17.8953,19.0771,18.865,2.31195,2501,5200182 +1069,insert,43.308,-1.0,-1.0,-1.0,-1.0,42.254,2.32665,2512,5230883 +1070,search,5068.16,0.91397,0.113417,17.8262,19.0047,18.722,2.32665,2512,5230883 +1071,insert,28.614,-1.0,-1.0,-1.0,-1.0,27.136,2.33653,2514,5255659 +1072,search,5094.65,0.91383,0.113667,17.8333,19.0344,25.961,2.33667,2515,5255659 +1073,insert,31.024,-1.0,-1.0,-1.0,-1.0,25.62,2.34727,2517,5281693 +1074,search,5133.32,0.91415,0.113427,17.7959,19.0096,19.194,2.34727,2517,5281693 +1075,insert,7.761,-1.0,-1.0,-1.0,-1.0,16.706,2.34937,2517,5290081 +1076,search,5120.54,0.91438,0.113194,17.7884,19.0034,18.756,2.34937,2517,5290081 +1077,insert,31.624,-1.0,-1.0,-1.0,-1.0,29.662,2.36121,2519,5316166 +1078,search,5119.42,0.91449,0.113005,17.7763,19.0135,18.511,2.36121,2519,5316166 +1079,insert,21.543,-1.0,-1.0,-1.0,-1.0,28.496,2.36936,2523,5335531 +1080,search,5047.65,0.91445,0.112787,17.7544,18.99,18.987,2.36936,2523,5335531 +1081,insert,432.024,-1.0,-1.0,-1.0,-1.0,170.085,2.51951,2551,5678781 +1082,search,5301.27,0.91543,0.112503,17.69,19.0044,46.327,2.51961,2555,5678781 +1083,insert,1.924,-1.0,-1.0,-1.0,-1.0,17.47,2.52014,2555,5679848 +1084,search,5294.24,0.91525,0.112546,17.6934,19.0017,28.2,2.52014,2556,5679848 +1085,insert,28.976,-1.0,-1.0,-1.0,-1.0,21.162,2.52964,2558,5702205 +1086,search,5267.88,0.91542,0.112034,17.6601,18.9734,19.273,2.52964,2558,5702205 +1087,insert,16.826,-1.0,-1.0,-1.0,-1.0,19.142,2.53445,2559,5715514 +1088,search,5254.87,0.91571,0.111869,17.6662,18.9802,19.027,2.53445,2559,5715514 +1089,insert,426.914,-1.0,-1.0,-1.0,-1.0,326.002,2.70573,2637,6103499 +1090,search,5518.19,0.91714,0.111186,17.3868,18.7838,48.836,2.70633,2645,6103499 +1091,insert,44.888,-1.0,-1.0,-1.0,-1.0,69.574,2.72744,2660,6152505 +1092,search,5566.01,0.91797,0.11044,17.2752,18.7237,30.337,2.72768,2662,6152505 +1093,insert,20.913,-1.0,-1.0,-1.0,-1.0,26.908,2.7348,2663,6171104 +1094,search,5499.81,0.91813,0.110245,17.2579,18.7142,23.214,2.7348,2663,6171104 +1095,insert,391.868,-1.0,-1.0,-1.0,-1.0,388.492,2.90807,2741,6557850 +1096,search,5828.15,0.92046,0.109151,16.8991,18.5694,94.406,2.90879,2756,6557850 +1097,insert,24.048,-1.0,-1.0,-1.0,-1.0,25.176,2.91719,2758,6579289 +1098,search,5792.53,0.92069,0.109092,16.8669,18.5409,37.405,2.91726,2761,6579289 +1099,insert,16.458,-1.0,-1.0,-1.0,-1.0,19.826,2.92425,2761,6591442 +1100,search,5781.48,0.92095,0.108892,16.8495,18.532,21.294,2.92425,2761,6591442 +1101,insert,43.094,-1.0,-1.0,-1.0,-1.0,24.319,2.94168,2762,6633739 +1102,search,5800.34,0.92112,0.108663,16.8068,18.5041,21.206,2.94168,2762,6633739 +1103,insert,83.717,-1.0,-1.0,-1.0,-1.0,118.034,2.98162,2793,6724357 +1104,search,5940.96,0.92246,0.107863,16.6399,18.4617,35.607,2.98166,2797,6724357 +1105,insert,3.211,-1.0,-1.0,-1.0,-1.0,20.138,2.98251,2797,6727059 +1106,search,5940.84,0.92231,0.108008,16.6636,18.4858,22.128,2.98251,2797,6727059 +1107,insert,257.033,-1.0,-1.0,-1.0,-1.0,196.608,3.08522,2833,6950869 +1108,search,9365.57,0.92306,0.107471,16.5731,18.4539,58.889,3.0858,2840,6950869 +1109,insert,6.469,-1.0,-1.0,-1.0,-1.0,20.534,3.08751,2840,6958173 +1110,search,6092.1,0.92313,0.107461,16.5663,18.4511,21.843,3.08751,2840,6958173 +1111,insert,44.122,-1.0,-1.0,-1.0,-1.0,62.184,3.10605,2855,6995273 +1112,search,6073.85,0.92307,0.107621,16.5703,18.4566,22.277,3.10605,2855,6995273 +1113,insert,58.335,-1.0,-1.0,-1.0,-1.0,71.491,3.1277,2865,7047862 +1114,search,6074.86,0.9236,0.106938,16.537,18.4476,22.92,3.1277,2865,7047862 +1115,insert,67.682,-1.0,-1.0,-1.0,-1.0,57.548,3.15131,2876,7110727 +1116,search,6103.01,0.9239,0.106545,16.4752,18.3969,23.033,3.15131,2876,7110727 +1117,insert,17.916,-1.0,-1.0,-1.0,-1.0,23.874,3.15631,2877,7125591 +1118,search,6204.25,0.92398,0.106504,16.459,18.3808,23.119,3.15631,2877,7125591 +1119,insert,12.005,-1.0,-1.0,-1.0,-1.0,26.639,3.16037,2879,7134442 +1120,search,6151.81,0.92406,0.106709,16.4402,18.3703,22.78,3.16037,2879,7134442 +1121,insert,29.259,-1.0,-1.0,-1.0,-1.0,53.768,3.17153,2886,7162109 +1122,search,6208.09,0.92447,0.106345,16.3967,18.3416,21.94,3.17153,2886,7162109 +1123,insert,15.107,-1.0,-1.0,-1.0,-1.0,23.71,3.17607,2887,7179509 +1124,search,6146.65,0.92441,0.106426,16.3901,18.337,21.986,3.17607,2887,7179509 +1125,insert,6.756,-1.0,-1.0,-1.0,-1.0,20.326,3.17851,2887,7186372 +1126,search,6146.12,0.92438,0.10647,16.3883,18.3385,22.373,3.17851,2887,7186372 +1127,insert,44.38,-1.0,-1.0,-1.0,-1.0,37.817,3.19583,2890,7231450 +1128,search,6234.93,0.92438,0.106451,16.3843,18.3552,22.583,3.19583,2890,7231450 +1129,insert,15.729,-1.0,-1.0,-1.0,-1.0,20.174,3.20059,2890,7243215 +1130,search,6161.13,0.9245,0.106305,16.3783,18.352,22.666,3.20059,2890,7243215 +1131,insert,44.392,-1.0,-1.0,-1.0,-1.0,49.878,3.21665,2895,7276107 +1132,search,6275.21,0.92475,0.106297,16.3711,18.3642,22.314,3.21665,2895,7276107 +1133,insert,19.686,-1.0,-1.0,-1.0,-1.0,34.447,3.22355,2899,7292577 +1134,search,6246.01,0.92474,0.106423,16.3348,18.318,21.937,3.22355,2899,7292577 +1135,insert,6.659,-1.0,-1.0,-1.0,-1.0,24.98,3.2259,2900,7299867 +1136,search,6177.37,0.92503,0.106206,16.3301,18.3251,28.294,3.22604,2902,7299867 +1137,insert,28.152,-1.0,-1.0,-1.0,-1.0,50.958,3.2367,2910,7323342 +1138,search,6274.75,0.92526,0.105948,16.3131,18.3193,25.507,3.23679,2911,7323342 +1139,insert,2.787,-1.0,-1.0,-1.0,-1.0,28.877,3.23795,2913,7325844 +1140,search,6255.28,0.92511,0.106092,16.3229,18.3253,23.067,3.23795,2913,7325844 +1141,insert,225.828,-1.0,-1.0,-1.0,-1.0,249.523,3.33369,2959,7540761 +1142,search,6412.92,0.9254,0.105969,16.2104,18.2427,63.501,3.33434,2970,7540761 +1143,insert,18.852,-1.0,-1.0,-1.0,-1.0,43.294,3.34149,2975,7561926 +1144,search,6375.32,0.92557,0.106085,16.2134,18.2724,28.86,3.34156,2977,7561926 +1145,insert,8.962,-1.0,-1.0,-1.0,-1.0,33.61,3.34651,2980,7571861 +1146,search,6319.11,0.92556,0.10613,16.2044,18.2664,23.323,3.34651,2980,7571861 +1147,insert,16.13,-1.0,-1.0,-1.0,-1.0,46.11,3.35419,2985,7590881 +1148,search,6410.73,0.92579,0.106091,16.1649,18.2505,26.368,3.35426,2986,7590881 +1149,insert,5.951,-1.0,-1.0,-1.0,-1.0,29.397,3.35611,2988,7595507 +1150,search,6671.49,0.92569,0.106119,16.1731,18.2576,23.963,3.35611,2988,7595507 +1151,insert,8.242,-1.0,-1.0,-1.0,-1.0,27.237,3.35899,2989,7601664 +1152,search,6444.46,0.9257,0.106054,16.1687,18.2483,24.195,3.35899,2989,7601664 +1153,delete,21.542,-1.0,-1.0,-1.0,-1.0,21.936,3.35707,2989,7566707 +1154,search,6453.71,0.92562,0.106001,16.1764,18.2615,23.837,3.35707,2989,7566707 +1155,delete,107.154,-1.0,-1.0,-1.0,-1.0,175.353,3.32401,2971,7396192 +1156,search,6337.92,0.92516,0.106319,16.2627,18.2974,23.278,3.32401,2971,7396192 +1157,delete,219.903,-1.0,-1.0,-1.0,-1.0,361.786,3.24518,2921,7079041 +1158,search,6180.1,0.92406,0.106962,16.378,18.3713,35.726,3.24538,2923,7079041 +1159,delete,319.29,-1.0,-1.0,-1.0,-1.0,448.667,3.08636,2863,6597115 +1160,search,5925.14,0.92213,0.107465,16.7798,18.5013,52.873,3.08705,2867,6597115 +1161,delete,230.195,-1.0,-1.0,-1.0,-1.0,277.615,2.9973,2825,6308902 +1162,search,5806.93,0.92063,0.108264,17.0826,18.7185,42.532,2.99776,2828,6308902 +1163,delete,108.427,-1.0,-1.0,-1.0,-1.0,184.509,2.94359,2805,6164514 +1164,search,5785.36,0.92027,0.108281,17.0775,18.6527,23.145,2.94359,2805,6164514 +1165,delete,27.301,-1.0,-1.0,-1.0,-1.0,49.737,2.93685,2802,6133412 +1166,search,5572.1,0.9201,0.108481,17.1098,18.7054,22.743,2.93685,2802,6133412 +1167,delete,56.243,-1.0,-1.0,-1.0,-1.0,20.6,2.92432,2802,6057218 +1168,search,5686.36,0.91981,0.108599,17.1916,18.796,28.415,2.92433,2803,6057218 +1169,delete,16.685,-1.0,-1.0,-1.0,-1.0,57.757,2.91271,2795,6028053 +1170,search,5650.04,0.91906,0.108971,17.3081,18.8725,22.585,2.91271,2795,6028053 +1171,delete,23.641,-1.0,-1.0,-1.0,-1.0,25.744,2.90549,2794,6001880 +1172,search,5627.36,0.91906,0.109127,17.3106,18.8609,22.714,2.90549,2794,6001880 +1173,delete,38.149,-1.0,-1.0,-1.0,-1.0,45.383,2.89563,2791,5957180 +1174,search,5608.24,0.91875,0.109627,17.3728,18.9137,23.021,2.89563,2791,5957180 +1175,delete,16.602,-1.0,-1.0,-1.0,-1.0,20.015,2.8935,2791,5939326 +1176,search,5615.73,0.91876,0.109577,17.3761,18.9135,22.767,2.8935,2791,5939326 +1177,delete,11.088,-1.0,-1.0,-1.0,-1.0,24.527,2.88879,2790,5924145 +1178,search,5595.47,0.91882,0.109487,17.3826,18.9252,22.579,2.88879,2790,5924145 +1179,delete,54.481,-1.0,-1.0,-1.0,-1.0,56.433,2.86326,2785,5863334 +1180,search,5598.37,0.91834,0.109822,17.4288,18.9442,22.104,2.86326,2785,5863334 +1181,delete,32.588,-1.0,-1.0,-1.0,-1.0,20.154,2.8536,2785,5822703 +1182,search,5571.27,0.91804,0.11025,17.4944,18.9938,21.951,2.8536,2785,5822703 +1183,delete,39.642,-1.0,-1.0,-1.0,-1.0,43.138,2.83969,2782,5775807 +1184,search,5563.45,0.91714,0.110871,17.5849,19.0212,22.958,2.83969,2782,5775807 +1185,delete,22.768,-1.0,-1.0,-1.0,-1.0,24.798,2.8358,2781,5750627 +1186,search,7299.77,0.9169,0.1108,17.6265,19.0393,23.789,2.8358,2781,5750627 +1187,delete,142.981,-1.0,-1.0,-1.0,-1.0,127.417,2.78844,2765,5601207 +1188,search,5497,0.91672,0.111148,17.6646,19.1089,22.983,2.78844,2765,5601207 +1189,delete,17.989,-1.0,-1.0,-1.0,-1.0,19.732,2.78304,2765,5586425 +1190,search,5479.37,0.91639,0.111279,17.6848,19.109,22.567,2.78304,2765,5586425 +1191,delete,11.647,-1.0,-1.0,-1.0,-1.0,28.199,2.77953,2764,5574023 +1192,search,5443.65,0.91633,0.111268,17.7014,19.1097,21.996,2.77953,2764,5574023 +1193,delete,179.02,-1.0,-1.0,-1.0,-1.0,316.363,2.67069,2726,5356157 +1194,search,5323.69,0.91586,0.111972,17.8002,19.2239,38.794,2.67108,2728,5356157 +1195,delete,17.324,-1.0,-1.0,-1.0,-1.0,19.609,2.66719,2728,5340404 +1196,search,5313.23,0.91612,0.11185,17.786,19.2294,22.14,2.66719,2728,5340404 +1197,delete,54.451,-1.0,-1.0,-1.0,-1.0,19.687,2.65219,2728,5293259 +1198,search,5324.93,0.91597,0.112259,17.8006,19.2318,22.505,2.65219,2728,5293259 +1199,delete,13.247,-1.0,-1.0,-1.0,-1.0,19.301,2.64688,2728,5279787 +1200,search,5307.9,0.91599,0.112244,17.7818,19.2127,22.813,2.64688,2728,5279787 +1201,delete,66.267,-1.0,-1.0,-1.0,-1.0,65.086,2.61457,2721,5216567 +1202,search,5272.48,0.91519,0.112454,17.9187,19.2792,22.001,2.61457,2721,5216567 +1203,delete,21.084,-1.0,-1.0,-1.0,-1.0,19.08,2.60765,2721,5196947 +1204,search,5249.69,0.9151,0.112774,17.9017,19.256,21.576,2.60765,2721,5196947 +1205,delete,20.822,-1.0,-1.0,-1.0,-1.0,44.778,2.59503,2717,5168447 +1206,search,5223.93,0.91512,0.112785,17.9099,19.2371,21.314,2.59503,2717,5168447 +1207,delete,20.4,-1.0,-1.0,-1.0,-1.0,36.002,2.58558,2715,5146927 +1208,search,5204.36,0.91447,0.113131,17.9912,19.2332,22.351,2.58558,2715,5146927 +1209,delete,254.866,-1.0,-1.0,-1.0,-1.0,308.057,2.45342,2672,4887612 +1210,search,5106.48,0.9141,0.113471,18.0622,19.2556,21.827,2.45342,2672,4887612 +1211,delete,6.858,-1.0,-1.0,-1.0,-1.0,18.453,2.45164,2672,4882301 +1212,search,5149.37,0.91387,0.113654,18.0714,19.2774,22.304,2.45164,2672,4882301 +1213,delete,107.26,-1.0,-1.0,-1.0,-1.0,79.648,2.40332,2661,4780915 +1214,search,5086.78,0.91326,0.113942,18.2268,19.4099,21.598,2.40332,2661,4780915 +1215,delete,12.553,-1.0,-1.0,-1.0,-1.0,18.483,2.40026,2661,4765751 +1216,search,4945.48,0.91319,0.114145,18.2247,19.4115,21.113,2.40026,2661,4765751 +1217,delete,293.166,-1.0,-1.0,-1.0,-1.0,367.354,2.23106,2581,4452306 +1218,search,4833.48,0.91099,0.115409,18.4955,19.5476,19.618,2.23106,2581,4452306 +1219,delete,75.392,-1.0,-1.0,-1.0,-1.0,134.245,2.18104,2556,4362045 +1220,search,4764.95,0.91023,0.116076,18.6339,19.6852,20.667,2.18104,2556,4362045 +1221,delete,37.259,-1.0,-1.0,-1.0,-1.0,35.716,2.16834,2551,4326917 +1222,search,4775.58,0.91006,0.116405,18.6905,19.6921,20.459,2.16834,2551,4326917 +1223,delete,205.112,-1.0,-1.0,-1.0,-1.0,257.206,2.05993,2502,4109781 +1224,search,4633.11,0.90782,0.117655,19.0711,19.9174,19.611,2.05993,2502,4109781 +1225,delete,30.218,-1.0,-1.0,-1.0,-1.0,32.839,2.04406,2497,4075879 +1226,search,4636.58,0.90721,0.118166,19.1032,19.9167,19.952,2.04406,2497,4075879 +1227,delete,68.257,-1.0,-1.0,-1.0,-1.0,68.967,2.01382,2490,4006503 +1228,search,4578.32,0.90715,0.118335,19.1436,19.9212,30.336,2.01427,2492,4006503 +1229,delete,84.968,-1.0,-1.0,-1.0,-1.0,52.243,1.97686,2487,3922625 +1230,search,4557.35,0.9064,0.118962,19.1807,19.9425,19.09,1.97686,2487,3922625 +1231,delete,65.599,-1.0,-1.0,-1.0,-1.0,133.682,1.93415,2450,3844860 +1232,search,4476.93,0.90534,0.120033,19.2537,19.9752,18.436,1.93415,2450,3844860 +1233,delete,11.795,-1.0,-1.0,-1.0,-1.0,19.326,1.92943,2448,3831414 +1234,search,4460.31,0.90475,0.120514,19.2819,19.9871,18.115,1.92943,2448,3831414 +1235,delete,132.636,-1.0,-1.0,-1.0,-1.0,106.587,1.86125,2432,3684234 +1236,search,4378.22,0.90346,0.120601,19.4914,20.1409,19.257,1.86125,2432,3684234 +1237,delete,20.277,-1.0,-1.0,-1.0,-1.0,16.216,1.85446,2432,3669074 +1238,search,4408.47,0.9036,0.120419,19.5018,20.1533,18.953,1.85446,2432,3669074 +1239,delete,63.54,-1.0,-1.0,-1.0,-1.0,17.639,1.83477,2431,3620984 +1240,search,4407.67,0.90383,0.12043,19.4768,20.1593,19.214,1.83477,2431,3620984 +1241,delete,43.807,-1.0,-1.0,-1.0,-1.0,23.164,1.81546,2430,3580113 +1242,search,4384.65,0.90352,0.12043,19.5896,20.2535,19.057,1.81546,2430,3580113 +1243,delete,50.022,-1.0,-1.0,-1.0,-1.0,32.961,1.79251,2425,3533909 +1244,search,4351.95,0.90284,0.12086,19.6866,20.3083,18.989,1.79251,2425,3533909 +1245,delete,15.993,-1.0,-1.0,-1.0,-1.0,16.284,1.78537,2425,3519590 +1246,search,4340.42,0.90268,0.121375,19.7009,20.3521,18.335,1.78537,2425,3519590 +1247,delete,48.155,-1.0,-1.0,-1.0,-1.0,94.621,1.74831,2388,3452537 +1248,search,4263.75,0.90121,0.1217,20.0605,20.5228,21.031,1.7484,2389,3452537 +1249,delete,33.484,-1.0,-1.0,-1.0,-1.0,31.095,1.73126,2387,3416711 +1250,search,4270.29,0.90092,0.121829,20.0936,20.5462,18.042,1.73126,2387,3416711 +1251,delete,28.05,-1.0,-1.0,-1.0,-1.0,30.297,1.71824,2383,3387653 +1252,search,4233.17,0.90063,0.121753,20.1729,20.6226,19.307,1.71824,2383,3387653 +1253,delete,19.776,-1.0,-1.0,-1.0,-1.0,21.892,1.70978,2382,3372157 +1254,search,4286.63,0.90075,0.121695,20.175,20.648,18.999,1.70978,2382,3372157 +1255,delete,104.067,-1.0,-1.0,-1.0,-1.0,106.524,1.64867,2338,3258215 +1256,search,4180.35,0.89863,0.122703,20.5796,20.9086,26.768,1.64867,2338,3258215 +1257,delete,38.372,-1.0,-1.0,-1.0,-1.0,46.929,1.62971,2330,3223657 +1258,search,4143.72,0.89812,0.122966,20.7238,21.0369,26.412,1.62971,2330,3223657 +1259,delete,50.26,-1.0,-1.0,-1.0,-1.0,50.412,1.60536,2325,3174982 +1260,search,4146.18,0.89795,0.122765,20.8811,21.186,26.516,1.60536,2325,3174982 +1261,delete,18.25,-1.0,-1.0,-1.0,-1.0,23.791,1.59939,2325,3157038 +1262,search,4146.65,0.89783,0.122617,20.9483,21.2339,25.966,1.59939,2325,3157038 +1263,delete,98.253,-1.0,-1.0,-1.0,-1.0,171.521,1.54396,2306,3058847 +1264,search,4060.34,0.89737,0.122726,21.0697,21.3257,34.282,1.54401,2307,3058847 +1265,delete,220.385,-1.0,-1.0,-1.0,-1.0,253.927,1.41204,2202,2833531 +1266,search,3858.02,0.89452,0.124569,21.4105,21.3276,23.525,1.41204,2202,2833531 +1267,delete,65.939,-1.0,-1.0,-1.0,-1.0,56.966,1.38404,2195,2763793 +1268,search,3866.22,0.89307,0.125898,21.6084,21.3697,23.465,1.38404,2195,2763793 +1269,delete,130.881,-1.0,-1.0,-1.0,-1.0,171.78,1.31519,2164,2627384 +1270,search,3809.89,0.89328,0.12557,21.5303,21.3216,24.057,1.31519,2164,2627384 +1271,delete,81.5,-1.0,-1.0,-1.0,-1.0,56.285,1.282,2159,2553476 +1272,search,3815.04,0.89273,0.125503,21.7382,21.4696,24.129,1.282,2159,2553476 +1273,delete,120.712,-1.0,-1.0,-1.0,-1.0,141.902,1.22198,2129,2443067 +1274,search,3735.09,0.89186,0.125312,21.9827,21.562,23.625,1.22198,2129,2443067 +1275,delete,25.49,-1.0,-1.0,-1.0,-1.0,27.169,1.21432,2128,2421654 +1276,search,3741.5,0.89114,0.125439,22.1054,21.6495,23.043,1.21432,2128,2421654 +1277,delete,29.033,-1.0,-1.0,-1.0,-1.0,44.226,1.20299,2122,2391396 +1278,search,3718.06,0.89045,0.125808,22.2342,21.7367,23.123,1.20299,2122,2391396 +1279,delete,25.79,-1.0,-1.0,-1.0,-1.0,20.858,1.19363,2122,2367854 +1280,search,3687.91,0.89038,0.125743,22.2709,21.7681,23.173,1.19363,2122,2367854 diff --git a/scripts/big_ann_perf_numbers/scan_0.15_no_aps_refinment_wma_delete.csv b/scripts/big_ann_perf_numbers/scan_0.15_no_aps_refinment_wma_delete.csv new file mode 100644 index 00000000..2a550f66 --- /dev/null +++ b/scripts/big_ann_perf_numbers/scan_0.15_no_aps_refinment_wma_delete.csv @@ -0,0 +1 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors diff --git a/scripts/big_ann_perf_numbers/scan_0.1_no_aps_refinment_wma.csv b/scripts/big_ann_perf_numbers/scan_0.1_no_aps_refinment_wma.csv new file mode 100644 index 00000000..f44212e9 --- /dev/null +++ b/scripts/big_ann_perf_numbers/scan_0.1_no_aps_refinment_wma.csv @@ -0,0 +1,307 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,2474.44,0.78814,0.157167,27.1243,17.7481,5.575,0.0184881,1024,38806 +3,insert,14.943,-1.0,-1.0,-1.0,-1.0,11.622,0.0226305,1020,49334 +4,search,1936.45,0.80572,0.156545,25.2691,17.2559,24.277,0.0226346,1017,49334 +5,insert,40.838,-1.0,-1.0,-1.0,-1.0,155.556,0.0332047,1008,73557 +6,search,1840.04,0.88145,0.133136,18.3181,16.1764,108.067,0.0333193,1011,73557 +7,insert,26.377,-1.0,-1.0,-1.0,-1.0,97.27,0.0429624,1011,95766 +8,search,1852.77,0.87507,0.134743,18.3914,15.6164,78.654,0.043044,1013,95766 +9,insert,8.297,-1.0,-1.0,-1.0,-1.0,68.645,0.0449143,1013,100025 +10,search,1852.78,0.87503,0.133335,18.2295,15.3042,71.031,0.0449171,1014,100025 +11,insert,10.258,-1.0,-1.0,-1.0,-1.0,74.512,0.0496414,1016,111417 +12,search,1770.43,0.8755,0.13347,18.0777,15.1491,73.301,0.0496671,1017,111417 +13,insert,23.314,-1.0,-1.0,-1.0,-1.0,73.03,0.0608246,1018,137139 +14,search,1847.34,0.87032,0.137551,18.2122,14.9817,13.688,0.0611184,1019,137139 +15,insert,17.826,-1.0,-1.0,-1.0,-1.0,9.192,0.0696652,1021,156710 +16,search,1902.5,0.86659,0.138558,18.302,14.6745,18.616,0.0696684,1023,156710 +17,insert,7.79,-1.0,-1.0,-1.0,-1.0,8.297,0.0717321,1025,161773 +18,search,1917.43,0.86518,0.138967,18.2843,14.6284,14.618,0.0719741,1029,161773 +19,insert,33.733,-1.0,-1.0,-1.0,-1.0,8.601,0.0790512,1031,177161 +20,search,1897.46,0.86927,0.134747,18.1524,14.4929,7.507,0.079052,1032,177161 +21,insert,356.684,-1.0,-1.0,-1.0,-1.0,169.807,0.200689,1081,448898 +22,search,2122.75,0.85509,0.139029,19.9947,15.3547,50.861,0.200685,1095,448898 +23,insert,393.898,-1.0,-1.0,-1.0,-1.0,250.916,0.327744,1152,733493 +24,search,2275.95,0.84922,0.140953,20.8617,15.957,96.691,0.328842,1182,733493 +25,insert,47.875,-1.0,-1.0,-1.0,-1.0,25.103,0.345984,1188,777443 +26,search,2371.65,0.84568,0.143666,21.2454,16.1756,16.276,0.345987,1191,777443 +27,insert,29.855,-1.0,-1.0,-1.0,-1.0,22.008,0.358023,1195,805394 +28,search,2384.53,0.84718,0.14228,21.217,16.1749,14.681,0.358152,1198,805394 +29,insert,57.885,-1.0,-1.0,-1.0,-1.0,49.159,0.386003,1211,866391 +30,search,2435.04,0.85166,0.141473,20.8367,16.1121,22.93,0.386032,1216,866391 +31,insert,524.088,-1.0,-1.0,-1.0,-1.0,323.368,0.553558,1287,1239816 +32,search,2672.22,0.84472,0.146027,21.7483,16.8256,34.923,0.553557,1294,1239816 +33,insert,52.224,-1.0,-1.0,-1.0,-1.0,32.113,0.573287,1303,1291211 +34,search,2711.1,0.84689,0.144118,21.8038,16.9059,7.72,0.573287,1303,1291211 +35,insert,32.721,-1.0,-1.0,-1.0,-1.0,21.36,0.585495,1308,1324978 +36,search,2671.48,0.84891,0.142213,21.6701,16.8286,7.382,0.585495,1308,1324978 +37,insert,49.162,-1.0,-1.0,-1.0,-1.0,33.544,0.606042,1314,1371885 +38,search,2749.11,0.85002,0.141407,21.8092,17.1229,20.051,0.606086,1317,1371885 +39,insert,16.78,-1.0,-1.0,-1.0,-1.0,12.528,0.609949,1318,1379335 +40,search,2762.19,0.84972,0.141774,21.8719,17.1904,10.901,0.60995,1319,1379335 +41,insert,51.874,-1.0,-1.0,-1.0,-1.0,18.261,0.623571,1322,1413312 +42,search,2729.04,0.85201,0.14091,21.8075,17.2453,11.334,0.623689,1323,1413312 +43,insert,12.207,-1.0,-1.0,-1.0,-1.0,13.67,0.627857,1326,1423411 +44,search,2781.39,0.85154,0.140795,21.8802,17.2859,7.572,0.627857,1326,1423411 +45,insert,225.454,-1.0,-1.0,-1.0,-1.0,20.159,0.642389,1331,1457559 +46,search,2748.25,0.8539,0.13963,21.7753,17.307,9.709,0.642389,1331,1457559 +47,insert,294.348,-1.0,-1.0,-1.0,-1.0,280.479,0.748614,1378,1688593 +48,search,2848.64,0.85676,0.141666,21.3603,17.3262,82.146,0.748714,1389,1688593 +49,insert,37.629,-1.0,-1.0,-1.0,-1.0,32.657,0.761819,1396,1723505 +50,search,2819.92,0.85785,0.141257,21.3848,17.4382,25.806,0.761833,1398,1723505 +51,insert,419.726,-1.0,-1.0,-1.0,-1.0,300.188,0.903659,1457,2035040 +52,search,3160.86,0.86258,0.139367,21.1566,17.7708,51.847,0.903666,1465,2035040 +53,insert,57.602,-1.0,-1.0,-1.0,-1.0,26.42,0.920326,1468,2078341 +54,search,3203.74,0.86312,0.138866,21.1651,17.7897,17.336,0.920539,1469,2078341 +55,insert,398.497,-1.0,-1.0,-1.0,-1.0,409.142,1.06886,1529,2399905 +56,search,3307.74,0.86598,0.138903,20.8097,17.9137,106.556,1.06975,1550,2399905 +57,insert,24.266,-1.0,-1.0,-1.0,-1.0,30.39,1.07846,1555,2419836 +58,search,3378.47,0.86491,0.139268,21.0153,18.057,32.327,1.07847,1557,2419836 +59,insert,245.099,-1.0,-1.0,-1.0,-1.0,151.647,1.17751,1578,2643083 +60,search,3483.57,0.85878,0.145052,22.3401,19.5367,56.963,1.17805,1587,2643083 +61,insert,14.836,-1.0,-1.0,-1.0,-1.0,15.744,1.18214,1589,2653064 +62,search,3523.8,0.85835,0.145686,22.4059,19.6398,16.362,1.18214,1590,2653064 +63,insert,5.228,-1.0,-1.0,-1.0,-1.0,8.636,1.18347,1590,2656118 +64,search,3542.63,0.85918,0.145274,22.3934,19.6431,17.117,1.18356,1592,2656118 +65,insert,98.241,-1.0,-1.0,-1.0,-1.0,68.465,1.20998,1608,2723424 +66,search,3608.58,0.8598,0.14485,22.3636,19.6494,13.138,1.20998,1609,2723424 +67,insert,53.68,-1.0,-1.0,-1.0,-1.0,69.099,1.22946,1624,2769021 +68,search,3625.45,0.86142,0.143871,22.2031,19.5608,13.308,1.22946,1625,2769021 +69,insert,57.788,-1.0,-1.0,-1.0,-1.0,69.951,1.25255,1637,2821777 +70,search,3650.21,0.86296,0.143051,21.9621,19.4246,15.579,1.25273,1638,2821777 +71,insert,17.468,-1.0,-1.0,-1.0,-1.0,17.826,1.25823,1640,2834921 +72,search,3687.49,0.86411,0.142111,21.8769,19.3657,10.379,1.25823,1640,2834921 +73,insert,9.043,-1.0,-1.0,-1.0,-1.0,9.037,1.26175,1640,2842044 +74,search,3622.89,0.86421,0.142082,21.8687,19.36,10.593,1.26175,1640,2842044 +75,insert,25.854,-1.0,-1.0,-1.0,-1.0,26.221,1.26836,1643,2858890 +76,search,3673.33,0.86424,0.141994,21.7892,19.3139,10.378,1.26836,1643,2858890 +77,insert,36.67,-1.0,-1.0,-1.0,-1.0,16.391,1.27777,1644,2882818 +78,search,3515.92,0.86404,0.141792,21.8563,19.3335,10.448,1.27777,1644,2882818 +79,insert,516.736,-1.0,-1.0,-1.0,-1.0,32.008,1.30594,1648,2947806 +80,search,3644.34,0.86364,0.142101,21.945,19.4665,10.085,1.30594,1648,2947806 +81,insert,41.13,-1.0,-1.0,-1.0,-1.0,32.416,1.31784,1652,2976895 +82,search,3618.55,0.86536,0.141395,21.7271,19.3549,23.194,1.31822,1655,2976895 +83,insert,16.364,-1.0,-1.0,-1.0,-1.0,26.413,1.32418,1658,2990682 +84,search,3585.86,0.86536,0.141607,21.7303,19.3555,9.955,1.32418,1658,2990682 +85,insert,40.194,-1.0,-1.0,-1.0,-1.0,15.823,1.33888,1659,3025613 +86,search,3749.05,0.86622,0.141088,21.5963,19.3407,11.276,1.33888,1659,3025613 +87,insert,667.543,-1.0,-1.0,-1.0,-1.0,702.849,1.54517,1771,3468632 +88,search,4125.22,0.87163,0.137083,21.0748,19.0563,116.053,1.54609,1788,3468632 +89,insert,434.835,-1.0,-1.0,-1.0,-1.0,559.573,1.70351,1881,3831692 +90,search,4379.66,0.87643,0.133152,20.5457,18.6692,60.419,1.70362,1889,3831692 +91,insert,41.588,-1.0,-1.0,-1.0,-1.0,32.255,1.71402,1891,3864067 +92,search,4446.58,0.87675,0.132666,20.5604,18.6668,22.044,1.71419,1893,3864067 +93,insert,269.092,-1.0,-1.0,-1.0,-1.0,274.718,1.81794,1934,4103827 +94,search,4567.02,0.87736,0.133347,20.505,18.6745,73.326,1.81856,1944,4103827 +95,insert,19.518,-1.0,-1.0,-1.0,-1.0,25.466,1.82449,1946,4119898 +96,search,4592.71,0.87754,0.133017,20.4252,18.6205,20.469,1.82449,1946,4119898 +97,insert,12.393,-1.0,-1.0,-1.0,-1.0,32.21,1.8272,1948,4129367 +98,search,4438.91,0.87748,0.132938,20.4399,18.6349,19.884,1.8272,1948,4129367 +99,insert,6.19,-1.0,-1.0,-1.0,-1.0,18.924,1.8287,1948,4134502 +100,search,4572.13,0.87752,0.132966,20.4259,18.6289,19.844,1.8287,1948,4134502 +101,insert,42.201,-1.0,-1.0,-1.0,-1.0,74.621,1.84239,1958,4164482 +102,search,4624.09,0.87805,0.132783,20.3931,18.6031,27.773,1.84243,1959,4164482 +103,insert,17.344,-1.0,-1.0,-1.0,-1.0,25.205,1.84798,1960,4178601 +104,search,4629.28,0.87887,0.132246,20.3717,18.6031,37.265,1.84839,1963,4178601 +105,insert,14.028,-1.0,-1.0,-1.0,-1.0,19.603,1.8523,1963,4187774 +106,search,4625.17,0.8788,0.132205,20.364,18.5929,26.858,1.85234,1964,4187774 +107,insert,24.552,-1.0,-1.0,-1.0,-1.0,26.947,1.85767,1965,4200109 +108,search,4626.83,0.87878,0.132318,20.3727,18.6062,21.665,1.85767,1965,4200109 +109,insert,588.079,-1.0,-1.0,-1.0,-1.0,448.028,2.01751,2040,4557867 +110,search,4929.65,0.88226,0.129651,20.0292,18.5651,62.383,2.01827,2046,4557867 +111,insert,41.996,-1.0,-1.0,-1.0,-1.0,52.224,2.03033,2053,4587252 +112,search,4932.13,0.88255,0.129516,19.9377,18.5272,23.869,2.03033,2053,4587252 +113,insert,17.388,-1.0,-1.0,-1.0,-1.0,22.481,2.03487,2053,4600056 +114,search,4923.82,0.88282,0.129248,19.8918,18.5044,23.383,2.03487,2053,4600056 +115,insert,40.076,-1.0,-1.0,-1.0,-1.0,51.11,2.0448,2058,4627422 +116,search,4912.78,0.88314,0.129103,19.8187,18.4959,23.431,2.0448,2058,4627422 +117,insert,19.72,-1.0,-1.0,-1.0,-1.0,22.324,2.05058,2058,4639292 +118,search,4925.89,0.8832,0.128945,19.8098,18.4701,24.69,2.05058,2058,4639292 +119,insert,7.489,-1.0,-1.0,-1.0,-1.0,22.544,2.05277,2058,4643423 +120,search,4936.77,0.88329,0.128863,19.8028,18.4657,24.165,2.05277,2058,4643423 +121,insert,20.916,-1.0,-1.0,-1.0,-1.0,34.72,2.05829,2060,4657252 +122,search,4981.14,0.88396,0.128376,19.7944,18.4832,28.731,2.05834,2061,4657252 +123,insert,440.351,-1.0,-1.0,-1.0,-1.0,393.619,2.18802,2110,4938477 +124,search,5154.4,0.88517,0.128615,19.5603,18.3541,138.387,2.1892,2126,4938477 +125,insert,62.888,-1.0,-1.0,-1.0,-1.0,98.223,2.20625,2137,4980492 +126,search,5165.66,0.88528,0.128865,19.4884,18.3759,42.501,2.20626,2139,4980492 +127,insert,74.395,-1.0,-1.0,-1.0,-1.0,95.393,2.22756,2154,5030515 +128,search,5190.51,0.88593,0.128553,19.4324,18.3591,41.906,2.22788,2156,5030515 +129,delete,17.913,-1.0,-1.0,-1.0,-1.0,25.557,2.22739,2156,5009391 +130,search,5178.89,0.88585,0.128609,19.43,18.3786,26.26,2.22739,2156,5009391 +131,delete,4.61,-1.0,-1.0,-1.0,-1.0,26.161,2.22739,2156,5004093 +132,search,5161.23,0.88598,0.128629,19.4055,18.3651,27.925,2.22739,2156,5004093 +133,delete,12.353,-1.0,-1.0,-1.0,-1.0,25.46,2.22723,2156,4991787 +134,search,4934.01,0.88589,0.128658,19.4177,18.3638,27.523,2.22723,2156,4991787 +135,delete,18.448,-1.0,-1.0,-1.0,-1.0,25.642,2.22623,2156,4974857 +136,search,4962.35,0.88602,0.12844,19.4134,18.3911,27.346,2.22623,2156,4974857 +137,delete,4.441,-1.0,-1.0,-1.0,-1.0,25.743,2.22623,2156,4971386 +138,search,4918.04,0.88587,0.128664,19.4149,18.3973,27.667,2.22623,2156,4971386 +139,delete,9.117,-1.0,-1.0,-1.0,-1.0,25.365,2.22623,2156,4963487 +140,search,4822.54,0.88574,0.128627,19.4484,18.4051,27.066,2.22623,2156,4963487 +141,delete,20.855,-1.0,-1.0,-1.0,-1.0,25.411,2.22477,2156,4941830 +142,search,5159.78,0.8858,0.128558,19.4561,18.4336,26.745,2.22477,2156,4941830 +143,delete,18.68,-1.0,-1.0,-1.0,-1.0,25.542,2.22286,2156,4925657 +144,search,5141.05,0.88574,0.12858,19.4585,18.4319,27.7,2.22286,2156,4925657 +145,delete,2.57,-1.0,-1.0,-1.0,-1.0,25.388,2.22258,2156,4922686 +146,search,5150.9,0.88559,0.12859,19.4607,18.4243,27.894,2.22258,2156,4922686 +147,delete,11.208,-1.0,-1.0,-1.0,-1.0,25.241,2.22237,2156,4913636 +148,search,5156.6,0.88575,0.128589,19.4718,18.4394,27.872,2.22237,2156,4913636 +149,delete,226.035,-1.0,-1.0,-1.0,-1.0,26.204,2.19594,2156,4717250 +150,search,4828.65,0.88448,0.129753,19.5149,18.5196,28.129,2.19594,2156,4717250 +151,delete,190.593,-1.0,-1.0,-1.0,-1.0,26.209,2.15983,2156,4560269 +152,search,4760.54,0.88354,0.130214,19.6133,18.5502,28.061,2.15983,2156,4560269 +153,delete,40.378,-1.0,-1.0,-1.0,-1.0,25.797,2.14888,2156,4521936 +154,search,4701.02,0.88231,0.131004,19.7578,18.628,27.56,2.14888,2156,4521936 +155,delete,21.538,-1.0,-1.0,-1.0,-1.0,25.562,2.1453,2156,4502222 +156,search,4623.31,0.88226,0.131292,19.789,18.6663,27.56,2.1453,2156,4502222 +157,delete,44.056,-1.0,-1.0,-1.0,-1.0,25.672,2.13806,2156,4456039 +158,search,4622.15,0.88186,0.131449,19.8555,18.7279,27.541,2.13806,2156,4456039 +159,delete,306.839,-1.0,-1.0,-1.0,-1.0,24.69,2.0221,2156,4143922 +160,search,4763,0.88064,0.131761,20.0619,18.8865,28.948,2.0221,2156,4143922 +161,delete,57.895,-1.0,-1.0,-1.0,-1.0,26.154,2.01502,2156,4101445 +162,search,4757.49,0.88061,0.131996,20.0804,18.9124,28.731,2.01502,2156,4101445 +163,delete,32.376,-1.0,-1.0,-1.0,-1.0,25.569,2.00505,2156,4080175 +164,search,4780.71,0.88001,0.13261,20.123,18.9419,28.213,2.00505,2156,4080175 +165,delete,44.832,-1.0,-1.0,-1.0,-1.0,25.764,1.99596,2156,4047313 +166,search,4890.05,0.87977,0.13286,20.116,18.9171,28.59,1.99596,2156,4047313 +167,delete,11.397,-1.0,-1.0,-1.0,-1.0,25.566,1.99405,2156,4040738 +168,search,4833.99,0.87941,0.133098,20.1303,18.9118,27.724,1.99405,2156,4040738 +169,delete,24.823,-1.0,-1.0,-1.0,-1.0,25.367,1.98666,2156,4022674 +170,search,4885.51,0.87895,0.133293,20.2177,18.9867,27.508,1.98666,2156,4022674 +171,delete,10.251,-1.0,-1.0,-1.0,-1.0,25.533,1.98329,2156,4015835 +172,search,4856.18,0.87889,0.133397,20.2252,18.9875,28.831,1.98329,2156,4015835 +173,delete,46.534,-1.0,-1.0,-1.0,-1.0,25.729,1.9715,2156,3986424 +174,search,4830.26,0.87768,0.134187,20.3609,19.0836,28.745,1.9715,2156,3986424 +175,delete,223.824,-1.0,-1.0,-1.0,-1.0,25.469,1.90142,2156,3804693 +176,search,4718.66,0.87645,0.134148,20.6133,19.1572,28.607,1.90142,2156,3804693 +177,delete,28.814,-1.0,-1.0,-1.0,-1.0,25.554,1.89195,2156,3786352 +178,search,4676.78,0.87598,0.134221,20.6848,19.1789,28.407,1.89195,2156,3786352 +179,delete,311.522,-1.0,-1.0,-1.0,-1.0,25.727,1.78583,2156,3556994 +180,search,4458.28,0.87173,0.135348,21.2028,19.3133,38.785,1.78591,2157,3556994 +181,delete,42.346,-1.0,-1.0,-1.0,-1.0,25.885,1.77456,2157,3526824 +182,search,4508.32,0.87141,0.135263,21.2739,19.3705,28.013,1.77456,2157,3526824 +183,delete,183.158,-1.0,-1.0,-1.0,-1.0,25.121,1.70473,2157,3350245 +184,search,4587.8,0.86489,0.139895,22.0921,19.7246,28.144,1.70473,2157,3350245 +185,delete,19.294,-1.0,-1.0,-1.0,-1.0,25.646,1.69876,2157,3340234 +186,search,4614.79,0.86483,0.139711,22.0941,19.7163,29.058,1.69876,2157,3340234 +187,delete,247.624,-1.0,-1.0,-1.0,-1.0,26.262,1.61039,2157,3150857 +188,search,4542.1,0.86825,0.137815,21.541,19.1875,29.323,1.61039,2157,3150857 +189,delete,16.291,-1.0,-1.0,-1.0,-1.0,25.62,1.60613,2157,3143609 +190,search,4626.18,0.86793,0.137925,21.5527,19.1642,28.667,1.60613,2157,3143609 +191,delete,4.211,-1.0,-1.0,-1.0,-1.0,25.356,1.60532,2157,3141038 +192,search,4626.87,0.86799,0.137911,21.5548,19.1613,28.39,1.60532,2157,3141038 +193,delete,88.542,-1.0,-1.0,-1.0,-1.0,27.029,1.57657,2157,3081734 +194,search,4603.9,0.86648,0.138724,21.7186,19.2356,28.375,1.57657,2157,3081734 +195,delete,53.738,-1.0,-1.0,-1.0,-1.0,26.248,1.55782,2157,3043966 +196,search,4520.8,0.86567,0.13938,21.8145,19.2978,27.817,1.55782,2157,3043966 +197,delete,48.247,-1.0,-1.0,-1.0,-1.0,26.284,1.54167,2157,3008145 +198,search,4545.97,0.86506,0.139286,21.9121,19.3272,27.87,1.54167,2157,3008145 +199,delete,13.242,-1.0,-1.0,-1.0,-1.0,25.998,1.53776,2157,2999384 +200,search,4557.1,0.86463,0.139577,21.9858,19.3904,27.883,1.53776,2157,2999384 +201,delete,8.886,-1.0,-1.0,-1.0,-1.0,25.75,1.53387,2157,2993528 +202,search,4573.25,0.86454,0.139724,22.0115,19.3967,29.247,1.53387,2157,2993528 +203,delete,23.835,-1.0,-1.0,-1.0,-1.0,25.537,1.52663,2157,2980973 +204,search,4284.43,0.86439,0.139713,22.019,19.3891,29.465,1.52663,2157,2980973 +205,delete,39.989,-1.0,-1.0,-1.0,-1.0,25.914,1.51751,2157,2960291 +206,search,4489.24,0.86422,0.139675,22.0462,19.4043,28.42,1.51751,2157,2960291 +207,delete,58.631,-1.0,-1.0,-1.0,-1.0,25.753,1.49751,2157,2920620 +208,search,4509.48,0.8639,0.139818,22.0269,19.3626,28.69,1.49751,2157,2920620 +209,delete,33.556,-1.0,-1.0,-1.0,-1.0,25.962,1.48663,2157,2900501 +210,search,4546.77,0.86317,0.140219,22.1533,19.4234,28.657,1.48663,2157,2900501 +211,delete,13.992,-1.0,-1.0,-1.0,-1.0,25.445,1.48207,2157,2891660 +212,search,4462.44,0.86283,0.140591,22.1774,19.4375,27.871,1.48207,2157,2891660 +213,delete,22.484,-1.0,-1.0,-1.0,-1.0,25.73,1.4725,2157,2873257 +214,search,4546.73,0.86182,0.141173,22.2549,19.4645,27.851,1.4725,2157,2873257 +215,delete,360.116,-1.0,-1.0,-1.0,-1.0,24.555,1.30946,2157,2537571 +216,search,4373.68,0.85573,0.14394,23.03,19.7104,29.605,1.30946,2157,2537571 +217,delete,277.173,-1.0,-1.0,-1.0,-1.0,24.817,1.19384,2157,2308194 +218,search,4280.43,0.85041,0.146946,23.7204,20.157,28.968,1.19384,2157,2308194 +219,delete,37.195,-1.0,-1.0,-1.0,-1.0,26.382,1.18208,2157,2285820 +220,search,4327.78,0.84915,0.147933,23.835,20.2401,29.06,1.18208,2157,2285820 +221,delete,292.139,-1.0,-1.0,-1.0,-1.0,25.55,1.07904,2157,2082858 +222,search,4258.14,0.84847,0.148016,24.0135,20.3769,29.155,1.07904,2157,2082858 +223,delete,19.551,-1.0,-1.0,-1.0,-1.0,25.836,1.07244,2157,2071207 +224,search,4363.44,0.8466,0.148741,24.1423,20.4217,28.428,1.07244,2157,2071207 +225,delete,11.46,-1.0,-1.0,-1.0,-1.0,25.866,1.06874,2157,2065400 +226,search,4248.87,0.84634,0.148911,24.1557,20.4035,28.535,1.06874,2157,2065400 +227,delete,5.468,-1.0,-1.0,-1.0,-1.0,25.665,1.06695,2157,2062105 +228,search,4344.44,0.84636,0.148823,24.1913,20.4172,28.231,1.06695,2157,2062105 +229,delete,29.337,-1.0,-1.0,-1.0,-1.0,26.043,1.05691,2157,2039610 +230,search,4296.81,0.84453,0.149053,24.5457,20.6348,28.377,1.05691,2157,2039610 +231,delete,16.441,-1.0,-1.0,-1.0,-1.0,25.779,1.05112,2157,2027795 +232,search,4355.52,0.84377,0.149205,24.6821,20.6599,28.263,1.05112,2157,2027795 +233,delete,7.141,-1.0,-1.0,-1.0,-1.0,25.871,1.04822,2157,2022859 +234,search,4327.33,0.84359,0.149064,24.7145,20.6756,30.656,1.04822,2157,2022859 +235,delete,13.511,-1.0,-1.0,-1.0,-1.0,25.81,1.04464,2157,2016264 +236,search,4261.65,0.84358,0.149063,24.7298,20.6732,29.706,1.04464,2157,2016264 +237,delete,369.253,-1.0,-1.0,-1.0,-1.0,26.72,0.924536,2157,1782866 +238,search,4306.43,0.838,0.153124,25.4226,21.0895,29.824,0.924536,2157,1782866 +239,delete,36.379,-1.0,-1.0,-1.0,-1.0,26.451,0.911847,2157,1758931 +240,search,4269.73,0.83609,0.153589,25.6725,21.1656,28.901,0.911847,2157,1758931 +241,delete,13.639,-1.0,-1.0,-1.0,-1.0,26.134,0.906906,2157,1751275 +242,search,4185.14,0.8354,0.1538,25.7673,21.2139,29.259,0.906906,2157,1751275 +243,delete,29.147,-1.0,-1.0,-1.0,-1.0,26.402,0.897618,2157,1733264 +244,search,4121.92,0.83143,0.155579,26.9636,22.4739,29.077,0.897618,2157,1733264 +245,delete,15.71,-1.0,-1.0,-1.0,-1.0,26.127,0.891938,2157,1722658 +246,search,4260.98,0.83066,0.155937,27.1447,22.6701,28.446,0.891938,2157,1722658 +247,delete,5.46,-1.0,-1.0,-1.0,-1.0,25.767,0.889939,2157,1719601 +248,search,4270.59,0.82993,0.156119,27.337,22.8898,28.564,0.889939,2157,1719601 +249,delete,12.204,-1.0,-1.0,-1.0,-1.0,25.862,0.885132,2157,1711648 +250,search,4184.71,0.82928,0.156483,27.4542,22.9667,30.992,0.885132,2157,1711648 +251,delete,334.872,-1.0,-1.0,-1.0,-1.0,39.362,0.762469,2158,1470624 +252,search,4225.45,0.82819,0.156192,28.6022,24.2926,29.991,0.762469,2158,1470624 +253,delete,52.671,-1.0,-1.0,-1.0,-1.0,27.002,0.747616,2158,1442070 +254,search,4230.89,0.82654,0.156397,29.0905,24.701,30.157,0.747616,2158,1442070 +255,delete,59.152,-1.0,-1.0,-1.0,-1.0,26.542,0.729171,2158,1405549 +256,search,4103.03,0.82083,0.159156,29.8269,25.0357,29.2,0.729171,2158,1405549 +257,insert,14.114,-1.0,-1.0,-1.0,-1.0,25.679,0.730899,2158,1416288 +258,search,4135.56,0.82131,0.159031,29.7062,24.9671,29.262,0.730899,2158,1416288 +259,insert,27.946,-1.0,-1.0,-1.0,-1.0,26.187,0.736486,2158,1443010 +260,search,4189.15,0.82249,0.159485,29.4463,24.9074,28.325,0.736486,2158,1443010 +261,insert,3.34,-1.0,-1.0,-1.0,-1.0,25.739,0.736822,2158,1447529 +262,search,4115.37,0.82316,0.159063,29.3614,24.8817,28.28,0.736822,2158,1447529 +263,insert,170.884,-1.0,-1.0,-1.0,-1.0,83.175,0.773697,2160,1569110 +264,search,4224.27,0.82722,0.159735,28.468,24.4792,85.44,0.773695,2164,1569110 +265,insert,76.676,-1.0,-1.0,-1.0,-1.0,26.255,0.789744,2164,1636119 +266,search,4270.27,0.8328,0.156434,27.8924,24.2468,78.455,0.789748,2169,1636119 +267,insert,41.389,-1.0,-1.0,-1.0,-1.0,42.387,0.799427,2171,1665152 +268,search,4254.76,0.83496,0.155953,27.5911,24.2011,29.731,0.799427,2171,1665152 +269,insert,413.543,-1.0,-1.0,-1.0,-1.0,175.557,0.89276,2185,1916952 +270,search,4343.85,0.83758,0.154804,26.9966,23.8127,110.868,0.892758,2192,1916952 +271,insert,6.068,-1.0,-1.0,-1.0,-1.0,35.509,0.893753,2193,1920898 +272,search,4325.44,0.83857,0.154145,26.9142,23.8152,48.69,0.893754,2195,1920898 +273,insert,31.617,-1.0,-1.0,-1.0,-1.0,26.411,0.901482,2195,1942757 +274,search,4318.62,0.84122,0.153066,26.4468,23.4535,29.105,0.901482,2195,1942757 +275,insert,648.039,-1.0,-1.0,-1.0,-1.0,269.614,1.06044,2223,2328372 +276,search,4488.44,0.84589,0.150628,25.7651,23.1174,83.691,1.06043,2230,2328372 +277,insert,48.368,-1.0,-1.0,-1.0,-1.0,26.281,1.07206,2230,2363257 +278,search,4490.63,0.84772,0.149327,25.5889,23.0383,27.286,1.07206,2230,2363257 +279,insert,35.264,-1.0,-1.0,-1.0,-1.0,25.798,1.08104,2230,2388933 +280,search,4463.48,0.84853,0.149108,25.5018,23.0004,28.277,1.08104,2230,2388933 +281,insert,427.199,-1.0,-1.0,-1.0,-1.0,131.256,1.21004,2250,2684315 +282,search,3376.62,0.8499,0.149353,25.1426,22.8688,46.759,1.21032,2254,2684315 +283,insert,342.288,-1.0,-1.0,-1.0,-1.0,135.081,1.39481,2281,3109157 +284,search,3551.58,0.85713,0.144358,24.2364,22.4297,64.495,1.39484,2290,3109157 +285,insert,11.289,-1.0,-1.0,-1.0,-1.0,18.136,1.39958,2290,3122751 +286,search,3574.38,0.85761,0.144192,24.1713,22.4137,28.887,1.39958,2292,3122751 +287,insert,28.112,-1.0,-1.0,-1.0,-1.0,22.159,1.41475,2293,3163679 +288,search,3568.48,0.85826,0.144268,24.0742,22.3686,18.851,1.41475,2293,3163679 +289,insert,240.918,-1.0,-1.0,-1.0,-1.0,140.547,1.55095,2317,3472031 +290,search,3684.91,0.86252,0.141948,23.4338,22.0721,49.971,1.55096,2323,3472031 +291,insert,19.492,-1.0,-1.0,-1.0,-1.0,18.164,1.56276,2323,3495577 +292,search,3688.63,0.86469,0.141022,23.2741,22.0288,28.027,1.56276,2325,3495577 +293,insert,11.627,-1.0,-1.0,-1.0,-1.0,26.919,1.57188,2327,3515193 +294,search,3713.54,0.86469,0.141355,23.1969,22.0472,18.952,1.57188,2327,3515193 +295,insert,144.894,-1.0,-1.0,-1.0,-1.0,70.941,1.65897,2337,3723063 +296,search,3787.19,0.86775,0.139383,22.7609,21.8177,25.848,1.65897,2338,3723063 +297,insert,2.889,-1.0,-1.0,-1.0,-1.0,22.252,1.66035,2339,3727399 +298,search,3754.52,0.86773,0.139319,22.7574,21.8104,19.883,1.66035,2339,3727399 +299,insert,56.873,-1.0,-1.0,-1.0,-1.0,22.49,1.68617,2340,3788002 +300,search,3819.59,0.87095,0.137514,22.4226,21.5306,19.794,1.68617,2340,3788002 +301,insert,475.938,-1.0,-1.0,-1.0,-1.0,262.819,1.94647,2410,4370866 +302,search,4107.62,0.89534,0.121218,19.3653,19.9771,47.656,1.94659,2424,4370866 +303,insert,38.254,-1.0,-1.0,-1.0,-1.0,10.167,1.96889,2424,4420306 +304,search,4117.14,0.8949,0.121793,19.4215,20.0113,10.593,1.96889,2424,4420306 +305,insert,243.639,-1.0,-1.0,-1.0,-1.0,267.654,2.13559,2463,4789669 +306,search,4286.64,0.89617,0.121268,19.1775,19.8657,82.34,2.13568,2474,4789669 +307,insert,19.163,-1.0,-1.0,-1.0,-1.0,12,2.14405,2475,4815550 diff --git a/scripts/big_ann_perf_numbers/scan_0.1_no_aps_refinment_wma_delete.csv b/scripts/big_ann_perf_numbers/scan_0.1_no_aps_refinment_wma_delete.csv new file mode 100644 index 00000000..429108ea --- /dev/null +++ b/scripts/big_ann_perf_numbers/scan_0.1_no_aps_refinment_wma_delete.csv @@ -0,0 +1,1280 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,2510.87,0.78885,0.157032,27.0527,17.7427,5.294,0.0184873,1024,38806 +3,insert,17.992,-1.0,-1.0,-1.0,-1.0,12.211,0.0226399,1020,49334 +4,search,1897.33,0.80641,0.155898,25.2135,17.2457,16.15,0.0226509,1017,49334 +5,insert,27.331,-1.0,-1.0,-1.0,-1.0,161.442,0.0331879,1008,73557 +6,search,1836.14,0.8825,0.132204,18.2191,16.1598,126.261,0.0332585,1010,73557 +7,insert,26.905,-1.0,-1.0,-1.0,-1.0,132.605,0.0429522,1009,95766 +8,search,1853.5,0.87644,0.134428,18.092,15.5519,99.892,0.0430787,1012,95766 +9,insert,4.133,-1.0,-1.0,-1.0,-1.0,90.646,0.0448135,1012,100025 +10,search,1790.94,0.87833,0.131411,17.9568,15.225,95.3,0.0448135,1012,100025 +11,insert,18.763,-1.0,-1.0,-1.0,-1.0,83.347,0.0498246,1016,111417 +12,search,1824.12,0.87698,0.132463,17.9426,15.0717,77.56,0.0498241,1017,111417 +13,insert,23.346,-1.0,-1.0,-1.0,-1.0,79.317,0.0609413,1019,137139 +14,search,1829.88,0.86956,0.13772,18.1671,14.9146,18.292,0.0613979,1021,137139 +15,insert,18.767,-1.0,-1.0,-1.0,-1.0,11.578,0.0700046,1024,156710 +16,search,1862.19,0.86521,0.138489,18.3854,14.6427,14.23,0.0700071,1026,156710 +17,insert,4.574,-1.0,-1.0,-1.0,-1.0,11.715,0.0722099,1030,161773 +18,search,1867.57,0.86589,0.138037,18.336,14.6029,15.972,0.07242,1034,161773 +19,insert,32.365,-1.0,-1.0,-1.0,-1.0,9.038,0.0790778,1036,177161 +20,search,1879.51,0.86879,0.135162,18.2326,14.4954,13.795,0.079081,1038,177161 +21,insert,395.862,-1.0,-1.0,-1.0,-1.0,216.196,0.201797,1100,448898 +22,search,2093.61,0.85609,0.137369,20.1353,15.4665,65.331,0.201875,1119,448898 +23,insert,389.892,-1.0,-1.0,-1.0,-1.0,290.344,0.3288,1186,733493 +24,search,2311.41,0.84861,0.140954,21.0543,16.1878,116.111,0.329953,1226,733493 +25,insert,37.211,-1.0,-1.0,-1.0,-1.0,30.434,0.347036,1234,777443 +26,search,2350.33,0.84505,0.143622,21.5492,16.4151,20.538,0.347098,1239,777443 +27,insert,28.429,-1.0,-1.0,-1.0,-1.0,23.696,0.358705,1244,805394 +28,search,2414.18,0.84439,0.144252,21.5598,16.4382,15.145,0.358809,1246,805394 +29,insert,69.083,-1.0,-1.0,-1.0,-1.0,72.422,0.385949,1266,866391 +30,search,2466.8,0.84816,0.143571,21.2681,16.4132,32.437,0.385959,1272,866391 +31,insert,490.382,-1.0,-1.0,-1.0,-1.0,330.674,0.554488,1355,1239816 +32,search,2685.18,0.84428,0.146244,21.9635,17.0705,58.568,0.55463,1371,1239816 +33,insert,30.903,-1.0,-1.0,-1.0,-1.0,32.743,0.574816,1383,1291211 +34,search,2714.91,0.84521,0.145599,22.1202,17.2401,23.198,0.574899,1388,1291211 +35,insert,32.543,-1.0,-1.0,-1.0,-1.0,31.841,0.586482,1398,1324978 +36,search,2768.03,0.84496,0.145398,22.0964,17.2303,10.165,0.586482,1398,1324978 +37,insert,62.535,-1.0,-1.0,-1.0,-1.0,43.626,0.606693,1407,1371885 +38,search,2792.24,0.84707,0.144021,22.1626,17.499,22.124,0.606804,1411,1371885 +39,insert,12.952,-1.0,-1.0,-1.0,-1.0,7.515,0.60949,1411,1379335 +40,search,2822.45,0.84787,0.143573,22.1817,17.5559,8.231,0.60949,1411,1379335 +41,insert,41.932,-1.0,-1.0,-1.0,-1.0,26.331,0.623605,1416,1413312 +42,search,2797.33,0.84873,0.143042,22.1136,17.6173,8.04,0.623605,1416,1413312 +43,insert,12.656,-1.0,-1.0,-1.0,-1.0,19.089,0.62779,1421,1423411 +44,search,2819.4,0.84895,0.142384,22.1871,17.6613,10.247,0.62779,1421,1423411 +45,insert,233.263,-1.0,-1.0,-1.0,-1.0,39.909,0.642786,1432,1457559 +46,search,2844,0.85025,0.142214,22.1521,17.7282,8.364,0.642786,1432,1457559 +47,insert,267.434,-1.0,-1.0,-1.0,-1.0,287.766,0.7478,1481,1688593 +48,search,3002.87,0.85338,0.144828,21.7492,17.7945,99.084,0.747934,1495,1688593 +49,insert,55.199,-1.0,-1.0,-1.0,-1.0,40.902,0.763193,1504,1723505 +50,search,2981.46,0.8542,0.144603,21.688,17.8515,25.353,0.763202,1508,1723505 +51,insert,406.42,-1.0,-1.0,-1.0,-1.0,323.47,0.90266,1583,2035040 +52,search,3249.42,0.86035,0.143096,21.4189,18.2485,42.354,0.902663,1592,2035040 +53,insert,46.754,-1.0,-1.0,-1.0,-1.0,27.989,0.920688,1597,2078341 +54,search,3281.34,0.86001,0.142983,21.4531,18.2694,9.85,0.920688,1597,2078341 +55,insert,413.547,-1.0,-1.0,-1.0,-1.0,459.84,1.06742,1672,2399905 +56,search,3552.58,0.86391,0.141969,21.1968,18.328,128.988,1.0684,1699,2399905 +57,insert,24.698,-1.0,-1.0,-1.0,-1.0,15.742,1.07695,1701,2419836 +58,search,3569.92,0.86325,0.142288,21.2595,18.3659,38.284,1.07698,1708,2419836 +59,insert,269.239,-1.0,-1.0,-1.0,-1.0,159.421,1.1761,1729,2643083 +60,search,3638.98,0.85309,0.152819,22.8749,19.9851,44.941,1.17614,1735,2643083 +61,insert,16.19,-1.0,-1.0,-1.0,-1.0,15.403,1.18034,1736,2653064 +62,search,3655.61,0.85333,0.152467,22.9282,20.0064,16.153,1.18039,1737,2653064 +63,insert,4.151,-1.0,-1.0,-1.0,-1.0,12.124,1.18138,1737,2656118 +64,search,3633.9,0.85343,0.152435,22.9259,20.0216,13.174,1.18138,1737,2656118 +65,insert,86.234,-1.0,-1.0,-1.0,-1.0,109.786,1.21057,1768,2723424 +66,search,3716.87,0.85431,0.150963,22.9123,20.0327,14.347,1.21057,1768,2723424 +67,insert,49.027,-1.0,-1.0,-1.0,-1.0,83.741,1.23077,1788,2769021 +68,search,3760.49,0.85583,0.149979,22.7181,19.9901,22.765,1.23086,1790,2769021 +69,insert,60.273,-1.0,-1.0,-1.0,-1.0,72.098,1.25285,1805,2821777 +70,search,3808.74,0.85829,0.148571,22.4546,19.8502,20.195,1.25285,1806,2821777 +71,insert,22.349,-1.0,-1.0,-1.0,-1.0,28.494,1.25871,1810,2834921 +72,search,3848.06,0.85951,0.147675,22.3654,19.7941,28.525,1.2589,1812,2834921 +73,insert,9.799,-1.0,-1.0,-1.0,-1.0,15.601,1.26118,1812,2842044 +74,search,3862,0.85942,0.14784,22.3547,19.7868,16.648,1.26118,1812,2842044 +75,insert,30.891,-1.0,-1.0,-1.0,-1.0,27.062,1.26972,1815,2858890 +76,search,3845.98,0.85954,0.147692,22.2835,19.7363,23.779,1.2699,1816,2858890 +77,insert,33.306,-1.0,-1.0,-1.0,-1.0,22.667,1.27909,1817,2882818 +78,search,3925.13,0.85907,0.148255,22.3278,19.7485,16.603,1.27909,1817,2882818 +79,insert,469.443,-1.0,-1.0,-1.0,-1.0,31.291,1.30629,1821,2947806 +80,search,4394.29,0.85997,0.147338,22.3646,19.8183,11.012,1.30629,1821,2947806 +81,insert,18.05,-1.0,-1.0,-1.0,-1.0,14.638,1.31727,1823,2976895 +82,search,2979.97,0.86133,0.146238,22.1634,19.7422,11.299,1.31727,1823,2976895 +83,insert,9.38,-1.0,-1.0,-1.0,-1.0,25.596,1.32417,1829,2990682 +84,search,3114.54,0.86122,0.146055,22.1316,19.7434,11.634,1.32417,1829,2990682 +85,insert,24.363,-1.0,-1.0,-1.0,-1.0,32.79,1.33769,1837,3025613 +86,search,2998.78,0.8627,0.145252,22.0215,19.7358,14.857,1.33779,1838,3025613 +87,insert,339.628,-1.0,-1.0,-1.0,-1.0,428.053,1.54249,1956,3468632 +88,search,4326.23,0.86737,0.142602,21.3978,19.4815,102.349,1.5428,1968,3468632 +89,insert,572.502,-1.0,-1.0,-1.0,-1.0,790.31,1.70218,2080,3831692 +90,search,4595.32,0.87385,0.13641,20.801,19.034,108.454,1.70282,2093,3831692 +91,insert,43.126,-1.0,-1.0,-1.0,-1.0,62.001,1.71585,2102,3864067 +92,search,4613.09,0.87431,0.135531,20.7768,19.0553,46.046,1.71599,2106,3864067 +93,insert,336.949,-1.0,-1.0,-1.0,-1.0,256.014,1.82029,2147,4103827 +94,search,4736.66,0.87541,0.134931,20.7224,19.0464,60.971,1.82068,2152,4103827 +95,insert,20.053,-1.0,-1.0,-1.0,-1.0,49.179,1.82706,2158,4119898 +96,search,4734.31,0.8757,0.135004,20.6472,18.996,34.293,1.82725,2159,4119898 +97,insert,11.546,-1.0,-1.0,-1.0,-1.0,25.891,1.83071,2159,4129367 +98,search,4737.63,0.8758,0.13482,20.6529,18.9927,34.546,1.83091,2161,4129367 +99,insert,8.566,-1.0,-1.0,-1.0,-1.0,30.993,1.83401,2162,4134502 +100,search,4755.38,0.87601,0.134731,20.6395,18.9914,41.147,1.83431,2165,4134502 +101,insert,40.52,-1.0,-1.0,-1.0,-1.0,48.615,1.84703,2169,4164482 +102,search,4768.65,0.87619,0.134728,20.5941,18.9886,28.01,1.84703,2169,4164482 +103,insert,25.527,-1.0,-1.0,-1.0,-1.0,38.153,1.85371,2172,4178601 +104,search,4815.29,0.87643,0.134586,20.5674,18.972,28.654,1.85371,2172,4178601 +105,insert,14.549,-1.0,-1.0,-1.0,-1.0,42.168,1.85761,2175,4187774 +106,search,4820.41,0.87637,0.134725,20.5533,18.9576,28.761,1.85761,2175,4187774 +107,insert,22.995,-1.0,-1.0,-1.0,-1.0,44.912,1.86285,2177,4200109 +108,search,4596.47,0.87629,0.1348,20.5572,18.9789,27.758,1.86285,2177,4200109 +109,insert,566.298,-1.0,-1.0,-1.0,-1.0,551.967,2.01867,2268,4557867 +110,search,3894.16,0.87927,0.133266,20.2417,18.9546,56.973,2.01936,2280,4557867 +111,insert,17.108,-1.0,-1.0,-1.0,-1.0,30.656,2.03011,2286,4587252 +112,search,3910.36,0.87976,0.13298,20.1785,18.9401,26.966,2.03048,2288,4587252 +113,insert,6.74,-1.0,-1.0,-1.0,-1.0,22.023,2.03475,2289,4600056 +114,search,3871.89,0.87983,0.133041,20.1453,18.9389,19.401,2.03475,2289,4600056 +115,insert,18.266,-1.0,-1.0,-1.0,-1.0,38.622,2.0463,2297,4627422 +116,search,3912.43,0.88064,0.132247,20.059,18.8716,22.081,2.04634,2298,4627422 +117,insert,8.143,-1.0,-1.0,-1.0,-1.0,18.55,2.05045,2298,4639292 +118,search,3904.84,0.88056,0.132404,20.0593,18.8578,23.538,2.05045,2298,4639292 +119,insert,5.514,-1.0,-1.0,-1.0,-1.0,22.996,2.0539,2299,4643423 +120,search,3945.45,0.88049,0.132455,20.0604,18.8648,19.738,2.0539,2299,4643423 +121,insert,12.382,-1.0,-1.0,-1.0,-1.0,22.345,2.05979,2300,4657252 +122,search,3931.43,0.88123,0.131974,20.0423,18.8702,27.411,2.05997,2302,4657252 +123,insert,202.928,-1.0,-1.0,-1.0,-1.0,279.064,2.19066,2361,4938477 +124,search,4072.44,0.88257,0.132693,19.7338,18.6822,88.191,2.19211,2380,4938477 +125,insert,33.54,-1.0,-1.0,-1.0,-1.0,33.448,2.2076,2386,4980492 +126,search,4044.42,0.88261,0.132702,19.6938,18.6786,22.979,2.20761,2388,4980492 +127,insert,35.448,-1.0,-1.0,-1.0,-1.0,35.554,2.22594,2399,5030515 +128,search,4111.11,0.88355,0.132007,19.5949,18.6696,15.99,2.22605,2401,5030515 +129,delete,10.875,-1.0,-1.0,-1.0,-1.0,15.381,2.22518,2390,5009391 +130,search,4065.59,0.88353,0.132007,19.6651,18.7597,11.302,2.22518,2390,5009391 +131,delete,2.677,-1.0,-1.0,-1.0,-1.0,10.668,2.22518,2390,5004093 +132,search,4046.49,0.88367,0.131942,19.6413,18.7374,11.049,2.22518,2390,5004093 +133,delete,6.279,-1.0,-1.0,-1.0,-1.0,10.115,2.22491,2390,4991787 +134,search,4046.43,0.88351,0.131898,19.6427,18.7376,11.551,2.22491,2390,4991787 +135,delete,9.434,-1.0,-1.0,-1.0,-1.0,27.575,2.22258,2387,4974857 +136,search,4039.39,0.883,0.132265,19.6683,18.7623,12.185,2.22258,2387,4974857 +137,delete,2.362,-1.0,-1.0,-1.0,-1.0,10.36,2.22258,2387,4971386 +138,search,4057.48,0.88287,0.132469,19.6712,18.7573,11.958,2.22258,2387,4971386 +139,delete,5.447,-1.0,-1.0,-1.0,-1.0,10.429,2.22245,2387,4963487 +140,search,4062.17,0.88288,0.132341,19.6986,18.7787,12.053,2.22245,2387,4963487 +141,delete,13.308,-1.0,-1.0,-1.0,-1.0,20.167,2.21949,2386,4941830 +142,search,4037.05,0.88321,0.132407,19.697,18.7964,12.319,2.21949,2386,4941830 +143,delete,10.279,-1.0,-1.0,-1.0,-1.0,17.547,2.21729,2385,4925657 +144,search,4028.73,0.88324,0.132271,19.7095,18.8049,12.33,2.21729,2385,4925657 +145,delete,1.743,-1.0,-1.0,-1.0,-1.0,14.006,2.21658,2384,4922686 +146,search,4014.98,0.88303,0.13252,19.724,18.8127,12.185,2.21658,2384,4922686 +147,delete,5.1,-1.0,-1.0,-1.0,-1.0,10.464,2.21658,2384,4913636 +148,search,4012.84,0.88304,0.132429,19.7508,18.8428,11.787,2.21658,2384,4913636 +149,delete,117.69,-1.0,-1.0,-1.0,-1.0,133.638,2.17954,2362,4717250 +150,search,3930.48,0.88261,0.133304,19.8735,18.9888,13.71,2.17954,2362,4717250 +151,delete,101.013,-1.0,-1.0,-1.0,-1.0,196.393,2.13132,2331,4560269 +152,search,3865.91,0.88165,0.132922,19.9984,19.005,20.433,2.13132,2331,4560269 +153,delete,18.784,-1.0,-1.0,-1.0,-1.0,57.252,2.12073,2324,4521936 +154,search,3839.32,0.88037,0.133873,20.1779,19.1118,20.624,2.12073,2324,4521936 +155,delete,12.051,-1.0,-1.0,-1.0,-1.0,25.355,2.11791,2323,4502222 +156,search,3837.44,0.8804,0.133817,20.1913,19.1337,20.708,2.11791,2323,4502222 +157,delete,26.591,-1.0,-1.0,-1.0,-1.0,30.605,2.1101,2321,4456039 +158,search,3820.92,0.87991,0.134283,20.2717,19.214,20.866,2.1101,2321,4456039 +159,delete,208.357,-1.0,-1.0,-1.0,-1.0,453.862,1.97109,2238,4143922 +160,search,3635.29,0.87769,0.135237,20.4918,19.3176,20.514,1.97109,2238,4143922 +161,delete,33.627,-1.0,-1.0,-1.0,-1.0,28.623,1.96262,2236,4101445 +162,search,3628.3,0.87768,0.135404,20.5386,19.3597,20.361,1.96262,2236,4101445 +163,delete,12.006,-1.0,-1.0,-1.0,-1.0,18.079,1.96003,2236,4080175 +164,search,3601.03,0.87722,0.135556,20.5887,19.3723,20.028,1.96003,2236,4080175 +165,delete,21.628,-1.0,-1.0,-1.0,-1.0,18.438,1.95454,2236,4047313 +166,search,3607.15,0.87701,0.135766,20.6206,19.4205,19.911,1.95454,2236,4047313 +167,delete,5.064,-1.0,-1.0,-1.0,-1.0,18.025,1.95281,2236,4040738 +168,search,3615.71,0.87678,0.136059,20.6495,19.4505,19.906,1.95281,2236,4040738 +169,delete,14.479,-1.0,-1.0,-1.0,-1.0,29.901,1.94716,2234,4022674 +170,search,3580.43,0.87651,0.136134,20.7229,19.4958,20.262,1.94716,2234,4022674 +171,delete,5.534,-1.0,-1.0,-1.0,-1.0,18.005,1.94567,2234,4015835 +172,search,3604.81,0.87631,0.136348,20.7356,19.4954,20.262,1.94567,2234,4015835 +173,delete,27.388,-1.0,-1.0,-1.0,-1.0,18.51,1.9367,2234,3986424 +174,search,3563.9,0.87545,0.137163,20.8375,19.5712,20.1,1.9367,2234,3986424 +175,delete,126.439,-1.0,-1.0,-1.0,-1.0,321.052,1.85196,2168,3804693 +176,search,3496.29,0.8735,0.137228,21.2564,19.8103,24.443,1.85224,2169,3804693 +177,delete,11.721,-1.0,-1.0,-1.0,-1.0,18.031,1.8479,2169,3786352 +178,search,3468.17,0.87312,0.137254,21.317,19.8345,20.786,1.8479,2169,3786352 +179,delete,202.676,-1.0,-1.0,-1.0,-1.0,303.802,1.72733,2085,3556994 +180,search,3421.15,0.86702,0.138478,22.2013,20.0513,24.376,1.72759,2086,3556994 +181,delete,18.816,-1.0,-1.0,-1.0,-1.0,17.531,1.72174,2085,3526824 +182,search,3285.16,0.86674,0.138693,22.2484,20.1216,21.141,1.72179,2086,3526824 +183,delete,127.528,-1.0,-1.0,-1.0,-1.0,339.736,1.63432,2004,3350245 +184,search,3160.55,0.85619,0.147825,23.3857,20.7234,18.28,1.63432,2004,3350245 +185,delete,7.062,-1.0,-1.0,-1.0,-1.0,17.24,1.63146,2002,3340234 +186,search,3138.15,0.85866,0.145296,23.1659,20.6214,17.868,1.63146,2002,3340234 +187,delete,122.46,-1.0,-1.0,-1.0,-1.0,191.066,1.53657,1967,3150857 +188,search,3093.07,0.86246,0.142132,22.3533,19.9071,26.262,1.53672,1969,3150857 +189,delete,4.71,-1.0,-1.0,-1.0,-1.0,15.849,1.53573,1969,3143609 +190,search,3072.21,0.86151,0.142415,22.4986,20.0336,17.636,1.53573,1969,3143609 +191,delete,1.648,-1.0,-1.0,-1.0,-1.0,15.639,1.53544,1969,3141038 +192,search,3061.69,0.86152,0.142414,22.503,20.0285,18.518,1.53544,1969,3141038 +193,delete,49.868,-1.0,-1.0,-1.0,-1.0,20.851,1.51712,1968,3081734 +194,search,3067.98,0.85982,0.144074,22.7053,20.0646,17.784,1.51712,1968,3081734 +195,delete,30.833,-1.0,-1.0,-1.0,-1.0,42.124,1.50054,1962,3043966 +196,search,3047.56,0.85906,0.144593,22.8204,20.1718,17.827,1.50054,1962,3043966 +197,delete,27.39,-1.0,-1.0,-1.0,-1.0,43.851,1.4878,1959,3008145 +198,search,3060.06,0.85802,0.144847,22.8781,20.1934,18.21,1.4878,1959,3008145 +199,delete,9.807,-1.0,-1.0,-1.0,-1.0,15.645,1.48378,1959,2999384 +200,search,3045.74,0.85797,0.144687,22.9556,20.2378,17.403,1.48378,1959,2999384 +201,delete,5.271,-1.0,-1.0,-1.0,-1.0,15.576,1.48111,1959,2993528 +202,search,3020.62,0.85774,0.144815,22.9821,20.2325,17.54,1.48111,1959,2993528 +203,delete,10.097,-1.0,-1.0,-1.0,-1.0,15.46,1.47758,1959,2980973 +204,search,3032.81,0.85726,0.145024,22.9876,20.2275,17.555,1.47758,1959,2980973 +205,delete,18.217,-1.0,-1.0,-1.0,-1.0,15.917,1.46936,1959,2960291 +206,search,3016.76,0.85721,0.145069,23.0212,20.2673,17.644,1.46936,1959,2960291 +207,delete,35.144,-1.0,-1.0,-1.0,-1.0,35.479,1.44714,1945,2920620 +208,search,2981.37,0.85644,0.146025,23.0808,20.3003,17.75,1.44714,1945,2920620 +209,delete,15.106,-1.0,-1.0,-1.0,-1.0,19.778,1.44065,1944,2900501 +210,search,2987.89,0.85582,0.146046,23.1376,20.2996,17.621,1.44065,1944,2900501 +211,delete,6.934,-1.0,-1.0,-1.0,-1.0,15.277,1.43712,1944,2891660 +212,search,2993.39,0.85585,0.146071,23.1515,20.3094,17.213,1.43712,1944,2891660 +213,delete,12.804,-1.0,-1.0,-1.0,-1.0,24.594,1.42917,1941,2873257 +214,search,2979.66,0.85506,0.146632,23.2227,20.3069,17.09,1.42917,1941,2873257 +215,delete,253.796,-1.0,-1.0,-1.0,-1.0,446.583,1.24234,1829,2537571 +216,search,2754.77,0.84921,0.14989,23.9383,20.441,26.915,1.24251,1831,2537571 +217,delete,155.398,-1.0,-1.0,-1.0,-1.0,281.1,1.13259,1761,2308194 +218,search,2609.24,0.84494,0.153393,24.4765,20.7676,15.126,1.13259,1761,2308194 +219,delete,14.009,-1.0,-1.0,-1.0,-1.0,20.575,1.12608,1759,2285820 +220,search,2630.88,0.84293,0.154444,24.6461,20.8375,15.547,1.12608,1759,2285820 +221,delete,148.965,-1.0,-1.0,-1.0,-1.0,268.937,1.01224,1666,2082858 +222,search,2477.86,0.8393,0.156557,25.1999,21.0657,14.019,1.01224,1666,2082858 +223,delete,9.23,-1.0,-1.0,-1.0,-1.0,14.901,1.0089,1665,2071207 +224,search,2472.94,0.83761,0.157449,25.2589,21.0267,14.529,1.0089,1665,2071207 +225,delete,4.836,-1.0,-1.0,-1.0,-1.0,12.31,1.00783,1665,2065400 +226,search,2471.69,0.8374,0.157636,25.2722,21.0276,14.312,1.00783,1665,2065400 +227,delete,3.22,-1.0,-1.0,-1.0,-1.0,12.337,1.00687,1665,2062105 +228,search,2470.23,0.83712,0.157576,25.3045,21.037,14.151,1.00687,1665,2062105 +229,delete,16.339,-1.0,-1.0,-1.0,-1.0,23.149,0.9991,1662,2039610 +230,search,2432.97,0.83571,0.15796,25.4911,21.0527,14.325,0.9991,1662,2039610 +231,delete,9.41,-1.0,-1.0,-1.0,-1.0,22.53,0.992544,1660,2027795 +232,search,2442.6,0.83513,0.157973,25.5951,21.0602,14.21,0.992544,1660,2027795 +233,delete,3.819,-1.0,-1.0,-1.0,-1.0,12.34,0.991211,1660,2022859 +234,search,2413.68,0.83482,0.158083,25.6066,21.0523,14.153,0.991211,1660,2022859 +235,delete,5.122,-1.0,-1.0,-1.0,-1.0,14.374,0.989339,1656,2016264 +236,search,2398.28,0.83417,0.158355,25.6888,21.0969,13.852,0.989339,1656,2016264 +237,delete,177.277,-1.0,-1.0,-1.0,-1.0,296.172,0.865572,1557,1782866 +238,search,2297.55,0.83006,0.161128,26.1586,21.3209,30.543,0.866272,1560,1782866 +239,delete,14.333,-1.0,-1.0,-1.0,-1.0,32.925,0.855528,1552,1758931 +240,search,2272.19,0.82854,0.161613,26.4253,21.4717,13.116,0.855528,1552,1758931 +241,delete,4.268,-1.0,-1.0,-1.0,-1.0,16.808,0.853507,1551,1751275 +242,search,2251.2,0.82817,0.161747,26.4988,21.4934,13.107,0.853507,1551,1751275 +243,delete,11.7,-1.0,-1.0,-1.0,-1.0,11.653,0.850073,1551,1733264 +244,search,2282.77,0.82629,0.162439,26.6447,21.5022,13.045,0.850073,1551,1733264 +245,delete,7.354,-1.0,-1.0,-1.0,-1.0,14.793,0.846959,1550,1722658 +246,search,2257.05,0.82555,0.162322,26.7402,21.5217,13.493,0.846959,1550,1722658 +247,delete,2.493,-1.0,-1.0,-1.0,-1.0,11.622,0.846386,1550,1719601 +248,search,2281.74,0.82523,0.162438,26.7692,21.5252,12.995,0.846386,1550,1719601 +249,delete,5.416,-1.0,-1.0,-1.0,-1.0,14.756,0.843263,1549,1711648 +250,search,2297.42,0.82415,0.162777,26.7984,21.5352,13.516,0.843263,1549,1711648 +251,delete,184.749,-1.0,-1.0,-1.0,-1.0,273.886,0.715505,1441,1470624 +252,search,2124.82,0.82335,0.162223,27.1209,21.5951,23.327,0.71584,1443,1470624 +253,delete,18.787,-1.0,-1.0,-1.0,-1.0,29.352,0.705441,1437,1442070 +254,search,2096.86,0.82177,0.162752,27.2435,21.6241,12.284,0.705441,1437,1442070 +255,delete,31.416,-1.0,-1.0,-1.0,-1.0,28.923,0.687757,1424,1405549 +256,search,2064.03,0.81792,0.165585,27.6843,21.7469,11.957,0.687757,1424,1405549 +257,insert,5.925,-1.0,-1.0,-1.0,-1.0,10.222,0.689618,1424,1416288 +258,search,2107.34,0.8182,0.16529,27.6103,21.7173,11.901,0.689618,1424,1416288 +259,insert,18.096,-1.0,-1.0,-1.0,-1.0,19.107,0.696107,1426,1443010 +260,search,2082.32,0.81806,0.165817,27.5624,21.7689,12.094,0.696107,1426,1443010 +261,insert,2.277,-1.0,-1.0,-1.0,-1.0,10.256,0.696853,1426,1447529 +262,search,2100.92,0.81891,0.165462,27.473,21.7392,12.121,0.696853,1426,1447529 +263,insert,93.909,-1.0,-1.0,-1.0,-1.0,25.877,0.734676,1429,1569110 +264,search,2133.75,0.82069,0.167103,26.8997,21.6203,17.766,0.73476,1430,1569110 +265,insert,54.653,-1.0,-1.0,-1.0,-1.0,18.328,0.757392,1432,1636119 +266,search,2112.58,0.82597,0.164696,26.4671,21.5103,12.076,0.757392,1432,1636119 +267,insert,24.993,-1.0,-1.0,-1.0,-1.0,25.055,0.767888,1435,1665152 +268,search,2126.63,0.82768,0.16473,26.1749,21.4363,11.81,0.767888,1435,1665152 +269,insert,191.249,-1.0,-1.0,-1.0,-1.0,66.769,0.86483,1445,1916952 +270,search,2232.49,0.8273,0.165135,26.0742,21.4135,33.571,0.864829,1448,1916952 +271,insert,3.63,-1.0,-1.0,-1.0,-1.0,10.704,0.866337,1448,1920898 +272,search,2193.28,0.82736,0.16492,26.032,21.407,12.095,0.866337,1448,1920898 +273,insert,18.481,-1.0,-1.0,-1.0,-1.0,12.708,0.876632,1449,1942757 +274,search,2214.24,0.83006,0.16331,25.7995,21.3159,11.876,0.876632,1449,1942757 +275,insert,320.733,-1.0,-1.0,-1.0,-1.0,195.504,1.04179,1480,2328372 +276,search,2350.67,0.83448,0.160991,25.4694,21.2769,82.428,1.04185,1493,2328372 +277,insert,26.353,-1.0,-1.0,-1.0,-1.0,11.046,1.05501,1493,2363257 +278,search,2371.52,0.83605,0.160407,25.3155,21.2328,41.523,1.05511,1500,2363257 +279,insert,20.423,-1.0,-1.0,-1.0,-1.0,11.216,1.06561,1500,2388933 +280,search,2407.83,0.83718,0.159445,25.2312,21.1948,16.506,1.06561,1501,2388933 +281,insert,183.965,-1.0,-1.0,-1.0,-1.0,104.981,1.19524,1512,2684315 +282,search,2492.13,0.83677,0.160492,25.3616,21.5698,19.952,1.19556,1514,2684315 +283,insert,333.32,-1.0,-1.0,-1.0,-1.0,196.387,1.38706,1556,3109157 +284,search,2621.9,0.84381,0.156855,24.5354,21.1729,71.166,1.38706,1569,3109157 +285,insert,9.398,-1.0,-1.0,-1.0,-1.0,14.405,1.39244,1570,3122751 +286,search,2631.28,0.84452,0.156419,24.5279,21.1891,37.851,1.39254,1576,3122751 +287,insert,30.937,-1.0,-1.0,-1.0,-1.0,17.856,1.40912,1577,3163679 +288,search,2656.42,0.84439,0.156573,24.4413,21.1451,17.482,1.40912,1578,3163679 +289,insert,227.542,-1.0,-1.0,-1.0,-1.0,181.255,1.54573,1611,3472031 +290,search,2770.36,0.8488,0.15505,23.9579,21.0016,65.804,1.54591,1619,3472031 +291,insert,21.224,-1.0,-1.0,-1.0,-1.0,14.031,1.55666,1620,3495577 +292,search,2772.04,0.85099,0.153805,23.7766,20.9749,36.807,1.55666,1624,3495577 +293,insert,12.77,-1.0,-1.0,-1.0,-1.0,25.178,1.56465,1627,3515193 +294,search,2819.03,0.8508,0.153968,23.7452,21.0188,18.263,1.56465,1628,3515193 +295,insert,151.958,-1.0,-1.0,-1.0,-1.0,162.832,1.65793,1660,3723063 +296,search,2874.47,0.855,0.151723,23.2139,20.781,39.295,1.65811,1665,3723063 +297,insert,3.472,-1.0,-1.0,-1.0,-1.0,11.936,1.65967,1665,3727399 +298,search,2880.43,0.85481,0.151958,23.2004,20.7771,20.402,1.65967,1666,3727399 +299,insert,52.73,-1.0,-1.0,-1.0,-1.0,29.312,1.68545,1672,3788002 +300,search,2935.55,0.85791,0.149034,22.9831,20.6083,17.687,1.68551,1673,3788002 +301,insert,443.376,-1.0,-1.0,-1.0,-1.0,522.936,1.94992,1812,4370866 +302,search,3293.47,0.88338,0.133219,20.2272,19.6253,91.483,1.95055,1843,4370866 +303,insert,27.242,-1.0,-1.0,-1.0,-1.0,59.855,1.97069,1853,4420306 +304,search,3299.07,0.88193,0.133859,20.3161,19.6364,17.859,1.97098,1854,4420306 +305,insert,194.903,-1.0,-1.0,-1.0,-1.0,309.532,2.13859,1911,4789669 +306,search,3450.71,0.88433,0.132829,19.9728,19.497,138.796,2.13988,1936,4789669 +307,insert,16.021,-1.0,-1.0,-1.0,-1.0,16.602,2.1481,1938,4815550 +308,search,3469.02,0.88473,0.132234,19.891,19.4585,42.407,2.1482,1944,4815550 +309,insert,5.55,-1.0,-1.0,-1.0,-1.0,12.971,2.15137,1944,4823345 +310,search,3485.4,0.88533,0.13201,19.8575,19.4432,16.576,2.15151,1945,4823345 +311,insert,9.633,-1.0,-1.0,-1.0,-1.0,12.662,2.15751,1945,4843493 +312,search,3525.93,0.88493,0.132405,19.9153,19.4709,13.858,2.15751,1945,4843493 +313,insert,31.596,-1.0,-1.0,-1.0,-1.0,61.231,2.17661,1954,4885489 +314,search,3560.63,0.88554,0.132065,19.8348,19.4153,17.747,2.17661,1955,4885489 +315,insert,5.255,-1.0,-1.0,-1.0,-1.0,12.472,2.17981,1955,4892335 +316,search,3510.67,0.88539,0.132098,19.8837,19.4734,13.594,2.17981,1955,4892335 +317,insert,35.303,-1.0,-1.0,-1.0,-1.0,44.096,2.19613,1964,4932996 +318,search,3535.19,0.88565,0.13189,19.8342,19.4305,21.391,2.19645,1966,4932996 +319,insert,390.584,-1.0,-1.0,-1.0,-1.0,420.137,2.4086,2050,5430684 +320,search,3765.28,0.89008,0.127945,19.2677,19.1523,114.015,2.4105,2079,5430684 +321,insert,29.723,-1.0,-1.0,-1.0,-1.0,70.109,2.42658,2099,5471239 +322,search,3818.29,0.89035,0.127552,19.1785,19.1051,42.486,2.42705,2108,5471239 +323,insert,70.88,-1.0,-1.0,-1.0,-1.0,118.492,2.47349,2137,5575560 +324,search,3870.29,0.89114,0.126874,19.1313,19.1053,28.493,2.47371,2141,5575560 +325,insert,936.469,-1.0,-1.0,-1.0,-1.0,544.041,2.70261,2221,6069499 +326,search,4085.11,0.89535,0.124076,18.5299,18.7791,185.349,2.70398,2258,6069499 +327,insert,41.762,-1.0,-1.0,-1.0,-1.0,54.66,2.7317,2270,6139647 +328,search,4153.75,0.89631,0.123221,18.3642,18.6525,69.543,2.7319,2282,6139647 +329,insert,5.247,-1.0,-1.0,-1.0,-1.0,17.599,2.73442,2282,6146884 +330,search,4200.29,0.89651,0.123116,18.332,18.6469,18.758,2.73442,2282,6146884 +331,insert,29.805,-1.0,-1.0,-1.0,-1.0,61.897,2.75343,2294,6188816 +332,search,4232.27,0.89644,0.123271,18.3477,18.6497,28.423,2.75362,2296,6188816 +333,insert,1.355,-1.0,-1.0,-1.0,-1.0,17.572,2.75399,2296,6190810 +334,search,4230.47,0.89637,0.123338,18.345,18.6488,19.015,2.75399,2296,6190810 +335,insert,35.778,-1.0,-1.0,-1.0,-1.0,39.598,2.77145,2304,6230108 +336,search,4251.48,0.89704,0.122932,18.2916,18.6219,18.681,2.77145,2304,6230108 +337,insert,14.422,-1.0,-1.0,-1.0,-1.0,28.879,2.7796,2306,6249699 +338,search,4241.69,0.89756,0.122546,18.2252,18.604,23.028,2.7796,2307,6249699 +339,insert,38.176,-1.0,-1.0,-1.0,-1.0,49.526,2.79706,2316,6290226 +340,search,4270.92,0.89803,0.121979,18.1315,18.5398,21.836,2.79714,2317,6290226 +341,insert,7.023,-1.0,-1.0,-1.0,-1.0,22.975,2.80047,2319,6297556 +342,search,4273.91,0.89797,0.122105,18.1255,18.5533,18.356,2.80047,2319,6297556 +343,insert,23.493,-1.0,-1.0,-1.0,-1.0,52.485,2.81354,2328,6330843 +344,search,4243.22,0.89822,0.121976,18.099,18.5404,24.531,2.81379,2330,6330843 +345,insert,25.659,-1.0,-1.0,-1.0,-1.0,50.888,2.82906,2340,6370265 +346,search,4296.43,0.89919,0.121283,18.0581,18.5243,30.94,2.82947,2344,6370265 +347,insert,325.928,-1.0,-1.0,-1.0,-1.0,564.027,3.03133,2469,6831086 +348,search,4596.63,0.90249,0.118953,17.587,18.2818,107.995,3.03407,2494,6831086 +349,insert,34.934,-1.0,-1.0,-1.0,-1.0,44.402,3.05232,2503,6876154 +350,search,4628.88,0.90311,0.118635,17.5586,18.2489,28.937,3.0526,2508,6876154 +351,insert,279.276,-1.0,-1.0,-1.0,-1.0,391.172,3.23727,2593,7288266 +352,search,4838.8,0.90164,0.121343,17.5257,18.1502,129.658,3.23869,2621,7288266 +353,insert,34.803,-1.0,-1.0,-1.0,-1.0,43.909,3.25699,2630,7333132 +354,search,4864.58,0.9023,0.120799,17.491,18.1536,55.557,3.25741,2640,7333132 +355,insert,269.038,-1.0,-1.0,-1.0,-1.0,500.907,3.42642,2747,7696838 +356,search,5095.65,0.90502,0.119472,17.1148,17.9863,135.12,3.42836,2773,7696838 +357,insert,12.483,-1.0,-1.0,-1.0,-1.0,36.753,3.4355,2777,7714498 +358,search,5045.1,0.90509,0.119502,17.0812,17.9673,31.587,3.43588,2781,7714498 +359,insert,7.134,-1.0,-1.0,-1.0,-1.0,28.24,3.43964,2784,7725755 +360,search,5079.92,0.90557,0.119212,17.025,17.9236,20.278,3.43964,2785,7725755 +361,insert,41.042,-1.0,-1.0,-1.0,-1.0,52.03,3.46528,2794,7789312 +362,search,5106.99,0.90652,0.118452,16.9221,17.8621,31.767,3.46565,2798,7789312 +363,insert,14.71,-1.0,-1.0,-1.0,-1.0,20.818,3.47075,2799,7803163 +364,search,5192.12,0.90633,0.118703,16.909,17.849,17.471,3.47075,2799,7803163 +365,insert,45.39,-1.0,-1.0,-1.0,-1.0,82.239,3.49651,2814,7863722 +366,search,5197.53,0.90691,0.118265,16.8294,17.7693,26.044,3.49681,2816,7863722 +367,insert,25.397,-1.0,-1.0,-1.0,-1.0,43.454,3.50868,2825,7895271 +368,search,5219.27,0.90752,0.117739,16.7723,17.7433,20.653,3.50868,2826,7895271 +369,insert,53.127,-1.0,-1.0,-1.0,-1.0,75.696,3.53592,2842,7961194 +370,search,5231.85,0.90804,0.117275,16.7122,17.7048,31.6,3.53645,2847,7961194 +371,insert,7.917,-1.0,-1.0,-1.0,-1.0,20.509,3.54012,2849,7970239 +372,search,5216.28,0.90803,0.1176,16.6961,17.7058,16.976,3.54012,2849,7970239 +373,insert,31.52,-1.0,-1.0,-1.0,-1.0,48.699,3.55836,2859,8010868 +374,search,5242.86,0.90859,0.116914,16.6505,17.6737,40.26,3.55922,2865,8010868 +375,insert,19.462,-1.0,-1.0,-1.0,-1.0,33.972,3.56874,2869,8040698 +376,search,5371.04,0.90871,0.117064,16.6138,17.6514,21.775,3.56887,2870,8040698 +377,insert,43.733,-1.0,-1.0,-1.0,-1.0,89.043,3.59598,2888,8104748 +378,search,5397.43,0.90915,0.116873,16.5022,17.5713,38.519,3.59637,2894,8104748 +379,insert,5.354,-1.0,-1.0,-1.0,-1.0,24.032,3.59877,2896,8109755 +380,search,5360.47,0.90921,0.116794,16.4999,17.5681,23.258,3.59899,2898,8109755 +381,insert,14.142,-1.0,-1.0,-1.0,-1.0,40.459,3.60637,2903,8130382 +382,search,5356.35,0.9093,0.11681,16.4846,17.5567,26.357,3.60677,2905,8130382 +383,insert,21.75,-1.0,-1.0,-1.0,-1.0,56.739,3.6157,2915,8153290 +384,search,5358.06,0.90963,0.116699,16.444,17.5315,21.717,3.6157,2916,8153290 +385,delete,9.798,-1.0,-1.0,-1.0,-1.0,16.976,3.61566,2916,8138873 +386,search,5374.49,0.90978,0.116655,16.4404,17.5435,18.319,3.61566,2916,8138873 +387,delete,13.178,-1.0,-1.0,-1.0,-1.0,17.678,3.61453,2916,8115069 +388,search,5386.07,0.91003,0.116394,16.4206,17.5232,20.608,3.61453,2916,8115069 +389,delete,11.745,-1.0,-1.0,-1.0,-1.0,23.327,3.61354,2915,8101157 +390,search,5378.98,0.90978,0.116603,16.4185,17.5218,18.638,3.61354,2915,8101157 +391,delete,51.951,-1.0,-1.0,-1.0,-1.0,46.455,3.59335,2914,8014644 +392,search,5386.79,0.9091,0.11651,16.5601,17.5774,37.071,3.59337,2917,8014644 +393,delete,22.653,-1.0,-1.0,-1.0,-1.0,17.479,3.59271,2917,7978358 +394,search,5361.96,0.90882,0.116643,16.5858,17.579,25.172,3.59271,2918,7978358 +395,delete,14.391,-1.0,-1.0,-1.0,-1.0,47.477,3.58671,2914,7957094 +396,search,5392.39,0.90873,0.116631,16.6155,17.5884,22.974,3.58684,2915,7957094 +397,delete,77.025,-1.0,-1.0,-1.0,-1.0,145.573,3.55177,2907,7823916 +398,search,5220.47,0.90866,0.116875,16.6141,17.5791,43.278,3.55207,2911,7823916 +399,delete,3.408,-1.0,-1.0,-1.0,-1.0,17.243,3.55165,2911,7818430 +400,search,5268.8,0.90873,0.116914,16.6373,17.6121,19.476,3.55165,2911,7818430 +401,delete,11.631,-1.0,-1.0,-1.0,-1.0,20.425,3.54919,2910,7800702 +402,search,5288.18,0.90833,0.117373,16.669,17.629,19.469,3.54919,2910,7800702 +403,delete,203.733,-1.0,-1.0,-1.0,-1.0,502.199,3.45965,2852,7517858 +404,search,5115.8,0.90769,0.117617,16.849,17.7301,24.768,3.45985,2853,7517858 +405,delete,44.31,-1.0,-1.0,-1.0,-1.0,17.122,3.45599,2853,7451637 +406,search,5072.64,0.90798,0.117583,16.828,17.7858,18.541,3.45599,2853,7451637 +407,delete,66.374,-1.0,-1.0,-1.0,-1.0,132.239,3.4263,2840,7360417 +408,search,5237.95,0.90762,0.117703,16.9023,17.8276,18.974,3.4263,2840,7360417 +409,delete,135.721,-1.0,-1.0,-1.0,-1.0,262.562,3.33262,2817,7131620 +410,search,5122.4,0.90787,0.117324,16.777,17.6487,38.608,3.33262,2820,7131620 +411,delete,175.167,-1.0,-1.0,-1.0,-1.0,345.101,3.2481,2772,6867617 +412,search,4979.09,0.90702,0.117955,16.8449,17.6781,23.975,3.24815,2773,6867617 +413,delete,10.001,-1.0,-1.0,-1.0,-1.0,22.323,3.24529,2772,6850098 +414,search,4962.54,0.90677,0.118137,16.8754,17.7056,23.694,3.24545,2773,6850098 +415,delete,47.317,-1.0,-1.0,-1.0,-1.0,48.782,3.22679,2770,6782921 +416,search,4957.11,0.90664,0.118503,16.8746,17.6942,18.324,3.22679,2770,6782921 +417,delete,150.89,-1.0,-1.0,-1.0,-1.0,320.965,3.15803,2739,6562039 +418,search,4893.02,0.90583,0.118968,16.9232,17.6831,48.758,3.15825,2742,6562039 +419,delete,20.116,-1.0,-1.0,-1.0,-1.0,16.415,3.15572,2742,6536285 +420,search,4848.14,0.9061,0.118681,16.9328,17.7013,35.592,3.15572,2744,6536285 +421,delete,16.786,-1.0,-1.0,-1.0,-1.0,67.442,3.14431,2738,6510820 +422,search,4889.31,0.906,0.118949,16.8834,17.6589,29.537,3.14459,2740,6510820 +423,delete,130.123,-1.0,-1.0,-1.0,-1.0,318.034,3.06726,2686,6334168 +424,search,4764.71,0.90425,0.119959,17.0926,17.8105,17.513,3.06726,2686,6334168 +425,delete,15.666,-1.0,-1.0,-1.0,-1.0,15.349,3.06427,2686,6316443 +426,search,4742.42,0.90402,0.120161,17.1117,17.8367,17.285,3.06427,2686,6316443 +427,delete,37.862,-1.0,-1.0,-1.0,-1.0,15.818,3.05068,2686,6266345 +428,search,4758.29,0.90359,0.120538,17.1783,17.8742,17.783,3.05068,2686,6266345 +429,delete,375.527,-1.0,-1.0,-1.0,-1.0,606.975,2.84772,2579,5883847 +430,search,4505.51,0.89414,0.126702,18.2726,18.2846,76.02,2.84867,2590,5883847 +431,delete,65.004,-1.0,-1.0,-1.0,-1.0,150.07,2.81391,2570,5801380 +432,search,4455.75,0.89388,0.126593,18.4275,18.404,24.268,2.81407,2571,5801380 +433,delete,160.422,-1.0,-1.0,-1.0,-1.0,385.991,2.72084,2517,5607567 +434,search,4342.06,0.89227,0.127489,18.6718,18.4994,41.668,2.72158,2520,5607567 +435,delete,80.821,-1.0,-1.0,-1.0,-1.0,45.387,2.68848,2515,5512176 +436,search,4363.23,0.89153,0.127892,18.6768,18.4943,22.607,2.68847,2516,5512176 +437,delete,7.875,-1.0,-1.0,-1.0,-1.0,31.306,2.68302,2514,5501141 +438,search,4329.37,0.8917,0.12784,18.6575,18.4784,16.468,2.68302,2514,5501141 +439,delete,81.165,-1.0,-1.0,-1.0,-1.0,278.715,2.6205,2482,5369822 +440,search,4241.24,0.89179,0.127894,18.5716,18.4056,27.865,2.6209,2484,5369822 +441,delete,23.624,-1.0,-1.0,-1.0,-1.0,26.268,2.60815,2483,5338985 +442,search,4224.32,0.89208,0.127643,18.5909,18.4407,17.191,2.60815,2483,5338985 +443,delete,25.95,-1.0,-1.0,-1.0,-1.0,74.761,2.5917,2474,5307279 +444,search,4231.09,0.89219,0.127654,18.4896,18.2809,27.07,2.5919,2476,5307279 +445,delete,33.688,-1.0,-1.0,-1.0,-1.0,29.165,2.57727,2474,5268382 +446,search,4273.79,0.89205,0.127456,18.5394,18.3365,16.607,2.57727,2474,5268382 +447,delete,305.14,-1.0,-1.0,-1.0,-1.0,408.007,2.40473,2384,4922787 +448,search,4035.58,0.88853,0.129326,19.0401,18.5828,35.831,2.40473,2386,4922787 +449,delete,27.367,-1.0,-1.0,-1.0,-1.0,20.099,2.39553,2385,4887227 +450,search,4046.61,0.88836,0.129109,18.9766,18.5188,24.275,2.39553,2386,4887227 +451,delete,79.166,-1.0,-1.0,-1.0,-1.0,114.116,2.35118,2364,4795082 +452,search,4000.52,0.88733,0.130036,19.0369,18.5393,15.855,2.35118,2364,4795082 +453,delete,241.265,-1.0,-1.0,-1.0,-1.0,681.014,2.19765,2293,4500175 +454,search,3826.53,0.88509,0.131616,19.4417,18.7688,89.293,2.19834,2301,4500175 +455,delete,23.764,-1.0,-1.0,-1.0,-1.0,31.695,2.1877,2299,4462648 +456,search,3801.9,0.88428,0.131685,19.5595,18.8299,53.411,2.1877,2304,4462648 +457,delete,4.787,-1.0,-1.0,-1.0,-1.0,18.335,2.18621,2304,4456761 +458,search,3826.78,0.88462,0.131657,19.5572,18.8342,25.667,2.18621,2305,4456761 +459,delete,30.993,-1.0,-1.0,-1.0,-1.0,79.939,2.1664,2294,4419992 +460,search,3841.33,0.88475,0.131457,19.5558,18.8702,19.864,2.1664,2294,4419992 +461,delete,4.952,-1.0,-1.0,-1.0,-1.0,17.569,2.16549,2294,4415725 +462,search,3786,0.88479,0.131467,19.5512,18.8872,19.688,2.16549,2294,4415725 +463,delete,37.318,-1.0,-1.0,-1.0,-1.0,42.93,2.14616,2282,4370609 +464,search,3748.1,0.88378,0.132141,19.6813,18.9521,19.791,2.14616,2282,4370609 +465,delete,18.155,-1.0,-1.0,-1.0,-1.0,39.159,2.13432,2276,4345244 +466,search,3731.18,0.88289,0.132492,19.8063,19.0151,19.516,2.13432,2276,4345244 +467,delete,21.692,-1.0,-1.0,-1.0,-1.0,26.877,2.12506,2273,4319734 +468,search,3732.37,0.88245,0.132865,19.8885,19.0692,19.749,2.12506,2273,4319734 +469,delete,15.921,-1.0,-1.0,-1.0,-1.0,49.651,2.11321,2264,4298361 +470,search,3714.1,0.88186,0.13321,19.9841,19.1063,19.391,2.11321,2264,4298361 +471,delete,92.319,-1.0,-1.0,-1.0,-1.0,146.717,2.05214,2232,4188315 +472,search,3631.51,0.88072,0.133855,20.0749,19.1223,18.917,2.05214,2232,4188315 +473,delete,96.901,-1.0,-1.0,-1.0,-1.0,131.474,2.00174,2216,4074567 +474,search,3596.58,0.87985,0.134741,20.2281,19.2537,18.806,2.00174,2216,4074567 +475,delete,300.107,-1.0,-1.0,-1.0,-1.0,588.142,1.78892,2101,3691698 +476,search,3386.41,0.87521,0.137182,20.8152,19.4593,24.032,1.78894,2102,3691698 +477,delete,40.539,-1.0,-1.0,-1.0,-1.0,77.175,1.7694,2094,3642619 +478,search,3340.16,0.87431,0.137864,20.88,19.4957,17.811,1.7694,2094,3642619 +479,delete,164.335,-1.0,-1.0,-1.0,-1.0,296.187,1.67867,2044,3421368 +480,search,3289.26,0.87244,0.139121,21.3095,19.7615,34.925,1.67885,2046,3421368 +481,delete,35.287,-1.0,-1.0,-1.0,-1.0,54.585,1.66212,2039,3378034 +482,search,3248.44,0.87289,0.138361,21.1966,19.66,24.656,1.66212,2040,3378034 +483,delete,214.34,-1.0,-1.0,-1.0,-1.0,469.66,1.51134,1941,3115682 +484,search,3075.59,0.86953,0.14094,21.6468,19.8255,16.616,1.51134,1941,3115682 +485,delete,19.291,-1.0,-1.0,-1.0,-1.0,14.663,1.50569,1941,3094629 +486,search,3069.06,0.86903,0.141207,21.7076,19.8791,16.523,1.50569,1941,3094629 +487,delete,6.363,-1.0,-1.0,-1.0,-1.0,14.357,1.50403,1941,3086025 +488,search,3082.58,0.86866,0.141379,21.7621,19.8796,16.394,1.50403,1941,3086025 +489,delete,26.507,-1.0,-1.0,-1.0,-1.0,20.661,1.49275,1940,3052105 +490,search,3075.6,0.86866,0.14089,21.7672,19.883,16.599,1.49275,1940,3052105 +491,delete,11.702,-1.0,-1.0,-1.0,-1.0,19.967,1.48652,1938,3038974 +492,search,3028.85,0.86797,0.140919,21.8737,19.9392,16.545,1.48652,1938,3038974 +493,delete,110.131,-1.0,-1.0,-1.0,-1.0,176.11,1.42234,1907,2911063 +494,search,2954.92,0.86685,0.14165,22.0065,20.0249,16.337,1.42234,1907,2911063 +495,delete,15.755,-1.0,-1.0,-1.0,-1.0,34.623,1.41361,1901,2888785 +496,search,2941.3,0.86628,0.142241,22.0527,20.0192,24.877,1.41386,1902,2888785 +497,delete,41.916,-1.0,-1.0,-1.0,-1.0,39.484,1.39202,1892,2829718 +498,search,2905.18,0.8655,0.142387,22.2043,20.0793,16.034,1.39202,1892,2829718 +499,delete,12.524,-1.0,-1.0,-1.0,-1.0,19.069,1.38597,1891,2816082 +500,search,2915.36,0.86551,0.142159,22.2222,20.0618,16.155,1.38597,1891,2816082 +501,delete,18.402,-1.0,-1.0,-1.0,-1.0,46.901,1.37424,1883,2790880 +502,search,2915.64,0.86523,0.141898,22.279,20.1062,17.378,1.37424,1883,2790880 +503,delete,15.081,-1.0,-1.0,-1.0,-1.0,22.541,1.37007,1882,2774459 +504,search,2917.76,0.8646,0.142237,22.3702,20.148,16.731,1.37007,1882,2774459 +505,delete,55.938,-1.0,-1.0,-1.0,-1.0,80.466,1.3356,1864,2714683 +506,search,2893.46,0.86302,0.142756,22.5755,20.2315,17.044,1.3356,1864,2714683 +507,delete,36.531,-1.0,-1.0,-1.0,-1.0,43.923,1.32156,1858,2680986 +508,search,2880.52,0.86233,0.142898,22.6363,20.2393,16.202,1.32156,1858,2680986 +509,delete,16.613,-1.0,-1.0,-1.0,-1.0,20.136,1.31292,1857,2662966 +510,search,2876.1,0.86207,0.143116,22.6534,20.2435,16.404,1.31292,1857,2662966 +511,delete,18.441,-1.0,-1.0,-1.0,-1.0,23.095,1.3044,1854,2642855 +512,search,2844.64,0.86103,0.143997,22.734,20.2679,16.158,1.3044,1854,2642855 +513,insert,226.702,-1.0,-1.0,-1.0,-1.0,98.407,1.38807,1866,2906838 +514,search,2903.64,0.86074,0.144675,22.8497,20.4903,16.149,1.38807,1866,2906838 +515,insert,4.398,-1.0,-1.0,-1.0,-1.0,13.809,1.3896,1866,2913746 +516,search,2895.5,0.86075,0.144615,22.8256,20.4754,16.222,1.3896,1866,2913746 +517,insert,12.943,-1.0,-1.0,-1.0,-1.0,13.997,1.39305,1866,2936862 +518,search,2905.4,0.86168,0.143768,22.7599,20.4582,16.076,1.39305,1866,2936862 +519,insert,62.976,-1.0,-1.0,-1.0,-1.0,56.086,1.41868,1870,3009732 +520,search,2937.74,0.86321,0.143986,22.4502,20.3824,48.763,1.41867,1875,3009732 +521,insert,9.933,-1.0,-1.0,-1.0,-1.0,14.867,1.4213,1875,3025906 +522,search,2938.74,0.86342,0.144172,22.4002,20.3754,15.978,1.4213,1875,3025906 +523,insert,29.497,-1.0,-1.0,-1.0,-1.0,25.176,1.4323,1876,3061899 +524,search,2951.83,0.86389,0.14448,22.3327,20.3585,25.519,1.4323,1877,3061899 +525,insert,30.619,-1.0,-1.0,-1.0,-1.0,26.46,1.44215,1879,3101209 +526,search,2940.7,0.86425,0.144369,22.2443,20.3239,23.69,1.44215,1880,3101209 +527,insert,383.457,-1.0,-1.0,-1.0,-1.0,155.913,1.59995,1907,3522334 +528,search,3265.91,0.86981,0.140251,21.5881,20.0377,66.204,1.60025,1915,3522334 +529,insert,225.708,-1.0,-1.0,-1.0,-1.0,104.501,1.71173,1928,3797294 +530,search,3280.68,0.87783,0.137689,20.6472,19.8543,58.913,1.71173,1934,3797294 +531,insert,51.493,-1.0,-1.0,-1.0,-1.0,33.504,1.73056,1937,3853781 +532,search,3286.94,0.87823,0.137104,20.5741,19.8052,42.256,1.73056,1941,3853781 +533,insert,17.215,-1.0,-1.0,-1.0,-1.0,15.269,1.73739,1941,3873352 +534,search,3287.58,0.87869,0.136745,20.5544,19.7852,37.195,1.73739,1944,3873352 +535,insert,18.031,-1.0,-1.0,-1.0,-1.0,15.248,1.74403,1944,3893308 +536,search,3296.11,0.87929,0.136206,20.4915,19.7577,16.784,1.74403,1944,3893308 +537,insert,7.38,-1.0,-1.0,-1.0,-1.0,27.586,1.74716,1946,3901271 +538,search,3307.59,0.87921,0.136278,20.4746,19.7442,16.814,1.74716,1946,3901271 +539,insert,26.122,-1.0,-1.0,-1.0,-1.0,29.763,1.7559,1949,3925164 +540,search,3303.03,0.87923,0.136182,20.4656,19.7295,16.862,1.7559,1949,3925164 +541,insert,393.762,-1.0,-1.0,-1.0,-1.0,258.019,1.92991,1997,4320915 +542,search,3472.1,0.88355,0.133304,19.8984,19.432,46.829,1.93063,2003,4320915 +543,insert,32.552,-1.0,-1.0,-1.0,-1.0,23.851,1.94589,2005,4360814 +544,search,3428.97,0.8843,0.132638,19.7949,19.3498,16.847,1.94589,2005,4360814 +545,insert,8.399,-1.0,-1.0,-1.0,-1.0,22.464,1.94941,2007,4370220 +546,search,3536.27,0.88408,0.132616,19.8112,19.3642,17.454,1.94941,2007,4370220 +547,insert,137.117,-1.0,-1.0,-1.0,-1.0,51.325,2.00967,2021,4510675 +548,search,3572.68,0.88764,0.130017,19.3981,19.1179,33.97,2.01017,2025,4510675 +549,insert,194.508,-1.0,-1.0,-1.0,-1.0,110.612,2.10866,2044,4744845 +550,search,3680.62,0.88456,0.133421,19.8405,19.5803,37.516,2.10866,2046,4744845 +551,insert,23.56,-1.0,-1.0,-1.0,-1.0,40.777,2.12098,2049,4771433 +552,search,3704.53,0.88449,0.133336,19.8301,19.5831,24.721,2.12098,2050,4771433 +553,insert,11.84,-1.0,-1.0,-1.0,-1.0,18.27,2.12521,2051,4782909 +554,search,3642.94,0.88555,0.132377,19.7639,19.5546,22.701,2.12546,2052,4782909 +555,insert,368.727,-1.0,-1.0,-1.0,-1.0,234.89,2.29619,2121,5180322 +556,search,3874.59,0.89188,0.126575,19.2378,19.3081,23.425,2.2963,2123,5180322 +557,insert,19.801,-1.0,-1.0,-1.0,-1.0,25.338,2.30548,2126,5203220 +558,search,3872.67,0.89214,0.126527,19.2401,19.3286,19.076,2.30548,2126,5203220 +559,insert,5.447,-1.0,-1.0,-1.0,-1.0,17.401,2.30801,2126,5210154 +560,search,3867.27,0.89217,0.126684,19.2168,19.324,18.817,2.30801,2126,5210154 +561,insert,4.657,-1.0,-1.0,-1.0,-1.0,17.252,2.31014,2126,5215426 +562,search,3871.54,0.89224,0.12664,19.2117,19.3236,18.695,2.31014,2126,5215426 +563,insert,34.53,-1.0,-1.0,-1.0,-1.0,17.517,2.32673,2126,5252726 +564,search,3867.41,0.89234,0.126655,19.1556,19.287,19.164,2.32673,2126,5252726 +565,insert,179.868,-1.0,-1.0,-1.0,-1.0,108.143,2.40853,2144,5447274 +566,search,3952.7,0.89332,0.126078,19.0104,19.1908,40.241,2.40892,2147,5447274 +567,insert,13.644,-1.0,-1.0,-1.0,-1.0,20.919,2.41548,2147,5463135 +568,search,4006.31,0.89318,0.126489,19.0306,19.2027,26.785,2.41548,2148,5463135 +569,insert,16.512,-1.0,-1.0,-1.0,-1.0,23.096,2.42168,2149,5478168 +570,search,3966.77,0.89309,0.126627,19.0007,19.1778,22.648,2.4217,2150,5478168 +571,insert,8.824,-1.0,-1.0,-1.0,-1.0,27.129,2.42793,2151,5492309 +572,search,4019.06,0.89313,0.126721,19.0561,19.2415,21.494,2.428,2152,5492309 +573,insert,60.995,-1.0,-1.0,-1.0,-1.0,53.374,2.454,2162,5554599 +574,search,4045.22,0.89363,0.126193,19.0527,19.2777,19.289,2.454,2162,5554599 +575,insert,43.614,-1.0,-1.0,-1.0,-1.0,34.826,2.47261,2166,5604500 +576,search,4057.98,0.89362,0.12633,18.9889,19.1845,19.297,2.47261,2166,5604500 +577,insert,31.881,-1.0,-1.0,-1.0,-1.0,45.1,2.48631,2176,5638834 +578,search,4070.56,0.89364,0.126416,18.9556,19.19,20.107,2.48631,2176,5638834 +579,insert,502.5,-1.0,-1.0,-1.0,-1.0,641.7,2.76371,2312,6229778 +580,search,4377.44,0.89888,0.121645,18.4571,18.996,51.336,2.76424,2321,6229778 +581,insert,21.059,-1.0,-1.0,-1.0,-1.0,40.014,2.777,2326,6261240 +582,search,4398.27,0.89856,0.122021,18.4502,19.0041,21.331,2.777,2326,6261240 +583,insert,18.862,-1.0,-1.0,-1.0,-1.0,25.617,2.78765,2327,6283315 +584,search,4406.86,0.8988,0.122078,18.4242,19.0089,21.442,2.78765,2327,6283315 +585,insert,199.902,-1.0,-1.0,-1.0,-1.0,244.257,2.91305,2370,6556101 +586,search,4534.72,0.90032,0.121445,18.1676,18.8065,93.278,2.91386,2388,6556101 +587,insert,27.464,-1.0,-1.0,-1.0,-1.0,34.071,2.92782,2394,6587187 +588,search,4567.33,0.90049,0.121461,18.1065,18.7614,25.828,2.92801,2397,6587187 +589,insert,40.529,-1.0,-1.0,-1.0,-1.0,39.769,2.94626,2402,6631526 +590,search,4600.99,0.90081,0.120997,18.0705,18.7435,27.996,2.94631,2405,6631526 +591,insert,387.594,-1.0,-1.0,-1.0,-1.0,533.804,3.15635,2524,7108942 +592,search,4879.61,0.90399,0.119053,17.6732,18.6006,131.864,3.1578,2551,7108942 +593,insert,125.141,-1.0,-1.0,-1.0,-1.0,172.211,3.24643,2583,7303976 +594,search,4925.48,0.9059,0.118243,17.3578,18.4301,99.22,3.24769,2603,7303976 +595,insert,3.312,-1.0,-1.0,-1.0,-1.0,26.825,3.24946,2606,7309244 +596,search,4974.94,0.90594,0.118156,17.3405,18.4192,41.208,3.24948,2612,7309244 +597,insert,149.842,-1.0,-1.0,-1.0,-1.0,164.764,3.3521,2648,7542477 +598,search,5093.58,0.90756,0.117446,17.1181,18.3228,55.248,3.35266,2658,7542477 +599,insert,94.56,-1.0,-1.0,-1.0,-1.0,140.768,3.39766,2690,7654438 +600,search,5168.46,0.90831,0.116343,17.0572,18.2854,61.027,3.39852,2702,7654438 +601,insert,49.013,-1.0,-1.0,-1.0,-1.0,73.539,3.42586,2717,7717936 +602,search,5222.05,0.90855,0.1164,16.9957,18.2647,28.535,3.42592,2720,7717936 +603,insert,5.726,-1.0,-1.0,-1.0,-1.0,18.779,3.42841,2721,7723521 +604,search,5224.4,0.9087,0.1162,16.9956,18.2687,15.915,3.42841,2721,7723521 +605,insert,41.193,-1.0,-1.0,-1.0,-1.0,40.059,3.44661,2726,7767404 +606,search,5219.67,0.90912,0.11591,16.9197,18.2198,15.349,3.44661,2726,7767404 +607,insert,41.362,-1.0,-1.0,-1.0,-1.0,29.268,3.47052,2729,7826115 +608,search,5201.58,0.90862,0.116517,16.9264,18.2016,19.971,3.47065,2730,7826115 +609,insert,150.895,-1.0,-1.0,-1.0,-1.0,267.585,3.56125,2778,8036351 +610,search,5314.38,0.91005,0.115757,16.7115,18.1222,80.299,3.56248,2795,8036351 +611,insert,23.817,-1.0,-1.0,-1.0,-1.0,44.373,3.57504,2802,8060364 +612,search,5371.44,0.91033,0.115551,16.6926,18.0989,35.349,3.57545,2807,8060364 +613,insert,150.313,-1.0,-1.0,-1.0,-1.0,206.299,3.65928,2851,8256083 +614,search,5522.31,0.91153,0.114417,16.5839,18.0471,48.865,3.65964,2862,8256083 +615,insert,283.372,-1.0,-1.0,-1.0,-1.0,397.878,3.8261,2936,8625707 +616,search,5732.69,0.9132,0.113324,16.2279,17.8244,165.204,3.82973,2970,8625707 +617,insert,238.803,-1.0,-1.0,-1.0,-1.0,415.194,3.97977,3045,8949227 +618,search,5875.59,0.91572,0.111523,15.8575,17.6005,202.706,3.98199,3083,8949227 +619,insert,165.242,-1.0,-1.0,-1.0,-1.0,223.894,4.08513,3124,9195551 +620,search,5931.33,0.91648,0.111119,15.7509,17.5468,181.414,4.08869,3165,9195551 +621,insert,26.516,-1.0,-1.0,-1.0,-1.0,47.317,4.10373,3172,9232372 +622,search,6087.17,0.91689,0.110726,15.7224,17.5541,34.178,4.1041,3177,9232372 +623,insert,17.292,-1.0,-1.0,-1.0,-1.0,35.862,4.1141,3183,9253561 +624,search,6080.87,0.91721,0.110697,15.6772,17.5332,18.731,4.1141,3183,9253561 +625,insert,61.393,-1.0,-1.0,-1.0,-1.0,80.545,4.14533,3197,9331230 +626,search,6115.02,0.91766,0.110446,15.6341,17.5351,45.473,4.14569,3201,9331230 +627,insert,37.687,-1.0,-1.0,-1.0,-1.0,78.684,4.16271,3215,9374063 +628,search,6181.94,0.91786,0.11059,15.5467,17.4575,36.998,4.16293,3219,9374063 +629,insert,13.206,-1.0,-1.0,-1.0,-1.0,36.825,4.16859,3222,9387598 +630,search,6131.13,0.91815,0.110229,15.5335,17.4561,37.764,4.16886,3226,9387598 +631,insert,25.155,-1.0,-1.0,-1.0,-1.0,53.723,4.1833,3233,9421724 +632,search,6168.71,0.9184,0.110059,15.5077,17.4277,32.721,4.18384,3237,9421724 +633,insert,39.062,-1.0,-1.0,-1.0,-1.0,67.418,4.19877,3250,9461430 +634,search,6222.2,0.91887,0.109593,15.4601,17.4151,25.313,4.19884,3252,9461430 +635,insert,19.927,-1.0,-1.0,-1.0,-1.0,39.76,4.20729,3258,9479469 +636,search,6254.73,0.91879,0.109911,15.4385,17.4062,19.384,4.20729,3258,9479469 +637,insert,362.618,-1.0,-1.0,-1.0,-1.0,621.587,4.38799,3384,9885997 +638,search,6527.72,0.92098,0.108362,15.1207,17.243,150.441,4.39094,3418,9885997 +639,insert,333.22,-1.0,-1.0,-1.0,-1.0,589.433,4.57139,3545,10292043 +640,search,6759.48,0.92332,0.106422,14.7372,17.0475,136.434,4.57397,3577,10292043 +641,delete,122.727,-1.0,-1.0,-1.0,-1.0,434.973,4.52078,3544,10096584 +642,search,6676.62,0.92304,0.106762,14.8085,17.1377,29.705,4.52085,3546,10096584 +643,delete,8.653,-1.0,-1.0,-1.0,-1.0,19.427,4.52023,3546,10084840 +644,search,6695.52,0.92316,0.106465,14.8071,17.1308,22.49,4.52023,3546,10084840 +645,delete,11.856,-1.0,-1.0,-1.0,-1.0,19.385,4.52,3546,10069802 +646,search,6732.2,0.92308,0.106535,14.8316,17.1546,22.716,4.52,3546,10069802 +647,delete,55.593,-1.0,-1.0,-1.0,-1.0,189.337,4.49645,3531,9990953 +648,search,6669.23,0.92278,0.106784,14.8754,17.1761,28.247,4.4967,3532,9990953 +649,delete,29.03,-1.0,-1.0,-1.0,-1.0,32.533,4.49465,3531,9951625 +650,search,6623.74,0.92281,0.106751,14.8731,17.1785,22.72,4.49465,3531,9951625 +651,delete,15.329,-1.0,-1.0,-1.0,-1.0,19.698,4.49232,3531,9927065 +652,search,6599.73,0.92271,0.106643,14.9199,17.2022,22.834,4.49232,3531,9927065 +653,delete,72.129,-1.0,-1.0,-1.0,-1.0,167.464,4.46289,3512,9831098 +654,search,6620.68,0.92228,0.107075,15.0115,17.2753,22.99,4.46289,3512,9831098 +655,delete,292.469,-1.0,-1.0,-1.0,-1.0,611.422,4.35904,3437,9462172 +656,search,6389.1,0.92024,0.108472,15.329,17.4523,37.668,4.35961,3441,9462172 +657,delete,157.741,-1.0,-1.0,-1.0,-1.0,265.813,4.27969,3386,9218902 +658,search,6230.84,0.91711,0.110136,15.8094,17.714,51.03,4.27988,3391,9218902 +659,delete,85.642,-1.0,-1.0,-1.0,-1.0,239.917,4.24242,3366,9106546 +660,search,6156.19,0.91634,0.11075,15.9291,17.792,31.226,4.24242,3367,9106546 +661,delete,35.449,-1.0,-1.0,-1.0,-1.0,47.332,4.23524,3365,9059776 +662,search,6265.21,0.91652,0.110644,15.9454,17.808,21.963,4.23524,3365,9059776 +663,delete,65.992,-1.0,-1.0,-1.0,-1.0,135.543,4.21456,3352,8990616 +664,search,6143.69,0.91607,0.110823,16.0211,17.8334,22.354,4.21456,3352,8990616 +665,delete,42.631,-1.0,-1.0,-1.0,-1.0,109.394,4.19639,3344,8937437 +666,search,6136.57,0.91591,0.110722,16.0556,17.8663,28.325,4.1967,3345,8937437 +667,delete,151.552,-1.0,-1.0,-1.0,-1.0,293.107,4.12794,3316,8766390 +668,search,6037.03,0.9155,0.110676,16.0924,17.8631,25.318,4.12801,3317,8766390 +669,delete,168.075,-1.0,-1.0,-1.0,-1.0,504.727,4.04164,3268,8532008 +670,search,5876.3,0.91467,0.111121,16.1992,17.9248,24.954,4.04188,3269,8532008 +671,delete,27.981,-1.0,-1.0,-1.0,-1.0,54.794,4.03206,3266,8489178 +672,search,5861.97,0.91451,0.111419,16.2472,17.9674,21.572,4.03206,3266,8489178 +673,delete,50.536,-1.0,-1.0,-1.0,-1.0,113.024,4.01281,3256,8421866 +674,search,5820.65,0.91432,0.111606,16.305,17.9975,21.545,4.01281,3256,8421866 +675,delete,104.337,-1.0,-1.0,-1.0,-1.0,70.059,3.98826,3250,8288319 +676,search,5815.02,0.91399,0.111765,16.4211,18.064,42.705,3.98896,3253,8288319 +677,delete,150.541,-1.0,-1.0,-1.0,-1.0,154.929,3.90664,3235,8072197 +678,search,5738.73,0.91561,0.10989,15.9958,17.645,21.228,3.90664,3235,8072197 +679,delete,27.516,-1.0,-1.0,-1.0,-1.0,48.077,3.89613,3233,8036947 +680,search,5721.08,0.91549,0.110015,16.0062,17.6381,20.387,3.89613,3233,8036947 +681,delete,6.878,-1.0,-1.0,-1.0,-1.0,17.796,3.89475,3233,8026197 +682,search,5706.62,0.91553,0.109975,16.0095,17.6378,20.798,3.89475,3233,8026197 +683,delete,258.106,-1.0,-1.0,-1.0,-1.0,220.927,3.79586,3200,7744229 +684,search,5588.21,0.91369,0.112145,16.2274,17.7987,26.081,3.79588,3201,7744229 +685,delete,217.073,-1.0,-1.0,-1.0,-1.0,130.506,3.70409,3182,7541066 +686,search,5485.97,0.91064,0.114894,16.5758,18.1433,36.062,3.7044,3185,7541066 +687,delete,16.961,-1.0,-1.0,-1.0,-1.0,17.8,3.69923,3185,7522190 +688,search,5612.29,0.91037,0.115025,16.6039,18.1775,20.843,3.69923,3185,7522190 +689,delete,98.621,-1.0,-1.0,-1.0,-1.0,260.812,3.64017,3158,7409400 +690,search,5419.38,0.9096,0.115458,16.7361,18.2871,37.357,3.64063,3161,7409400 +691,delete,36.815,-1.0,-1.0,-1.0,-1.0,17.997,3.62601,3161,7375648 +692,search,5688.53,0.90954,0.115487,16.7431,18.2998,19.937,3.62601,3161,7375648 +693,delete,144.25,-1.0,-1.0,-1.0,-1.0,200.663,3.55434,3138,7222068 +694,search,5364.12,0.90827,0.116378,16.8797,18.3344,26.663,3.55443,3139,7222068 +695,delete,23.708,-1.0,-1.0,-1.0,-1.0,153.341,3.53938,3133,7196331 +696,search,5767.41,0.90921,0.115519,16.7216,18.1959,36.723,3.53966,3135,7196331 +697,delete,26.322,-1.0,-1.0,-1.0,-1.0,33.465,3.52835,3133,7170716 +698,search,5786.02,0.90884,0.115838,16.7886,18.2398,20.884,3.52835,3133,7170716 +699,delete,10.736,-1.0,-1.0,-1.0,-1.0,63.66,3.52059,3131,7155255 +700,search,6211.35,0.90917,0.115368,16.7569,18.2154,49.498,3.52085,3132,7155255 +701,delete,89.001,-1.0,-1.0,-1.0,-1.0,34.093,3.50085,3132,7097733 +702,search,6257.01,0.90886,0.115227,16.7988,18.2282,19.535,3.50085,3132,7097733 +703,delete,154.684,-1.0,-1.0,-1.0,-1.0,316.567,3.4098,3087,6924172 +704,search,5255.84,0.90899,0.115976,16.7216,18.2641,26.788,3.41001,3088,6924172 +705,delete,33.472,-1.0,-1.0,-1.0,-1.0,23.986,3.39899,3087,6896276 +706,search,5323.04,0.90861,0.115881,16.7709,18.2524,20.164,3.39899,3087,6896276 +707,delete,516.352,-1.0,-1.0,-1.0,-1.0,946.962,3.10596,2923,6385507 +708,search,4802.83,0.90513,0.118804,16.9913,18.2016,28.879,3.10612,2925,6385507 +709,delete,155.422,-1.0,-1.0,-1.0,-1.0,540.973,3.0098,2865,6207658 +710,search,4834.78,0.90281,0.120682,17.2774,18.3161,34.073,3.01015,2868,6207658 +711,delete,39.513,-1.0,-1.0,-1.0,-1.0,59.247,2.99421,2864,6162720 +712,search,4745.23,0.90268,0.120821,17.3169,18.3644,24.481,2.99445,2865,6162720 +713,delete,204.44,-1.0,-1.0,-1.0,-1.0,407.72,2.88055,2813,5927803 +714,search,4674.16,0.90113,0.122537,17.525,18.5304,36.587,2.8808,2815,5927803 +715,delete,30.966,-1.0,-1.0,-1.0,-1.0,48.725,2.86724,2810,5895666 +716,search,4672.94,0.90081,0.122835,17.5584,18.5396,18.292,2.86724,2810,5895666 +717,delete,24.541,-1.0,-1.0,-1.0,-1.0,39.826,2.86086,2808,5868774 +718,search,4596.58,0.90018,0.123358,17.5894,18.5694,18.001,2.86086,2808,5868774 +719,delete,307.949,-1.0,-1.0,-1.0,-1.0,704.747,2.69211,2679,5533540 +720,search,4401,0.8967,0.126217,17.9225,18.6437,33.855,2.69253,2682,5533540 +721,delete,92.503,-1.0,-1.0,-1.0,-1.0,149.374,2.64318,2656,5418387 +722,search,4327.78,0.89642,0.12597,18.0273,18.7036,28.142,2.64362,2658,5418387 +723,delete,18.027,-1.0,-1.0,-1.0,-1.0,24.699,2.63612,2657,5398291 +724,search,4364.43,0.8961,0.126228,18.0383,18.7047,16.592,2.63612,2657,5398291 +725,delete,91.701,-1.0,-1.0,-1.0,-1.0,167.764,2.58455,2646,5273546 +726,search,4332.69,0.89541,0.126657,18.0928,18.6888,39.621,2.58455,2648,5273546 +727,delete,78.149,-1.0,-1.0,-1.0,-1.0,119.219,2.5427,2634,5180937 +728,search,4277.19,0.89484,0.12754,18.1689,18.7268,33.562,2.5427,2636,5180937 +729,delete,105.762,-1.0,-1.0,-1.0,-1.0,136.641,2.49521,2615,5079524 +730,search,4244.68,0.89392,0.128506,18.2827,18.7702,16.935,2.49521,2615,5079524 +731,delete,66.057,-1.0,-1.0,-1.0,-1.0,14.791,2.47665,2615,5019265 +732,search,4215.41,0.89256,0.129541,18.4127,18.8229,16.823,2.47665,2615,5019265 +733,delete,71.625,-1.0,-1.0,-1.0,-1.0,75.857,2.44644,2608,4955565 +734,search,4180.59,0.89226,0.130061,18.4737,18.9343,16.608,2.44644,2608,4955565 +735,delete,142.668,-1.0,-1.0,-1.0,-1.0,507.775,2.35695,2552,4776948 +736,search,4107.62,0.89902,0.122935,18.0878,18.8437,73.36,2.35869,2561,4776948 +737,delete,105.46,-1.0,-1.0,-1.0,-1.0,94.595,2.3107,2550,4654026 +738,search,4048.45,0.89703,0.124588,18.2876,18.9293,41.696,2.31083,2554,4654026 +739,delete,57.945,-1.0,-1.0,-1.0,-1.0,157.982,2.26999,2538,4589888 +740,search,4017.48,0.897,0.123818,18.2721,18.8944,20.915,2.26998,2539,4589888 +741,delete,138.158,-1.0,-1.0,-1.0,-1.0,195.887,2.19686,2508,4430862 +742,search,3944.58,0.89575,0.125152,18.3497,18.8476,16.514,2.19686,2508,4430862 +743,delete,162.406,-1.0,-1.0,-1.0,-1.0,355.449,2.09781,2456,4237049 +744,search,3844.92,0.89526,0.125837,18.5328,19.0562,16.252,2.09781,2456,4237049 +745,delete,225.247,-1.0,-1.0,-1.0,-1.0,540.07,1.96723,2380,4000195 +746,search,3747.27,0.89273,0.127346,18.9081,19.2737,44.197,1.96809,2384,4000195 +747,delete,176.922,-1.0,-1.0,-1.0,-1.0,252.939,1.87543,2339,3803954 +748,search,3662.45,0.89127,0.127865,19.0317,19.237,37.242,1.87536,2341,3803954 +749,delete,69.892,-1.0,-1.0,-1.0,-1.0,66.473,1.84543,2333,3733214 +750,search,3638.24,0.89095,0.127571,19.2037,19.3349,22.389,1.84543,2333,3733214 +751,delete,19.774,-1.0,-1.0,-1.0,-1.0,51.178,1.83349,2327,3709324 +752,search,3637.91,0.89003,0.128186,19.2978,19.3776,22.513,1.83349,2327,3709324 +753,delete,52.249,-1.0,-1.0,-1.0,-1.0,130.821,1.80289,2313,3649123 +754,search,3592.77,0.88923,0.128572,19.4685,19.5279,22.025,1.80289,2313,3649123 +755,delete,24.082,-1.0,-1.0,-1.0,-1.0,32.733,1.79243,2311,3621475 +756,search,3581.15,0.88905,0.128907,19.4456,19.5015,31.533,1.79273,2312,3621475 +757,delete,23.534,-1.0,-1.0,-1.0,-1.0,37.809,1.78053,2308,3594479 +758,search,3570.82,0.88897,0.128537,19.4496,19.4771,21.802,1.78053,2308,3594479 +759,delete,20.864,-1.0,-1.0,-1.0,-1.0,48.661,1.76757,2304,3569055 +760,search,3564.14,0.88876,0.128606,19.5026,19.4777,21.67,1.76757,2304,3569055 +761,delete,32.787,-1.0,-1.0,-1.0,-1.0,41.354,1.75132,2299,3527896 +762,search,3553.82,0.88786,0.129361,19.604,19.5377,22.429,1.75132,2299,3527896 +763,delete,23.587,-1.0,-1.0,-1.0,-1.0,36.292,1.74271,2296,3509970 +764,search,3554.09,0.88776,0.129346,19.6463,19.5931,22.746,1.74271,2296,3509970 +765,delete,324.974,-1.0,-1.0,-1.0,-1.0,495.119,1.55885,2168,3187549 +766,search,3337.92,0.88482,0.131058,20.0447,19.7647,20.313,1.55885,2168,3187549 +767,delete,345.902,-1.0,-1.0,-1.0,-1.0,701.792,1.34987,1978,2822906 +768,search,3036.31,0.87664,0.136429,20.8919,20.0749,18.732,1.34987,1978,2822906 +769,insert,9.685,-1.0,-1.0,-1.0,-1.0,15.85,1.35187,1978,2832899 +770,search,3052.35,0.87715,0.136058,20.8717,20.0547,17.984,1.35187,1978,2832899 +771,insert,34.008,-1.0,-1.0,-1.0,-1.0,23.248,1.36176,1979,2863523 +772,search,3062.2,0.87739,0.136314,20.8203,20.0761,18.128,1.36176,1979,2863523 +773,insert,47.991,-1.0,-1.0,-1.0,-1.0,25.79,1.37782,1981,2925169 +774,search,3089.32,0.87922,0.134884,20.6937,20.0304,18.194,1.37782,1981,2925169 +775,insert,37.288,-1.0,-1.0,-1.0,-1.0,38.859,1.39007,1984,2961356 +776,search,3085.71,0.87914,0.134858,20.6521,20.0045,17.938,1.39007,1984,2961356 +777,insert,28.21,-1.0,-1.0,-1.0,-1.0,16.057,1.39827,1984,2998678 +778,search,3081.53,0.88016,0.134193,20.5498,19.9156,18.029,1.39827,1984,2998678 +779,insert,13.461,-1.0,-1.0,-1.0,-1.0,22.763,1.40208,1985,3011024 +780,search,3096.71,0.88032,0.134011,20.5229,19.9172,17.916,1.40208,1985,3011024 +781,insert,6.622,-1.0,-1.0,-1.0,-1.0,15.756,1.40335,1985,3019515 +782,search,3095.76,0.88048,0.133866,20.5205,19.9224,17.664,1.40335,1985,3019515 +783,insert,47.05,-1.0,-1.0,-1.0,-1.0,16.128,1.41718,1985,3082298 +784,search,3114.09,0.88139,0.133223,20.3805,19.8343,17.787,1.41718,1985,3082298 +785,insert,39.595,-1.0,-1.0,-1.0,-1.0,64.008,1.43369,1991,3122090 +786,search,3139.53,0.8827,0.132825,20.1985,19.7658,43.526,1.43369,1995,3122090 +787,insert,21.126,-1.0,-1.0,-1.0,-1.0,16.502,1.4394,1995,3140294 +788,search,3156,0.8829,0.132518,20.1396,19.7392,24.732,1.4394,1996,3140294 +789,insert,6.972,-1.0,-1.0,-1.0,-1.0,16.264,1.44104,1996,3147819 +790,search,3154.77,0.88271,0.132592,20.1426,19.7307,18.275,1.44104,1996,3147819 +791,insert,32.587,-1.0,-1.0,-1.0,-1.0,16.638,1.45033,1996,3180668 +792,search,3173.86,0.88292,0.132709,20.1053,19.7152,18.255,1.45033,1996,3180668 +793,insert,1.74,-1.0,-1.0,-1.0,-1.0,16.109,1.45072,1996,3183235 +794,search,3163.79,0.88293,0.132689,20.1072,19.7162,18.166,1.45072,1996,3183235 +795,insert,60.475,-1.0,-1.0,-1.0,-1.0,22.414,1.46941,1997,3242089 +796,search,3168.57,0.88401,0.131886,19.9988,19.648,18.336,1.46941,1997,3242089 +797,insert,5.638,-1.0,-1.0,-1.0,-1.0,16.042,1.47101,1997,3247192 +798,search,3181.61,0.88396,0.131911,19.9989,19.6463,18.414,1.47101,1997,3247192 +799,insert,5.692,-1.0,-1.0,-1.0,-1.0,16.1,1.47244,1997,3254626 +800,search,3191.15,0.88397,0.131863,19.9685,19.6184,18.231,1.47244,1997,3254626 +801,insert,17.007,-1.0,-1.0,-1.0,-1.0,21.565,1.47727,1998,3274953 +802,search,3195.71,0.8845,0.131566,19.8736,19.5652,18.112,1.47727,1998,3274953 +803,insert,828.268,-1.0,-1.0,-1.0,-1.0,544.966,1.80826,2090,4032688 +804,search,3532.73,0.89223,0.125314,19.0017,19.1737,132.281,1.80866,2112,4032688 +805,insert,18.291,-1.0,-1.0,-1.0,-1.0,17.535,1.81519,2112,4050220 +806,search,3550.06,0.8918,0.125815,19.0258,19.1689,44.888,1.81547,2117,4050220 +807,insert,15.536,-1.0,-1.0,-1.0,-1.0,17.859,1.82099,2117,4067940 +808,search,3568.07,0.89173,0.125797,19.0408,19.1715,19.85,1.82099,2117,4067940 +809,insert,40.525,-1.0,-1.0,-1.0,-1.0,27.962,1.83531,2119,4099369 +810,search,3573.09,0.89221,0.125512,18.9758,19.1589,19.805,1.83531,2119,4099369 +811,insert,34.081,-1.0,-1.0,-1.0,-1.0,17.342,1.84704,2119,4129509 +812,search,3584.85,0.89253,0.125256,18.9621,19.1273,19.92,1.84704,2119,4129509 +813,insert,51.838,-1.0,-1.0,-1.0,-1.0,21.628,1.86653,2120,4178669 +814,search,3599.21,0.89407,0.124337,18.7448,18.9716,20.054,1.86653,2120,4178669 +815,insert,3.528,-1.0,-1.0,-1.0,-1.0,17.341,1.86751,2120,4180960 +816,search,3614.21,0.89404,0.12439,18.7506,18.978,19.78,1.86751,2120,4180960 +817,insert,81.388,-1.0,-1.0,-1.0,-1.0,87.215,1.89561,2132,4255359 +818,search,3619.14,0.8944,0.124324,18.6601,18.9064,19.997,1.89561,2132,4255359 +819,insert,17.41,-1.0,-1.0,-1.0,-1.0,22.648,1.90139,2134,4272706 +820,search,3631.25,0.89481,0.123747,18.6251,18.8746,19.759,1.90139,2134,4272706 +821,insert,16.241,-1.0,-1.0,-1.0,-1.0,32.628,1.90758,2136,4287851 +822,search,3632.92,0.89471,0.123836,18.6069,18.8704,20.727,1.90758,2136,4287851 +823,insert,36.521,-1.0,-1.0,-1.0,-1.0,30.921,1.92273,2137,4322788 +824,search,3648.2,0.89461,0.123863,18.5713,18.8394,26.461,1.92286,2138,4322788 +825,insert,91.997,-1.0,-1.0,-1.0,-1.0,35.617,1.95437,2141,4396408 +826,search,3701.76,0.89506,0.122885,18.6684,18.9726,19.701,1.95437,2141,4396408 +827,insert,23.251,-1.0,-1.0,-1.0,-1.0,23.378,1.96323,2142,4415721 +828,search,3679.18,0.89488,0.123023,18.7113,19.0177,19.496,1.96323,2142,4415721 +829,insert,465.379,-1.0,-1.0,-1.0,-1.0,231.052,2.14502,2182,4832079 +830,search,3861.48,0.8971,0.122098,18.3135,18.8263,69.372,2.14595,2192,4832079 +831,insert,36.684,-1.0,-1.0,-1.0,-1.0,30.309,2.15941,2194,4864525 +832,search,3841.76,0.89766,0.121877,18.2841,18.8333,26.266,2.15943,2195,4864525 +833,insert,7.831,-1.0,-1.0,-1.0,-1.0,28.831,2.16191,2198,4874443 +834,search,3863,0.89737,0.12222,18.3049,18.862,20.432,2.16191,2198,4874443 +835,insert,68.444,-1.0,-1.0,-1.0,-1.0,38.492,2.18926,2203,4938267 +836,search,3898.74,0.89818,0.121769,18.1888,18.7928,29.569,2.18931,2205,4938267 +837,insert,23.933,-1.0,-1.0,-1.0,-1.0,24.809,2.1978,2206,4961147 +838,search,3997.69,0.89817,0.121699,18.195,18.7946,20.416,2.1978,2206,4961147 +839,insert,72.922,-1.0,-1.0,-1.0,-1.0,48.335,2.22722,2211,5033740 +840,search,3940.39,0.89928,0.120994,18.0729,18.7598,20.807,2.22722,2211,5033740 +841,insert,57.168,-1.0,-1.0,-1.0,-1.0,28.293,2.24701,2213,5078736 +842,search,3958.4,0.89936,0.12092,18.0747,18.7694,24.064,2.24712,2214,5078736 +843,insert,434.884,-1.0,-1.0,-1.0,-1.0,273.51,2.41921,2262,5467376 +844,search,4145.5,0.90217,0.11931,17.7952,18.6723,98.512,2.42005,2280,5467376 +845,insert,346.726,-1.0,-1.0,-1.0,-1.0,303.791,2.57716,2335,5815547 +846,search,4311.07,0.9048,0.117792,17.4489,18.5286,65.407,2.578,2343,5815547 +847,insert,7.233,-1.0,-1.0,-1.0,-1.0,20.608,2.58083,2343,5821782 +848,search,4316.64,0.90501,0.117471,17.4286,18.5178,35.502,2.58108,2346,5821782 +849,insert,1.977,-1.0,-1.0,-1.0,-1.0,25.173,2.58169,2347,5825594 +850,search,4319.03,0.90493,0.117602,17.4296,18.5248,22.046,2.58169,2347,5825594 +851,insert,11.462,-1.0,-1.0,-1.0,-1.0,23.476,2.58657,2348,5834880 +852,search,4303.76,0.90502,0.117599,17.4082,18.5194,22.028,2.58657,2348,5834880 +853,insert,25.055,-1.0,-1.0,-1.0,-1.0,37.358,2.59676,2351,5860274 +854,search,4320.9,0.90541,0.117419,17.3735,18.5008,34.166,2.59698,2354,5860274 +855,insert,7.47,-1.0,-1.0,-1.0,-1.0,25.76,2.59935,2356,5866064 +856,search,4330.42,0.90532,0.117607,17.38,18.5059,22.16,2.59935,2356,5866064 +857,insert,22.392,-1.0,-1.0,-1.0,-1.0,28.92,2.60535,2358,5884456 +858,search,4344.36,0.90544,0.117618,17.3165,18.4343,20.431,2.60535,2358,5884456 +859,insert,39.592,-1.0,-1.0,-1.0,-1.0,51.083,2.61823,2366,5916786 +860,search,4350.52,0.9057,0.117615,17.2488,18.3804,15.666,2.61823,2366,5916786 +861,insert,8.984,-1.0,-1.0,-1.0,-1.0,17.822,2.62101,2367,5923467 +862,search,4349.63,0.90563,0.117694,17.2416,18.3789,15.698,2.62101,2367,5923467 +863,insert,77.222,-1.0,-1.0,-1.0,-1.0,75.839,2.65345,2379,5997844 +864,search,4369.28,0.9042,0.119152,17.3102,18.3882,15.392,2.65345,2379,5997844 +865,insert,14.313,-1.0,-1.0,-1.0,-1.0,16.505,2.65821,2380,6009561 +866,search,4362.81,0.90465,0.118985,17.3113,18.4005,22.434,2.65824,2381,6009561 +867,insert,47.338,-1.0,-1.0,-1.0,-1.0,45.016,2.67867,2389,6052876 +868,search,4386.73,0.90433,0.119397,17.2814,18.3977,20.614,2.67879,2390,6052876 +869,insert,5.392,-1.0,-1.0,-1.0,-1.0,26.261,2.68106,2392,6059234 +870,search,4395.8,0.90469,0.119117,17.2875,18.4225,15.583,2.68106,2392,6059234 +871,insert,38.407,-1.0,-1.0,-1.0,-1.0,39.744,2.69695,2397,6098225 +872,search,4408.44,0.90472,0.119178,17.2696,18.4256,15.498,2.69695,2397,6098225 +873,insert,41.511,-1.0,-1.0,-1.0,-1.0,43.746,2.71274,2402,6137308 +874,search,4435.01,0.90498,0.118829,17.3054,18.4544,15.828,2.71274,2402,6137308 +875,insert,19.454,-1.0,-1.0,-1.0,-1.0,19.305,2.71829,2403,6154089 +876,search,4474.45,0.90492,0.119009,17.3234,18.4822,15.474,2.71829,2403,6154089 +877,insert,2.846,-1.0,-1.0,-1.0,-1.0,13.451,2.71906,2403,6156061 +878,search,4455.55,0.90499,0.118935,17.3185,18.4919,15.712,2.71906,2403,6156061 +879,insert,296.348,-1.0,-1.0,-1.0,-1.0,309.423,2.8535,2458,6446004 +880,search,4558.95,0.90638,0.11819,17.0673,18.3504,88.378,2.85476,2473,6446004 +881,insert,437.451,-1.0,-1.0,-1.0,-1.0,396.505,3.08658,2556,6969764 +882,search,4797.43,0.90838,0.116372,16.8808,18.2394,134.222,3.08758,2582,6969764 +883,insert,373.834,-1.0,-1.0,-1.0,-1.0,503.402,3.29059,2689,7422501 +884,search,5089.04,0.9114,0.114367,16.4222,17.996,120.445,3.29253,2716,7422501 +885,insert,6.907,-1.0,-1.0,-1.0,-1.0,23.924,3.29513,2718,7432053 +886,search,5138.38,0.91164,0.114411,16.3837,17.9817,35.741,3.29556,2724,7432053 +887,insert,293.583,-1.0,-1.0,-1.0,-1.0,378.336,3.45252,2801,7789886 +888,search,5316.11,0.91338,0.113557,16.1025,17.9509,108.805,3.45412,2825,7789886 +889,insert,343.116,-1.0,-1.0,-1.0,-1.0,501.432,3.64403,2923,8210883 +890,search,5511.19,0.91563,0.111811,15.8296,17.8311,152.837,3.6461,2957,8210883 +891,insert,25.78,-1.0,-1.0,-1.0,-1.0,40.357,3.65751,2964,8240504 +892,search,5540.18,0.91561,0.111946,15.8071,17.8085,33.553,3.65765,2968,8240504 +893,insert,39.118,-1.0,-1.0,-1.0,-1.0,74.205,3.67896,2981,8299693 +894,search,5560.97,0.91606,0.111493,15.7691,17.7964,32.533,3.67918,2984,8299693 +895,insert,48.801,-1.0,-1.0,-1.0,-1.0,74.968,3.69614,3001,8349126 +896,search,5622.35,0.91637,0.111213,15.7345,17.7894,27.952,3.69648,3003,8349126 +897,delete,50.157,-1.0,-1.0,-1.0,-1.0,124.839,3.67597,2995,8268842 +898,search,5611.44,0.91622,0.11103,15.7485,17.8063,25.659,3.67621,2996,8268842 +899,delete,10.982,-1.0,-1.0,-1.0,-1.0,16.994,3.67621,2996,8250527 +900,search,5611.47,0.91613,0.111158,15.7833,17.8247,18.93,3.67621,2996,8250527 +901,delete,40.461,-1.0,-1.0,-1.0,-1.0,77.902,3.66581,2986,8185565 +902,search,5557.15,0.91572,0.111389,15.8264,17.8364,25.714,3.66602,2987,8185565 +903,delete,40.624,-1.0,-1.0,-1.0,-1.0,103.32,3.64473,2973,8126915 +904,search,5516.94,0.91467,0.111785,16.0078,17.9142,24.4,3.64474,2974,8126915 +905,delete,16.851,-1.0,-1.0,-1.0,-1.0,17.061,3.64426,2974,8098533 +906,search,5507.88,0.91457,0.111878,16.011,17.9316,19.03,3.64426,2974,8098533 +907,delete,17.638,-1.0,-1.0,-1.0,-1.0,38.226,3.63997,2972,8074833 +908,search,5540.36,0.91446,0.111872,16.0199,17.9116,19.14,3.63997,2972,8074833 +909,delete,28.39,-1.0,-1.0,-1.0,-1.0,17.02,3.6389,2972,8035274 +910,search,5517.63,0.91432,0.112098,16.0486,17.9332,19.518,3.6389,2972,8035274 +911,delete,59.417,-1.0,-1.0,-1.0,-1.0,38.217,3.63326,2970,7957223 +912,search,5474.48,0.9137,0.112717,16.1267,18.0162,19.79,3.63326,2970,7957223 +913,delete,27.417,-1.0,-1.0,-1.0,-1.0,73.935,3.61963,2960,7917554 +914,search,5448.11,0.91302,0.112863,16.191,18.003,18.792,3.61963,2960,7917554 +915,delete,47.54,-1.0,-1.0,-1.0,-1.0,108.381,3.60322,2946,7861503 +916,search,5397.55,0.91268,0.112959,16.2246,17.9967,18.891,3.60322,2946,7861503 +917,delete,10.413,-1.0,-1.0,-1.0,-1.0,16.232,3.60244,2946,7844487 +918,search,5405.81,0.91257,0.112929,16.2323,18.007,18.955,3.60244,2946,7844487 +919,delete,21.103,-1.0,-1.0,-1.0,-1.0,29.126,3.60063,2945,7814993 +920,search,5396.75,0.91258,0.113152,16.2579,18.0381,19.288,3.60063,2945,7814993 +921,delete,18.015,-1.0,-1.0,-1.0,-1.0,51.813,3.59541,2942,7793855 +922,search,5427.73,0.91257,0.112938,16.2784,18.0504,19.223,3.59541,2942,7793855 +923,delete,43.377,-1.0,-1.0,-1.0,-1.0,27.879,3.58937,2940,7736054 +924,search,5387.24,0.91248,0.112868,16.3158,18.0607,19.178,3.58937,2940,7736054 +925,delete,112.837,-1.0,-1.0,-1.0,-1.0,201.146,3.53752,2907,7587942 +926,search,5286,0.91135,0.113971,16.4392,18.1148,53.132,3.53862,2913,7587942 +927,delete,18.358,-1.0,-1.0,-1.0,-1.0,22.175,3.53554,2912,7565670 +928,search,5259.41,0.91127,0.113929,16.4682,18.1377,25.525,3.53554,2913,7565670 +929,delete,23.941,-1.0,-1.0,-1.0,-1.0,16.393,3.53278,2913,7529948 +930,search,5270.27,0.91096,0.114224,16.4859,18.1796,18.724,3.53278,2913,7529948 +931,delete,397.775,-1.0,-1.0,-1.0,-1.0,966.565,3.34873,2786,7016700 +932,search,4989.6,0.90943,0.115662,16.5599,18.1023,57.01,3.34968,2796,7016700 +933,delete,21.73,-1.0,-1.0,-1.0,-1.0,58.655,3.33751,2790,6980979 +934,search,5002.37,0.90927,0.115759,16.5909,18.0971,23.285,3.33772,2791,6980979 +935,delete,16.819,-1.0,-1.0,-1.0,-1.0,69.246,3.3284,2787,6954250 +936,search,4986.3,0.90886,0.1161,16.6094,18.1146,18.132,3.3284,2787,6954250 +937,delete,23.926,-1.0,-1.0,-1.0,-1.0,37.839,3.31908,2785,6924596 +938,search,4967.88,0.90859,0.116313,16.6639,18.1457,17.541,3.31908,2785,6924596 +939,delete,120.655,-1.0,-1.0,-1.0,-1.0,60.179,3.29039,2779,6784831 +940,search,4901.97,0.90776,0.117657,16.705,18.1626,17.77,3.29039,2779,6784831 +941,delete,32.685,-1.0,-1.0,-1.0,-1.0,15.301,3.28281,2779,6745819 +942,search,4890.73,0.90761,0.117691,16.7227,18.1425,17.814,3.28281,2779,6745819 +943,delete,3.376,-1.0,-1.0,-1.0,-1.0,15.333,3.28237,2779,6740944 +944,search,4906.98,0.90757,0.117696,16.7338,18.1532,20.426,3.28237,2779,6740944 +945,delete,106.167,-1.0,-1.0,-1.0,-1.0,311.478,3.22578,2745,6627202 +946,search,4822.83,0.90709,0.117855,16.8465,18.2016,34.511,3.22631,2749,6627202 +947,delete,19.296,-1.0,-1.0,-1.0,-1.0,15.307,3.22214,2749,6607194 +948,search,4817.79,0.90655,0.11817,16.8884,18.2178,17.715,3.22214,2749,6607194 +949,delete,49.202,-1.0,-1.0,-1.0,-1.0,68.406,3.20143,2740,6551065 +950,search,4776.78,0.90647,0.118308,16.9079,18.1958,17.41,3.20143,2740,6551065 +951,delete,30.241,-1.0,-1.0,-1.0,-1.0,50.028,3.18016,2729,6506798 +952,search,4749.12,0.90657,0.118039,16.9431,18.1965,22.49,3.18027,2730,6506798 +953,delete,37.547,-1.0,-1.0,-1.0,-1.0,72.968,3.16403,2727,6459301 +954,search,4743.47,0.90653,0.117898,16.953,18.2148,17.536,3.16403,2727,6459301 +955,delete,13.276,-1.0,-1.0,-1.0,-1.0,29.106,3.15594,2723,6440069 +956,search,4760.15,0.9066,0.117794,16.999,18.2937,17.996,3.15594,2723,6440069 +957,delete,308.972,-1.0,-1.0,-1.0,-1.0,410.632,3.00458,2663,6132470 +958,search,4617.65,0.90451,0.119081,17.2853,18.5269,17.339,3.00458,2663,6132470 +959,delete,46.403,-1.0,-1.0,-1.0,-1.0,44.842,2.98572,2658,6086241 +960,search,4572.99,0.9034,0.119893,17.4475,18.6325,17.288,2.98572,2658,6086241 +961,delete,21.752,-1.0,-1.0,-1.0,-1.0,14.402,2.98031,2658,6064734 +962,search,4573.74,0.90353,0.119779,17.4439,18.6402,16.8,2.98031,2658,6064734 +963,delete,102.242,-1.0,-1.0,-1.0,-1.0,61.091,2.93351,2650,5967543 +964,search,4522.71,0.90317,0.119733,17.5223,18.6698,16.353,2.93351,2650,5967543 +965,delete,50.943,-1.0,-1.0,-1.0,-1.0,83.093,2.90742,2637,5906381 +966,search,4486.16,0.90253,0.12039,17.5545,18.6878,17.026,2.90742,2637,5906381 +967,delete,44.546,-1.0,-1.0,-1.0,-1.0,52.479,2.88715,2632,5851849 +968,search,4470.37,0.9025,0.120504,17.6045,18.7245,21.756,2.88731,2633,5851849 +969,delete,61.005,-1.0,-1.0,-1.0,-1.0,54.392,2.85719,2626,5783023 +970,search,4457.69,0.90174,0.120959,17.7137,18.7821,16.996,2.85719,2626,5783023 +971,delete,276.489,-1.0,-1.0,-1.0,-1.0,393.143,2.71357,2571,5496517 +972,search,4320.59,0.90044,0.122013,17.9114,18.8624,22.583,2.71383,2572,5496517 +973,delete,239.77,-1.0,-1.0,-1.0,-1.0,324.439,2.5779,2514,5237981 +974,search,4165.48,0.8972,0.12466,18.3128,19.0926,26.537,2.57828,2516,5237981 +975,delete,118.201,-1.0,-1.0,-1.0,-1.0,250.328,2.50254,2455,5099794 +976,search,4059.75,0.89496,0.126095,18.627,19.1827,15.197,2.50254,2455,5099794 +977,delete,51.007,-1.0,-1.0,-1.0,-1.0,121.567,2.47305,2441,5032210 +978,search,4023.9,0.89434,0.126473,18.741,19.2642,15.251,2.47305,2441,5032210 +979,delete,9.19,-1.0,-1.0,-1.0,-1.0,17.181,2.46955,2440,5020211 +980,search,4009.43,0.89428,0.126642,18.7422,19.2532,15.738,2.46955,2440,5020211 +981,delete,102.001,-1.0,-1.0,-1.0,-1.0,173.118,2.41247,2415,4902291 +982,search,3972.27,0.89279,0.127392,18.9343,19.3171,22.684,2.41248,2416,4902291 +983,delete,35.717,-1.0,-1.0,-1.0,-1.0,13.441,2.40354,2416,4872577 +984,search,3973.35,0.89278,0.127613,18.9553,19.3352,15.583,2.40354,2416,4872577 +985,delete,38.861,-1.0,-1.0,-1.0,-1.0,23.371,2.39136,2415,4836730 +986,search,3953.26,0.89235,0.127828,18.9956,19.3682,15.757,2.39136,2415,4836730 +987,delete,41.455,-1.0,-1.0,-1.0,-1.0,24.636,2.37774,2413,4797164 +988,search,3921.03,0.89207,0.1282,19.095,19.4159,30.222,2.3784,2414,4797164 +989,delete,13.554,-1.0,-1.0,-1.0,-1.0,13.35,2.37433,2414,4784789 +990,search,3920.91,0.89222,0.127989,19.0587,19.3921,15.401,2.37433,2414,4784789 +991,delete,66.729,-1.0,-1.0,-1.0,-1.0,226.377,2.33598,2391,4705420 +992,search,3901.93,0.89522,0.125494,18.8565,19.4593,49.407,2.33668,2396,4705420 +993,delete,74.828,-1.0,-1.0,-1.0,-1.0,102.404,2.29672,2385,4621807 +994,search,3853.21,0.89396,0.125974,18.956,19.4514,23.592,2.29677,2387,4621807 +995,delete,58.369,-1.0,-1.0,-1.0,-1.0,71.282,2.26522,2378,4555849 +996,search,3826.49,0.89349,0.126335,19.0319,19.47,15.494,2.26522,2378,4555849 +997,delete,28.017,-1.0,-1.0,-1.0,-1.0,50.617,2.2524,2375,4526975 +998,search,3847.7,0.8935,0.126484,19.0376,19.462,15.373,2.2524,2375,4526975 +999,delete,110.466,-1.0,-1.0,-1.0,-1.0,154.198,2.19358,2351,4408448 +1000,search,3800.63,0.89326,0.126231,19.08,19.4691,28.113,2.19382,2352,4408448 +1001,delete,115.589,-1.0,-1.0,-1.0,-1.0,130.855,2.13752,2337,4303257 +1002,search,3764.08,0.8928,0.126275,19.1727,19.5384,22.456,2.13752,2337,4303257 +1003,delete,41.725,-1.0,-1.0,-1.0,-1.0,54.85,2.11493,2331,4259551 +1004,search,3720.43,0.8926,0.126544,19.2315,19.5582,22.222,2.11493,2331,4259551 +1005,delete,18.607,-1.0,-1.0,-1.0,-1.0,26.14,2.10764,2330,4241434 +1006,search,3730.14,0.89263,0.126606,19.2302,19.5676,22.249,2.10764,2330,4241434 +1007,delete,171.133,-1.0,-1.0,-1.0,-1.0,326.898,2.00702,2281,4052722 +1008,search,3627.38,0.89065,0.128185,19.5065,19.7274,21.684,2.00702,2281,4052722 +1009,delete,273.552,-1.0,-1.0,-1.0,-1.0,379.394,1.84592,2204,3738432 +1010,search,3486.49,0.88807,0.12956,19.6378,19.6654,29.83,1.84624,2205,3738432 +1011,delete,332.766,-1.0,-1.0,-1.0,-1.0,525.162,1.65252,2090,3372539 +1012,search,3272.8,0.88334,0.13124,20.0682,19.7994,36.887,1.65305,2093,3372539 +1013,delete,9.458,-1.0,-1.0,-1.0,-1.0,39.702,1.64766,2088,3361432 +1014,search,3261.47,0.88255,0.131645,20.193,19.8993,26.807,1.64773,2089,3361432 +1015,delete,231.539,-1.0,-1.0,-1.0,-1.0,360.837,1.51355,2011,3091692 +1016,search,3167.21,0.87943,0.133227,20.7062,20.0308,18.446,1.51355,2011,3091692 +1017,delete,300.616,-1.0,-1.0,-1.0,-1.0,473.47,1.34699,1923,2775418 +1018,search,2969.91,0.87768,0.13406,20.8793,20.0382,28.194,1.34744,1924,2775418 +1019,delete,18.823,-1.0,-1.0,-1.0,-1.0,36.362,1.33951,1921,2754073 +1020,search,2981.47,0.87713,0.134475,20.9658,20.1365,17.751,1.33951,1921,2754073 +1021,delete,104.465,-1.0,-1.0,-1.0,-1.0,73.467,1.30063,1907,2638917 +1022,search,2922.3,0.87535,0.135607,21.1329,20.2157,17.42,1.30063,1907,2638917 +1023,delete,75.146,-1.0,-1.0,-1.0,-1.0,101.163,1.26316,1879,2556494 +1024,search,2857.85,0.87207,0.138725,21.4336,20.4014,16.957,1.26316,1879,2556494 +1025,insert,22.274,-1.0,-1.0,-1.0,-1.0,27.721,1.26968,1881,2594934 +1026,search,2887.49,0.87297,0.138201,21.2935,20.3187,16.793,1.26968,1881,2594934 +1027,insert,182.326,-1.0,-1.0,-1.0,-1.0,85.044,1.3307,1889,2802105 +1028,search,2942.79,0.87323,0.138801,21.2512,20.4404,41.655,1.3307,1892,2802105 +1029,insert,377.993,-1.0,-1.0,-1.0,-1.0,212.263,1.47454,1923,3203975 +1030,search,3116.16,0.87935,0.134747,20.6259,20.1683,102.333,1.47465,1938,3203975 +1031,insert,551.468,-1.0,-1.0,-1.0,-1.0,308.514,1.73082,1968,3830566 +1032,search,3339.87,0.88214,0.13567,20.2031,20.2685,87.47,1.73082,1979,3830566 +1033,insert,415.119,-1.0,-1.0,-1.0,-1.0,270.063,1.91549,2027,4252669 +1034,search,3504.2,0.88779,0.132399,19.2201,19.5834,93.276,1.91553,2039,4252669 +1035,insert,216.563,-1.0,-1.0,-1.0,-1.0,234.989,2.02793,2072,4512107 +1036,search,3614.11,0.8892,0.132134,19.0263,19.5445,89.305,2.02858,2085,4512107 +1037,insert,17.062,-1.0,-1.0,-1.0,-1.0,21.569,2.03365,2086,4525815 +1038,search,3626.07,0.8894,0.132015,18.9895,19.5214,18.28,2.03365,2086,4525815 +1039,insert,85.733,-1.0,-1.0,-1.0,-1.0,32.089,2.06173,2091,4599215 +1040,search,3637.27,0.89003,0.13175,18.8562,19.4363,18.374,2.06173,2091,4599215 +1041,insert,12.391,-1.0,-1.0,-1.0,-1.0,16.547,2.06583,2091,4609851 +1042,search,3663.58,0.89025,0.131838,18.813,19.4291,18.388,2.06583,2091,4609851 +1043,insert,21.397,-1.0,-1.0,-1.0,-1.0,26.088,2.07265,2093,4627411 +1044,search,3657.55,0.89031,0.131778,18.7959,19.4187,18.256,2.07265,2093,4627411 +1045,insert,50.111,-1.0,-1.0,-1.0,-1.0,18.903,2.08969,2094,4670607 +1046,search,3654.09,0.89069,0.132007,18.729,19.4353,18.276,2.08969,2094,4670607 +1047,insert,6.353,-1.0,-1.0,-1.0,-1.0,21.549,2.09164,2095,4675386 +1048,search,3666.22,0.89086,0.131947,18.711,19.4208,18.113,2.09164,2095,4675386 +1049,insert,18.079,-1.0,-1.0,-1.0,-1.0,22.218,2.09808,2096,4689998 +1050,search,3661.99,0.89095,0.131895,18.6839,19.4094,26.067,2.09828,2097,4689998 +1051,insert,102.614,-1.0,-1.0,-1.0,-1.0,69.583,2.13841,2111,4783971 +1052,search,3724.22,0.89131,0.131895,18.6613,19.4049,21.888,2.13848,2112,4783971 +1053,insert,50.467,-1.0,-1.0,-1.0,-1.0,37.465,2.15505,2117,4826219 +1054,search,3735,0.89113,0.132139,18.6362,19.365,18.268,2.15505,2117,4826219 +1055,insert,45.192,-1.0,-1.0,-1.0,-1.0,21.102,2.16831,2118,4862355 +1056,search,3757.83,0.89172,0.132032,18.5487,19.3058,18.302,2.16831,2118,4862355 +1057,insert,9.664,-1.0,-1.0,-1.0,-1.0,16.709,2.17047,2118,4869564 +1058,search,3754.78,0.89182,0.132035,18.5548,19.3001,18.423,2.17047,2118,4869564 +1059,insert,30.88,-1.0,-1.0,-1.0,-1.0,16.741,2.17903,2118,4898570 +1060,search,3752.78,0.89208,0.131799,18.5221,19.2767,18.381,2.17903,2118,4898570 +1061,insert,12.267,-1.0,-1.0,-1.0,-1.0,19.299,2.18265,2119,4907501 +1062,search,3776.47,0.892,0.13183,18.4976,19.2457,24.722,2.18281,2120,4907501 +1063,insert,8.82,-1.0,-1.0,-1.0,-1.0,28.889,2.1861,2122,4916386 +1064,search,3769.11,0.89237,0.131491,18.4965,19.2376,21.784,2.18615,2123,4916386 +1065,insert,313.719,-1.0,-1.0,-1.0,-1.0,255.608,2.3092,2168,5194839 +1066,search,3882.72,0.8942,0.129415,18.2529,19.0722,56.868,2.3093,2174,5194839 +1067,insert,5.91,-1.0,-1.0,-1.0,-1.0,18.037,2.31135,2174,5200182 +1068,search,3889.32,0.89465,0.12905,18.2186,19.0463,32.087,2.31152,2177,5200182 +1069,insert,32.787,-1.0,-1.0,-1.0,-1.0,42.549,2.32452,2187,5230883 +1070,search,3915.69,0.89494,0.128397,18.121,18.9842,19.363,2.32452,2187,5230883 +1071,insert,24.659,-1.0,-1.0,-1.0,-1.0,53.328,2.33532,2193,5255659 +1072,search,3930.59,0.8954,0.128218,18.0972,18.9911,19.413,2.33532,2193,5255659 +1073,insert,42.067,-1.0,-1.0,-1.0,-1.0,17.914,2.34888,2193,5281693 +1074,search,3937.12,0.8956,0.128056,18.0648,18.9683,21.51,2.34888,2193,5281693 +1075,insert,11.252,-1.0,-1.0,-1.0,-1.0,17.953,2.35235,2193,5290081 +1076,search,3949.57,0.89567,0.128075,18.0346,18.9325,19.773,2.35235,2193,5290081 +1077,insert,30.592,-1.0,-1.0,-1.0,-1.0,23.574,2.36256,2194,5316166 +1078,search,3942.28,0.89562,0.127978,18.0559,18.9651,19.435,2.36256,2194,5316166 +1079,insert,24.457,-1.0,-1.0,-1.0,-1.0,21.262,2.37079,2195,5335531 +1080,search,3967.7,0.89599,0.127799,18.0337,18.9715,19.339,2.37079,2195,5335531 +1081,insert,383.642,-1.0,-1.0,-1.0,-1.0,218.852,2.52355,2232,5678781 +1082,search,4083.12,0.89744,0.126754,17.8416,18.9768,88.64,2.52489,2245,5678781 +1083,insert,1.048,-1.0,-1.0,-1.0,-1.0,19.194,2.52527,2245,5679848 +1084,search,4096.89,0.89729,0.126928,17.8642,18.9911,31.717,2.52527,2247,5679848 +1085,insert,21.33,-1.0,-1.0,-1.0,-1.0,33.108,2.53382,2251,5702205 +1086,search,4102.89,0.89756,0.12679,17.8317,18.9699,20.479,2.53382,2251,5702205 +1087,insert,16.803,-1.0,-1.0,-1.0,-1.0,26.963,2.5393,2253,5715514 +1088,search,4117.02,0.89767,0.126701,17.8003,18.9587,26.169,2.53957,2254,5715514 +1089,insert,428.356,-1.0,-1.0,-1.0,-1.0,353.092,2.70722,2336,6103499 +1090,search,4328.15,0.89983,0.124974,17.5956,18.8381,69.448,2.70755,2348,6103499 +1091,insert,53.621,-1.0,-1.0,-1.0,-1.0,58.113,2.7293,2362,6152505 +1092,search,4349.66,0.90017,0.125037,17.5816,18.8423,15.698,2.7293,2362,6152505 +1093,insert,19.306,-1.0,-1.0,-1.0,-1.0,23.251,2.73674,2364,6171104 +1094,search,4339.63,0.90027,0.12497,17.5631,18.8404,25.543,2.73716,2365,6171104 +1095,insert,371.748,-1.0,-1.0,-1.0,-1.0,490.563,2.91159,2457,6557850 +1096,search,4546.45,0.90316,0.123043,17.1815,18.6211,97.648,2.91264,2474,6557850 +1097,insert,17.73,-1.0,-1.0,-1.0,-1.0,25.696,2.92036,2477,6579289 +1098,search,4581.86,0.90356,0.122644,17.1444,18.6058,23.224,2.92036,2479,6579289 +1099,insert,10.733,-1.0,-1.0,-1.0,-1.0,14.064,2.92478,2479,6591442 +1100,search,4571.91,0.90372,0.122582,17.111,18.5635,15.636,2.92478,2479,6591442 +1101,insert,39.951,-1.0,-1.0,-1.0,-1.0,33.897,2.94152,2484,6633739 +1102,search,4580.93,0.9042,0.122161,17.0778,18.5504,26.955,2.94182,2487,6633739 +1103,insert,96.978,-1.0,-1.0,-1.0,-1.0,112.674,2.98458,2519,6724357 +1104,search,4675.59,0.90587,0.121029,16.9194,18.4708,26.567,2.98471,2522,6724357 +1105,insert,2.349,-1.0,-1.0,-1.0,-1.0,14.585,2.98561,2522,6727059 +1106,search,4698.91,0.90601,0.120891,16.9345,18.4836,23.88,2.98597,2523,6727059 +1107,insert,224.06,-1.0,-1.0,-1.0,-1.0,258.676,3.08338,2576,6950869 +1108,search,4816.99,0.90637,0.1204,16.8188,18.4172,41.445,3.08358,2581,6950869 +1109,insert,5.113,-1.0,-1.0,-1.0,-1.0,21.805,3.08526,2582,6958173 +1110,search,4817.32,0.90653,0.120358,16.7917,18.3984,16.263,3.08526,2582,6958173 +1111,insert,33.787,-1.0,-1.0,-1.0,-1.0,55.106,3.09831,2593,6995273 +1112,search,4791.35,0.90675,0.120214,16.774,18.3817,16.071,3.09831,2593,6995273 +1113,insert,46.542,-1.0,-1.0,-1.0,-1.0,65.197,3.12207,2608,7047862 +1114,search,4832.1,0.90676,0.120178,16.7418,18.351,34.381,3.12224,2611,7047862 +1115,insert,62.041,-1.0,-1.0,-1.0,-1.0,98.248,3.14732,2632,7110727 +1116,search,4881.35,0.90771,0.119961,16.6864,18.3606,31.113,3.14772,2635,7110727 +1117,insert,16.247,-1.0,-1.0,-1.0,-1.0,28.918,3.15306,2639,7125591 +1118,search,4922.02,0.9077,0.120119,16.6926,18.3704,16.777,3.15306,2639,7125591 +1119,insert,9.446,-1.0,-1.0,-1.0,-1.0,14.745,3.15605,2639,7134442 +1120,search,4942.14,0.9077,0.120202,16.6672,18.3585,16.631,3.15605,2639,7134442 +1121,insert,35.046,-1.0,-1.0,-1.0,-1.0,40.613,3.16933,2644,7162109 +1122,search,4957.82,0.90802,0.120078,16.6191,18.3359,20.4,3.16934,2645,7162109 +1123,insert,20.104,-1.0,-1.0,-1.0,-1.0,36.842,3.17753,2650,7179509 +1124,search,4943.4,0.90849,0.119653,16.6069,18.3227,22.328,3.17764,2651,7179509 +1125,insert,6.846,-1.0,-1.0,-1.0,-1.0,23.349,3.18061,2652,7186372 +1126,search,4942.22,0.90839,0.119742,16.6007,18.3128,24.004,3.18081,2653,7186372 +1127,insert,42.876,-1.0,-1.0,-1.0,-1.0,61.358,3.20147,2663,7231450 +1128,search,4950.53,0.90835,0.119727,16.6012,18.3197,31.878,3.20183,2667,7231450 +1129,insert,13.438,-1.0,-1.0,-1.0,-1.0,29.213,3.20663,2670,7243215 +1130,search,5005.94,0.9085,0.119535,16.5827,18.3082,23.793,3.20673,2672,7243215 +1131,insert,39.691,-1.0,-1.0,-1.0,-1.0,43.999,3.22105,2678,7276107 +1132,search,5005.45,0.90816,0.119882,16.5835,18.3448,16.985,3.22105,2678,7276107 +1133,insert,19.035,-1.0,-1.0,-1.0,-1.0,22.783,3.22762,2680,7292577 +1134,search,5025.78,0.90862,0.11929,16.5698,18.3275,25.24,3.22779,2682,7292577 +1135,insert,8.844,-1.0,-1.0,-1.0,-1.0,30.699,3.23105,2685,7299867 +1136,search,5015.91,0.90851,0.119314,16.5772,18.32,16.927,3.23105,2685,7299867 +1137,insert,25.923,-1.0,-1.0,-1.0,-1.0,30.626,3.24003,2689,7323342 +1138,search,5022.79,0.90839,0.119415,16.5786,18.3219,16.425,3.24003,2689,7323342 +1139,insert,3.958,-1.0,-1.0,-1.0,-1.0,26.955,3.24209,2691,7325844 +1140,search,5020.71,0.90873,0.119092,16.5687,18.3054,23.781,3.24223,2693,7325844 +1141,insert,206.014,-1.0,-1.0,-1.0,-1.0,275.949,3.33407,2750,7540761 +1142,search,5156.6,0.91005,0.117957,16.3558,18.1282,65.615,3.33526,2764,7540761 +1143,insert,23.708,-1.0,-1.0,-1.0,-1.0,34.327,3.3451,2769,7561926 +1144,search,5183.69,0.91002,0.117766,16.3447,18.1386,22.657,3.34515,2770,7561926 +1145,insert,10.215,-1.0,-1.0,-1.0,-1.0,28.445,3.34922,2773,7571861 +1146,search,5186.87,0.91027,0.117572,16.3464,18.141,23.551,3.34943,2774,7571861 +1147,insert,19.864,-1.0,-1.0,-1.0,-1.0,31.191,3.35688,2777,7590881 +1148,search,5184.25,0.91023,0.117736,16.3253,18.1129,20.241,3.35692,2778,7590881 +1149,insert,6.132,-1.0,-1.0,-1.0,-1.0,20.823,3.35917,2779,7595507 +1150,search,5175.79,0.91021,0.117807,16.3184,18.1037,17.245,3.35917,2779,7595507 +1151,insert,5.772,-1.0,-1.0,-1.0,-1.0,19.855,3.36091,2780,7601664 +1152,search,5174.16,0.91048,0.117638,16.3059,18.0976,38.971,3.36168,2785,7601664 +1153,delete,17.335,-1.0,-1.0,-1.0,-1.0,24.442,3.35959,2784,7566707 +1154,search,5166.86,0.91016,0.117801,16.3365,18.1327,17.792,3.35959,2784,7566707 +1155,delete,104.003,-1.0,-1.0,-1.0,-1.0,202.038,3.32428,2765,7396192 +1156,search,5125.75,0.90961,0.117762,16.3816,18.1447,23.796,3.32431,2766,7396192 +1157,delete,237.989,-1.0,-1.0,-1.0,-1.0,444.279,3.24074,2705,7079041 +1158,search,4998.14,0.90829,0.118976,16.5685,18.2335,45.227,3.24144,2709,7079041 +1159,delete,331.938,-1.0,-1.0,-1.0,-1.0,450.006,3.07665,2648,6597115 +1160,search,4841.62,0.90564,0.119313,17.0006,18.38,72.404,3.07762,2655,6597115 +1161,delete,212.053,-1.0,-1.0,-1.0,-1.0,306.323,2.98165,2605,6308902 +1162,search,4655.65,0.90393,0.120306,17.2842,18.5806,33.739,2.98186,2608,6308902 +1163,delete,93.166,-1.0,-1.0,-1.0,-1.0,233.264,2.9251,2579,6164514 +1164,search,4583.41,0.90354,0.12074,17.3274,18.5752,25.827,2.92532,2580,6164514 +1165,delete,28.361,-1.0,-1.0,-1.0,-1.0,37.398,2.91859,2577,6133412 +1166,search,4549.93,0.90347,0.120642,17.3554,18.6026,17.213,2.91859,2577,6133412 +1167,delete,52.193,-1.0,-1.0,-1.0,-1.0,15.396,2.90749,2577,6057218 +1168,search,4547.02,0.9028,0.121199,17.4523,18.7013,17.341,2.90749,2577,6057218 +1169,delete,18.006,-1.0,-1.0,-1.0,-1.0,85.726,2.89318,2566,6028053 +1170,search,4540.28,0.90054,0.12207,17.8459,18.893,17.225,2.89318,2566,6028053 +1171,delete,24.687,-1.0,-1.0,-1.0,-1.0,21.219,2.88651,2565,6001880 +1172,search,4506.82,0.90043,0.122081,17.8574,18.9027,17.383,2.88651,2565,6001880 +1173,delete,38.019,-1.0,-1.0,-1.0,-1.0,41.562,2.87506,2562,5957180 +1174,search,4505.01,0.90013,0.122476,17.8923,18.9304,17.261,2.87506,2562,5957180 +1175,delete,15.351,-1.0,-1.0,-1.0,-1.0,14.571,2.87324,2562,5939326 +1176,search,4516.26,0.90019,0.122519,17.8759,18.9134,17.097,2.87324,2562,5939326 +1177,delete,12.221,-1.0,-1.0,-1.0,-1.0,35.182,2.86878,2561,5924145 +1178,search,4477.91,0.90028,0.122537,17.867,18.9081,16.83,2.86878,2561,5924145 +1179,delete,53.033,-1.0,-1.0,-1.0,-1.0,93.176,2.84409,2550,5863334 +1180,search,4435.8,0.90017,0.122549,17.9083,18.9173,16.755,2.84409,2550,5863334 +1181,delete,34.875,-1.0,-1.0,-1.0,-1.0,14.79,2.83161,2550,5822703 +1182,search,4450.71,0.89995,0.123176,17.9215,18.9417,17.193,2.83161,2550,5822703 +1183,delete,40.245,-1.0,-1.0,-1.0,-1.0,50.732,2.81737,2545,5775807 +1184,search,4422.6,0.89901,0.124013,17.9807,18.9527,17.081,2.81737,2545,5775807 +1185,delete,23.714,-1.0,-1.0,-1.0,-1.0,28.102,2.81091,2543,5750627 +1186,search,4436.42,0.89881,0.124075,18.0253,18.9551,17.173,2.81091,2543,5750627 +1187,delete,137.605,-1.0,-1.0,-1.0,-1.0,191.88,2.75749,2518,5601207 +1188,search,4361.04,0.89813,0.124507,18.1217,19.0574,22.841,2.75751,2519,5601207 +1189,delete,13.255,-1.0,-1.0,-1.0,-1.0,14.364,2.75516,2519,5586425 +1190,search,4323.71,0.89791,0.124741,18.1179,19.0514,16.676,2.75516,2519,5586425 +1191,delete,10.119,-1.0,-1.0,-1.0,-1.0,21.575,2.75248,2518,5574023 +1192,search,4355.59,0.89785,0.124629,18.1584,19.0781,21.909,2.75259,2519,5574023 +1193,delete,179.87,-1.0,-1.0,-1.0,-1.0,319.61,2.64866,2481,5356157 +1194,search,4223.98,0.89695,0.124666,18.3255,19.1348,16.314,2.64866,2481,5356157 +1195,delete,13.026,-1.0,-1.0,-1.0,-1.0,14.073,2.64523,2481,5340404 +1196,search,4231.56,0.89682,0.124881,18.305,19.1347,16.517,2.64523,2481,5340404 +1197,delete,49.841,-1.0,-1.0,-1.0,-1.0,14.398,2.62958,2481,5293259 +1198,search,4213.5,0.89649,0.125362,18.3267,19.1591,16.493,2.62958,2481,5293259 +1199,delete,11.574,-1.0,-1.0,-1.0,-1.0,19.664,2.62421,2480,5279787 +1200,search,4212.39,0.89662,0.125124,18.3056,19.1277,16.723,2.62421,2480,5279787 +1201,delete,71.12,-1.0,-1.0,-1.0,-1.0,104.345,2.58937,2468,5216567 +1202,search,4199.9,0.89557,0.125814,18.4141,19.2112,16.567,2.58937,2468,5216567 +1203,delete,20.718,-1.0,-1.0,-1.0,-1.0,13.902,2.58403,2468,5196947 +1204,search,4153.9,0.89563,0.125794,18.4381,19.2321,16.291,2.58403,2468,5196947 +1205,delete,20.735,-1.0,-1.0,-1.0,-1.0,39.023,2.57214,2464,5168447 +1206,search,4160.14,0.89539,0.125812,18.46,19.205,16.328,2.57214,2464,5168447 +1207,delete,20.536,-1.0,-1.0,-1.0,-1.0,48.044,2.56042,2460,5146927 +1208,search,4119.18,0.89535,0.125383,18.5226,19.1917,32.15,2.56087,2463,5146927 +1209,delete,234.793,-1.0,-1.0,-1.0,-1.0,318.21,2.42836,2416,4887612 +1210,search,4039.75,0.89423,0.126295,18.5014,19.0551,21.657,2.4286,2417,4887612 +1211,delete,4.279,-1.0,-1.0,-1.0,-1.0,13.693,2.42684,2417,4882301 +1212,search,4032.7,0.89435,0.126289,18.5046,19.0499,16.38,2.42684,2417,4882301 +1213,delete,107.114,-1.0,-1.0,-1.0,-1.0,103.29,2.37985,2405,4780915 +1214,search,4008.04,0.89339,0.126939,18.6922,19.1902,15.994,2.37985,2405,4780915 +1215,delete,14.959,-1.0,-1.0,-1.0,-1.0,13.598,2.3758,2405,4765751 +1216,search,4015.37,0.89326,0.126658,18.6929,19.1685,16.034,2.3758,2405,4765751 +1217,delete,313.116,-1.0,-1.0,-1.0,-1.0,425.142,2.20907,2317,4452306 +1218,search,3836.16,0.89072,0.127679,19.0472,19.3778,29.321,2.2092,2318,4452306 +1219,delete,79.884,-1.0,-1.0,-1.0,-1.0,168.953,2.15866,2285,4362045 +1220,search,3776.47,0.88921,0.128834,19.1923,19.4092,25.45,2.15866,2286,4362045 +1221,delete,31.035,-1.0,-1.0,-1.0,-1.0,34.881,2.14681,2283,4326917 +1222,search,3763.19,0.889,0.129175,19.2281,19.4299,25.602,2.14687,2284,4326917 +1223,delete,181.218,-1.0,-1.0,-1.0,-1.0,287.705,2.03944,2228,4109781 +1224,search,3642.64,0.88613,0.131328,19.5099,19.5291,20.628,2.03944,2228,4109781 +1225,delete,25.601,-1.0,-1.0,-1.0,-1.0,47.126,2.02571,2224,4075879 +1226,search,3630.47,0.88581,0.131756,19.5161,19.484,20.535,2.02571,2224,4075879 +1227,delete,62.548,-1.0,-1.0,-1.0,-1.0,97.999,1.99317,2212,4006503 +1228,search,3599.72,0.88549,0.131615,19.613,19.5174,26.709,1.99321,2213,4006503 +1229,delete,72.632,-1.0,-1.0,-1.0,-1.0,82.775,1.95396,2204,3922625 +1230,search,3537.87,0.88456,0.132527,19.703,19.5672,21.218,1.95396,2204,3922625 +1231,delete,77.212,-1.0,-1.0,-1.0,-1.0,153.205,1.91059,2165,3844860 +1232,search,3512.75,0.88255,0.134077,19.858,19.5616,20.028,1.91059,2165,3844860 +1233,delete,12.817,-1.0,-1.0,-1.0,-1.0,21.937,1.90637,2164,3831414 +1234,search,3487.01,0.88235,0.134089,19.9051,19.5939,20.243,1.90637,2164,3831414 +1235,delete,144.989,-1.0,-1.0,-1.0,-1.0,151.89,1.83465,2143,3684234 +1236,search,3424.41,0.88158,0.134207,20.0265,19.6125,20.176,1.83465,2143,3684234 +1237,delete,13.084,-1.0,-1.0,-1.0,-1.0,23.67,1.8308,2142,3669074 +1238,search,3433.55,0.88144,0.134108,20.0168,19.5975,19.668,1.8308,2142,3669074 +1239,delete,52.075,-1.0,-1.0,-1.0,-1.0,18.085,1.81247,2142,3620984 +1240,search,3411.88,0.88111,0.134549,20.0523,19.609,19.72,1.81247,2142,3620984 +1241,delete,37.731,-1.0,-1.0,-1.0,-1.0,17.727,1.79797,2142,3580113 +1242,search,3397.82,0.88072,0.13503,20.0874,19.6343,19.646,1.79797,2142,3580113 +1243,delete,46.127,-1.0,-1.0,-1.0,-1.0,40.026,1.774,2138,3533909 +1244,search,3403.08,0.88046,0.135635,20.0601,19.6079,19.583,1.774,2138,3533909 +1245,delete,13.76,-1.0,-1.0,-1.0,-1.0,21.248,1.7675,2137,3519590 +1246,search,3371.48,0.88008,0.136104,20.0736,19.6238,19.297,1.7675,2137,3519590 +1247,delete,48.734,-1.0,-1.0,-1.0,-1.0,77.926,1.73201,2107,3452537 +1248,search,3313.37,0.88094,0.134287,20.4095,19.8576,22.041,1.73206,2108,3452537 +1249,delete,33.433,-1.0,-1.0,-1.0,-1.0,39.434,1.71709,2105,3416711 +1250,search,3302.45,0.88027,0.134608,20.4307,19.8369,19.113,1.71709,2105,3416711 +1251,delete,25.444,-1.0,-1.0,-1.0,-1.0,34.571,1.70445,2101,3387653 +1252,search,3303.63,0.87985,0.134845,20.4647,19.832,19.745,1.70445,2101,3387653 +1253,delete,14.49,-1.0,-1.0,-1.0,-1.0,25.488,1.69821,2099,3372157 +1254,search,3312.22,0.87951,0.135044,20.4681,19.8386,19.931,1.69821,2099,3372157 +1255,delete,105.78,-1.0,-1.0,-1.0,-1.0,95.31,1.63936,2065,3258215 +1256,search,3255.41,0.87782,0.135303,20.8373,20.0105,18.762,1.63936,2065,3258215 +1257,delete,34.914,-1.0,-1.0,-1.0,-1.0,70.5,1.62005,2052,3223657 +1258,search,3216.7,0.87735,0.135522,20.9455,20.0295,33.574,1.62025,2054,3223657 +1259,delete,43.552,-1.0,-1.0,-1.0,-1.0,71.192,1.59594,2043,3174982 +1260,search,3199.14,0.87676,0.136062,20.9941,20.0345,18.486,1.59594,2043,3174982 +1261,delete,20.632,-1.0,-1.0,-1.0,-1.0,16.292,1.58728,2043,3157038 +1262,search,3201.68,0.87655,0.136242,21.0488,20.068,18.659,1.58728,2043,3157038 +1263,delete,96.074,-1.0,-1.0,-1.0,-1.0,172.066,1.5309,2021,3058847 +1264,search,3154.73,0.87607,0.135894,21.1419,20.1005,18.239,1.5309,2021,3058847 +1265,delete,221.662,-1.0,-1.0,-1.0,-1.0,292.673,1.39603,1911,2833531 +1266,search,2986.09,0.87245,0.138987,21.5975,20.2781,44.185,1.39651,1914,2833531 +1267,delete,59.181,-1.0,-1.0,-1.0,-1.0,48.174,1.37352,1906,2763793 +1268,search,2946.84,0.87096,0.139493,21.7849,20.3687,17.029,1.37352,1906,2763793 +1269,delete,128.328,-1.0,-1.0,-1.0,-1.0,143.012,1.3081,1870,2627384 +1270,search,2888.27,0.87091,0.139068,21.9489,20.5068,16.75,1.3081,1870,2627384 +1271,delete,69.914,-1.0,-1.0,-1.0,-1.0,85.781,1.27188,1860,2553476 +1272,search,2852.46,0.87026,0.139427,22.1786,20.7333,16.189,1.27188,1860,2553476 +1273,delete,99.231,-1.0,-1.0,-1.0,-1.0,156.533,1.21056,1826,2443067 +1274,search,2805.45,0.86805,0.141061,22.4193,20.8485,15.942,1.21056,1826,2443067 +1275,delete,19.117,-1.0,-1.0,-1.0,-1.0,28.127,1.20158,1824,2421654 +1276,search,2810.13,0.86736,0.141163,22.4812,20.8669,17.037,1.20158,1824,2421654 +1277,delete,35.325,-1.0,-1.0,-1.0,-1.0,52.197,1.18811,1816,2391396 +1278,search,2770.39,0.86736,0.140894,22.5549,20.9303,16.479,1.18811,1816,2391396 +1279,delete,29.439,-1.0,-1.0,-1.0,-1.0,14.178,1.17976,1816,2367854 +1280,search,2790.09,0.86689,0.141322,22.5976,20.9709,16.123,1.17976,1816,2367854 diff --git a/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.8_search_0.1.csv b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.8_search_0.1.csv new file mode 100644 index 00000000..2be61257 --- /dev/null +++ b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.8_search_0.1.csv @@ -0,0 +1,29 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,mainteance_ms,index_mem_gb +2,search,1966.88,0.71332,0.182665,1.162,0.0183453 +3,insert,22.103,-1.0,-1.0,0.0,0.0223841 +4,search,1398.1,0.72815,0.180415,1.184,0.0223841 +5,insert,31.968,-1.0,-1.0,0.0,0.0328105 +6,search,1389.93,0.799,0.180539,177.015,0.032773 +7,insert,34.503,-1.0,-1.0,0.0,0.0427686 +8,search,1399.31,0.81574,0.16815,133.372,0.0427686 +9,insert,4.49,-1.0,-1.0,0.0,0.044299 +10,search,1407.64,0.82142,0.165197,134.381,0.044299 +11,insert,13.766,-1.0,-1.0,0.0,0.0500224 +12,search,1431.51,0.82763,0.16291,180.121,0.0500224 +13,insert,52.451,-1.0,-1.0,0.0,0.0608752 +14,search,1446.61,0.82239,0.165688,170.954,0.0605223 +15,insert,21.708,-1.0,-1.0,0.0,0.0686713 +16,search,1425.14,0.83243,0.15671,135.156,0.0688671 +17,insert,3.033,-1.0,-1.0,0.0,0.0705889 +18,search,1456.43,0.83221,0.15824,159.039,0.071309 +19,insert,40.076,-1.0,-1.0,0.0,0.0784319 +20,search,1487.27,0.84655,0.148915,158.331,0.0782972 +21,insert,491.279,-1.0,-1.0,0.0,0.197168 +22,search,1818.86,0.85366,0.143039,430.002,0.197955 +23,insert,376.862,-1.0,-1.0,0.0,0.321783 +24,search,2471.47,0.85815,0.14255,1096.25,0.320824 +25,insert,45.479,-1.0,-1.0,0.0,0.338688 +26,search,2243.45,0.81484,0.162522,854.74,0.34008 +27,insert,43.335,-1.0,-1.0,0.0,0.353591 +28,search,2075.38,0.78549,0.174883,730.448,0.352456 +29,insert,84.975,-1.0,-1.0,0.0,0.379445 diff --git a/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.01.csv b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.01.csv new file mode 100644 index 00000000..829e94e3 --- /dev/null +++ b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.01.csv @@ -0,0 +1,47 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,mainteance_ms,index_mem_gb +2,search,585.299,0.30402,0.225896,0.511,0.0183457 +3,insert,10.04,-1.0,-1.0,0.0,0.0223845 +4,search,415.75,0.30883,0.227605,0.559,0.0223845 +5,insert,14.88,-1.0,-1.0,0.0,0.032811 +6,search,402.922,0.33185,0.257537,55.708,0.0331737 +7,insert,22.003,-1.0,-1.0,0.0,0.0421803 +8,search,540.134,0.45891,0.279876,120.639,0.0421925 +9,insert,3.849,-1.0,-1.0,0.0,0.0445059 +10,search,471.401,0.48103,0.267652,100.67,0.0442721 +11,insert,7.19,-1.0,-1.0,0.0,0.0493354 +12,search,482.474,0.46609,0.261136,109.657,0.049213 +13,insert,19.952,-1.0,-1.0,0.0,0.0602477 +14,search,424.731,0.42959,0.256302,85.101,0.0596525 +15,insert,17.45,-1.0,-1.0,0.0,0.0691225 +16,search,535.665,0.42391,0.248077,118.329,0.0688002 +17,insert,3.757,-1.0,-1.0,0.0,0.0712164 +18,search,557.768,0.42416,0.250314,146.088,0.0713127 +19,insert,55.46,-1.0,-1.0,0.0,0.0778346 +20,search,508.566,0.46841,0.253801,154.774,0.0778554 +21,insert,344.779,-1.0,-1.0,0.0,0.197099 +22,search,632.27,0.45878,0.247826,615.33,0.1968 +23,insert,257.694,-1.0,-1.0,0.0,0.315962 +24,search,787.558,0.44987,0.248761,1127.29,0.320233 +25,insert,29.082,-1.0,-1.0,0.0,0.339114 +26,search,740.793,0.4214,0.246488,656.74,0.33744 +27,insert,22.399,-1.0,-1.0,0.0,0.351942 +28,search,630.734,0.40953,0.248731,559.74,0.352849 +29,insert,34.986,-1.0,-1.0,0.0,0.37803 +30,search,632.673,0.41583,0.25098,631.984,0.37854 +31,insert,211.411,-1.0,-1.0,0.0,0.542229 +32,search,870.634,0.42194,0.250413,2233.28,0.543381 +33,insert,42.188,-1.0,-1.0,0.0,0.563387 +34,search,672.622,0.4093,0.250446,963.042,0.563182 +35,insert,26.475,-1.0,-1.0,0.0,0.579811 +36,search,735.102,0.40922,0.250803,911.636,0.578841 +37,insert,41.002,-1.0,-1.0,0.0,0.6011 +38,search,706.686,0.41145,0.251707,0.82,0.6011 +39,insert,5.553,-1.0,-1.0,0.0,0.603488 +40,search,772.152,0.41287,0.252508,846.74,0.602592 +41,insert,33.772,-1.0,-1.0,0.0,0.616597 +42,search,710.203,0.41421,0.252721,711.144,0.615995 +43,insert,8.314,-1.0,-1.0,0.0,0.621732 +44,search,720.162,0.44295,0.251208,1436.69,0.62276 +45,insert,221.623,-1.0,-1.0,0.0,0.637596 +46,search,801.289,0.4398,0.251661,697.218,0.638071 +47,insert,193.781,-1.0,-1.0,0.0,0.739662 diff --git a/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.05.csv b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.05.csv new file mode 100644 index 00000000..40c50321 --- /dev/null +++ b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.05.csv @@ -0,0 +1,30 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,mainteance_ms,index_mem_gb +2,search,1212.99,0.57318,0.215132,0.692,0.0183457 +3,insert,12.785,-1.0,-1.0,0.0,0.0223845 +4,search,886.228,0.58479,0.214352,0.713,0.0223845 +5,insert,28.288,-1.0,-1.0,0.0,0.032811 +6,search,921.25,0.6501,0.234824,52.991,0.0331737 +7,insert,31.242,-1.0,-1.0,0.0,0.0421803 +8,search,938.611,0.71684,0.210044,138.231,0.0421803 +9,insert,9.044,-1.0,-1.0,0.0,0.044432 +10,search,923.082,0.72108,0.207619,140.308,0.044432 +11,insert,51.868,-1.0,-1.0,0.0,0.0500424 +12,search,950.845,0.7306,0.208018,180.701,0.0500424 +13,insert,33.462,-1.0,-1.0,0.0,0.0608618 +14,search,962.175,0.72104,0.210593,319.951,0.0609393 +15,insert,22.704,-1.0,-1.0,0.0,0.0695069 +16,search,983.141,0.73115,0.202415,298.381,0.0693388 +17,insert,4.653,-1.0,-1.0,0.0,0.0717288 +18,search,966.999,0.73309,0.205165,305.028,0.0708937 +19,insert,43.22,-1.0,-1.0,0.0,0.0785575 +20,search,990.416,0.74476,0.195827,132.452,0.0786428 +21,insert,444.208,-1.0,-1.0,0.0,0.197663 +22,search,1275.96,0.7618,0.184556,483.58,0.196977 +23,insert,373.099,-1.0,-1.0,0.0,0.3222 +24,search,1701.5,0.74563,0.192421,1089,0.321832 +25,insert,66.437,-1.0,-1.0,0.0,0.342217 +26,search,1452.44,0.69651,0.207331,861.978,0.342148 +27,insert,44.298,-1.0,-1.0,0.0,0.353195 +28,search,1261.29,0.6582,0.217089,585.985,0.352812 +29,insert,93.125,-1.0,-1.0,0.0,0.379974 +30,search,1289.35,0.66812,0.215953,427.994,0.379034 diff --git a/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.15.csv b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.15.csv new file mode 100644 index 00000000..b2e68761 --- /dev/null +++ b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.15.csv @@ -0,0 +1,1280 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,mainteance_ms,index_mem_gb +2,search,2370.43,0.79264,0.15727,1.745,0.0183465 +3,insert,20.312,-1.0,-1.0,0.0,0.0223853 +4,search,1849.53,0.80591,0.154693,1.568,0.0223853 +5,insert,26.312,-1.0,-1.0,0.0,0.0328118 +6,search,1806.6,0.86827,0.143601,170.432,0.0327742 +7,insert,33.965,-1.0,-1.0,0.0,0.0427698 +8,search,1837.66,0.86926,0.140821,211.307,0.0423957 +9,insert,4.234,-1.0,-1.0,0.0,0.0443512 +10,search,1872,0.87893,0.134219,236.555,0.044146 +11,insert,14.287,-1.0,-1.0,0.0,0.0492725 +12,search,1909.27,0.89952,0.119695,178.682,0.0491815 +13,insert,27.952,-1.0,-1.0,0.0,0.0605692 +14,search,1865.12,0.89471,0.123545,137.244,0.0605256 +15,insert,23.923,-1.0,-1.0,0.0,0.0695469 +16,search,1934.97,0.90145,0.117855,157.477,0.0691939 +17,insert,4.199,-1.0,-1.0,0.0,0.07132 +18,search,1958.84,0.90297,0.117948,153.911,0.0712368 +19,insert,43.037,-1.0,-1.0,0.0,0.0782773 +20,search,2032.41,0.91147,0.111212,117.065,0.0782683 +21,insert,482.318,-1.0,-1.0,0.0,0.197889 +22,search,2391.27,0.91268,0.107618,540.626,0.197268 +23,insert,367.092,-1.0,-1.0,0.0,0.322254 +24,search,3112.75,0.91586,0.105317,1270.93,0.321147 +25,insert,51.624,-1.0,-1.0,0.0,0.341713 +26,search,2946.23,0.89096,0.120569,933.838,0.341172 +27,insert,30.927,-1.0,-1.0,0.0,0.3528 +28,search,2878.29,0.86545,0.135159,591.642,0.353767 +29,insert,85.583,-1.0,-1.0,0.0,0.379818 +30,search,2912.4,0.86581,0.135647,436.711,0.379959 +31,insert,454.665,-1.0,-1.0,0.0,0.541888 +32,search,3562,0.87398,0.131974,1769.48,0.541311 +33,insert,71.333,-1.0,-1.0,0.0,0.564693 +34,search,3449.56,0.85727,0.141115,1305.06,0.566473 +35,insert,42.006,-1.0,-1.0,0.0,0.580664 +36,search,3434.88,0.85308,0.143894,418.574,0.580322 +37,insert,54.78,-1.0,-1.0,0.0,0.600903 +38,search,3437.14,0.85261,0.144287,315.559,0.601657 +39,insert,4.759,-1.0,-1.0,0.0,0.604079 +40,search,2894.42,0.85691,0.142132,320.784,0.603973 +41,insert,44.326,-1.0,-1.0,0.0,0.618961 +42,search,3486.31,0.85956,0.140792,513.456,0.618796 +43,insert,10.257,-1.0,-1.0,0.0,0.623914 +44,search,3480.71,0.85911,0.140584,446.153,0.624144 +45,insert,257.317,-1.0,-1.0,0.0,0.638205 +46,search,3462.88,0.86065,0.139706,499.457,0.63791 +47,insert,303.829,-1.0,-1.0,0.0,0.741015 +48,search,3558.95,0.8649,0.13903,1306.45,0.740241 +49,insert,42.084,-1.0,-1.0,0.0,0.75487 +50,search,3796.34,0.86379,0.141246,890.549,0.75556 +51,insert,495.022,-1.0,-1.0,0.0,0.891335 +52,search,3945.57,0.87665,0.132782,1925.26,0.890553 +53,insert,61.582,-1.0,-1.0,0.0,0.909738 +54,search,4196.81,0.86639,0.139038,1272.46,0.909348 +55,insert,402.291,-1.0,-1.0,0.0,1.04916 +56,search,4498.11,0.87953,0.133721,3073.49,1.04993 +57,insert,33.233,-1.0,-1.0,0.0,1.05899 +58,search,4535.96,0.87348,0.137471,1580.18,1.05987 +59,insert,295.398,-1.0,-1.0,0.0,1.15773 +60,search,4644.69,0.87461,0.135688,2346.47,1.1571 +61,insert,23.623,-1.0,-1.0,0.0,1.16123 +62,search,4522.09,0.86913,0.141141,1087.59,1.16131 +63,insert,5.186,-1.0,-1.0,0.0,1.16269 +64,search,4473.07,0.86846,0.141798,1028.11,1.16244 +65,insert,184.46,-1.0,-1.0,0.0,1.19188 +66,search,4570.56,0.87171,0.139973,1547.37,1.19192 +67,insert,63.577,-1.0,-1.0,0.0,1.21131 +68,search,4528.54,0.87268,0.139645,1110.3,1.2113 +69,insert,74.782,-1.0,-1.0,0.0,1.23668 +70,search,4825.38,0.87426,0.137943,1406.35,1.23644 +71,insert,21.709,-1.0,-1.0,0.0,1.242 +72,search,4706.48,0.87457,0.137918,816.532,1.24103 +73,insert,9.165,-1.0,-1.0,0.0,1.24542 +74,search,4647.51,0.8747,0.137987,6.208,1.24542 +75,insert,26.83,-1.0,-1.0,0.0,1.25231 +76,search,4748.15,0.87493,0.137693,717.19,1.25222 +77,insert,32.082,-1.0,-1.0,0.0,1.26245 +78,search,4808.37,0.87439,0.137938,792.425,1.26191 +79,insert,680.873,-1.0,-1.0,0.0,1.29251 +80,search,4695.98,0.87394,0.137836,1079.18,1.29272 +81,insert,40.776,-1.0,-1.0,0.0,1.30371 +82,search,4739.84,0.87515,0.136642,1101.36,1.3041 +83,insert,22.124,-1.0,-1.0,0.0,1.31046 +84,search,4775.22,0.87618,0.135781,1161.59,1.31139 +85,insert,49.498,-1.0,-1.0,0.0,1.3266 +86,search,4838.35,0.87705,0.135723,1192.89,1.32693 +87,insert,751.012,-1.0,-1.0,0.0,1.51832 +88,search,4875.43,0.88738,0.127872,5076.96,1.51677 +89,insert,572.394,-1.0,-1.0,0.0,1.67498 +90,search,4925.56,0.88854,0.126729,3663.53,1.6757 +91,insert,47.421,-1.0,-1.0,0.0,1.69296 +92,search,4803.1,0.88502,0.129077,1345.4,1.69345 +93,insert,396.746,-1.0,-1.0,0.0,1.79526 +94,search,4968.18,0.88594,0.128361,1599.27,1.7949 +95,insert,16.004,-1.0,-1.0,0.0,1.80045 +96,search,4963.7,0.88555,0.128435,1122.98,1.8018 +97,insert,9.724,-1.0,-1.0,0.0,1.8061 +98,search,4954.49,0.88538,0.128785,1043.43,1.8063 +99,insert,5.957,-1.0,-1.0,0.0,1.80903 +100,search,5036.91,0.88547,0.128725,3.57,1.80903 +101,insert,42.547,-1.0,-1.0,0.0,1.82065 +102,search,5074.71,0.88613,0.12837,1213.36,1.82145 +103,insert,18.565,-1.0,-1.0,0.0,1.82798 +104,search,5841.38,0.88609,0.128253,5.714,1.82798 +105,insert,17.381,-1.0,-1.0,0.0,1.83237 +106,search,5818.99,0.88637,0.128023,1335.4,1.8323 +107,insert,18.866,-1.0,-1.0,0.0,1.8371 +108,search,5839.12,0.88646,0.127926,3.77,1.8371 +109,insert,733.292,-1.0,-1.0,0.0,1.9988 +110,search,6164.55,0.88955,0.125923,4735.24,2.00004 +111,insert,38.372,-1.0,-1.0,0.0,2.01247 +112,search,6103.05,0.88945,0.12652,1937.94,2.01379 +113,insert,14.636,-1.0,-1.0,0.0,2.0189 +114,search,6310.7,0.88945,0.126346,1381.47,2.01878 +115,insert,44.495,-1.0,-1.0,0.0,2.02936 +116,search,6244.65,0.89028,0.125654,6.779,2.02936 +117,insert,23.195,-1.0,-1.0,0.0,2.03489 +118,search,6327.02,0.89043,0.125243,7.15,2.03489 +119,insert,10.161,-1.0,-1.0,0.0,2.03665 +120,search,6119.13,0.89048,0.12524,1319.98,2.03681 +121,insert,24.167,-1.0,-1.0,0.0,2.04243 +122,search,6288.69,0.89048,0.1252,1314.92,2.04179 +123,insert,591.925,-1.0,-1.0,0.0,2.16494 +124,search,6752.83,0.89348,0.124014,2990.92,2.16236 +125,insert,75.521,-1.0,-1.0,0.0,2.18253 +126,search,5704.28,0.89371,0.123867,1846.69,2.18363 +127,insert,76.199,-1.0,-1.0,0.0,2.20661 +128,search,6330.02,0.89507,0.123169,2172.15,2.20556 +129,delete,10.361,-1.0,-1.0,0.0,2.20475 +130,search,6444.44,0.89481,0.123189,2002.84,2.20509 +131,delete,3.57,-1.0,-1.0,0.0,2.20505 +132,search,6676.43,0.89513,0.123071,9.029,2.20505 +133,delete,14.905,-1.0,-1.0,0.0,2.20505 +134,search,6523.89,0.8952,0.1231,8.679,2.20505 +135,delete,25.164,-1.0,-1.0,0.0,2.20361 +136,search,6718.7,0.89665,0.121675,8.589,2.20361 +137,delete,4.641,-1.0,-1.0,0.0,2.20324 +138,search,6734.1,0.89661,0.121687,8.782,2.20324 +139,delete,9.82,-1.0,-1.0,0.0,2.20122 +140,search,6477.88,0.89667,0.121626,8.582,2.20122 +141,delete,22.458,-1.0,-1.0,0.0,2.19894 +142,search,6548.58,0.89691,0.121608,8.575,2.19894 +143,delete,16.012,-1.0,-1.0,0.0,2.19751 +144,search,6703.05,0.89714,0.12144,8.571,2.19751 +145,delete,3.161,-1.0,-1.0,0.0,2.19666 +146,search,6548.61,0.89669,0.121652,8.618,2.19666 +147,delete,9.508,-1.0,-1.0,0.0,2.19632 +148,search,6644.73,0.89683,0.121682,8.586,2.19632 +149,delete,217.473,-1.0,-1.0,0.0,2.15297 +150,search,6651.4,0.90159,0.117581,6.822,2.15297 +151,delete,116.943,-1.0,-1.0,0.0,2.11581 +152,search,5838.79,0.90153,0.117719,5.858,2.11581 +153,delete,29.835,-1.0,-1.0,0.0,2.10282 +154,search,6308.56,0.90122,0.118108,8.811,2.10282 +155,delete,30.911,-1.0,-1.0,0.0,2.09805 +156,search,6093.38,0.90118,0.11805,9.087,2.09805 +157,delete,53.782,-1.0,-1.0,0.0,2.08773 +158,search,6124.56,0.90091,0.118504,8.743,2.08773 +159,delete,362.993,-1.0,-1.0,0.0,1.97016 +160,search,6411.57,0.90044,0.118671,8.644,1.97016 +161,delete,56.72,-1.0,-1.0,0.0,1.95529 +162,search,6090.25,0.9005,0.118557,8.892,1.95529 +163,delete,30.217,-1.0,-1.0,0.0,1.94444 +164,search,6373.51,0.90029,0.118643,11.121,1.94444 +165,delete,40.536,-1.0,-1.0,0.0,1.93449 +166,search,5896.14,0.90042,0.118582,5.746,1.93449 +167,delete,4.756,-1.0,-1.0,0.0,1.93286 +168,search,5445.44,0.90041,0.118524,5.719,1.93286 +169,delete,12.693,-1.0,-1.0,0.0,1.92527 +170,search,5320.26,0.90017,0.11866,5.731,1.92527 +171,delete,6,-1.0,-1.0,0.0,1.92215 +172,search,5619.92,0.89991,0.11887,5.729,1.92215 +173,delete,24.528,-1.0,-1.0,0.0,1.91119 +174,search,6016.14,0.89875,0.120062,8.809,1.91119 +175,delete,187.373,-1.0,-1.0,0.0,1.83751 +176,search,6197.75,0.89881,0.119869,8.636,1.83751 +177,delete,18.243,-1.0,-1.0,0.0,1.83167 +178,search,6036.82,0.89839,0.119965,9.487,1.83167 +179,delete,277.993,-1.0,-1.0,0.0,1.72953 +180,search,6040.89,0.89511,0.1217,8.866,1.72953 +181,delete,37.367,-1.0,-1.0,0.0,1.71671 +182,search,5816.15,0.89504,0.121898,8.845,1.71671 +183,delete,185.154,-1.0,-1.0,0.0,1.64609 +184,search,5900.02,0.89018,0.124488,9.281,1.64609 +185,delete,18.741,-1.0,-1.0,0.0,1.64284 +186,search,5709.96,0.89038,0.124311,9.385,1.64284 +187,delete,243.213,-1.0,-1.0,0.0,1.55335 +188,search,5861.75,0.89384,0.12119,8.846,1.55335 +189,delete,13.405,-1.0,-1.0,0.0,1.54912 +190,search,5679.7,0.89361,0.121327,8.965,1.54912 +191,delete,5.429,-1.0,-1.0,0.0,1.54841 +192,search,5886.29,0.89365,0.121283,8.8,1.54841 +193,delete,78.305,-1.0,-1.0,0.0,1.52053 +194,search,5698.73,0.89092,0.123385,8.73,1.52053 +195,delete,45.966,-1.0,-1.0,0.0,1.50421 +196,search,5465.24,0.89003,0.123804,8.928,1.50421 +197,delete,46.866,-1.0,-1.0,0.0,1.49285 +198,search,5438.47,0.88996,0.123851,8.797,1.49285 +199,delete,15.619,-1.0,-1.0,0.0,1.48822 +200,search,5574.72,0.88935,0.124253,8.764,1.48822 +201,delete,8.344,-1.0,-1.0,0.0,1.48599 +202,search,5725.68,0.88915,0.124342,8.8,1.48599 +203,delete,16.788,-1.0,-1.0,0.0,1.48215 +204,search,5558.64,0.88882,0.124659,8.833,1.48215 +205,delete,28.878,-1.0,-1.0,0.0,1.47233 +206,search,5694.39,0.88905,0.124582,8.825,1.47233 +207,delete,59.744,-1.0,-1.0,0.0,1.4524 +208,search,5668.62,0.88934,0.124024,8.814,1.4524 +209,delete,30.581,-1.0,-1.0,0.0,1.43977 +210,search,5645.99,0.88906,0.124042,9.525,1.43977 +211,delete,24.687,-1.0,-1.0,0.0,1.43462 +212,search,5667.02,0.88889,0.12418,9.006,1.43462 +213,delete,25.319,-1.0,-1.0,0.0,1.42514 +214,search,5450,0.88823,0.124256,9.246,1.42514 +215,delete,408.223,-1.0,-1.0,0.0,1.26758 +216,search,5429.99,0.88258,0.129093,8.993,1.26758 +217,delete,281.247,-1.0,-1.0,0.0,1.16842 +218,search,5372.85,0.87823,0.132255,9.007,1.16842 +219,delete,42.548,-1.0,-1.0,0.0,1.15408 +220,search,5513.66,0.87674,0.133329,8.978,1.15408 +221,delete,235.925,-1.0,-1.0,0.0,1.05579 +222,search,5407.92,0.87681,0.133623,8.994,1.05579 +223,delete,19.145,-1.0,-1.0,0.0,1.0512 +224,search,5347.71,0.87538,0.135072,9.063,1.0512 +225,delete,10.507,-1.0,-1.0,0.0,1.04769 +226,search,5350.19,0.87525,0.135197,8.79,1.04769 +227,delete,5.905,-1.0,-1.0,0.0,1.04609 +228,search,5257.85,0.87509,0.135253,8.965,1.04609 +229,delete,30.699,-1.0,-1.0,0.0,1.03552 +230,search,5341.46,0.87409,0.135644,9.141,1.03552 +231,delete,16.536,-1.0,-1.0,0.0,1.02886 +232,search,5316.52,0.87332,0.135818,8.837,1.02886 +233,delete,8.661,-1.0,-1.0,0.0,1.02533 +234,search,5322.72,0.87292,0.136189,9.005,1.02533 +235,delete,10.637,-1.0,-1.0,0.0,1.02217 +236,search,5317.07,0.87301,0.13615,8.789,1.02217 +237,delete,287.863,-1.0,-1.0,0.0,0.912288 +238,search,5177.42,0.87081,0.137601,8.837,0.912288 +239,delete,27.627,-1.0,-1.0,0.0,0.898865 +240,search,5170.96,0.86924,0.138058,8.98,0.898865 +241,delete,10.716,-1.0,-1.0,0.0,0.894298 +242,search,5150.72,0.86902,0.138144,8.84,0.894298 +243,delete,24.795,-1.0,-1.0,0.0,0.884985 +244,search,5196.43,0.86713,0.13941,8.889,0.884985 +245,delete,17.391,-1.0,-1.0,0.0,0.878583 +246,search,5132.12,0.8667,0.139723,10.996,0.878583 +247,delete,7.15,-1.0,-1.0,0.0,0.87733 +248,search,5141.86,0.86651,0.139753,8.955,0.87733 +249,delete,13.223,-1.0,-1.0,0.0,0.873851 +250,search,5144.65,0.86582,0.140295,9.24,0.873851 +251,delete,317.227,-1.0,-1.0,0.0,0.758392 +252,search,5070.06,0.86618,0.139086,8.905,0.758392 +253,delete,42.376,-1.0,-1.0,0.0,0.741415 +254,search,4821.72,0.86449,0.139571,8.884,0.741415 +255,delete,54.052,-1.0,-1.0,0.0,0.722807 +256,search,5018.51,0.8609,0.142023,8.867,0.722807 +257,insert,12.038,-1.0,-1.0,0.0,0.723916 +258,search,5016.29,0.86076,0.142187,8.998,0.723916 +259,insert,42.693,-1.0,-1.0,0.0,0.730189 +260,search,5022.22,0.86179,0.141523,8.817,0.730189 +261,insert,3.396,-1.0,-1.0,0.0,0.730437 +262,search,5022.49,0.86256,0.140984,8.835,0.730437 +263,insert,259.267,-1.0,-1.0,0.0,0.772004 +264,search,5014.95,0.86646,0.140574,1166.41,0.772104 +265,insert,103.202,-1.0,-1.0,0.0,0.789935 +266,search,5025.99,0.86985,0.138825,825.798,0.790054 +267,insert,50.407,-1.0,-1.0,0.0,0.800599 +268,search,5014.8,0.8713,0.138488,847.96,0.799977 +269,insert,513.745,-1.0,-1.0,0.0,0.891504 +270,search,5130.13,0.87429,0.136771,1279.28,0.890334 +271,insert,5.331,-1.0,-1.0,0.0,0.891911 +272,search,5149.88,0.87361,0.137422,1258.84,0.891877 +273,insert,41.612,-1.0,-1.0,0.0,0.901871 +274,search,5045.29,0.87421,0.137165,989.221,0.901719 +275,insert,791.315,-1.0,-1.0,0.0,1.05598 +276,search,5346.1,0.87704,0.136278,2098.19,1.0565 +277,insert,57.582,-1.0,-1.0,0.0,1.0677 +278,search,5375,0.87769,0.135739,1580.51,1.06678 +279,insert,40.624,-1.0,-1.0,0.0,1.07585 +280,search,5480.81,0.8793,0.134535,10.506,1.07585 +281,insert,604.308,-1.0,-1.0,0.0,1.1981 +282,search,5611.5,0.88164,0.134186,1788.17,1.19842 +283,insert,784.395,-1.0,-1.0,0.0,1.37921 +284,search,5932.94,0.88467,0.131886,2576.28,1.37934 +285,insert,21.556,-1.0,-1.0,0.0,1.38514 +286,search,5797.82,0.88506,0.131404,1866.46,1.38421 +287,insert,62.823,-1.0,-1.0,0.0,1.39938 +288,search,5830.48,0.8851,0.131078,1763.53,1.39929 +289,insert,543.531,-1.0,-1.0,0.0,1.5316 +290,search,6166.93,0.89045,0.128085,2400.47,1.52969 +291,insert,38.436,-1.0,-1.0,0.0,1.54102 +292,search,6112.56,0.89099,0.128259,1700.03,1.53999 +293,insert,27.261,-1.0,-1.0,0.0,1.54861 +294,search,6208.77,0.89093,0.128322,1684.94,1.54823 +295,insert,353.553,-1.0,-1.0,0.0,1.63759 +296,search,6424.34,0.89386,0.12716,2325.26,1.63953 +297,insert,3.891,-1.0,-1.0,0.0,1.64104 +298,search,6415.45,0.89321,0.127759,1747.3,1.64031 +299,insert,135.65,-1.0,-1.0,0.0,1.66604 +300,search,6523.38,0.89705,0.123993,12.311,1.66604 +301,insert,1160.8,-1.0,-1.0,0.0,1.91598 +302,search,6960.7,0.91789,0.107698,4464.85,1.9183 +303,insert,85.993,-1.0,-1.0,0.0,1.94002 +304,search,7195.47,0.91638,0.109388,4328.16,1.93932 +305,insert,746.137,-1.0,-1.0,0.0,2.10153 +306,search,7351.41,0.91533,0.110426,3434.2,2.10152 +307,insert,48.995,-1.0,-1.0,0.0,2.11233 +308,search,7419.37,0.91623,0.109571,2555.2,2.1128 +309,insert,14.309,-1.0,-1.0,0.0,2.11684 +310,search,7051.92,0.91685,0.109342,2183.21,2.117 +311,insert,26.961,-1.0,-1.0,0.0,2.12568 +312,search,7564.24,0.91726,0.109026,13.673,2.12568 +313,insert,83.784,-1.0,-1.0,0.0,2.14206 +314,search,7592.21,0.91748,0.108923,2312.42,2.14163 +315,insert,14.04,-1.0,-1.0,0.0,2.14306 +316,search,7420.44,0.91743,0.108927,13.91,2.14306 +317,insert,122.815,-1.0,-1.0,0.0,2.1608 +318,search,7338.61,0.91784,0.108682,2384.86,2.16105 +319,insert,1146.96,-1.0,-1.0,0.0,2.38195 +320,search,7852.97,0.92089,0.106402,5372.35,2.37786 +321,insert,82.775,-1.0,-1.0,0.0,2.39468 +322,search,7618.29,0.92003,0.107538,2980.58,2.39472 +323,insert,166.946,-1.0,-1.0,0.0,2.44016 +324,search,7866.62,0.9208,0.107211,3002.53,2.4388 +325,insert,2226.89,-1.0,-1.0,0.0,2.65967 +326,search,8381.94,0.92228,0.106334,4897.01,2.66044 +327,insert,147.069,-1.0,-1.0,0.0,2.69139 +328,search,8357.83,0.92279,0.10566,3627.67,2.69166 +329,insert,15.328,-1.0,-1.0,0.0,2.69491 +330,search,8189.53,0.92329,0.105514,3530.85,2.69474 +331,insert,83.476,-1.0,-1.0,0.0,2.71384 +332,search,8352.53,0.92356,0.105284,2776.4,2.71408 +333,insert,3.494,-1.0,-1.0,0.0,2.71484 +334,search,8473.44,0.92347,0.105287,15.048,2.71484 +335,insert,78.487,-1.0,-1.0,0.0,2.73121 +336,search,8363.5,0.92409,0.104486,2714.03,2.73168 +337,insert,37.908,-1.0,-1.0,0.0,2.73911 +338,search,8502.58,0.92431,0.103918,2439.03,2.73891 +339,insert,88.943,-1.0,-1.0,0.0,2.75685 +340,search,8585.06,0.92473,0.103492,2555.65,2.75659 +341,insert,12.339,-1.0,-1.0,0.0,2.76027 +342,search,8715.8,0.9248,0.103535,2655.13,2.7606 +343,insert,84.997,-1.0,-1.0,0.0,2.77526 +344,search,8682.78,0.92537,0.103092,2583.91,2.77513 +345,insert,90.266,-1.0,-1.0,0.0,2.79192 +346,search,8649.29,0.92583,0.102763,3132.62,2.7914 +347,insert,1041.98,-1.0,-1.0,0.0,2.99312 +348,search,9087.48,0.92839,0.100279,7424.27,2.99149 +349,insert,71.412,-1.0,-1.0,0.0,3.01042 +350,search,9040.91,0.92869,0.0997187,3883.62,3.01079 +351,insert,764.418,-1.0,-1.0,0.0,3.19679 +352,search,9336.98,0.9295,0.10007,6952.37,3.19772 +353,insert,124.962,-1.0,-1.0,0.0,3.21686 +354,search,9287.24,0.92861,0.101091,3964.58,3.21663 +355,insert,807.433,-1.0,-1.0,0.0,3.37221 +356,search,9718.97,0.93041,0.099033,7143.5,3.37682 +357,insert,43.448,-1.0,-1.0,0.0,3.38417 +358,search,9671.05,0.93058,0.0990375,4168.5,3.38394 +359,insert,24.307,-1.0,-1.0,0.0,3.39016 +360,search,9940.82,0.93098,0.0988698,3368.02,3.39082 +361,insert,123.511,-1.0,-1.0,0.0,3.42146 +362,search,10074.7,0.93159,0.0978674,3238.78,3.42064 +363,insert,42.904,-1.0,-1.0,0.0,3.42586 +364,search,9835.23,0.9316,0.0978978,18.044,3.42586 +365,insert,155.149,-1.0,-1.0,0.0,3.45207 +366,search,9866,0.93189,0.0976154,3439.17,3.45261 +367,insert,76.222,-1.0,-1.0,0.0,3.46529 +368,search,9807.51,0.9323,0.0973215,4079.88,3.46452 +369,insert,153.032,-1.0,-1.0,0.0,3.48997 +370,search,9899.71,0.93249,0.0971594,4308.65,3.49143 +371,insert,30.13,-1.0,-1.0,0.0,3.49764 +372,search,10031.6,0.93299,0.096737,3415.49,3.49774 +373,insert,125.685,-1.0,-1.0,0.0,3.51628 +374,search,9875.67,0.93317,0.0964727,4445.13,3.51512 +375,insert,80.979,-1.0,-1.0,0.0,3.52836 +376,search,9953.98,0.93327,0.0965464,3741.86,3.52838 +377,insert,142.22,-1.0,-1.0,0.0,3.55477 +378,search,10211.2,0.93369,0.0962969,4467.36,3.55376 +379,insert,15.817,-1.0,-1.0,0.0,3.55607 +380,search,10139.8,0.93377,0.0964232,3482.1,3.55603 +381,insert,51.693,-1.0,-1.0,0.0,3.56491 +382,search,10060.1,0.9338,0.0964065,18.252,3.56491 +383,insert,59.078,-1.0,-1.0,0.0,3.57517 +384,search,10100.9,0.93368,0.0964881,3571.42,3.57391 +385,delete,9.837,-1.0,-1.0,0.0,3.57373 +386,search,10174.2,0.93386,0.0962398,3481.62,3.57394 +387,delete,18.541,-1.0,-1.0,0.0,3.57256 +388,search,10109.2,0.93396,0.0961754,18.505,3.57256 +389,delete,22.434,-1.0,-1.0,0.0,3.57159 +390,search,10188.2,0.934,0.096091,18.557,3.57159 +391,delete,122.187,-1.0,-1.0,0.0,3.55045 +392,search,9910.3,0.93367,0.0963981,18.353,3.55045 +393,delete,40.525,-1.0,-1.0,0.0,3.54915 +394,search,10064.5,0.93354,0.0964366,18.3,3.54915 +395,delete,26.297,-1.0,-1.0,0.0,3.54175 +396,search,10008.6,0.93351,0.0964171,18.382,3.54175 +397,delete,142.452,-1.0,-1.0,0.0,3.50887 +398,search,10104.2,0.93343,0.096543,19.096,3.50887 +399,delete,12.871,-1.0,-1.0,0.0,3.5085 +400,search,10135.8,0.93342,0.0965799,18.827,3.5085 +401,delete,33.415,-1.0,-1.0,0.0,3.50574 +402,search,9902.35,0.93325,0.0966256,18.579,3.50574 +403,delete,397.753,-1.0,-1.0,0.0,3.42029 +404,search,9787.99,0.93327,0.0966603,18.901,3.42029 +405,delete,86.374,-1.0,-1.0,0.0,3.41322 +406,search,9751.06,0.93442,0.0959388,18.943,3.41322 +407,delete,123.419,-1.0,-1.0,0.0,3.38609 +408,search,9631.36,0.93481,0.0952276,18.8,3.38609 +409,delete,270.525,-1.0,-1.0,0.0,3.30027 +410,search,9552.1,0.93481,0.0951436,18.363,3.30027 +411,delete,299.539,-1.0,-1.0,0.0,3.22306 +412,search,9402.47,0.9338,0.0965309,19.356,3.22306 +413,delete,41.105,-1.0,-1.0,0.0,3.21762 +414,search,9534.32,0.93368,0.096633,18.954,3.21762 +415,delete,111.542,-1.0,-1.0,0.0,3.19706 +416,search,9296.31,0.9336,0.096645,18.822,3.19706 +417,delete,278.283,-1.0,-1.0,0.0,3.13278 +418,search,9512.22,0.93392,0.0961475,18.832,3.13278 +419,delete,32.752,-1.0,-1.0,0.0,3.13066 +420,search,9296.51,0.93364,0.09643,18.587,3.13066 +421,delete,36.27,-1.0,-1.0,0.0,3.12104 +422,search,9490.53,0.9337,0.09639,18.437,3.12104 +423,delete,220.637,-1.0,-1.0,0.0,3.05721 +424,search,9424.6,0.93372,0.0959603,18.54,3.05721 +425,delete,29.558,-1.0,-1.0,0.0,3.04986 +426,search,9091.03,0.93388,0.0959724,18.363,3.04986 +427,delete,64.523,-1.0,-1.0,0.0,3.03756 +428,search,9122.95,0.93323,0.0969718,19.462,3.03756 +429,delete,581.073,-1.0,-1.0,0.0,2.88377 +430,search,8819.64,0.92593,0.104044,18.871,2.88377 +431,delete,129.953,-1.0,-1.0,0.0,2.84768 +432,search,8753.34,0.92595,0.104119,18.59,2.84768 +433,delete,287.576,-1.0,-1.0,0.0,2.76654 +434,search,8709.19,0.92545,0.104192,18.71,2.76654 +435,delete,162.44,-1.0,-1.0,0.0,2.71817 +436,search,8868.72,0.92478,0.104982,18.953,2.71817 +437,delete,24.708,-1.0,-1.0,0.0,2.71203 +438,search,8703.93,0.92484,0.104918,18.596,2.71203 +439,delete,173.496,-1.0,-1.0,0.0,2.6552 +440,search,8629.91,0.92454,0.104826,18.538,2.6552 +441,delete,42.827,-1.0,-1.0,0.0,2.6445 +442,search,8472.11,0.92422,0.104982,18.502,2.6445 +443,delete,38.133,-1.0,-1.0,0.0,2.63085 +444,search,8474.06,0.92454,0.104807,19.531,2.63085 +445,delete,74.5,-1.0,-1.0,0.0,2.6134 +446,search,8685.8,0.92438,0.104822,18.806,2.6134 +447,delete,528.064,-1.0,-1.0,0.0,2.4595 +448,search,8040.53,0.92388,0.104566,18.899,2.4595 +449,delete,62.439,-1.0,-1.0,0.0,2.44375 +450,search,8147.22,0.92325,0.105012,18.609,2.44375 +451,delete,134.091,-1.0,-1.0,0.0,2.40159 +452,search,8464.67,0.92208,0.105963,18.878,2.40159 +453,delete,399.294,-1.0,-1.0,0.0,2.28838 +454,search,8264.59,0.92124,0.106366,18.77,2.28838 +455,delete,69.72,-1.0,-1.0,0.0,2.27165 +456,search,8186.68,0.92079,0.106412,18.567,2.27165 +457,delete,10.357,-1.0,-1.0,0.0,2.26856 +458,search,8198.72,0.92073,0.106463,18.69,2.26856 +459,delete,55.975,-1.0,-1.0,0.0,2.24889 +460,search,8199.36,0.92083,0.106467,18.334,2.24889 +461,delete,9.344,-1.0,-1.0,0.0,2.24647 +462,search,8185.23,0.92094,0.10631,19.383,2.24647 +463,delete,95.873,-1.0,-1.0,0.0,2.22179 +464,search,8299.45,0.92059,0.106786,19.17,2.22179 +465,delete,40.723,-1.0,-1.0,0.0,2.21214 +466,search,8195.89,0.92024,0.107127,19.292,2.21214 +467,delete,59.915,-1.0,-1.0,0.0,2.19875 +468,search,8221.2,0.9199,0.107524,18.705,2.19875 +469,delete,38.87,-1.0,-1.0,0.0,2.18744 +470,search,8100.32,0.91945,0.10783,18.975,2.18744 +471,delete,169.402,-1.0,-1.0,0.0,2.13073 +472,search,8083.44,0.91839,0.108645,18.618,2.13073 +473,delete,186.419,-1.0,-1.0,0.0,2.08024 +474,search,8111.31,0.91764,0.10939,18.788,2.08024 +475,delete,489.547,-1.0,-1.0,0.0,1.89124 +476,search,7701.34,0.91494,0.111263,18.467,1.89124 +477,delete,72.402,-1.0,-1.0,0.0,1.86868 +478,search,7845.18,0.91472,0.111308,18.486,1.86868 +479,delete,264.49,-1.0,-1.0,0.0,1.77566 +480,search,8013.58,0.91639,0.109358,19.229,1.77566 +481,delete,89.231,-1.0,-1.0,0.0,1.756 +482,search,7888.54,0.91597,0.109544,19.184,1.756 +483,delete,384.857,-1.0,-1.0,0.0,1.62161 +484,search,7711.13,0.91422,0.110846,18.875,1.62161 +485,delete,48.874,-1.0,-1.0,0.0,1.61231 +486,search,7736.65,0.91406,0.111218,18.508,1.61231 +487,delete,19.229,-1.0,-1.0,0.0,1.6077 +488,search,7767.84,0.91364,0.111336,19.181,1.6077 +489,delete,55.771,-1.0,-1.0,0.0,1.5901 +490,search,7761.1,0.9134,0.111384,19.055,1.5901 +491,delete,32.457,-1.0,-1.0,0.0,1.58259 +492,search,7699.92,0.91321,0.111353,18.608,1.58259 +493,delete,199.825,-1.0,-1.0,0.0,1.52016 +494,search,7737.03,0.91315,0.111238,18.465,1.52016 +495,delete,26.272,-1.0,-1.0,0.0,1.50998 +496,search,7646.78,0.91258,0.111729,18.46,1.50998 +497,delete,86.569,-1.0,-1.0,0.0,1.47853 +498,search,7694.04,0.91197,0.112167,18.423,1.47853 +499,delete,21.136,-1.0,-1.0,0.0,1.47278 +500,search,7722.97,0.91142,0.112783,19.157,1.47278 +501,delete,54.462,-1.0,-1.0,0.0,1.46045 +502,search,7589.77,0.9112,0.113236,18.966,1.46045 +503,delete,42.572,-1.0,-1.0,0.0,1.45221 +504,search,7740.52,0.91103,0.113445,19.086,1.45221 +505,delete,100.246,-1.0,-1.0,0.0,1.42046 +506,search,7680.75,0.91057,0.113633,18.687,1.42046 +507,delete,66.136,-1.0,-1.0,0.0,1.40249 +508,search,7648.9,0.9103,0.113649,18.644,1.40249 +509,delete,30.629,-1.0,-1.0,0.0,1.39219 +510,search,7665.6,0.91026,0.113626,18.577,1.39219 +511,delete,44.483,-1.0,-1.0,0.0,1.38136 +512,search,7670.25,0.9098,0.113854,18.623,1.38136 +513,insert,606.419,-1.0,-1.0,0.0,1.46906 +514,search,7469.68,0.90994,0.114228,1672.39,1.46854 +515,insert,9.926,-1.0,-1.0,0.0,1.46912 +516,search,7623.43,0.91002,0.114107,1457.58,1.46947 +517,insert,29.901,-1.0,-1.0,0.0,1.47395 +518,search,7594.4,0.9108,0.113431,1202.48,1.47333 +519,insert,156.446,-1.0,-1.0,0.0,1.4973 +520,search,7449.24,0.91159,0.113803,18.838,1.4973 +521,insert,36.478,-1.0,-1.0,0.0,1.50124 +522,search,7502.1,0.91173,0.11379,18.94,1.50124 +523,insert,86.419,-1.0,-1.0,0.0,1.5117 +524,search,7491.89,0.91149,0.114405,18.634,1.5117 +525,insert,87.142,-1.0,-1.0,0.0,1.52068 +526,search,7514.79,0.91181,0.114132,18.704,1.52068 +527,insert,1041.38,-1.0,-1.0,0.0,1.6795 +528,search,7663.87,0.91329,0.113722,2617.73,1.67656 +529,insert,529.129,-1.0,-1.0,0.0,1.76951 +530,search,8086.74,0.91832,0.110638,2417.77,1.77148 +531,insert,98.448,-1.0,-1.0,0.0,1.78419 +532,search,7888.23,0.91964,0.109622,2077.5,1.78343 +533,insert,30.908,-1.0,-1.0,0.0,1.78967 +534,search,8077.57,0.91995,0.10919,2127.7,1.79036 +535,insert,37.665,-1.0,-1.0,0.0,1.79605 +536,search,8106.08,0.92023,0.108881,1947.31,1.79579 +537,insert,16.351,-1.0,-1.0,0.0,1.79866 +538,search,8042.77,0.9201,0.108968,12.543,1.79866 +539,insert,69.071,-1.0,-1.0,0.0,1.8085 +540,search,7886.23,0.92029,0.108928,2068.88,1.80905 +541,insert,898.759,-1.0,-1.0,0.0,1.96806 +542,search,8475.73,0.92346,0.10609,3340.06,1.9679 +543,insert,69.665,-1.0,-1.0,0.0,1.98096 +544,search,8449.07,0.92369,0.106315,2021.76,1.98147 +545,insert,11.551,-1.0,-1.0,0.0,1.98324 +546,search,8411.9,0.92375,0.106178,12.603,1.98324 +547,insert,275.414,-1.0,-1.0,0.0,2.03775 +548,search,8491.34,0.92608,0.104222,2298.13,2.03785 +549,insert,575.872,-1.0,-1.0,0.0,2.13247 +550,search,8587.87,0.92413,0.106531,2660.57,2.13477 +551,insert,46.837,-1.0,-1.0,0.0,2.14489 +552,search,8598.71,0.92439,0.106099,2713.38,2.14476 +553,insert,29.178,-1.0,-1.0,0.0,2.15031 +554,search,8758.4,0.92534,0.105505,2682.65,2.15041 +555,insert,917.151,-1.0,-1.0,0.0,2.31594 +556,search,8980.65,0.93136,0.0986931,3009.83,2.31594 +557,insert,49.232,-1.0,-1.0,0.0,2.32667 +558,search,8932.33,0.93152,0.0985567,2277.57,2.32665 +559,insert,10.14,-1.0,-1.0,0.0,2.32892 +560,search,9144.37,0.93165,0.0983563,12.914,2.32892 +561,insert,7.646,-1.0,-1.0,0.0,2.32974 +562,search,9014.16,0.93164,0.0983779,12.536,2.32974 +563,insert,141.584,-1.0,-1.0,0.0,2.34584 +564,search,9159.04,0.9321,0.097889,2558.38,2.34623 +565,insert,546.319,-1.0,-1.0,0.0,2.42876 +566,search,9058.34,0.93181,0.0984864,3100.28,2.42946 +567,insert,33.436,-1.0,-1.0,0.0,2.43492 +568,search,9064.41,0.93181,0.0985574,2565.55,2.43484 +569,insert,34.155,-1.0,-1.0,0.0,2.44206 +570,search,8067.59,0.93196,0.0984996,1788.61,2.4419 +571,insert,24.772,-1.0,-1.0,0.0,2.44743 +572,search,9185.15,0.93164,0.0988141,2408.62,2.44722 +573,insert,135.294,-1.0,-1.0,0.0,2.47192 +574,search,8703.94,0.93271,0.0976571,12.839,2.47192 +575,insert,152.525,-1.0,-1.0,0.0,2.49139 +576,search,8568.19,0.9325,0.0979077,12.415,2.49139 +577,insert,95.869,-1.0,-1.0,0.0,2.50722 +578,search,9151.63,0.93289,0.0976847,2732.69,2.50691 +579,insert,1525.79,-1.0,-1.0,0.0,2.76571 +580,search,9741.21,0.9371,0.0939612,7412.18,2.76528 +581,insert,41.865,-1.0,-1.0,0.0,2.77226 +582,search,9734.44,0.93626,0.0953554,4427.44,2.77442 +583,insert,32.47,-1.0,-1.0,0.0,2.78131 +584,search,9752.42,0.93596,0.0956456,2968.42,2.78087 +585,insert,574.049,-1.0,-1.0,0.0,2.90131 +586,search,9971.74,0.93687,0.0946458,3278.01,2.90037 +587,insert,99.654,-1.0,-1.0,0.0,2.91372 +588,search,9940.95,0.93735,0.0943432,3502.11,2.91342 +589,insert,157.965,-1.0,-1.0,0.0,2.93531 +590,search,10031.3,0.93758,0.0940884,2943.7,2.93624 +591,insert,1128.71,-1.0,-1.0,0.0,3.14169 +592,search,10443.6,0.93983,0.0918912,6641.51,3.14491 +593,insert,340.693,-1.0,-1.0,0.0,3.22742 +594,search,10540.2,0.9396,0.0924035,5020.26,3.23087 +595,insert,12.502,-1.0,-1.0,0.0,3.23284 +596,search,10610.2,0.93979,0.0921867,3293.24,3.23036 +597,insert,541.161,-1.0,-1.0,0.0,3.33021 +598,search,10762,0.94105,0.0909527,4614.32,3.33044 +599,insert,215.304,-1.0,-1.0,0.0,3.37714 +600,search,10825.8,0.9417,0.0902752,4777.77,3.37657 +601,insert,111.947,-1.0,-1.0,0.0,3.40341 +602,search,10950.1,0.94217,0.0900294,3381.29,3.40189 +603,insert,13.102,-1.0,-1.0,0.0,3.40459 +604,search,10933.8,0.94228,0.0899401,3484.57,3.405 +605,insert,160.949,-1.0,-1.0,0.0,3.42483 +606,search,10979.3,0.94226,0.0900049,4028.65,3.42445 +607,insert,139.761,-1.0,-1.0,0.0,3.43876 +608,search,10928.6,0.94198,0.0904536,13.074,3.43876 +609,insert,517.408,-1.0,-1.0,0.0,3.5293 +610,search,10999.8,0.94234,0.0903041,4378.48,3.5307 +611,insert,43.353,-1.0,-1.0,0.0,3.54038 +612,search,10360.7,0.94297,0.0898629,2555.83,3.54067 +613,insert,440.276,-1.0,-1.0,0.0,3.62649 +614,search,10726.6,0.94332,0.0896098,3718.33,3.62575 +615,insert,729.793,-1.0,-1.0,0.0,3.78343 +616,search,11521.3,0.94409,0.0888817,8273.68,3.78143 +617,insert,725.668,-1.0,-1.0,0.0,3.92426 +618,search,11811,0.94501,0.0880444,5062.44,3.92492 +619,insert,449.278,-1.0,-1.0,0.0,4.03573 +620,search,11913.9,0.94528,0.088171,5877.11,4.03532 +621,insert,88.709,-1.0,-1.0,0.0,4.05046 +622,search,11928.3,0.94573,0.0876028,4757.1,4.05196 +623,insert,49.561,-1.0,-1.0,0.0,4.06222 +624,search,11826.5,0.94585,0.087622,3696.12,4.06263 +625,insert,184.936,-1.0,-1.0,0.0,4.09456 +626,search,12014.5,0.94575,0.0878827,4100.38,4.09495 +627,insert,122.69,-1.0,-1.0,0.0,4.11522 +628,search,10623.6,0.94628,0.0871616,3341.99,4.11648 +629,insert,24.294,-1.0,-1.0,0.0,4.12272 +630,search,11837.4,0.94645,0.0870205,2896.06,4.12184 +631,insert,96.006,-1.0,-1.0,0.0,4.13573 +632,search,11103.6,0.94657,0.0869753,3400.56,4.13638 +633,insert,122.806,-1.0,-1.0,0.0,4.15355 +634,search,11773,0.94687,0.0865742,3344.78,4.15559 +635,insert,46.435,-1.0,-1.0,0.0,4.16414 +636,search,11100.4,0.94685,0.0865742,16.477,4.16414 +637,insert,989.045,-1.0,-1.0,0.0,4.33947 +638,search,12395.5,0.94802,0.0847487,9483.3,4.34113 +639,insert,946.264,-1.0,-1.0,0.0,4.51501 +640,search,12781,0.9491,0.0836172,7876.76,4.51652 +641,delete,133.665,-1.0,-1.0,0.0,4.48952 +642,search,12061.5,0.94903,0.0836762,3839.6,4.48714 +643,delete,6.473,-1.0,-1.0,0.0,4.487 +644,search,11331.2,0.94935,0.0834897,3131.75,4.48708 +645,delete,9.697,-1.0,-1.0,0.0,4.48674 +646,search,12519.7,0.94934,0.0835006,13.295,4.48674 +647,delete,45.385,-1.0,-1.0,0.0,4.46796 +648,search,12087.6,0.94918,0.0834752,21.459,4.46796 +649,delete,81.786,-1.0,-1.0,0.0,4.46358 +650,search,12809.5,0.94919,0.0836432,21.043,4.46358 +651,delete,47.599,-1.0,-1.0,0.0,4.45937 +652,search,12613.4,0.9491,0.0836411,20.753,4.45937 +653,delete,183.824,-1.0,-1.0,0.0,4.42891 +654,search,12750.1,0.94894,0.0837811,20.255,4.42891 +655,delete,561.212,-1.0,-1.0,0.0,4.31865 +656,search,12465.1,0.94851,0.0832268,19.943,4.31865 +657,delete,326.615,-1.0,-1.0,0.0,4.23759 +658,search,12388.5,0.94646,0.0844983,19.821,4.23759 +659,delete,163.5,-1.0,-1.0,0.0,4.20045 +660,search,11957.1,0.94628,0.0845764,21.059,4.20045 +661,delete,111.485,-1.0,-1.0,0.0,4.1905 +662,search,12247,0.94609,0.0847802,20.463,4.1905 +663,delete,151.691,-1.0,-1.0,0.0,4.17113 +664,search,12220.6,0.94587,0.0850159,20.45,4.17113 +665,delete,99.059,-1.0,-1.0,0.0,4.15584 +666,search,12206.1,0.94583,0.0850366,20.142,4.15584 +667,delete,308.165,-1.0,-1.0,0.0,4.0896 +668,search,12231,0.94546,0.085491,19.947,4.0896 +669,delete,315.766,-1.0,-1.0,0.0,4.03 +670,search,11944.7,0.94517,0.0855174,19.783,4.03 +671,delete,52.335,-1.0,-1.0,0.0,4.02036 +672,search,11959.5,0.94496,0.0857042,21.119,4.02036 +673,delete,131.821,-1.0,-1.0,0.0,4.0012 +674,search,11932.6,0.94501,0.085673,20.561,4.0012 +675,delete,209.45,-1.0,-1.0,0.0,3.97016 +676,search,11908.2,0.94416,0.0866518,20.329,3.97016 +677,delete,301.152,-1.0,-1.0,0.0,3.88659 +678,search,11749.8,0.94472,0.0860051,20.263,3.88659 +679,delete,56.729,-1.0,-1.0,0.0,3.87584 +680,search,11737,0.94465,0.0860713,19.804,3.87584 +681,delete,20.461,-1.0,-1.0,0.0,3.87339 +682,search,11807.8,0.94464,0.0861835,19.93,3.87339 +683,delete,410.347,-1.0,-1.0,0.0,3.79015 +684,search,11573.5,0.94186,0.089433,20.714,3.79015 +685,delete,399.201,-1.0,-1.0,0.0,3.69625 +686,search,11349.6,0.93812,0.0931982,20.966,3.69625 +687,delete,44.049,-1.0,-1.0,0.0,3.68985 +688,search,11367,0.93807,0.0932882,20.391,3.68985 +689,delete,196.22,-1.0,-1.0,0.0,3.63962 +690,search,11355.4,0.93793,0.0932278,20.596,3.63962 +691,delete,90.646,-1.0,-1.0,0.0,3.62423 +692,search,11279.6,0.93749,0.0936147,20.04,3.62423 +693,delete,273.616,-1.0,-1.0,0.0,3.55997 +694,search,9800.17,0.93775,0.0935914,13.287,3.55997 +695,delete,20.401,-1.0,-1.0,0.0,3.54933 +696,search,10410.7,0.93786,0.0934303,15.775,3.54933 +697,delete,27.129,-1.0,-1.0,0.0,3.54065 +698,search,10172.7,0.93789,0.093436,20.851,3.54065 +699,delete,29.286,-1.0,-1.0,0.0,3.53419 +700,search,10903.5,0.93799,0.0933197,21.232,3.53419 +701,delete,124.8,-1.0,-1.0,0.0,3.51537 +702,search,10797.6,0.93771,0.0935422,20.373,3.51537 +703,delete,337.802,-1.0,-1.0,0.0,3.43972 +704,search,10576,0.93841,0.0928648,13.962,3.43972 +705,delete,43.118,-1.0,-1.0,0.0,3.42626 +706,search,9497.62,0.93803,0.0931741,13.811,3.42626 +707,delete,544.563,-1.0,-1.0,0.0,3.17575 +708,search,9750.32,0.93264,0.0990654,13.592,3.17575 +709,delete,143.313,-1.0,-1.0,0.0,3.09488 +710,search,10269.3,0.93141,0.0999181,20.164,3.09488 +711,delete,73.979,-1.0,-1.0,0.0,3.07488 +712,search,10620.8,0.93131,0.100214,19.98,3.07488 +713,delete,354.822,-1.0,-1.0,0.0,2.96643 +714,search,9442.79,0.93055,0.100926,14.346,2.96643 +715,delete,49.761,-1.0,-1.0,0.0,2.95109 +716,search,9836,0.93039,0.101083,14.346,2.95109 +717,delete,41.727,-1.0,-1.0,0.0,2.93654 +718,search,9316.86,0.93026,0.101185,13.852,2.93654 +719,delete,317.139,-1.0,-1.0,0.0,2.77923 +720,search,10235,0.92708,0.104737,14.53,2.77923 +721,delete,105.213,-1.0,-1.0,0.0,2.73318 +722,search,8939.64,0.92733,0.104416,19.818,2.73318 +723,delete,36.619,-1.0,-1.0,0.0,2.72293 +724,search,8926.69,0.92696,0.10484,13.621,2.72293 +725,delete,94.637,-1.0,-1.0,0.0,2.67626 +726,search,9935.44,0.92702,0.104875,13.403,2.67626 +727,delete,81.733,-1.0,-1.0,0.0,2.6275 +728,search,8544.32,0.9265,0.105254,14.371,2.6275 +729,delete,86.613,-1.0,-1.0,0.0,2.57527 +730,search,8836.84,0.92515,0.106486,14.237,2.57527 +731,delete,84.755,-1.0,-1.0,0.0,2.54735 +732,search,9300.29,0.92426,0.107475,20.795,2.54735 +733,delete,136.296,-1.0,-1.0,0.0,2.51602 +734,search,9674.46,0.92374,0.10772,20.723,2.51602 +735,delete,284.887,-1.0,-1.0,0.0,2.43589 +736,search,8936.2,0.92721,0.103164,15.277,2.43589 +737,delete,131.278,-1.0,-1.0,0.0,2.37699 +738,search,9350.6,0.9271,0.103136,13.345,2.37699 +739,delete,57.294,-1.0,-1.0,0.0,2.34728 +740,search,9863.04,0.92719,0.102904,13.55,2.34728 +741,delete,139.304,-1.0,-1.0,0.0,2.27506 +742,search,9850.31,0.92765,0.102621,20.457,2.27506 +743,delete,307.176,-1.0,-1.0,0.0,2.1823 +744,search,9493.86,0.92778,0.102272,13.177,2.1823 +745,delete,204.322,-1.0,-1.0,0.0,2.07413 +746,search,9683.31,0.92739,0.102535,21.283,2.07413 +747,delete,255.443,-1.0,-1.0,0.0,1.96997 +748,search,9901.76,0.9267,0.102416,14.564,1.96997 +749,delete,79.95,-1.0,-1.0,0.0,1.934 +750,search,10168.5,0.92619,0.102547,20.386,1.934 +751,delete,42.99,-1.0,-1.0,0.0,1.92225 +752,search,9871.71,0.92591,0.102781,20.198,1.92225 +753,delete,127.042,-1.0,-1.0,0.0,1.89104 +754,search,10085,0.9259,0.102737,20.464,1.89104 +755,delete,55.942,-1.0,-1.0,0.0,1.87483 +756,search,9783.78,0.92522,0.103377,20.086,1.87483 +757,delete,48.181,-1.0,-1.0,0.0,1.86127 +758,search,9927.9,0.92515,0.103486,20.089,1.86127 +759,delete,40.633,-1.0,-1.0,0.0,1.84845 +760,search,9183.86,0.92498,0.10368,20.053,1.84845 +761,delete,61.69,-1.0,-1.0,0.0,1.82818 +762,search,9912.19,0.9245,0.103975,21.203,1.82818 +763,delete,61.745,-1.0,-1.0,0.0,1.81926 +764,search,10010.3,0.92457,0.103889,20.438,1.81926 +765,delete,579.359,-1.0,-1.0,0.0,1.66301 +766,search,10101.7,0.92419,0.103928,20.235,1.66301 +767,delete,607.836,-1.0,-1.0,0.0,1.4741 +768,search,9377.71,0.92113,0.105288,20.827,1.4741 +769,insert,18.714,-1.0,-1.0,0.0,1.47506 +770,search,9910.57,0.9211,0.105184,20.206,1.47506 +771,insert,56.145,-1.0,-1.0,0.0,1.48227 +772,search,9454.47,0.92128,0.105154,20.365,1.48227 +773,insert,98.813,-1.0,-1.0,0.0,1.49283 +774,search,9839.8,0.92181,0.1049,20.089,1.49283 +775,insert,83.041,-1.0,-1.0,0.0,1.50213 +776,search,9327.65,0.92199,0.105095,1697.24,1.50195 +777,insert,63.219,-1.0,-1.0,0.0,1.50746 +778,search,9390.1,0.92218,0.104573,20.921,1.50746 +779,insert,48.379,-1.0,-1.0,0.0,1.51037 +780,search,9939.46,0.92238,0.104522,20.668,1.51037 +781,insert,16.792,-1.0,-1.0,0.0,1.5111 +782,search,9547.11,0.92236,0.104533,21.029,1.5111 +783,insert,124.802,-1.0,-1.0,0.0,1.52341 +784,search,9913.93,0.92291,0.104191,20.687,1.52341 +785,insert,109.617,-1.0,-1.0,0.0,1.53745 +786,search,9345.31,0.92419,0.103793,19.965,1.53745 +787,insert,34.438,-1.0,-1.0,0.0,1.54025 +788,search,9650.86,0.92437,0.10353,20.016,1.54025 +789,insert,11.404,-1.0,-1.0,0.0,1.54126 +790,search,9380.38,0.92443,0.103574,20.058,1.54126 +791,insert,59.626,-1.0,-1.0,0.0,1.5481 +792,search,9790.42,0.92497,0.103097,20.107,1.5481 +793,insert,2.595,-1.0,-1.0,0.0,1.54834 +794,search,9478.27,0.92495,0.103081,21.438,1.54834 +795,insert,173.76,-1.0,-1.0,0.0,1.5649 +796,search,9791.58,0.92552,0.102455,20.816,1.5649 +797,insert,9.45,-1.0,-1.0,0.0,1.56518 +798,search,9435.71,0.92561,0.1024,20.484,1.56518 +799,insert,10.09,-1.0,-1.0,0.0,1.5657 +800,search,9856.55,0.92564,0.102411,20.503,1.5657 +801,insert,38.605,-1.0,-1.0,0.0,1.56826 +802,search,9520.11,0.92581,0.102251,20.224,1.56826 +803,insert,2066.61,-1.0,-1.0,0.0,1.85996 +804,search,9819.41,0.93053,0.0985602,3693.18,1.86054 +805,insert,31.84,-1.0,-1.0,0.0,1.86579 +806,search,10000.5,0.93097,0.0984954,2753.69,1.86658 +807,insert,38.733,-1.0,-1.0,0.0,1.87177 +808,search,10146.2,0.93078,0.0987695,2147.63,1.87169 +809,insert,91.074,-1.0,-1.0,0.0,1.8841 +810,search,10178.2,0.93109,0.0985241,2096.44,1.88408 +811,insert,89.769,-1.0,-1.0,0.0,1.89535 +812,search,10065.1,0.93171,0.0977382,20.582,1.89535 +813,insert,142.866,-1.0,-1.0,0.0,1.91287 +814,search,10162.1,0.933,0.09656,20.513,1.91287 +815,insert,4.218,-1.0,-1.0,0.0,1.91338 +816,search,10032.4,0.93301,0.0965816,20.078,1.91338 +817,insert,243.129,-1.0,-1.0,0.0,1.94399 +818,search,10081.2,0.93316,0.0965896,2399.3,1.94306 +819,insert,47.365,-1.0,-1.0,0.0,1.94967 +820,search,10156.4,0.9337,0.0960053,20.628,1.94967 +821,insert,31.652,-1.0,-1.0,0.0,1.9538 +822,search,10334.5,0.9336,0.0961158,20.505,1.9538 +823,insert,104.64,-1.0,-1.0,0.0,1.96726 +824,search,10121.6,0.93377,0.0960281,20.555,1.96726 +825,insert,277.918,-1.0,-1.0,0.0,1.99787 +826,search,10348.6,0.93391,0.0958361,2544.09,1.99681 +827,insert,44.017,-1.0,-1.0,0.0,2.00347 +828,search,10148,0.93353,0.0961304,20.081,2.00347 +829,insert,1102.69,-1.0,-1.0,0.0,2.17002 +830,search,10351.4,0.93543,0.0937886,2833.2,2.16875 +831,insert,61.806,-1.0,-1.0,0.0,2.17934 +832,search,10593.9,0.93551,0.0942154,2375.53,2.17938 +833,insert,25.091,-1.0,-1.0,0.0,2.18336 +834,search,10421.2,0.93564,0.0941307,20.994,2.18336 +835,insert,171.632,-1.0,-1.0,0.0,2.20683 +836,search,10701.1,0.93631,0.0933472,20.488,2.20683 +837,insert,73.279,-1.0,-1.0,0.0,2.21652 +838,search,10521.5,0.93636,0.0932796,20.517,2.21652 +839,insert,228.299,-1.0,-1.0,0.0,2.24323 +840,search,10570.6,0.93703,0.0928895,2687.41,2.24332 +841,insert,110.814,-1.0,-1.0,0.0,2.25978 +842,search,10607.2,0.93717,0.092616,19.674,2.25978 +843,insert,1025.86,-1.0,-1.0,0.0,2.42052 +844,search,10981.9,0.9388,0.0907831,3179.49,2.42103 +845,insert,854.433,-1.0,-1.0,0.0,2.57111 +846,search,11016.8,0.93953,0.0905515,3596.34,2.57167 +847,insert,8.653,-1.0,-1.0,0.0,2.57232 +848,search,11179.7,0.93959,0.0905889,3406.43,2.57239 +849,insert,10.078,-1.0,-1.0,0.0,2.57415 +850,search,10994.9,0.93942,0.0908013,20.437,2.57415 +851,insert,31.947,-1.0,-1.0,0.0,2.57844 +852,search,10976.9,0.93942,0.0908453,20.389,2.57844 +853,insert,68.769,-1.0,-1.0,0.0,2.58789 +854,search,10950.8,0.93949,0.0908109,20.235,2.58789 +855,insert,18.889,-1.0,-1.0,0.0,2.59032 +856,search,11078,0.93953,0.090706,21.376,2.59032 +857,insert,62.061,-1.0,-1.0,0.0,2.59773 +858,search,11281.2,0.93971,0.0905357,2915.12,2.59751 +859,insert,119.684,-1.0,-1.0,0.0,2.61279 +860,search,11168.6,0.93984,0.0902744,20.687,2.61279 +861,insert,33.772,-1.0,-1.0,0.0,2.61558 +862,search,11214,0.93985,0.090195,20.626,2.61558 +863,insert,207.826,-1.0,-1.0,0.0,2.64567 +864,search,11084.1,0.93892,0.0915658,3281.35,2.64613 +865,insert,32.595,-1.0,-1.0,0.0,2.65181 +866,search,10991.2,0.93898,0.0915263,2837.59,2.65129 +867,insert,102.809,-1.0,-1.0,0.0,2.66844 +868,search,11082.9,0.93919,0.0913342,3156.36,2.66871 +869,insert,18.044,-1.0,-1.0,0.0,2.67104 +870,search,11177.5,0.93913,0.0912949,21.027,2.67104 +871,insert,110.664,-1.0,-1.0,0.0,2.68713 +872,search,11039.8,0.9391,0.0912988,3159.51,2.6875 +873,insert,112.825,-1.0,-1.0,0.0,2.70475 +874,search,11443,0.93919,0.0912576,3281.63,2.70396 +875,insert,55.843,-1.0,-1.0,0.0,2.71153 +876,search,11156.2,0.93917,0.0912327,20.289,2.71153 +877,insert,4.718,-1.0,-1.0,0.0,2.7124 +878,search,11176.3,0.93919,0.0912247,20.361,2.7124 +879,insert,674.385,-1.0,-1.0,0.0,2.83785 +880,search,11510.7,0.93966,0.09099,4459.48,2.83779 +881,insert,1464.45,-1.0,-1.0,0.0,3.06579 +882,search,11785,0.94123,0.0893865,6439.14,3.06579 +883,insert,1123.26,-1.0,-1.0,0.0,3.26232 +884,search,12142.1,0.94323,0.0879599,5916.77,3.26188 +885,insert,17.598,-1.0,-1.0,0.0,3.26586 +886,search,11957.7,0.94321,0.0882404,3837.97,3.26598 +887,insert,733.579,-1.0,-1.0,0.0,3.42247 +888,search,12308.6,0.94464,0.0874738,5621.16,3.422 +889,insert,1009.45,-1.0,-1.0,0.0,3.60575 +890,search,12703,0.94578,0.0865391,7548.34,3.60807 +891,insert,85.319,-1.0,-1.0,0.0,3.61957 +892,search,12855.5,0.9459,0.0862532,4354.38,3.61832 +893,insert,106.678,-1.0,-1.0,0.0,3.64619 +894,search,12755.2,0.94617,0.0858636,4237.99,3.64616 +895,insert,139.086,-1.0,-1.0,0.0,3.66895 +896,search,12895,0.94623,0.0858522,3907.81,3.66861 +897,delete,73.14,-1.0,-1.0,0.0,3.65386 +898,search,12791,0.9462,0.0858761,22.506,3.65386 +899,delete,27.02,-1.0,-1.0,0.0,3.65241 +900,search,12578,0.94622,0.0858374,22.526,3.65241 +901,delete,77.959,-1.0,-1.0,0.0,3.6465 +902,search,12756.5,0.94594,0.0862095,22.018,3.6465 +903,delete,70.406,-1.0,-1.0,0.0,3.6272 +904,search,12625.2,0.94549,0.0863827,22.917,3.6272 +905,delete,46.12,-1.0,-1.0,0.0,3.62462 +906,search,12584.1,0.9455,0.0864214,23.226,3.62462 +907,delete,57.406,-1.0,-1.0,0.0,3.62133 +908,search,12979.2,0.9454,0.0864007,23.15,3.62133 +909,delete,63.056,-1.0,-1.0,0.0,3.61818 +910,search,12480.6,0.94526,0.0864108,22.674,3.61818 +911,delete,124.857,-1.0,-1.0,0.0,3.60783 +912,search,12587.7,0.94495,0.086919,22.323,3.60783 +913,delete,50.255,-1.0,-1.0,0.0,3.59805 +914,search,12552.8,0.94499,0.0871997,22.061,3.59805 +915,delete,75.901,-1.0,-1.0,0.0,3.58357 +916,search,12633.2,0.94463,0.0875453,22.482,3.58357 +917,delete,17.566,-1.0,-1.0,0.0,3.58198 +918,search,12490.9,0.94467,0.087274,23.592,3.58198 +919,delete,63.852,-1.0,-1.0,0.0,3.57931 +920,search,12383.8,0.94466,0.0873742,23.031,3.57931 +921,delete,43.826,-1.0,-1.0,0.0,3.57491 +922,search,12381.4,0.94473,0.0872857,22.885,3.57491 +923,delete,109.17,-1.0,-1.0,0.0,3.56247 +924,search,12491.4,0.9445,0.0874321,22.484,3.56247 +925,delete,224.453,-1.0,-1.0,0.0,3.51015 +926,search,12592.2,0.94325,0.0888414,22.095,3.51015 +927,delete,25.336,-1.0,-1.0,0.0,3.50639 +928,search,12400.6,0.94306,0.0890213,22.646,3.50639 +929,delete,40.226,-1.0,-1.0,0.0,3.49994 +930,search,12513.5,0.94307,0.0890122,24.249,3.49994 +931,delete,724.863,-1.0,-1.0,0.0,3.37018 +932,search,11907.5,0.9414,0.0907115,22.705,3.37018 +933,delete,52.065,-1.0,-1.0,0.0,3.35814 +934,search,11825.4,0.94142,0.0906305,23.015,3.35814 +935,delete,44.775,-1.0,-1.0,0.0,3.35047 +936,search,11794,0.94141,0.0906065,21.907,3.35047 +937,delete,48.988,-1.0,-1.0,0.0,3.3394 +938,search,11517,0.94122,0.0908628,22.383,3.3394 +939,delete,211.334,-1.0,-1.0,0.0,3.30743 +940,search,11768.4,0.93993,0.0924315,22.649,3.30743 +941,delete,62.808,-1.0,-1.0,0.0,3.29344 +942,search,11790.7,0.93904,0.0932671,23.616,3.29344 +943,delete,12.581,-1.0,-1.0,0.0,3.29256 +944,search,11883.9,0.93897,0.0933302,23.044,3.29256 +945,delete,210.372,-1.0,-1.0,0.0,3.24109 +946,search,12051.8,0.93875,0.0934368,22.77,3.24109 +947,delete,52.558,-1.0,-1.0,0.0,3.23337 +948,search,11744.2,0.93836,0.0937092,22.906,3.23337 +949,delete,93.754,-1.0,-1.0,0.0,3.2136 +950,search,11695.6,0.93844,0.0938044,23.1,3.2136 +951,delete,53.623,-1.0,-1.0,0.0,3.19276 +952,search,11773.5,0.93833,0.0937667,22.159,3.19276 +953,delete,67.418,-1.0,-1.0,0.0,3.1791 +954,search,11767.3,0.93807,0.0940783,23.522,3.1791 +955,delete,40.679,-1.0,-1.0,0.0,3.17088 +956,search,11885.5,0.93842,0.0937677,26.798,3.17088 +957,delete,545.324,-1.0,-1.0,0.0,3.04551 +958,search,11551.5,0.9376,0.0946303,22.589,3.04551 +959,delete,96.787,-1.0,-1.0,0.0,3.02536 +960,search,11657.9,0.93722,0.0951841,22.53,3.02536 +961,delete,60.08,-1.0,-1.0,0.0,3.01517 +962,search,11697,0.93693,0.095406,22.312,3.01517 +963,delete,187.591,-1.0,-1.0,0.0,2.96868 +964,search,11476.8,0.93601,0.0962363,22.338,2.96868 +965,delete,96.005,-1.0,-1.0,0.0,2.93849 +966,search,11574.1,0.93572,0.0964939,24.392,2.93849 +967,delete,92.308,-1.0,-1.0,0.0,2.91801 +968,search,11534.9,0.93545,0.0967102,22.796,2.91801 +969,delete,121.226,-1.0,-1.0,0.0,2.88856 +970,search,11434.8,0.93523,0.0967639,22.803,2.88856 +971,delete,509.202,-1.0,-1.0,0.0,2.7672 +972,search,11301,0.93413,0.097536,22.693,2.7672 +973,delete,456.644,-1.0,-1.0,0.0,2.64443 +974,search,11126.1,0.93264,0.0996393,22.267,2.64443 +975,delete,223.373,-1.0,-1.0,0.0,2.5802 +976,search,10979.5,0.9301,0.10169,22.108,2.5802 +977,delete,97.631,-1.0,-1.0,0.0,2.54584 +978,search,10945.3,0.92993,0.101778,22.176,2.54584 +979,delete,21.081,-1.0,-1.0,0.0,2.53934 +980,search,11003.8,0.92983,0.101813,23.49,2.53934 +981,delete,220.891,-1.0,-1.0,0.0,2.48699 +982,search,11134.5,0.92923,0.102142,23.036,2.48699 +983,delete,80.775,-1.0,-1.0,0.0,2.47241 +984,search,10901.2,0.92899,0.10242,22.441,2.47241 +985,delete,82.049,-1.0,-1.0,0.0,2.45261 +986,search,10646.2,0.92851,0.10285,23.334,2.45261 +987,delete,103.729,-1.0,-1.0,0.0,2.43129 +988,search,10481.5,0.92812,0.103712,22.373,2.43129 +989,delete,28.82,-1.0,-1.0,0.0,2.42451 +990,search,11073.3,0.92799,0.103774,22.855,2.42451 +991,delete,112.849,-1.0,-1.0,0.0,2.39021 +992,search,10759.5,0.93008,0.10105,22.595,2.39021 +993,delete,149.641,-1.0,-1.0,0.0,2.34855 +994,search,10783.4,0.92991,0.101013,23.865,2.34855 +995,delete,163.514,-1.0,-1.0,0.0,2.31388 +996,search,10858.4,0.92924,0.10195,23.203,2.31388 +997,delete,66.565,-1.0,-1.0,0.0,2.30315 +998,search,11032.9,0.92916,0.101977,23.033,2.30315 +999,delete,214.043,-1.0,-1.0,0.0,2.24696 +1000,search,10655.2,0.92907,0.101788,22.787,2.24696 +1001,delete,224.534,-1.0,-1.0,0.0,2.1951 +1002,search,10190.4,0.92906,0.101759,15.351,2.1951 +1003,delete,48.697,-1.0,-1.0,0.0,2.17226 +1004,search,10917.9,0.92887,0.101921,22.58,2.17226 +1005,delete,30.945,-1.0,-1.0,0.0,2.16246 +1006,search,10572.9,0.92851,0.102245,22.461,2.16246 +1007,delete,276.454,-1.0,-1.0,0.0,2.07703 +1008,search,10393.5,0.92823,0.102227,23.271,2.07703 +1009,delete,568.339,-1.0,-1.0,0.0,1.92822 +1010,search,10027.6,0.92791,0.102236,23.186,1.92822 +1011,delete,684.891,-1.0,-1.0,0.0,1.74666 +1012,search,11111.4,0.92572,0.104457,23.011,1.74666 +1013,delete,22.364,-1.0,-1.0,0.0,1.74125 +1014,search,10798.1,0.92568,0.104534,22.552,1.74125 +1015,delete,468.331,-1.0,-1.0,0.0,1.60232 +1016,search,10307,0.92413,0.105038,14.857,1.60232 +1017,delete,292.475,-1.0,-1.0,0.0,1.445 +1018,search,10801.6,0.92424,0.104801,22.446,1.445 +1019,delete,32.295,-1.0,-1.0,0.0,1.43507 +1020,search,10987,0.92409,0.104878,22.57,1.43507 +1021,delete,200.335,-1.0,-1.0,0.0,1.37596 +1022,search,11026,0.92379,0.104947,23.711,1.37596 +1023,delete,200.849,-1.0,-1.0,0.0,1.33407 +1024,search,10267.7,0.92237,0.105907,23.145,1.33407 +1025,insert,85.641,-1.0,-1.0,0.0,1.34118 +1026,search,10658.7,0.92241,0.105962,22.978,1.34118 +1027,insert,503.002,-1.0,-1.0,0.0,1.3933 +1028,search,10230.2,0.92343,0.105728,1706.48,1.39281 +1029,insert,1039.21,-1.0,-1.0,0.0,1.52543 +1030,search,10529.8,0.9266,0.103746,2132.18,1.52562 +1031,insert,1589.92,-1.0,-1.0,0.0,1.775 +1032,search,10628.2,0.92841,0.103865,3446.6,1.77208 +1033,insert,1049.79,-1.0,-1.0,0.0,1.94074 +1034,search,10859.6,0.9306,0.102189,3545.68,1.94258 +1035,insert,632.119,-1.0,-1.0,0.0,2.04904 +1036,search,11245.5,0.93115,0.102103,3499.63,2.05049 +1037,insert,29.547,-1.0,-1.0,0.0,2.05364 +1038,search,11224.5,0.93075,0.102592,2254.2,2.05342 +1039,insert,171.083,-1.0,-1.0,0.0,2.07994 +1040,search,11009.8,0.93132,0.101962,2394.57,2.07948 +1041,insert,23.613,-1.0,-1.0,0.0,2.08296 +1042,search,11211,0.93142,0.101849,23.419,2.08296 +1043,insert,44.315,-1.0,-1.0,0.0,2.08955 +1044,search,10747.2,0.9316,0.101617,24.846,2.08955 +1045,insert,126.196,-1.0,-1.0,0.0,2.10402 +1046,search,11171.2,0.93217,0.10103,24.062,2.10402 +1047,insert,9.708,-1.0,-1.0,0.0,2.10496 +1048,search,11047.5,0.93221,0.101026,25.218,2.10496 +1049,insert,51.301,-1.0,-1.0,0.0,2.11091 +1050,search,11211.3,0.9321,0.101283,24.193,2.11091 +1051,insert,287.207,-1.0,-1.0,0.0,2.14832 +1052,search,11103,0.93242,0.10075,23.834,2.14832 +1053,insert,115.384,-1.0,-1.0,0.0,2.16468 +1054,search,11198.4,0.93276,0.10028,23.166,2.16468 +1055,insert,86.219,-1.0,-1.0,0.0,2.17834 +1056,search,11021.9,0.93309,0.100119,24.336,2.17834 +1057,insert,18.574,-1.0,-1.0,0.0,2.18037 +1058,search,11395.8,0.93308,0.100213,24.129,2.18037 +1059,insert,77.037,-1.0,-1.0,0.0,2.18645 +1060,search,11441.2,0.93334,0.100035,23.746,2.18645 +1061,insert,31.635,-1.0,-1.0,0.0,2.19008 +1062,search,11179.3,0.93324,0.100149,26.335,2.19008 +1063,insert,32.77,-1.0,-1.0,0.0,2.19361 +1064,search,11186.2,0.93314,0.100314,2802.58,2.19314 +1065,insert,803.164,-1.0,-1.0,0.0,2.31336 +1066,search,11578.8,0.93427,0.0991946,3318.46,2.3137 +1067,insert,13.328,-1.0,-1.0,0.0,2.3171 +1068,search,11318.1,0.93426,0.0990603,2503.94,2.31595 +1069,insert,84.039,-1.0,-1.0,0.0,2.32832 +1070,search,11370.8,0.93521,0.0977099,2440.07,2.3289 +1071,insert,62.896,-1.0,-1.0,0.0,2.33927 +1072,search,11441.2,0.9352,0.0978654,24.007,2.33927 +1073,insert,81.88,-1.0,-1.0,0.0,2.3487 +1074,search,11406.6,0.93524,0.0979667,23.731,2.3487 +1075,insert,29.977,-1.0,-1.0,0.0,2.35163 +1076,search,11394.9,0.93556,0.0975113,24.333,2.35163 +1077,insert,73.133,-1.0,-1.0,0.0,2.36166 +1078,search,11367.6,0.93535,0.0979435,23.597,2.36166 +1079,insert,53.91,-1.0,-1.0,0.0,2.36913 +1080,search,11444.4,0.93556,0.0976445,24.759,2.36913 +1081,insert,1074.45,-1.0,-1.0,0.0,2.50923 +1082,search,11457.4,0.93557,0.0982474,3076.81,2.50971 +1083,insert,3.903,-1.0,-1.0,0.0,2.51037 +1084,search,11737.7,0.93562,0.0982153,1683.93,2.50986 +1085,insert,70.002,-1.0,-1.0,0.0,2.51992 +1086,search,11745,0.93583,0.0980727,23.643,2.51992 +1087,insert,53.956,-1.0,-1.0,0.0,2.52598 +1088,search,11634.6,0.93585,0.0981136,23.624,2.52598 +1089,insert,1078.11,-1.0,-1.0,0.0,2.69203 +1090,search,11929,0.93776,0.0963279,3929.28,2.6923 +1091,insert,107.929,-1.0,-1.0,0.0,2.71349 +1092,search,12059.6,0.93855,0.0956363,3247.95,2.71375 +1093,insert,49.724,-1.0,-1.0,0.0,2.72085 +1094,search,12030.4,0.93859,0.0955644,2902.68,2.72033 +1095,insert,1112.55,-1.0,-1.0,0.0,2.88354 +1096,search,12321.6,0.9402,0.0941433,4904.11,2.88522 +1097,insert,48.363,-1.0,-1.0,0.0,2.89475 +1098,search,12479.5,0.94059,0.0940135,3176.52,2.89393 +1099,insert,23.424,-1.0,-1.0,0.0,2.89866 +1100,search,12635.9,0.9405,0.0941726,3042.82,2.89891 +1101,insert,113.03,-1.0,-1.0,0.0,2.91645 +1102,search,12562.1,0.94087,0.0937852,25.293,2.91645 +1103,insert,261.194,-1.0,-1.0,0.0,2.95628 +1104,search,12251.4,0.94257,0.0917806,3694.84,2.95602 +1105,insert,5.014,-1.0,-1.0,0.0,2.95682 +1106,search,12461.3,0.94279,0.0914158,24.404,2.95682 +1107,insert,599.639,-1.0,-1.0,0.0,3.05417 +1108,search,12368.8,0.94347,0.0904353,3954.2,3.05337 +1109,insert,14.675,-1.0,-1.0,0.0,3.05641 +1110,search,12327.7,0.94337,0.0906025,3374.02,3.05647 +1111,insert,122.996,-1.0,-1.0,0.0,3.07384 +1112,search,12435.3,0.94361,0.0903466,3651.05,3.07365 +1113,insert,176.768,-1.0,-1.0,0.0,3.098 +1114,search,12672.1,0.9439,0.0899554,3803.27,3.0976 +1115,insert,174.405,-1.0,-1.0,0.0,3.12309 +1116,search,12883.7,0.94453,0.0890904,24.432,3.12309 +1117,insert,35.816,-1.0,-1.0,0.0,3.12926 +1118,search,12521.5,0.94464,0.0888804,24.537,3.12926 +1119,insert,23.354,-1.0,-1.0,0.0,3.13339 +1120,search,12592,0.94475,0.0886946,26.172,3.13339 +1121,insert,79.876,-1.0,-1.0,0.0,3.14545 +1122,search,12727.2,0.94464,0.0889367,3742.9,3.1452 +1123,insert,59.258,-1.0,-1.0,0.0,3.15439 +1124,search,12752.6,0.94451,0.089301,25.418,3.15439 +1125,insert,32.67,-1.0,-1.0,0.0,3.15839 +1126,search,12763.2,0.94453,0.089281,3861.12,3.15885 +1127,insert,98.321,-1.0,-1.0,0.0,3.17502 +1128,search,12522.6,0.94447,0.0894965,24.488,3.17502 +1129,insert,39.447,-1.0,-1.0,0.0,3.18185 +1130,search,12705.4,0.94444,0.0895772,3500.85,3.18151 +1131,insert,116.574,-1.0,-1.0,0.0,3.1951 +1132,search,12802.4,0.94448,0.0893989,25.347,3.1951 +1133,insert,61.56,-1.0,-1.0,0.0,3.20091 +1134,search,12951.2,0.94445,0.0894529,25.215,3.20091 +1135,insert,28.393,-1.0,-1.0,0.0,3.20356 +1136,search,12742.9,0.94448,0.0893877,24.908,3.20356 +1137,insert,83.852,-1.0,-1.0,0.0,3.21235 +1138,search,12791.1,0.94444,0.0894094,24.729,3.21235 +1139,insert,6.562,-1.0,-1.0,0.0,3.21328 +1140,search,12587.4,0.94448,0.0893989,24.715,3.21328 +1141,insert,665.513,-1.0,-1.0,0.0,3.30944 +1142,search,12691.1,0.9447,0.0893043,4800.33,3.30872 +1143,insert,62.792,-1.0,-1.0,0.0,3.31725 +1144,search,13049.7,0.94466,0.0893208,3995.88,3.31735 +1145,insert,20.478,-1.0,-1.0,0.0,3.32176 +1146,search,10695.2,0.9449,0.0891558,2408.92,3.32264 +1147,insert,49.148,-1.0,-1.0,0.0,3.33275 +1148,search,12888.8,0.94489,0.0892655,25.008,3.33275 +1149,insert,14.385,-1.0,-1.0,0.0,3.33456 +1150,search,12911.6,0.9449,0.0892455,24.823,3.33456 +1151,insert,12.767,-1.0,-1.0,0.0,3.33669 +1152,search,12806.2,0.94491,0.0892601,26.449,3.33669 +1153,delete,56.175,-1.0,-1.0,0.0,3.33444 +1154,search,12714.6,0.94484,0.0894021,26.068,3.33444 +1155,delete,215.555,-1.0,-1.0,0.0,3.31012 +1156,search,12865.2,0.94445,0.0895979,25.992,3.31012 +1157,delete,407.24,-1.0,-1.0,0.0,3.24417 +1158,search,12891.6,0.94394,0.0901793,25.171,3.24417 +1159,delete,579.351,-1.0,-1.0,0.0,3.08195 +1160,search,12242.1,0.94346,0.0896556,24.771,3.08195 +1161,delete,350.144,-1.0,-1.0,0.0,2.99927 +1162,search,12195.4,0.94267,0.0899649,24.737,2.99927 +1163,delete,157.559,-1.0,-1.0,0.0,2.94686 +1164,search,12259.7,0.94262,0.0899356,26.665,2.94686 +1165,delete,79.921,-1.0,-1.0,0.0,2.94128 +1166,search,12333.4,0.9424,0.0900969,25.812,2.94128 +1167,delete,127.375,-1.0,-1.0,0.0,2.92531 +1168,search,12137.4,0.94205,0.0902859,25.447,2.92531 +1169,delete,43.005,-1.0,-1.0,0.0,2.91273 +1170,search,12358.4,0.94162,0.0904572,25.093,2.91273 +1171,delete,56.445,-1.0,-1.0,0.0,2.90197 +1172,search,12381.1,0.94145,0.0908488,24.942,2.90197 +1173,delete,56.882,-1.0,-1.0,0.0,2.89364 +1174,search,12170.8,0.94107,0.0914135,25.019,2.89364 +1175,delete,24.017,-1.0,-1.0,0.0,2.89006 +1176,search,11982.4,0.94081,0.0916731,26.677,2.89006 +1177,delete,33.123,-1.0,-1.0,0.0,2.88361 +1178,search,12119,0.94078,0.091706,26.022,2.88361 +1179,delete,116.65,-1.0,-1.0,0.0,2.86579 +1180,search,11976,0.94041,0.0920352,25.957,2.86579 +1181,delete,89.881,-1.0,-1.0,0.0,2.8524 +1182,search,11914.9,0.93984,0.0926575,25.187,2.8524 +1183,delete,73.489,-1.0,-1.0,0.0,2.83482 +1184,search,12102.3,0.9392,0.0932981,24.881,2.83482 +1185,delete,32.919,-1.0,-1.0,0.0,2.8279 +1186,search,11934.1,0.9391,0.0934209,24.7,2.8279 +1187,delete,196.126,-1.0,-1.0,0.0,2.7884 +1188,search,11731,0.93844,0.0939108,26.436,2.7884 +1189,delete,36.447,-1.0,-1.0,0.0,2.78282 +1190,search,11799.1,0.9384,0.0939366,25.998,2.78282 +1191,delete,27.037,-1.0,-1.0,0.0,2.77922 +1192,search,11819.3,0.93849,0.0939393,25.331,2.77922 +1193,delete,360.862,-1.0,-1.0,0.0,2.6865 +1194,search,11669.2,0.93854,0.0937826,25.114,2.6865 +1195,delete,43.601,-1.0,-1.0,0.0,2.68005 +1196,search,11692.4,0.93834,0.0939404,24.822,2.68005 +1197,delete,92.407,-1.0,-1.0,0.0,2.65755 +1198,search,11673,0.93726,0.0954506,24.901,2.65755 +1199,delete,18.254,-1.0,-1.0,0.0,2.65162 +1200,search,11711.7,0.93722,0.0954464,25.936,2.65162 +1201,delete,116.153,-1.0,-1.0,0.0,2.62378 +1202,search,11733.7,0.93692,0.0954684,25.799,2.62378 +1203,delete,61.731,-1.0,-1.0,0.0,2.6153 +1204,search,11909.2,0.93658,0.095761,25.872,2.6153 +1205,delete,46.47,-1.0,-1.0,0.0,2.60364 +1206,search,11914.3,0.93657,0.0956987,25.506,2.60364 +1207,delete,53.522,-1.0,-1.0,0.0,2.59517 +1208,search,11752.8,0.93632,0.0961171,25.29,2.59517 +1209,delete,421.044,-1.0,-1.0,0.0,2.48365 +1210,search,11557.8,0.93645,0.0959836,25.585,2.48365 +1211,delete,9.201,-1.0,-1.0,0.0,2.48123 +1212,search,11620,0.93638,0.096124,24.86,2.48123 +1213,delete,165.486,-1.0,-1.0,0.0,2.43512 +1214,search,11650.3,0.93583,0.0970165,26.395,2.43512 +1215,delete,48.201,-1.0,-1.0,0.0,2.42722 +1216,search,11736.3,0.93571,0.0970698,25.846,2.42722 +1217,delete,573.38,-1.0,-1.0,0.0,2.27596 +1218,search,11526.2,0.93372,0.0983995,25.704,2.27596 +1219,delete,168.957,-1.0,-1.0,0.0,2.23087 +1220,search,11540.9,0.9318,0.100043,25.329,2.23087 +1221,delete,61.17,-1.0,-1.0,0.0,2.21446 +1222,search,11554.6,0.93172,0.100313,25.252,2.21446 +1223,delete,345.553,-1.0,-1.0,0.0,2.11115 +1224,search,11584.3,0.93041,0.101151,24.958,2.11115 +1225,delete,54.708,-1.0,-1.0,0.0,2.0922 +1226,search,11552.4,0.92997,0.101481,27.014,2.0922 +1227,delete,153.984,-1.0,-1.0,0.0,2.05692 +1228,search,11277.9,0.92999,0.101195,25.846,2.05692 +1229,delete,181.601,-1.0,-1.0,0.0,2.01934 +1230,search,11020.3,0.92932,0.101863,26.114,2.01934 +1231,delete,131.621,-1.0,-1.0,0.0,1.98088 +1232,search,11000,0.92665,0.103937,25.831,1.98088 +1233,delete,35.55,-1.0,-1.0,0.0,1.97416 +1234,search,11264,0.92668,0.103882,25.304,1.97416 +1235,delete,255.213,-1.0,-1.0,0.0,1.90024 +1236,search,11307.8,0.92642,0.103978,24.959,1.90024 +1237,delete,24.611,-1.0,-1.0,0.0,1.89271 +1238,search,11414.7,0.92624,0.104009,25.532,1.89271 +1239,delete,86.599,-1.0,-1.0,0.0,1.86959 +1240,search,11525,0.92551,0.10459,26.448,1.86959 +1241,delete,100.431,-1.0,-1.0,0.0,1.84945 +1242,search,11162.2,0.9249,0.105235,26.223,1.84945 +1243,delete,103.553,-1.0,-1.0,0.0,1.82668 +1244,search,10695.9,0.92415,0.106189,25.484,1.82668 +1245,delete,36.071,-1.0,-1.0,0.0,1.81732 +1246,search,11234.7,0.92373,0.106643,25.81,1.81732 +1247,delete,104.959,-1.0,-1.0,0.0,1.78334 +1248,search,11296.9,0.92649,0.10287,25.242,1.78334 +1249,delete,64.177,-1.0,-1.0,0.0,1.76659 +1250,search,11299.8,0.92646,0.103012,25.063,1.76659 +1251,delete,53.82,-1.0,-1.0,0.0,1.751 +1252,search,11403.2,0.92612,0.103364,25.295,1.751 +1253,delete,23.088,-1.0,-1.0,0.0,1.74497 +1254,search,10732.6,0.92621,0.103341,26.578,1.74497 +1255,delete,217.329,-1.0,-1.0,0.0,1.69026 +1256,search,11197,0.92551,0.104101,25.943,1.69026 +1257,delete,73.87,-1.0,-1.0,0.0,1.67396 +1258,search,11244.1,0.92524,0.104361,25.69,1.67396 +1259,delete,106.66,-1.0,-1.0,0.0,1.64912 +1260,search,11419.3,0.92492,0.104796,25.408,1.64912 +1261,delete,48.538,-1.0,-1.0,0.0,1.63668 +1262,search,11466.6,0.92454,0.105141,25.627,1.63668 +1263,delete,168.893,-1.0,-1.0,0.0,1.58836 +1264,search,11578.5,0.92467,0.10493,25.42,1.58836 +1265,delete,403.826,-1.0,-1.0,0.0,1.46797 +1266,search,11650.7,0.92319,0.106073,25.677,1.46797 +1267,delete,153.056,-1.0,-1.0,0.0,1.43126 +1268,search,11592.5,0.92253,0.1069,26.013,1.43126 +1269,delete,288.938,-1.0,-1.0,0.0,1.3678 +1270,search,11606.8,0.92315,0.105998,26.373,1.3678 +1271,delete,163.603,-1.0,-1.0,0.0,1.32805 +1272,search,11624.7,0.92247,0.10658,25.906,1.32805 +1273,delete,226.867,-1.0,-1.0,0.0,1.27181 +1274,search,11597.1,0.92252,0.106076,25.305,1.27181 +1275,delete,42.498,-1.0,-1.0,0.0,1.26047 +1276,search,11582.1,0.92225,0.106155,25.012,1.26047 +1277,delete,55.772,-1.0,-1.0,0.0,1.24434 +1278,search,11197.7,0.92227,0.105984,25.108,1.24434 +1279,delete,48.211,-1.0,-1.0,0.0,1.23193 +1280,search,10915.2,0.92219,0.106108,26.295,1.23193 diff --git a/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.15_0.05.csv b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.15_0.05.csv new file mode 100644 index 00000000..fd3b1527 --- /dev/null +++ b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.15_0.05.csv @@ -0,0 +1,1280 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,mainteance_ms,index_mem_gb +2,search,2478.36,0.79265,0.157278,1.573,0.0183461 +3,insert,12.534,-1.0,-1.0,0.0,0.0223849 +4,search,1841.16,0.80591,0.154693,1.644,0.0223849 +5,insert,24.405,-1.0,-1.0,0.0,0.0328114 +6,search,1853.36,0.86827,0.143601,64.501,0.0328114 +7,insert,50.438,-1.0,-1.0,0.0,0.042328 +8,search,1854.38,0.85514,0.150788,62.558,0.0427421 +9,insert,3.552,-1.0,-1.0,0.0,0.0441925 +10,search,1872.83,0.86976,0.14109,201.125,0.0445361 +11,insert,11.731,-1.0,-1.0,0.0,0.0496132 +12,search,1892.94,0.87701,0.136764,128.288,0.0493651 +13,insert,27.074,-1.0,-1.0,0.0,0.060446 +14,search,1918.72,0.87481,0.138646,122.849,0.0600482 +15,insert,23.592,-1.0,-1.0,0.0,0.0693449 +16,search,1946.37,0.87969,0.134277,116.483,0.0690895 +17,insert,4.149,-1.0,-1.0,0.0,0.0711776 +18,search,1948.86,0.88182,0.134675,150.359,0.0710969 +19,insert,43.253,-1.0,-1.0,0.0,0.0780043 +20,search,2025.2,0.89383,0.125522,164.391,0.0779207 +21,insert,481.439,-1.0,-1.0,0.0,0.195993 +22,search,2391.5,0.90447,0.114981,532.143,0.197184 +23,insert,362.683,-1.0,-1.0,0.0,0.322902 +24,search,3057.96,0.90553,0.115544,1048.4,0.321499 +25,insert,49.14,-1.0,-1.0,0.0,0.341429 +26,search,2978.97,0.88326,0.128743,864.672,0.33958 +27,insert,32.645,-1.0,-1.0,0.0,0.352045 +28,search,2828.43,0.85572,0.143934,581.507,0.353742 +29,insert,73.045,-1.0,-1.0,0.0,0.380439 +30,search,2932.48,0.85548,0.145531,431.951,0.381391 +31,insert,409.685,-1.0,-1.0,0.0,0.542076 +32,search,3635.2,0.86522,0.142505,2156.88,0.542371 +33,insert,86.042,-1.0,-1.0,0.0,0.564849 +34,search,3380.54,0.84203,0.156818,1340.2,0.566884 +35,insert,51.7,-1.0,-1.0,0.0,0.582861 +36,search,3444.97,0.84006,0.158726,505.095,0.582021 +37,insert,59.788,-1.0,-1.0,0.0,0.603509 +38,search,3483.23,0.83988,0.159201,487.089,0.604192 +39,insert,7.6,-1.0,-1.0,0.0,0.606748 +40,search,3362.4,0.84252,0.156966,536.257,0.605784 +41,insert,48.764,-1.0,-1.0,0.0,0.620256 +42,search,3466.98,0.84319,0.156288,521.115,0.621045 +43,insert,10.116,-1.0,-1.0,0.0,0.624561 +44,search,3396.18,0.84228,0.156299,496.466,0.624738 +45,insert,301.695,-1.0,-1.0,0.0,0.638764 +46,search,3493.55,0.84579,0.154272,883.896,0.638924 +47,insert,339.656,-1.0,-1.0,0.0,0.740057 +48,search,3800.23,0.85236,0.152054,1720.09,0.739191 +49,insert,77.807,-1.0,-1.0,0.0,0.75536 +50,search,3697.67,0.84931,0.154299,1145.76,0.755521 +51,insert,374.126,-1.0,-1.0,0.0,0.890766 +52,search,4064.79,0.86407,0.143891,2663.65,0.890432 +53,insert,55.108,-1.0,-1.0,0.0,0.909918 +54,search,4124.36,0.85025,0.151859,1558.59,0.91115 +55,insert,361.036,-1.0,-1.0,0.0,1.05393 +56,search,4586.64,0.86635,0.144103,2958.37,1.05086 +57,insert,32.196,-1.0,-1.0,0.0,1.05961 +58,search,4354.18,0.86148,0.146311,1520.23,1.06001 +59,insert,285.429,-1.0,-1.0,0.0,1.15585 +60,search,4560.34,0.86215,0.145231,1891.05,1.15471 +61,insert,23.193,-1.0,-1.0,0.0,1.15945 +62,search,4489.36,0.85717,0.149653,1009.45,1.15956 +63,insert,3.645,-1.0,-1.0,0.0,1.16042 +64,search,4490.85,0.85732,0.149697,8.561,1.16042 +65,insert,147.489,-1.0,-1.0,0.0,1.19046 +66,search,4514.34,0.8605,0.148254,2089.42,1.18982 +67,insert,64.344,-1.0,-1.0,0.0,1.21097 +68,search,4585.89,0.86106,0.148292,1361.3,1.21082 +69,insert,77.203,-1.0,-1.0,0.0,1.23493 +70,search,4453.36,0.86193,0.147393,1136.63,1.23453 +71,insert,20.693,-1.0,-1.0,0.0,1.24011 +72,search,4665.79,0.86302,0.146872,1073.98,1.23994 +73,insert,8.357,-1.0,-1.0,0.0,1.24274 +74,search,4398.68,0.86269,0.147179,8.77,1.24274 +75,insert,27.044,-1.0,-1.0,0.0,1.24926 +76,search,4625.14,0.8631,0.146792,9.465,1.24926 +77,insert,85.246,-1.0,-1.0,0.0,1.26026 +78,search,4679.16,0.86282,0.1471,1148.24,1.26015 +79,insert,728.392,-1.0,-1.0,0.0,1.28933 +80,search,4729.39,0.86233,0.147337,1329.57,1.28987 +81,insert,52.829,-1.0,-1.0,0.0,1.30123 +82,search,4649.69,0.86303,0.146853,9.057,1.30123 +83,insert,26.88,-1.0,-1.0,0.0,1.30624 +84,search,4537.01,0.86282,0.147059,1141.43,1.30697 +85,insert,68.899,-1.0,-1.0,0.0,1.3248 +86,search,4767.23,0.86486,0.146225,1235.05,1.3253 +87,insert,792.027,-1.0,-1.0,0.0,1.51798 +88,search,5050.8,0.87695,0.134579,5408.71,1.51853 +89,insert,474.304,-1.0,-1.0,0.0,1.67437 +90,search,5735.9,0.87769,0.133345,6802.8,1.67697 +91,insert,49.091,-1.0,-1.0,0.0,1.69194 +92,search,5622.94,0.87292,0.137766,2119.28,1.69232 +93,insert,417.36,-1.0,-1.0,0.0,1.79528 +94,search,5541.67,0.87452,0.136528,2110.43,1.79606 +95,insert,20.649,-1.0,-1.0,0.0,1.80208 +96,search,5561.98,0.87461,0.136577,1563.52,1.80175 +97,insert,17.552,-1.0,-1.0,0.0,1.80725 +98,search,5513.68,0.87443,0.136729,5.09,1.80725 +99,insert,13.31,-1.0,-1.0,0.0,1.80993 +100,search,5719.86,0.87457,0.136614,4.988,1.80993 +101,insert,58.03,-1.0,-1.0,0.0,1.82143 +102,search,5569.1,0.87553,0.135739,1752.98,1.82252 +103,insert,26.114,-1.0,-1.0,0.0,1.82831 +104,search,5701.71,0.87576,0.135659,5.021,1.82831 +105,insert,14.803,-1.0,-1.0,0.0,1.83187 +106,search,5566.32,0.87537,0.135924,5.158,1.83187 +107,insert,30.411,-1.0,-1.0,0.0,1.83832 +108,search,5755.71,0.87544,0.135823,1577.6,1.8375 +109,insert,807.522,-1.0,-1.0,0.0,1.99301 +110,search,6002.68,0.87866,0.133779,4739.76,1.99591 +111,insert,41.069,-1.0,-1.0,0.0,2.01051 +112,search,5854.25,0.8776,0.134592,2301.17,2.00976 +113,insert,13.825,-1.0,-1.0,0.0,2.01363 +114,search,5880.59,0.87749,0.134656,1768.14,2.01361 +115,insert,43.545,-1.0,-1.0,0.0,2.0251 +116,search,5817.8,0.87814,0.134078,1978.89,2.02517 +117,insert,20.328,-1.0,-1.0,0.0,2.03094 +118,search,5801.19,0.87785,0.133866,5.54,2.03094 +119,insert,4.48,-1.0,-1.0,0.0,2.0321 +120,search,6083.57,0.87794,0.133861,5.483,2.0321 +121,insert,48.315,-1.0,-1.0,0.0,2.03746 +122,search,5840.58,0.87813,0.133744,1892.19,2.03721 +123,insert,602.549,-1.0,-1.0,0.0,2.16106 +124,search,6128.66,0.88084,0.132932,3866.97,2.15906 +125,insert,85.81,-1.0,-1.0,0.0,2.17875 +126,search,6286.2,0.8804,0.133098,2672.51,2.18027 +127,insert,82.464,-1.0,-1.0,0.0,2.20135 +128,search,6192.76,0.88153,0.132764,2885,2.20137 +129,delete,15.611,-1.0,-1.0,0.0,2.20052 +130,search,6082.36,0.88058,0.133406,1980.45,2.20037 +131,delete,2.824,-1.0,-1.0,0.0,2.20037 +132,search,6188.3,0.88083,0.133338,5.665,2.20037 +133,delete,10.391,-1.0,-1.0,0.0,2.19995 +134,search,6146.93,0.88096,0.133275,6.29,2.19995 +135,delete,32.089,-1.0,-1.0,0.0,2.19862 +136,search,6034,0.88397,0.129876,5.809,2.19862 +137,delete,4.787,-1.0,-1.0,0.0,2.19862 +138,search,6193.91,0.88394,0.129951,5.846,2.19862 +139,delete,13.456,-1.0,-1.0,0.0,2.19704 +140,search,5950.32,0.88361,0.130279,5.99,2.19704 +141,delete,25.381,-1.0,-1.0,0.0,2.19455 +142,search,6104.23,0.88446,0.12972,6.044,2.19455 +143,delete,21.115,-1.0,-1.0,0.0,2.19209 +144,search,5985.06,0.88467,0.129552,5.787,2.19209 +145,delete,3.305,-1.0,-1.0,0.0,2.19153 +146,search,5989.38,0.88427,0.129744,5.899,2.19153 +147,delete,13.181,-1.0,-1.0,0.0,2.19074 +148,search,5939.73,0.88441,0.129778,5.671,2.19074 +149,delete,211.311,-1.0,-1.0,0.0,2.15126 +150,search,5920.76,0.88901,0.126316,5.763,2.15126 +151,delete,185.58,-1.0,-1.0,0.0,2.11187 +152,search,5809.26,0.88897,0.126617,5.806,2.11187 +153,delete,42.572,-1.0,-1.0,0.0,2.09711 +154,search,5815.99,0.8886,0.127082,5.762,2.09711 +155,delete,20.702,-1.0,-1.0,0.0,2.09221 +156,search,5907.66,0.88857,0.127081,5.719,2.09221 +157,delete,48.034,-1.0,-1.0,0.0,2.08237 +158,search,5773.87,0.88769,0.127709,5.777,2.08237 +159,delete,321.389,-1.0,-1.0,0.0,1.96167 +160,search,5692.61,0.88667,0.128574,6.021,1.96167 +161,delete,46.095,-1.0,-1.0,0.0,1.95016 +162,search,5480.32,0.88681,0.128456,6.476,1.95016 +163,delete,43.223,-1.0,-1.0,0.0,1.94007 +164,search,5219.5,0.88618,0.128767,6.186,1.94007 +165,delete,47.051,-1.0,-1.0,0.0,1.92987 +166,search,5596.24,0.88636,0.128854,6.074,1.92987 +167,delete,11.975,-1.0,-1.0,0.0,1.92722 +168,search,5434.19,0.88573,0.129239,5.963,1.92722 +169,delete,28.41,-1.0,-1.0,0.0,1.91789 +170,search,5377.57,0.88545,0.129615,5.874,1.91789 +171,delete,16.186,-1.0,-1.0,0.0,1.91382 +172,search,5590.21,0.88514,0.129925,5.838,1.91382 +173,delete,51.633,-1.0,-1.0,0.0,1.90235 +174,search,5403.09,0.88394,0.13065,5.832,1.90235 +175,delete,207.19,-1.0,-1.0,0.0,1.83231 +176,search,5494.86,0.88398,0.130233,5.931,1.83231 +177,delete,22.514,-1.0,-1.0,0.0,1.82606 +178,search,5397.71,0.88288,0.130668,6.038,1.82606 +179,delete,267.643,-1.0,-1.0,0.0,1.72411 +180,search,5034.55,0.87965,0.131832,6.232,1.72411 +181,delete,31.629,-1.0,-1.0,0.0,1.71091 +182,search,5287.41,0.87934,0.132458,6.236,1.71091 +183,delete,188.799,-1.0,-1.0,0.0,1.64133 +184,search,5057.33,0.87455,0.134677,6.135,1.64133 +185,delete,11.897,-1.0,-1.0,0.0,1.63781 +186,search,4909.34,0.87414,0.134872,5.931,1.63781 +187,delete,224.575,-1.0,-1.0,0.0,1.54994 +188,search,5121.41,0.87738,0.132038,6.132,1.54994 +189,delete,10.57,-1.0,-1.0,0.0,1.54637 +190,search,4926.5,0.87733,0.131965,6.229,1.54637 +191,delete,3.941,-1.0,-1.0,0.0,1.54521 +192,search,4937.78,0.87736,0.131998,6.225,1.54521 +193,delete,66.946,-1.0,-1.0,0.0,1.51521 +194,search,5087.8,0.87423,0.134004,6.461,1.51521 +195,delete,50.011,-1.0,-1.0,0.0,1.50018 +196,search,5223.3,0.87238,0.13519,6.312,1.50018 +197,delete,49.944,-1.0,-1.0,0.0,1.48792 +198,search,5121.57,0.87239,0.135136,1424.57,1.48475 +199,delete,43.656,-1.0,-1.0,0.0,1.47944 +200,search,4955.57,0.87175,0.135413,6.151,1.47944 +201,delete,10.797,-1.0,-1.0,0.0,1.47612 +202,search,5098.1,0.8716,0.13554,6.353,1.47612 +203,delete,19.562,-1.0,-1.0,0.0,1.47069 +204,search,4951.74,0.87137,0.135821,6.105,1.47069 +205,delete,33.241,-1.0,-1.0,0.0,1.46155 +206,search,4784,0.87109,0.136011,6.108,1.46155 +207,delete,64.913,-1.0,-1.0,0.0,1.44045 +208,search,4880.87,0.87141,0.135728,5.94,1.44045 +209,delete,28.569,-1.0,-1.0,0.0,1.42993 +210,search,5134.09,0.87107,0.13558,6.117,1.42993 +211,delete,12.444,-1.0,-1.0,0.0,1.42611 +212,search,4973.64,0.87092,0.135626,6.587,1.42611 +213,delete,24.888,-1.0,-1.0,0.0,1.41721 +214,search,5039.05,0.87047,0.135796,5.966,1.41721 +215,delete,389.371,-1.0,-1.0,0.0,1.26704 +216,search,4841.32,0.86368,0.140725,6.365,1.26704 +217,delete,257.97,-1.0,-1.0,0.0,1.16702 +218,search,4738.05,0.85877,0.14386,6.371,1.16702 +219,delete,30.419,-1.0,-1.0,0.0,1.15345 +220,search,4662.81,0.85727,0.144926,6.266,1.15345 +221,delete,238.093,-1.0,-1.0,0.0,1.05006 +222,search,4498.32,0.85745,0.145373,6.249,1.05006 +223,delete,14.601,-1.0,-1.0,0.0,1.04442 +224,search,4581.92,0.85577,0.146588,6.183,1.04442 +225,delete,6.764,-1.0,-1.0,0.0,1.0422 +226,search,4518.76,0.8548,0.147296,6.411,1.0422 +227,delete,9.336,-1.0,-1.0,0.0,1.04058 +228,search,4556.44,0.85459,0.147346,6.296,1.04058 +229,delete,34.257,-1.0,-1.0,0.0,1.02885 +230,search,4448.45,0.85282,0.149097,6.183,1.02885 +231,delete,18.31,-1.0,-1.0,0.0,1.02245 +232,search,4309.72,0.85219,0.149564,6.169,1.02245 +233,delete,7.978,-1.0,-1.0,0.0,1.02059 +234,search,4479.33,0.85183,0.149804,6.158,1.02059 +235,delete,14.012,-1.0,-1.0,0.0,1.01663 +236,search,4415.61,0.85113,0.150262,6.184,1.01663 +237,delete,297.671,-1.0,-1.0,0.0,0.904098 +238,search,4326.41,0.84896,0.151575,6.152,0.904098 +239,delete,28.519,-1.0,-1.0,0.0,0.891171 +240,search,4326.78,0.84719,0.151772,6.126,0.891171 +241,delete,11.374,-1.0,-1.0,0.0,0.887242 +242,search,4331.92,0.84669,0.152021,6.232,0.887242 +243,delete,23.768,-1.0,-1.0,0.0,0.879869 +244,search,4295.36,0.84377,0.153611,6.321,0.879869 +245,delete,17.427,-1.0,-1.0,0.0,0.874374 +246,search,4287.64,0.84324,0.153727,6.073,0.874374 +247,delete,5.966,-1.0,-1.0,0.0,0.872423 +248,search,4268.57,0.84296,0.153881,6.23,0.872423 +249,delete,10.142,-1.0,-1.0,0.0,0.868603 +250,search,4052.89,0.84231,0.154146,6.184,0.868603 +251,delete,303.096,-1.0,-1.0,0.0,0.759665 +252,search,3814.35,0.84161,0.153805,6.073,0.759665 +253,delete,38.52,-1.0,-1.0,0.0,0.745505 +254,search,3918.58,0.83925,0.154876,5.932,0.745505 +255,delete,47.826,-1.0,-1.0,0.0,0.726537 +256,search,3922.35,0.83482,0.157373,5.956,0.726537 +257,insert,10.63,-1.0,-1.0,0.0,0.727565 +258,search,3900.57,0.83479,0.157399,6.198,0.727565 +259,insert,34.249,-1.0,-1.0,0.0,0.733859 +260,search,3902.62,0.83625,0.157024,6.04,0.733859 +261,insert,3.358,-1.0,-1.0,0.0,0.7343 +262,search,4065.98,0.8372,0.156322,6.15,0.7343 +263,insert,249.528,-1.0,-1.0,0.0,0.776752 +264,search,4118.63,0.8454,0.150513,1066.57,0.776424 +265,insert,99.827,-1.0,-1.0,0.0,0.793373 +266,search,4076.73,0.84708,0.149798,927.14,0.793225 +267,insert,57.458,-1.0,-1.0,0.0,0.804204 +268,search,3956.49,0.84978,0.14915,821.557,0.804047 +269,insert,514.397,-1.0,-1.0,0.0,0.893202 +270,search,4076.99,0.85261,0.148562,1600.83,0.890283 +271,insert,4.607,-1.0,-1.0,0.0,0.891292 +272,search,4211.53,0.8515,0.149382,1452.06,0.894095 +273,insert,33.554,-1.0,-1.0,0.0,0.901891 +274,search,4086.99,0.85238,0.148462,1454.45,0.901183 +275,insert,801.61,-1.0,-1.0,0.0,1.05381 +276,search,4460.4,0.85611,0.146582,1841.74,1.05301 +277,insert,57.89,-1.0,-1.0,0.0,1.06426 +278,search,4277.09,0.85599,0.146656,1150.59,1.06564 +279,insert,36.155,-1.0,-1.0,0.0,1.07496 +280,search,4436.94,0.85759,0.145204,1206.83,1.07339 +281,insert,593.381,-1.0,-1.0,0.0,1.2014 +282,search,4539.95,0.86017,0.14521,1695.77,1.19509 +283,insert,683.588,-1.0,-1.0,0.0,1.37559 +284,search,4571.61,0.86273,0.144432,2125.09,1.3758 +285,insert,16.862,-1.0,-1.0,0.0,1.38118 +286,search,4750.93,0.86337,0.143932,2099.53,1.38302 +287,insert,57.602,-1.0,-1.0,0.0,1.39911 +288,search,4858.86,0.86405,0.143433,1519.61,1.3994 +289,insert,552.833,-1.0,-1.0,0.0,1.52845 +290,search,4861.1,0.86965,0.141111,2086.18,1.52879 +291,insert,41.152,-1.0,-1.0,0.0,1.53873 +292,search,5020.96,0.87025,0.140568,1780.85,1.53922 +293,insert,29.486,-1.0,-1.0,0.0,1.54695 +294,search,5037.55,0.87003,0.140637,1606.44,1.54692 +295,insert,387.603,-1.0,-1.0,0.0,1.63458 +296,search,5024.42,0.87296,0.139928,1689.35,1.63495 +297,insert,5.037,-1.0,-1.0,0.0,1.63632 +298,search,5190.75,0.87203,0.140411,7.515,1.63632 +299,insert,115.888,-1.0,-1.0,0.0,1.66174 +300,search,4890.32,0.87539,0.137518,7.446,1.66174 +301,insert,1099.41,-1.0,-1.0,0.0,1.9191 +302,search,5778.8,0.89804,0.121196,4792.43,1.91738 +303,insert,78.374,-1.0,-1.0,0.0,1.93856 +304,search,5776.28,0.89763,0.121678,3831.84,1.93833 +305,insert,559.415,-1.0,-1.0,0.0,2.10275 +306,search,6061.46,0.89691,0.122395,2967.63,2.10265 +307,insert,51.057,-1.0,-1.0,0.0,2.11286 +308,search,5838.62,0.89785,0.121993,2310.45,2.11243 +309,insert,15.716,-1.0,-1.0,0.0,2.11586 +310,search,5781.07,0.89813,0.121778,2070.58,2.11404 +311,insert,31.893,-1.0,-1.0,0.0,2.1226 +312,search,5812.5,0.89835,0.121659,2024.76,2.12238 +313,insert,96.441,-1.0,-1.0,0.0,2.14197 +314,search,5803.18,0.89853,0.121699,2019.09,2.14153 +315,insert,11.166,-1.0,-1.0,0.0,2.14316 +316,search,5713.62,0.89838,0.121768,9.597,2.14316 +317,insert,118.329,-1.0,-1.0,0.0,2.16223 +318,search,6010.4,0.89799,0.122315,9.655,2.16223 +319,insert,1011.75,-1.0,-1.0,0.0,2.37749 +320,search,6227.76,0.90141,0.1194,5437.54,2.38127 +321,insert,69.056,-1.0,-1.0,0.0,2.39988 +322,search,6260.98,0.89998,0.121042,2532.12,2.39935 +323,insert,168.189,-1.0,-1.0,0.0,2.44672 +324,search,6189.55,0.9013,0.120014,2768.02,2.4458 +325,insert,2188.15,-1.0,-1.0,0.0,2.66086 +326,search,6612.59,0.90248,0.11945,4239.24,2.66078 +327,insert,122.826,-1.0,-1.0,0.0,2.69141 +328,search,6710.61,0.9031,0.118616,3655.9,2.68907 +329,insert,22.665,-1.0,-1.0,0.0,2.69317 +330,search,6492.93,0.90348,0.118322,2578.82,2.69359 +331,insert,90.729,-1.0,-1.0,0.0,2.71268 +332,search,6664.61,0.90391,0.11823,2588.78,2.71361 +333,insert,5.227,-1.0,-1.0,0.0,2.71509 +334,search,6502.69,0.90445,0.117725,2333.28,2.71457 +335,insert,70.948,-1.0,-1.0,0.0,2.73203 +336,search,6778.49,0.90494,0.117192,2426.23,2.73198 +337,insert,41.013,-1.0,-1.0,0.0,2.74068 +338,search,6783.22,0.9053,0.116648,2393.09,2.74095 +339,insert,102.653,-1.0,-1.0,0.0,2.75847 +340,search,6594.18,0.90573,0.116359,2464.66,2.75952 +341,insert,17.651,-1.0,-1.0,0.0,2.76236 +342,search,6631.75,0.90541,0.116496,10.641,2.76236 +343,insert,79.342,-1.0,-1.0,0.0,2.77735 +344,search,6604.18,0.90578,0.116182,10.591,2.77735 +345,insert,103.404,-1.0,-1.0,0.0,2.79561 +346,search,6565.74,0.90632,0.115658,2508.59,2.79623 +347,insert,936.765,-1.0,-1.0,0.0,2.99042 +348,search,6960.63,0.90959,0.112568,8905.32,2.99036 +349,insert,70.745,-1.0,-1.0,0.0,3.01012 +350,search,7176.34,0.90941,0.112954,3603.63,3.00827 +351,insert,703.683,-1.0,-1.0,0.0,3.18858 +352,search,7293.73,0.90999,0.112715,5368.6,3.18805 +353,insert,121.815,-1.0,-1.0,0.0,3.20888 +354,search,7186.93,0.90767,0.114843,3504.05,3.21207 +355,insert,731.807,-1.0,-1.0,0.0,3.37289 +356,search,7371.98,0.90964,0.113621,5172.55,3.37221 +357,insert,38.819,-1.0,-1.0,0.0,3.37938 +358,search,7407.14,0.90948,0.113754,3304.22,3.37986 +359,insert,21.129,-1.0,-1.0,0.0,3.38425 +360,search,7305.46,0.90918,0.114063,11.087,3.38425 +361,insert,130.18,-1.0,-1.0,0.0,3.41329 +362,search,7081.85,0.91017,0.113252,3559.47,3.41311 +363,insert,31.731,-1.0,-1.0,0.0,3.41948 +364,search,7311.31,0.91013,0.113339,3301.06,3.4192 +365,insert,124.99,-1.0,-1.0,0.0,3.44465 +366,search,7602.32,0.91057,0.11274,3334.48,3.44562 +367,insert,60.259,-1.0,-1.0,0.0,3.45981 +368,search,7453.81,0.91086,0.112748,3127.46,3.45949 +369,insert,159.569,-1.0,-1.0,0.0,3.49044 +370,search,7424.27,0.91141,0.11202,3308.41,3.49014 +371,insert,24.101,-1.0,-1.0,0.0,3.49414 +372,search,7484.45,0.91143,0.112002,3320.36,3.4941 +373,insert,115.591,-1.0,-1.0,0.0,3.51237 +374,search,7547.67,0.91178,0.111454,3883.23,3.5131 +375,insert,70.585,-1.0,-1.0,0.0,3.52665 +376,search,7699.95,0.91166,0.111606,3309.3,3.52717 +377,insert,110.206,-1.0,-1.0,0.0,3.55181 +378,search,7707.21,0.91193,0.111842,4026.41,3.5521 +379,insert,12.54,-1.0,-1.0,0.0,3.55435 +380,search,7505.7,0.9118,0.11202,3269.23,3.55385 +381,insert,40.752,-1.0,-1.0,0.0,3.56187 +382,search,7502.26,0.91192,0.111874,11.409,3.56187 +383,insert,64.08,-1.0,-1.0,0.0,3.57171 +384,search,7497.77,0.9119,0.111979,3355.08,3.57211 +385,delete,9.179,-1.0,-1.0,0.0,3.57196 +386,search,7533.67,0.91185,0.112102,11.152,3.57196 +387,delete,21.734,-1.0,-1.0,0.0,3.57094 +388,search,7541.08,0.91201,0.111993,11.156,3.57094 +389,delete,18.691,-1.0,-1.0,0.0,3.56969 +390,search,7404.64,0.91157,0.112362,11.208,3.56969 +391,delete,98.044,-1.0,-1.0,0.0,3.54853 +392,search,7607.2,0.91148,0.112744,11.932,3.54853 +393,delete,59.058,-1.0,-1.0,0.0,3.54706 +394,search,7169.97,0.91133,0.112824,11.633,3.54706 +395,delete,33.747,-1.0,-1.0,0.0,3.53993 +396,search,7152.63,0.91125,0.112833,11.695,3.53993 +397,delete,163.238,-1.0,-1.0,0.0,3.50846 +398,search,7473.83,0.9109,0.113162,11.495,3.50846 +399,delete,8.314,-1.0,-1.0,0.0,3.50799 +400,search,7508.49,0.91088,0.113187,11.934,3.50799 +401,delete,24.518,-1.0,-1.0,0.0,3.50527 +402,search,7499.29,0.91067,0.113519,11.532,3.50527 +403,delete,358.375,-1.0,-1.0,0.0,3.42014 +404,search,7377.91,0.91058,0.112975,11.435,3.42014 +405,delete,71.045,-1.0,-1.0,0.0,3.41151 +406,search,7285.21,0.91202,0.112409,11.447,3.41151 +407,delete,125.299,-1.0,-1.0,0.0,3.38258 +408,search,7350.46,0.91245,0.111952,11.37,3.38258 +409,delete,251.026,-1.0,-1.0,0.0,3.29698 +410,search,7210.22,0.91224,0.112039,11.221,3.29698 +411,delete,283.096,-1.0,-1.0,0.0,3.22401 +412,search,6903.76,0.91102,0.113274,11.73,3.22401 +413,delete,33.28,-1.0,-1.0,0.0,3.21697 +414,search,6834.86,0.9109,0.113347,11.73,3.21697 +415,delete,102.637,-1.0,-1.0,0.0,3.19667 +416,search,6803.85,0.91072,0.113424,11.585,3.19667 +417,delete,251.516,-1.0,-1.0,0.0,3.13049 +418,search,6539.71,0.9107,0.113376,11.465,3.13049 +419,delete,32.951,-1.0,-1.0,0.0,3.12623 +420,search,6786.64,0.91031,0.113691,11.515,3.12623 +421,delete,32.175,-1.0,-1.0,0.0,3.116 +422,search,6887.87,0.91041,0.113577,11.477,3.116 +423,delete,201.995,-1.0,-1.0,0.0,3.05635 +424,search,6718.32,0.90985,0.113969,11.126,3.05635 +425,delete,23.994,-1.0,-1.0,0.0,3.05145 +426,search,6772.74,0.91019,0.113755,11.168,3.05145 +427,delete,62.614,-1.0,-1.0,0.0,3.04109 +428,search,6400.42,0.90947,0.114326,11.108,3.04109 +429,delete,558.157,-1.0,-1.0,0.0,2.88633 +430,search,5520.87,0.89995,0.12329,7.479,2.88633 +431,delete,53.899,-1.0,-1.0,0.0,2.84769 +432,search,6262.02,0.90008,0.123133,11.081,2.84769 +433,delete,230.931,-1.0,-1.0,0.0,2.77155 +434,search,5975.25,0.89983,0.123087,11.68,2.77155 +435,delete,151.502,-1.0,-1.0,0.0,2.72474 +436,search,6229.89,0.89889,0.124231,11.567,2.72474 +437,delete,21.18,-1.0,-1.0,0.0,2.7196 +438,search,6032.29,0.89845,0.124508,11.42,2.7196 +439,delete,169.906,-1.0,-1.0,0.0,2.66641 +440,search,6206,0.89797,0.124529,11.388,2.66641 +441,delete,49.703,-1.0,-1.0,0.0,2.65253 +442,search,5997.26,0.89785,0.124741,11.357,2.65253 +443,delete,44.388,-1.0,-1.0,0.0,2.63904 +444,search,6162.58,0.89761,0.124901,11.16,2.63904 +445,delete,53.999,-1.0,-1.0,0.0,2.62242 +446,search,5931.22,0.89757,0.124943,11.05,2.62242 +447,delete,468.763,-1.0,-1.0,0.0,2.46324 +448,search,5942.04,0.89747,0.124589,11.114,2.46324 +449,delete,45.661,-1.0,-1.0,0.0,2.45092 +450,search,5928.56,0.89633,0.125216,11.033,2.45092 +451,delete,118.521,-1.0,-1.0,0.0,2.40849 +452,search,5879.94,0.89476,0.126614,11.08,2.40849 +453,delete,368.603,-1.0,-1.0,0.0,2.2903 +454,search,5766.87,0.89381,0.126825,11.003,2.2903 +455,delete,45.083,-1.0,-1.0,0.0,2.27424 +456,search,5570.75,0.89301,0.127219,10.938,2.27424 +457,delete,8.377,-1.0,-1.0,0.0,2.27111 +458,search,5559.27,0.89238,0.127513,11.548,2.27111 +459,delete,67.453,-1.0,-1.0,0.0,2.2525 +460,search,5728.46,0.89234,0.12738,11.241,2.2525 +461,delete,12.261,-1.0,-1.0,0.0,2.25075 +462,search,5497.49,0.89233,0.127325,11.443,2.25075 +463,delete,93.478,-1.0,-1.0,0.0,2.2265 +464,search,5674.98,0.89164,0.12783,11.252,2.2265 +465,delete,39.379,-1.0,-1.0,0.0,2.21419 +466,search,5555.44,0.89083,0.128229,11.059,2.21419 +467,delete,44.995,-1.0,-1.0,0.0,2.20259 +468,search,5666.69,0.89038,0.12873,11.06,2.20259 +469,delete,30.135,-1.0,-1.0,0.0,2.19197 +470,search,5633.6,0.88928,0.129278,11.084,2.19197 +471,delete,173.173,-1.0,-1.0,0.0,2.13495 +472,search,5527.45,0.88853,0.130066,10.812,2.13495 +473,delete,157.439,-1.0,-1.0,0.0,2.083 +474,search,5513.45,0.88811,0.130488,10.951,2.083 +475,delete,486.715,-1.0,-1.0,0.0,1.89445 +476,search,5159.44,0.88494,0.132468,11.106,1.89445 +477,delete,70.517,-1.0,-1.0,0.0,1.87034 +478,search,5223.68,0.88425,0.132937,10.895,1.87034 +479,delete,252.924,-1.0,-1.0,0.0,1.77785 +480,search,5422.44,0.88593,0.13108,10.928,1.77785 +481,delete,65.339,-1.0,-1.0,0.0,1.75463 +482,search,5447.27,0.8853,0.131296,10.93,1.75463 +483,delete,329.813,-1.0,-1.0,0.0,1.625 +484,search,5278.61,0.88179,0.134075,11.253,1.625 +485,delete,42.313,-1.0,-1.0,0.0,1.61306 +486,search,5115.4,0.88156,0.134065,11.307,1.61306 +487,delete,20.76,-1.0,-1.0,0.0,1.60918 +488,search,5263.3,0.88113,0.134127,11.464,1.60918 +489,delete,59.88,-1.0,-1.0,0.0,1.59253 +490,search,5239.18,0.88027,0.134802,11.041,1.59253 +491,delete,27.561,-1.0,-1.0,0.0,1.58489 +492,search,5126.03,0.88012,0.134846,10.865,1.58489 +493,delete,188.058,-1.0,-1.0,0.0,1.52173 +494,search,5198.09,0.87971,0.135112,10.916,1.52173 +495,delete,31.484,-1.0,-1.0,0.0,1.50927 +496,search,5161.87,0.87844,0.13583,11.059,1.50927 +497,delete,86.138,-1.0,-1.0,0.0,1.47985 +498,search,5053.76,0.87731,0.136356,11.042,1.47985 +499,delete,24.964,-1.0,-1.0,0.0,1.47283 +500,search,5124.18,0.87681,0.136702,10.861,1.47283 +501,delete,38.414,-1.0,-1.0,0.0,1.45849 +502,search,5104.4,0.87587,0.137567,10.644,1.45849 +503,delete,25.81,-1.0,-1.0,0.0,1.45023 +504,search,4931.83,0.87591,0.137674,10.828,1.45023 +505,delete,85.753,-1.0,-1.0,0.0,1.41925 +506,search,4895.03,0.87517,0.137859,10.8,1.41925 +507,delete,50.484,-1.0,-1.0,0.0,1.40182 +508,search,5103.72,0.87497,0.137684,10.814,1.40182 +509,delete,26.673,-1.0,-1.0,0.0,1.39144 +510,search,5035.28,0.87433,0.13815,11.292,1.39144 +511,delete,48.287,-1.0,-1.0,0.0,1.3821 +512,search,5001.15,0.87361,0.138813,10.984,1.3821 +513,insert,550.844,-1.0,-1.0,0.0,1.469 +514,search,4970.43,0.87365,0.139377,11.196,1.469 +515,insert,10.574,-1.0,-1.0,0.0,1.46985 +516,search,4718.05,0.87329,0.139762,10.841,1.46985 +517,insert,38.752,-1.0,-1.0,0.0,1.4734 +518,search,4944.18,0.87447,0.139016,10.894,1.4734 +519,insert,172.481,-1.0,-1.0,0.0,1.49714 +520,search,4953.84,0.87584,0.138708,11.168,1.49714 +521,insert,25.967,-1.0,-1.0,0.0,1.49994 +522,search,4746.16,0.87537,0.139232,10.86,1.49994 +523,insert,75.319,-1.0,-1.0,0.0,1.51006 +524,search,4863.31,0.87546,0.13922,1653.15,1.50924 +525,insert,82.93,-1.0,-1.0,0.0,1.51822 +526,search,4688.25,0.87555,0.139243,10.975,1.51822 +527,insert,924.391,-1.0,-1.0,0.0,1.67234 +528,search,5064.02,0.87973,0.13713,2251.73,1.67247 +529,insert,513.679,-1.0,-1.0,0.0,1.76844 +530,search,5090.37,0.88615,0.134159,2668.67,1.76935 +531,insert,92.832,-1.0,-1.0,0.0,1.7825 +532,search,4993.24,0.88577,0.134676,2132.81,1.78165 +533,insert,25.034,-1.0,-1.0,0.0,1.78648 +534,search,5101.84,0.8864,0.134202,1793.69,1.7866 +535,insert,36.653,-1.0,-1.0,0.0,1.79229 +536,search,5256.83,0.88755,0.133159,1680.92,1.79189 +537,insert,12.704,-1.0,-1.0,0.0,1.79474 +538,search,5234.11,0.8876,0.133049,1702.3,1.79495 +539,insert,54.105,-1.0,-1.0,0.0,1.80496 +540,search,5210.01,0.88717,0.133382,1747.23,1.80506 +541,insert,807.721,-1.0,-1.0,0.0,1.96343 +542,search,5295.03,0.89095,0.13053,2491.73,1.96372 +543,insert,69.153,-1.0,-1.0,0.0,1.97743 +544,search,5242.39,0.89156,0.129811,2165.53,1.97625 +545,insert,10.18,-1.0,-1.0,0.0,1.9778 +546,search,5263.97,0.89121,0.130166,1810.8,1.97779 +547,insert,252.86,-1.0,-1.0,0.0,2.03051 +548,search,5198.2,0.89437,0.128148,2203.21,2.03049 +549,insert,514.123,-1.0,-1.0,0.0,2.13099 +550,search,5625.44,0.89184,0.129767,2632.66,2.12978 +551,insert,39.765,-1.0,-1.0,0.0,2.14031 +552,search,5630.17,0.89263,0.129071,2445.25,2.14044 +553,insert,17.588,-1.0,-1.0,0.0,2.14489 +554,search,5528.99,0.89243,0.129623,2235.13,2.14635 +555,insert,976.956,-1.0,-1.0,0.0,2.31797 +556,search,5756.25,0.89846,0.123712,2348.04,2.31776 +557,insert,52.245,-1.0,-1.0,0.0,2.32883 +558,search,5681.94,0.89875,0.12339,2175.08,2.32888 +559,insert,15.262,-1.0,-1.0,0.0,2.33169 +560,search,5689.1,0.89878,0.123375,11.303,2.33169 +561,insert,7.556,-1.0,-1.0,0.0,2.33253 +562,search,5887.35,0.89881,0.123364,11.164,2.33253 +563,insert,100.421,-1.0,-1.0,0.0,2.34852 +564,search,5651.61,0.89898,0.123284,10.961,2.34852 +565,insert,499.866,-1.0,-1.0,0.0,2.42994 +566,search,5614.79,0.89882,0.123784,2882.81,2.43048 +567,insert,22.466,-1.0,-1.0,0.0,2.43496 +568,search,5685.67,0.89884,0.12391,2497.65,2.43375 +569,insert,31.165,-1.0,-1.0,0.0,2.4413 +570,search,5719.84,0.89886,0.124073,11.597,2.4413 +571,insert,36.988,-1.0,-1.0,0.0,2.44673 +572,search,5962.47,0.89822,0.124756,11.266,2.44673 +573,insert,152.009,-1.0,-1.0,0.0,2.47152 +574,search,5974.85,0.89863,0.124411,11.231,2.47152 +575,insert,138.262,-1.0,-1.0,0.0,2.49136 +576,search,5754.43,0.89877,0.124438,11.184,2.49136 +577,insert,84.866,-1.0,-1.0,0.0,2.50686 +578,search,6027.73,0.89926,0.124211,2604.8,2.50725 +579,insert,1366.38,-1.0,-1.0,0.0,2.76007 +580,search,6354.11,0.9044,0.119568,5269.21,2.76287 +581,insert,39.387,-1.0,-1.0,0.0,2.77052 +582,search,6381.21,0.90444,0.119882,2771.86,2.77022 +583,insert,37.341,-1.0,-1.0,0.0,2.77907 +584,search,6358.35,0.90444,0.119682,11.124,2.77907 +585,insert,484.959,-1.0,-1.0,0.0,2.89806 +586,search,6245.25,0.90557,0.119019,3033.45,2.89875 +587,insert,58.185,-1.0,-1.0,0.0,2.91176 +588,search,6459.08,0.90564,0.119153,2936.78,2.91091 +589,insert,119.834,-1.0,-1.0,0.0,2.93013 +590,search,6277.04,0.90563,0.118988,11.696,2.93013 +591,insert,1058.21,-1.0,-1.0,0.0,3.13943 +592,search,6523.66,0.90819,0.116968,5283.25,3.13808 +593,insert,402.238,-1.0,-1.0,0.0,3.2268 +594,search,6662.17,0.90778,0.117751,3805.95,3.22583 +595,insert,7.376,-1.0,-1.0,0.0,3.22743 +596,search,6666.03,0.90769,0.117973,3347.18,3.22708 +597,insert,492.459,-1.0,-1.0,0.0,3.331 +598,search,6722.31,0.90869,0.117461,3684.26,3.32957 +599,insert,187.674,-1.0,-1.0,0.0,3.38614 +600,search,6767.3,0.90944,0.117083,4030.1,3.38361 +601,insert,102.521,-1.0,-1.0,0.0,3.40693 +602,search,6911.36,0.90968,0.116898,3344.42,3.40537 +603,insert,15.17,-1.0,-1.0,0.0,3.40791 +604,search,6765.48,0.90971,0.116784,12.423,3.40791 +605,insert,124.879,-1.0,-1.0,0.0,3.4262 +606,search,6810.94,0.90942,0.117134,11.974,3.4262 +607,insert,110.818,-1.0,-1.0,0.0,3.44181 +608,search,6764.51,0.90922,0.117431,12.305,3.44181 +609,insert,485.73,-1.0,-1.0,0.0,3.53301 +610,search,6844.09,0.90955,0.117511,4026.31,3.53305 +611,insert,60.348,-1.0,-1.0,0.0,3.54418 +612,search,6915.42,0.90963,0.117408,3596.67,3.54298 +613,insert,314.486,-1.0,-1.0,0.0,3.62683 +614,search,6928.42,0.91027,0.116786,4199.24,3.62699 +615,insert,651.142,-1.0,-1.0,0.0,3.79368 +616,search,7313,0.91158,0.115931,5466.07,3.79124 +617,insert,683.633,-1.0,-1.0,0.0,3.93388 +618,search,7243.53,0.9128,0.115124,7599.08,3.9323 +619,insert,534.1,-1.0,-1.0,0.0,4.04058 +620,search,7217.99,0.91289,0.115181,3565.28,4.04166 +621,insert,68.733,-1.0,-1.0,0.0,4.05881 +622,search,7318.37,0.91331,0.114983,12.233,4.05881 +623,insert,43.813,-1.0,-1.0,0.0,4.06868 +624,search,7368.86,0.91293,0.115343,12.066,4.06868 +625,insert,137.781,-1.0,-1.0,0.0,4.10167 +626,search,7369.02,0.91332,0.115004,3918.27,4.10273 +627,insert,90.246,-1.0,-1.0,0.0,4.12282 +628,search,7313.22,0.91374,0.11472,4059.91,4.12204 +629,insert,31.63,-1.0,-1.0,0.0,4.12834 +630,search,7363.74,0.9138,0.114582,3744.1,4.12938 +631,insert,91.514,-1.0,-1.0,0.0,4.14318 +632,search,7449.99,0.9139,0.114613,4187.73,4.14254 +633,insert,130.38,-1.0,-1.0,0.0,4.16204 +634,search,7313.98,0.91427,0.114115,4047.16,4.16146 +635,insert,69.295,-1.0,-1.0,0.0,4.17068 +636,search,7529.44,0.91397,0.114436,11.942,4.17068 +637,insert,900.742,-1.0,-1.0,0.0,4.34102 +638,search,7611.42,0.91559,0.113038,9246.49,4.34343 +639,insert,803.134,-1.0,-1.0,0.0,4.51737 +640,search,7710.3,0.91687,0.111696,10348.9,4.5175 +641,delete,177.283,-1.0,-1.0,0.0,4.49226 +642,search,7855.61,0.91652,0.112242,5676.94,4.4915 +643,delete,9.512,-1.0,-1.0,0.0,4.49112 +644,search,7689.23,0.91653,0.11211,4395.22,4.4907 +645,delete,12.116,-1.0,-1.0,0.0,4.49005 +646,search,7466.11,0.91613,0.112455,4217.52,4.48821 +647,delete,55.557,-1.0,-1.0,0.0,4.47236 +648,search,7532.85,0.91569,0.112865,12.93,4.47236 +649,delete,40.445,-1.0,-1.0,0.0,4.46946 +650,search,7558.39,0.91582,0.112905,12.91,4.46946 +651,delete,29.949,-1.0,-1.0,0.0,4.46569 +652,search,7636.57,0.91565,0.112897,13.878,4.46569 +653,delete,121.082,-1.0,-1.0,0.0,4.43522 +654,search,7728.73,0.91522,0.112876,13.888,4.43522 +655,delete,497.406,-1.0,-1.0,0.0,4.32935 +656,search,7567.26,0.91437,0.113381,13.465,4.32935 +657,delete,330.176,-1.0,-1.0,0.0,4.255 +658,search,7357.88,0.91022,0.115948,13.273,4.255 +659,delete,175.483,-1.0,-1.0,0.0,4.21948 +660,search,7424.98,0.91019,0.115777,13.179,4.21948 +661,delete,71.56,-1.0,-1.0,0.0,4.2099 +662,search,7375.38,0.91025,0.115845,11.974,4.2099 +663,delete,72.618,-1.0,-1.0,0.0,4.19336 +664,search,7241.95,0.90965,0.116418,12.967,4.19336 +665,delete,68.501,-1.0,-1.0,0.0,4.17802 +666,search,7361.54,0.90965,0.11641,12.747,4.17802 +667,delete,266.569,-1.0,-1.0,0.0,4.11425 +668,search,7247.61,0.90938,0.116366,12.869,4.11425 +669,delete,279.686,-1.0,-1.0,0.0,4.04729 +670,search,7128.99,0.9085,0.116843,13.682,4.04729 +671,delete,84.994,-1.0,-1.0,0.0,4.03901 +672,search,7002.1,0.90829,0.117035,13.387,4.03901 +673,delete,113.585,-1.0,-1.0,0.0,4.02027 +674,search,7097.33,0.90844,0.116834,13.446,4.02027 +675,delete,187.768,-1.0,-1.0,0.0,3.99687 +676,search,6727.01,0.90689,0.117993,13.157,3.99687 +677,delete,275.589,-1.0,-1.0,0.0,3.91368 +678,search,6745.9,0.90811,0.116966,12.017,3.91368 +679,delete,39.869,-1.0,-1.0,0.0,3.9023 +680,search,6676.53,0.90767,0.116992,12.866,3.9023 +681,delete,15.422,-1.0,-1.0,0.0,3.90012 +682,search,6391.92,0.90767,0.117052,13.054,3.90012 +683,delete,410.335,-1.0,-1.0,0.0,3.81113 +684,search,6486.17,0.90443,0.121256,12.689,3.81113 +685,delete,348.497,-1.0,-1.0,0.0,3.71703 +686,search,6345.94,0.89898,0.12644,13.848,3.71703 +687,delete,35.713,-1.0,-1.0,0.0,3.71177 +688,search,6034.65,0.89904,0.126478,13.458,3.71177 +689,delete,139.213,-1.0,-1.0,0.0,3.66147 +690,search,6005.73,0.8988,0.126503,11.646,3.66147 +691,delete,61,-1.0,-1.0,0.0,3.64485 +692,search,5994.03,0.89779,0.127366,11.283,3.64485 +693,delete,160.802,-1.0,-1.0,0.0,3.57767 +694,search,5928.87,0.89808,0.127162,11.334,3.57767 +695,delete,31.465,-1.0,-1.0,0.0,3.56676 +696,search,6057.83,0.89807,0.127314,11.285,3.56676 +697,delete,39.682,-1.0,-1.0,0.0,3.55401 +698,search,5957.35,0.89768,0.127724,12.686,3.55401 +699,delete,17.58,-1.0,-1.0,0.0,3.5473 +700,search,6379.11,0.89783,0.127511,12.607,3.5473 +701,delete,87.365,-1.0,-1.0,0.0,3.52798 +702,search,6110.18,0.89786,0.127315,12.689,3.52798 +703,delete,282.833,-1.0,-1.0,0.0,3.45272 +704,search,6143.3,0.89852,0.126387,12.601,3.45272 +705,delete,53.765,-1.0,-1.0,0.0,3.43882 +706,search,6253.98,0.89798,0.126772,13.524,3.43882 +707,delete,801.142,-1.0,-1.0,0.0,3.18566 +708,search,5805.07,0.8899,0.133624,13.031,3.18566 +709,delete,286.458,-1.0,-1.0,0.0,3.10387 +710,search,5950.46,0.88804,0.13478,13.221,3.10387 +711,delete,102.251,-1.0,-1.0,0.0,3.08141 +712,search,5644.02,0.88799,0.134978,12.918,3.08141 +713,delete,341.357,-1.0,-1.0,0.0,2.97453 +714,search,5825.99,0.88624,0.135836,15.509,2.97453 +715,delete,54.029,-1.0,-1.0,0.0,2.96095 +716,search,5567.69,0.88588,0.135893,12.614,2.96095 +717,delete,49.91,-1.0,-1.0,0.0,2.94929 +718,search,5809.84,0.88543,0.136285,12.635,2.94929 +719,delete,539.434,-1.0,-1.0,0.0,2.79039 +720,search,5374.68,0.88063,0.1416,12.553,2.79039 +721,delete,144.486,-1.0,-1.0,0.0,2.74261 +722,search,5579.21,0.88086,0.141401,12.607,2.74261 +723,delete,33.329,-1.0,-1.0,0.0,2.73255 +724,search,5406.81,0.88053,0.141406,12.537,2.73255 +725,delete,173.551,-1.0,-1.0,0.0,2.6806 +726,search,5500.03,0.87993,0.141509,12.485,2.6806 +727,delete,138.177,-1.0,-1.0,0.0,2.63424 +728,search,5448.55,0.87926,0.141842,13.307,2.63424 +729,delete,191.987,-1.0,-1.0,0.0,2.58508 +730,search,5399.36,0.87744,0.142715,12.887,2.58508 +731,delete,134.727,-1.0,-1.0,0.0,2.55171 +732,search,5396.81,0.8764,0.144012,12.82,2.55171 +733,delete,134.661,-1.0,-1.0,0.0,2.51523 +734,search,5192.72,0.87573,0.144294,12.543,2.51523 +735,delete,246.512,-1.0,-1.0,0.0,2.43798 +736,search,5274.53,0.88056,0.139043,13.034,2.43798 +737,delete,198.893,-1.0,-1.0,0.0,2.38472 +738,search,5293.73,0.88019,0.139207,12.658,2.38472 +739,delete,98.965,-1.0,-1.0,0.0,2.3557 +740,search,5294.44,0.88012,0.139088,12.534,2.3557 +741,delete,252.644,-1.0,-1.0,0.0,2.27662 +742,search,5454.36,0.87942,0.139461,12.395,2.27662 +743,delete,287.884,-1.0,-1.0,0.0,2.1817 +744,search,5250.82,0.87888,0.139431,12.366,2.1817 +745,delete,359.057,-1.0,-1.0,0.0,2.06655 +746,search,5256.36,0.87758,0.140345,12.332,2.06655 +747,delete,287.877,-1.0,-1.0,0.0,1.9647 +748,search,5247.35,0.87613,0.140669,12.345,1.9647 +749,delete,100.155,-1.0,-1.0,0.0,1.92874 +750,search,5016.19,0.87528,0.141165,13.078,1.92874 +751,delete,57.581,-1.0,-1.0,0.0,1.91626 +752,search,5102.81,0.87474,0.141216,12.928,1.91626 +753,delete,105.528,-1.0,-1.0,0.0,1.88665 +754,search,5244.79,0.87417,0.141297,12.608,1.88665 +755,delete,67.678,-1.0,-1.0,0.0,1.8708 +756,search,4995.93,0.87344,0.141984,12.158,1.8708 +757,delete,58.211,-1.0,-1.0,0.0,1.85706 +758,search,5181.87,0.8729,0.142188,12.119,1.85706 +759,delete,47.603,-1.0,-1.0,0.0,1.84432 +760,search,5171.65,0.8727,0.14225,12.313,1.84432 +761,delete,73.58,-1.0,-1.0,0.0,1.82521 +762,search,5007.22,0.87186,0.142639,12.407,1.82521 +763,delete,34.12,-1.0,-1.0,0.0,1.81629 +764,search,5116.44,0.87137,0.142805,12.367,1.81629 +765,delete,507.637,-1.0,-1.0,0.0,1.65949 +766,search,5051.17,0.86984,0.14348,12.158,1.65949 +767,delete,506.107,-1.0,-1.0,0.0,1.47454 +768,search,4919.41,0.86459,0.146133,7.993,1.47454 +769,insert,7.317,-1.0,-1.0,0.0,1.47518 +770,search,3853.14,0.8641,0.146356,8.035,1.47518 +771,insert,40.869,-1.0,-1.0,0.0,1.48031 +772,search,7737.46,0.86422,0.1464,17.543,1.48031 +773,insert,115.177,-1.0,-1.0,0.0,1.49027 +774,search,4734.72,0.8655,0.145446,1011.18,1.49058 +775,insert,62.863,-1.0,-1.0,0.0,1.4999 +776,search,4511.01,0.86512,0.146109,10.794,1.4999 +777,insert,46.22,-1.0,-1.0,0.0,1.50415 +778,search,4483.15,0.86596,0.144971,10.862,1.50415 +779,insert,25.008,-1.0,-1.0,0.0,1.50653 +780,search,4496.82,0.86624,0.144974,12.443,1.50653 +781,insert,12.768,-1.0,-1.0,0.0,1.50737 +782,search,4596.66,0.86589,0.14535,10.895,1.50737 +783,insert,103.64,-1.0,-1.0,0.0,1.51997 +784,search,4570.82,0.86666,0.144712,12.151,1.51997 +785,insert,98.292,-1.0,-1.0,0.0,1.53457 +786,search,4582.19,0.86936,0.143983,12.563,1.53457 +787,insert,46.972,-1.0,-1.0,0.0,1.53844 +788,search,4436.64,0.86882,0.144413,11.964,1.53844 +789,insert,10.271,-1.0,-1.0,0.0,1.53962 +790,search,4532.05,0.86891,0.144455,12.263,1.53962 +791,insert,56.065,-1.0,-1.0,0.0,1.54547 +792,search,4515.56,0.86907,0.144409,11.744,1.54547 +793,insert,3.247,-1.0,-1.0,0.0,1.54575 +794,search,4774.92,0.86906,0.144422,11.78,1.54575 +795,insert,124.219,-1.0,-1.0,0.0,1.56208 +796,search,4661.88,0.8697,0.143874,11.841,1.56208 +797,insert,5.75,-1.0,-1.0,0.0,1.56237 +798,search,4726.11,0.86928,0.144228,11.698,1.56237 +799,insert,7.213,-1.0,-1.0,0.0,1.56269 +800,search,4762.04,0.86933,0.144238,12.474,1.56269 +801,insert,36.198,-1.0,-1.0,0.0,1.56589 +802,search,4700.88,0.86934,0.144384,12.187,1.56589 +803,insert,2086.08,-1.0,-1.0,0.0,1.85626 +804,search,4893.93,0.87644,0.140297,2137.32,1.85954 +805,insert,29.767,-1.0,-1.0,0.0,1.86489 +806,search,4796.68,0.87676,0.139667,2044.66,1.86294 +807,insert,28.347,-1.0,-1.0,0.0,1.86827 +808,search,4829.87,0.8763,0.139742,10.453,1.86827 +809,insert,52.813,-1.0,-1.0,0.0,1.87851 +810,search,4684.62,0.87699,0.139333,12.125,1.87851 +811,insert,64.41,-1.0,-1.0,0.0,1.88905 +812,search,4980.87,0.87759,0.138851,11.999,1.88905 +813,insert,95.58,-1.0,-1.0,0.0,1.9071 +814,search,4981.14,0.87924,0.136944,10.531,1.9071 +815,insert,2.012,-1.0,-1.0,0.0,1.9076 +816,search,4719.81,0.87928,0.136923,10.654,1.9076 +817,insert,146.859,-1.0,-1.0,0.0,1.94059 +818,search,5016.45,0.87887,0.137395,11.834,1.94059 +819,insert,37.976,-1.0,-1.0,0.0,1.94673 +820,search,5042.95,0.87936,0.136751,12.395,1.94673 +821,insert,27.529,-1.0,-1.0,0.0,1.95041 +822,search,5039.18,0.8793,0.136733,12.916,1.95041 +823,insert,75.031,-1.0,-1.0,0.0,1.96298 +824,search,5015.98,0.87887,0.136702,12.371,1.96298 +825,insert,241.691,-1.0,-1.0,0.0,1.99253 +826,search,5031.16,0.87917,0.136564,12.268,1.99253 +827,insert,42.984,-1.0,-1.0,0.0,1.99896 +828,search,5050.2,0.87879,0.13657,12.23,1.99896 +829,insert,1126.19,-1.0,-1.0,0.0,2.16788 +830,search,5215.68,0.882,0.134552,2709.73,2.16781 +831,insert,73.74,-1.0,-1.0,0.0,2.17931 +832,search,5175.52,0.88216,0.134616,11.853,2.17931 +833,insert,18.667,-1.0,-1.0,0.0,2.18281 +834,search,5005.18,0.88229,0.134498,11.828,2.18281 +835,insert,138.155,-1.0,-1.0,0.0,2.20798 +836,search,5030.84,0.8829,0.134004,11.912,2.20798 +837,insert,45.138,-1.0,-1.0,0.0,2.21628 +838,search,5121.45,0.88305,0.134021,11.656,2.21628 +839,insert,160.068,-1.0,-1.0,0.0,2.24429 +840,search,5053.49,0.88414,0.133175,2077.42,2.24481 +841,insert,98.166,-1.0,-1.0,0.0,2.26104 +842,search,4985.46,0.88391,0.132993,8.659,2.26104 +843,insert,1009.86,-1.0,-1.0,0.0,2.42153 +844,search,5211.96,0.88643,0.131608,2844.65,2.42169 +845,insert,899.935,-1.0,-1.0,0.0,2.5745 +846,search,5341.47,0.88804,0.130712,2811.98,2.57394 +847,insert,4.581,-1.0,-1.0,0.0,2.57481 +848,search,4490.87,0.8878,0.130893,1681.78,2.57425 +849,insert,7.96,-1.0,-1.0,0.0,2.57893 +850,search,4458.84,0.88735,0.131249,7.961,2.57893 +851,insert,13.601,-1.0,-1.0,0.0,2.58263 +852,search,5285.59,0.88742,0.131269,11.834,2.58263 +853,insert,55.16,-1.0,-1.0,0.0,2.59048 +854,search,5399.2,0.88746,0.131271,11.867,2.59048 +855,insert,14.223,-1.0,-1.0,0.0,2.5929 +856,search,4878.39,0.88712,0.131528,10.948,2.5929 +857,insert,36.285,-1.0,-1.0,0.0,2.59988 +858,search,5279.62,0.88739,0.131472,12.577,2.59988 +859,insert,73.661,-1.0,-1.0,0.0,2.61329 +860,search,5308.67,0.8876,0.131392,12.591,2.61329 +861,insert,31.07,-1.0,-1.0,0.0,2.61677 +862,search,5265.05,0.88699,0.131793,9.054,2.61677 +863,insert,197.026,-1.0,-1.0,0.0,2.64645 +864,search,5425.94,0.88519,0.133514,12.371,2.64645 +865,insert,33.098,-1.0,-1.0,0.0,2.65107 +866,search,5210.81,0.88513,0.133582,12.384,2.65107 +867,insert,133.37,-1.0,-1.0,0.0,2.66938 +868,search,5195.67,0.88473,0.133805,11.174,2.66938 +869,insert,17.825,-1.0,-1.0,0.0,2.67232 +870,search,5262.91,0.8848,0.133755,11.975,2.67232 +871,insert,118.747,-1.0,-1.0,0.0,2.68899 +872,search,5164.23,0.88439,0.134013,11.504,2.68899 +873,insert,118.499,-1.0,-1.0,0.0,2.70755 +874,search,5380.57,0.88463,0.133954,12.089,2.70755 +875,insert,62.373,-1.0,-1.0,0.0,2.71399 +876,search,4747.5,0.88468,0.133961,7.897,2.71399 +877,insert,2.286,-1.0,-1.0,0.0,2.71499 +878,search,4424.94,0.88424,0.134353,8.04,2.71499 +879,insert,691.964,-1.0,-1.0,0.0,2.83636 +880,search,5473.75,0.88506,0.134079,3421.21,2.83566 +881,insert,1214.38,-1.0,-1.0,0.0,3.06275 +882,search,5675.83,0.88673,0.13301,4541.31,3.06151 +883,insert,1093.8,-1.0,-1.0,0.0,3.26334 +884,search,5849.48,0.88986,0.130746,4090.81,3.26281 +885,insert,25.147,-1.0,-1.0,0.0,3.26693 +886,search,5847.21,0.88991,0.130566,3388.24,3.26724 +887,insert,723.721,-1.0,-1.0,0.0,3.42099 +888,search,5875.58,0.89071,0.130779,4293.71,3.42124 +889,insert,722.376,-1.0,-1.0,0.0,3.60557 +890,search,6074.72,0.89243,0.13007,4877.87,3.60723 +891,insert,79.219,-1.0,-1.0,0.0,3.61987 +892,search,5913.9,0.8924,0.130168,3014.45,3.61916 +893,insert,131.396,-1.0,-1.0,0.0,3.64781 +894,search,5912.63,0.8927,0.130056,3538.32,3.64516 +895,insert,130.193,-1.0,-1.0,0.0,3.6675 +896,search,5943.87,0.89235,0.130308,9.403,3.6675 +897,delete,86.891,-1.0,-1.0,0.0,3.65521 +898,search,5937.36,0.89187,0.13054,8.829,3.65521 +899,delete,16.123,-1.0,-1.0,0.0,3.65498 +900,search,5873.52,0.89176,0.130647,8.754,3.65498 +901,delete,60.3,-1.0,-1.0,0.0,3.64741 +902,search,5839.46,0.89083,0.13119,8.753,3.64741 +903,delete,70.712,-1.0,-1.0,0.0,3.62754 +904,search,5779.23,0.88957,0.13132,8.638,3.62754 +905,delete,31.806,-1.0,-1.0,0.0,3.62418 +906,search,5634.8,0.88955,0.131258,10.431,3.62418 +907,delete,44.043,-1.0,-1.0,0.0,3.62065 +908,search,5817.26,0.88899,0.131522,9.36,3.62065 +909,delete,60.026,-1.0,-1.0,0.0,3.61659 +910,search,5684.07,0.88891,0.131766,8.313,3.61659 +911,delete,88.552,-1.0,-1.0,0.0,3.60887 +912,search,5854.99,0.88791,0.132469,9.166,3.60887 +913,delete,58.165,-1.0,-1.0,0.0,3.6 +914,search,5854.82,0.88842,0.132621,9.029,3.6 +915,delete,86.736,-1.0,-1.0,0.0,3.58595 +916,search,5841.65,0.88754,0.13338,8.928,3.58595 +917,delete,20.128,-1.0,-1.0,0.0,3.58574 +918,search,5750.84,0.88712,0.133625,5.751,3.58574 +919,delete,19.075,-1.0,-1.0,0.0,3.58368 +920,search,5615.77,0.88692,0.133694,8.637,3.58368 +921,delete,28.899,-1.0,-1.0,0.0,3.57837 +922,search,5554.98,0.88649,0.13402,8.922,3.57837 +923,delete,72.263,-1.0,-1.0,0.0,3.56558 +924,search,5556.02,0.88592,0.134411,8.447,3.56558 +925,delete,186.892,-1.0,-1.0,0.0,3.5162 +926,search,5574.52,0.88383,0.136458,8.548,3.5162 +927,delete,21.586,-1.0,-1.0,0.0,3.51463 +928,search,5421.46,0.88295,0.137095,8.21,3.51463 +929,delete,45.637,-1.0,-1.0,0.0,3.51001 +930,search,5632.87,0.88294,0.136991,9.38,3.51001 +931,delete,626.369,-1.0,-1.0,0.0,3.37267 +932,search,5262,0.88024,0.139908,6.419,3.37267 +933,delete,32.753,-1.0,-1.0,0.0,3.35932 +934,search,5261.25,0.87985,0.140319,9.002,3.35932 +935,delete,31.753,-1.0,-1.0,0.0,3.35205 +936,search,5302.32,0.87976,0.140327,8.953,3.35205 +937,delete,51.074,-1.0,-1.0,0.0,3.33867 +938,search,5218.61,0.87893,0.140567,5.848,3.33867 +939,delete,117.295,-1.0,-1.0,0.0,3.30522 +940,search,5070.9,0.87706,0.142577,8.852,3.30522 +941,delete,60.748,-1.0,-1.0,0.0,3.29361 +942,search,5206.34,0.87589,0.143312,8.685,3.29361 +943,delete,5.965,-1.0,-1.0,0.0,3.29276 +944,search,5089.78,0.87528,0.143644,8.911,3.29276 +945,delete,147.901,-1.0,-1.0,0.0,3.24265 +946,search,5073.63,0.87498,0.143788,8.754,3.24265 +947,delete,40.197,-1.0,-1.0,0.0,3.23709 +948,search,5135.96,0.87428,0.144476,8.825,3.23709 +949,delete,62.855,-1.0,-1.0,0.0,3.2194 +950,search,5005.22,0.87388,0.145077,9.451,3.2194 +951,delete,62.847,-1.0,-1.0,0.0,3.19959 +952,search,5045.04,0.87333,0.145234,9.064,3.19959 +953,delete,76.023,-1.0,-1.0,0.0,3.18954 +954,search,5010.29,0.87256,0.145546,9.795,3.18954 +955,delete,31.639,-1.0,-1.0,0.0,3.18065 +956,search,5080.68,0.87286,0.14535,9.53,3.18065 +957,delete,463.486,-1.0,-1.0,0.0,3.06202 +958,search,5028.17,0.8711,0.146812,9.32,3.06202 +959,delete,77.104,-1.0,-1.0,0.0,3.04476 +960,search,5017.82,0.86986,0.147618,9.076,3.04476 +961,delete,48.441,-1.0,-1.0,0.0,3.03563 +962,search,4927.94,0.86954,0.147887,6.084,3.03563 +963,delete,100.413,-1.0,-1.0,0.0,2.99043 +964,search,4047.98,0.8673,0.148951,5.504,2.99043 +965,delete,51.202,-1.0,-1.0,0.0,2.96299 +966,search,4047.5,0.86673,0.149177,5.397,2.96299 +967,delete,37.06,-1.0,-1.0,0.0,2.94278 +968,search,4020.02,0.86638,0.149376,5.483,2.94278 +969,delete,48.187,-1.0,-1.0,0.0,2.91184 +970,search,3979.33,0.86513,0.149958,5.448,2.91184 +971,delete,224.282,-1.0,-1.0,0.0,2.78632 +972,search,3896.36,0.86315,0.151453,5.555,2.78632 +973,delete,194.742,-1.0,-1.0,0.0,2.66581 +974,search,4672.31,0.86071,0.153874,7.428,2.66581 +975,delete,128.406,-1.0,-1.0,0.0,2.59885 +976,search,4445.22,0.85621,0.157798,7.669,2.59885 +977,delete,74.889,-1.0,-1.0,0.0,2.56284 +978,search,4440.25,0.85586,0.157831,10.214,2.56284 +979,delete,29.839,-1.0,-1.0,0.0,2.55741 +980,search,4432.47,0.85496,0.15842,9.554,2.55741 +981,delete,182.804,-1.0,-1.0,0.0,2.50316 +982,search,4552.03,0.85404,0.158745,9.578,2.50316 +983,delete,71.322,-1.0,-1.0,0.0,2.48808 +984,search,4427.2,0.8536,0.159241,9.359,2.48808 +985,delete,72.74,-1.0,-1.0,0.0,2.47091 +986,search,4510.24,0.85185,0.160648,9.164,2.47091 +987,delete,76.914,-1.0,-1.0,0.0,2.45173 +988,search,4482.64,0.85137,0.161231,9.179,2.45173 +989,delete,32.73,-1.0,-1.0,0.0,2.44432 +990,search,4478.85,0.85115,0.161431,8.745,2.44432 +991,delete,110.526,-1.0,-1.0,0.0,2.40929 +992,search,4464.43,0.85514,0.156573,9.055,2.40929 +993,delete,146.108,-1.0,-1.0,0.0,2.37182 +994,search,4534.73,0.85466,0.156757,6.078,2.37182 +995,delete,71.242,-1.0,-1.0,0.0,2.33899 +996,search,4548.5,0.85297,0.158104,8.553,2.33899 +997,delete,48.196,-1.0,-1.0,0.0,2.3246 +998,search,4897.73,0.85288,0.158189,8.954,2.3246 +999,delete,184.913,-1.0,-1.0,0.0,2.2681 +1000,search,4842.4,0.85191,0.158325,8.652,2.2681 +1001,delete,192.119,-1.0,-1.0,0.0,2.2139 +1002,search,4721.6,0.85083,0.159079,8.838,2.2139 +1003,delete,77.874,-1.0,-1.0,0.0,2.19173 +1004,search,4838.87,0.85056,0.159448,9.657,2.19173 +1005,delete,44.494,-1.0,-1.0,0.0,2.18165 +1006,search,4937.46,0.84988,0.159804,9.346,2.18165 +1007,delete,301.365,-1.0,-1.0,0.0,2.08453 +1008,search,4822.04,0.84812,0.160736,9.198,2.08453 +1009,delete,499.087,-1.0,-1.0,0.0,1.93794 +1010,search,4658.6,0.84779,0.160149,10.54,1.93794 +1011,delete,685.317,-1.0,-1.0,0.0,1.75694 +1012,search,4474.97,0.84243,0.162592,9.138,1.75694 +1013,delete,21.119,-1.0,-1.0,0.0,1.75079 +1014,search,4433.93,0.84233,0.162589,9.529,1.75079 +1015,delete,421.241,-1.0,-1.0,0.0,1.61054 +1016,search,5154.16,0.83983,0.163054,11.564,1.61054 +1017,delete,574.61,-1.0,-1.0,0.0,1.45285 +1018,search,4367.08,0.83867,0.163142,9.368,1.45285 +1019,delete,35.649,-1.0,-1.0,0.0,1.4406 +1020,search,4386.8,0.83855,0.162983,8.659,1.4406 +1021,delete,186.004,-1.0,-1.0,0.0,1.3834 +1022,search,4277.7,0.83628,0.164337,9.157,1.3834 +1023,delete,141.716,-1.0,-1.0,0.0,1.34051 +1024,search,4195.5,0.83442,0.165907,8.794,1.34051 +1025,insert,67.183,-1.0,-1.0,0.0,1.34772 +1026,search,4297.26,0.83477,0.165493,8.254,1.34772 +1027,insert,413.522,-1.0,-1.0,0.0,1.39918 +1028,search,4523.74,0.837,0.164666,9.98,1.39918 +1029,insert,1118.95,-1.0,-1.0,0.0,1.53221 +1030,search,4108.91,0.84216,0.16135,2500.39,1.53226 +1031,insert,1865.01,-1.0,-1.0,0.0,1.7822 +1032,search,4337.89,0.84574,0.161938,3319.96,1.78073 +1033,insert,1154.67,-1.0,-1.0,0.0,1.94648 +1034,search,4440.96,0.84954,0.160537,2606.85,1.94561 +1035,insert,686.512,-1.0,-1.0,0.0,2.05195 +1036,search,4509.97,0.84966,0.161371,2586.75,2.05221 +1037,insert,28.736,-1.0,-1.0,0.0,2.05679 +1038,search,4401.76,0.84996,0.161042,8.771,2.05679 +1039,insert,151.903,-1.0,-1.0,0.0,2.08479 +1040,search,4538.91,0.85054,0.160797,8.736,2.08479 +1041,insert,16.031,-1.0,-1.0,0.0,2.08751 +1042,search,4460.23,0.85016,0.161333,10.195,2.08751 +1043,insert,56.116,-1.0,-1.0,0.0,2.09493 +1044,search,4477.3,0.85032,0.161079,9.8,2.09493 +1045,insert,128.893,-1.0,-1.0,0.0,2.11048 +1046,search,4519.2,0.85158,0.16001,9.751,2.11048 +1047,insert,10.622,-1.0,-1.0,0.0,2.1117 +1048,search,4394.12,0.85102,0.16032,9.427,2.1117 +1049,insert,45.545,-1.0,-1.0,0.0,2.11761 +1050,search,4475.89,0.8511,0.160335,10.652,2.11761 +1051,insert,290.453,-1.0,-1.0,0.0,2.15328 +1052,search,4422.84,0.85199,0.159646,9.269,2.15328 +1053,insert,138.916,-1.0,-1.0,0.0,2.17072 +1054,search,4463.37,0.85218,0.159523,8.968,2.17072 +1055,insert,89.333,-1.0,-1.0,0.0,2.18452 +1056,search,4422.53,0.85294,0.158907,8.779,2.18452 +1057,insert,16.084,-1.0,-1.0,0.0,2.1864 +1058,search,4292.78,0.85237,0.159239,8.835,2.1864 +1059,insert,56.422,-1.0,-1.0,0.0,2.1913 +1060,search,4295.49,0.85274,0.159064,8.497,2.1913 +1061,insert,23.715,-1.0,-1.0,0.0,2.1948 +1062,search,4311.73,0.85285,0.159017,8.822,2.1948 +1063,insert,33.236,-1.0,-1.0,0.0,2.19881 +1064,search,4280.13,0.85218,0.159498,8.649,2.19881 +1065,insert,773.199,-1.0,-1.0,0.0,2.31456 +1066,search,4372.74,0.85367,0.158251,2790.62,2.31483 +1067,insert,11.875,-1.0,-1.0,0.0,2.31714 +1068,search,4347.21,0.85365,0.158007,8.901,2.31714 +1069,insert,90.968,-1.0,-1.0,0.0,2.33183 +1070,search,4352.5,0.85448,0.157106,9.307,2.33183 +1071,insert,68.97,-1.0,-1.0,0.0,2.34138 +1072,search,4359.06,0.8545,0.157207,9.442,2.34138 +1073,insert,76.055,-1.0,-1.0,0.0,2.34905 +1074,search,4296.23,0.85373,0.157726,9.001,2.34905 +1075,insert,31.955,-1.0,-1.0,0.0,2.35244 +1076,search,4345.81,0.85403,0.157443,8.606,2.35244 +1077,insert,75.621,-1.0,-1.0,0.0,2.36249 +1078,search,4346.85,0.85408,0.157559,8.878,2.36249 +1079,insert,68.06,-1.0,-1.0,0.0,2.36937 +1080,search,4315.77,0.85379,0.157635,8.937,2.36937 +1081,insert,1043.3,-1.0,-1.0,0.0,2.51008 +1082,search,4412,0.85588,0.157168,8.787,2.51008 +1083,insert,1.696,-1.0,-1.0,0.0,2.51043 +1084,search,4400.68,0.85586,0.157202,8.749,2.51043 +1085,insert,55.342,-1.0,-1.0,0.0,2.51919 +1086,search,4382.35,0.8562,0.156887,8.383,2.51919 +1087,insert,31.023,-1.0,-1.0,0.0,2.52385 +1088,search,4361.04,0.85641,0.156856,8.458,2.52385 +1089,insert,951.639,-1.0,-1.0,0.0,2.68921 +1090,search,4506.16,0.85922,0.154965,3679.32,2.69184 +1091,insert,116.785,-1.0,-1.0,0.0,2.71377 +1092,search,4536.92,0.86075,0.153837,2849.45,2.71456 +1093,insert,57.128,-1.0,-1.0,0.0,2.72434 +1094,search,4512.56,0.86031,0.15394,9.224,2.72434 +1095,insert,1008.01,-1.0,-1.0,0.0,2.89405 +1096,search,4495.01,0.86289,0.152949,3056.82,2.89374 +1097,insert,53.305,-1.0,-1.0,0.0,2.90153 +1098,search,4437.25,0.86285,0.153118,8.746,2.90153 +1099,insert,26.241,-1.0,-1.0,0.0,2.90558 +1100,search,4399.68,0.86245,0.153379,8.434,2.90558 +1101,insert,112.315,-1.0,-1.0,0.0,2.92325 +1102,search,4603.35,0.86302,0.153121,8.775,2.92325 +1103,insert,209.149,-1.0,-1.0,0.0,2.96313 +1104,search,4583.48,0.86541,0.151755,8.806,2.96313 +1105,insert,1.853,-1.0,-1.0,0.0,2.96337 +1106,search,4460.47,0.8654,0.151768,8.117,2.96337 +1107,insert,514.896,-1.0,-1.0,0.0,3.05956 +1108,search,4639.09,0.86724,0.15002,3601.57,3.06008 +1109,insert,14.304,-1.0,-1.0,0.0,3.06263 +1110,search,4614.89,0.86634,0.150637,8.157,3.06263 +1111,insert,97.494,-1.0,-1.0,0.0,3.0776 +1112,search,4529.16,0.86665,0.150321,9.51,3.0776 +1113,insert,126.464,-1.0,-1.0,0.0,3.10023 +1114,search,4683.18,0.86697,0.150288,9.221,3.10023 +1115,insert,174.275,-1.0,-1.0,0.0,3.12912 +1116,search,4676.36,0.86692,0.150054,9.264,3.12912 +1117,insert,50.009,-1.0,-1.0,0.0,3.13756 +1118,search,4545.07,0.86704,0.14995,3350.72,3.13722 +1119,insert,24.514,-1.0,-1.0,0.0,3.14077 +1120,search,4564.93,0.86664,0.150209,8.701,3.14077 +1121,insert,76.866,-1.0,-1.0,0.0,3.15244 +1122,search,4432.6,0.8661,0.15062,8.452,3.15244 +1123,insert,55.642,-1.0,-1.0,0.0,3.16084 +1124,search,4664.56,0.8661,0.150746,7.949,3.16084 +1125,insert,25.855,-1.0,-1.0,0.0,3.16443 +1126,search,4636.11,0.8652,0.151598,8.157,3.16443 +1127,insert,131.611,-1.0,-1.0,0.0,3.18547 +1128,search,4560.39,0.86541,0.151774,8.122,3.18547 +1129,insert,29.586,-1.0,-1.0,0.0,3.18998 +1130,search,4617.82,0.86541,0.151794,8.11,3.18998 +1131,insert,103.283,-1.0,-1.0,0.0,3.2014 +1132,search,4599.8,0.8647,0.152286,7.968,3.2014 +1133,insert,46.548,-1.0,-1.0,0.0,3.20834 +1134,search,4617.07,0.86475,0.152255,7.859,3.20834 +1135,insert,10.276,-1.0,-1.0,0.0,3.20963 +1136,search,4630.29,0.86474,0.152258,9.068,3.20963 +1137,insert,72.954,-1.0,-1.0,0.0,3.21792 +1138,search,4631.75,0.86408,0.152698,3669.24,3.21766 +1139,insert,8.587,-1.0,-1.0,0.0,3.21954 +1140,search,4628.07,0.86405,0.152733,8.765,3.21954 +1141,insert,624.502,-1.0,-1.0,0.0,3.31071 +1142,search,4644.65,0.8638,0.153372,3418.56,3.31093 +1143,insert,77.853,-1.0,-1.0,0.0,3.3217 +1144,search,4662.99,0.86383,0.153455,8.2,3.3217 +1145,insert,18.273,-1.0,-1.0,0.0,3.32469 +1146,search,4638.81,0.86379,0.153443,8.211,3.32469 +1147,insert,61.426,-1.0,-1.0,0.0,3.33225 +1148,search,4632.6,0.86309,0.153739,7.934,3.33225 +1149,insert,7.583,-1.0,-1.0,0.0,3.33277 +1150,search,4619.58,0.86313,0.15376,7.978,3.33277 +1151,insert,21.926,-1.0,-1.0,0.0,3.33676 +1152,search,4610.75,0.86315,0.153759,7.994,3.33676 +1153,delete,32.136,-1.0,-1.0,0.0,3.33486 +1154,search,4570.31,0.8624,0.154224,8.05,3.33486 +1155,delete,161.426,-1.0,-1.0,0.0,3.31668 +1156,search,4537.55,0.86151,0.154566,7.794,3.31668 +1157,delete,331.696,-1.0,-1.0,0.0,3.25435 +1158,search,4421.35,0.8595,0.155665,9.074,3.25435 +1159,delete,480.553,-1.0,-1.0,0.0,3.09651 +1160,search,4290.38,0.8571,0.155769,8.999,3.09651 +1161,delete,323.204,-1.0,-1.0,0.0,3.01376 +1162,search,4215.62,0.85492,0.15688,8.457,3.01376 +1163,delete,174.01,-1.0,-1.0,0.0,2.96792 +1164,search,4187.2,0.85388,0.157354,8.201,2.96792 +1165,delete,45.716,-1.0,-1.0,0.0,2.96335 +1166,search,4162.81,0.85349,0.157448,8.555,2.96335 +1167,delete,96.228,-1.0,-1.0,0.0,2.94549 +1168,search,4139.13,0.85316,0.157258,8.218,2.94549 +1169,delete,31.737,-1.0,-1.0,0.0,2.93301 +1170,search,4116.07,0.85073,0.158251,8.321,2.93301 +1171,delete,46.528,-1.0,-1.0,0.0,2.92267 +1172,search,4095.52,0.85032,0.158658,10.772,2.92267 +1173,delete,55.891,-1.0,-1.0,0.0,2.91458 +1174,search,4057.69,0.84885,0.159779,8.42,2.91458 +1175,delete,23.075,-1.0,-1.0,0.0,2.91149 +1176,search,4050.94,0.8485,0.160167,7.767,2.91149 +1177,delete,19.627,-1.0,-1.0,0.0,2.90696 +1178,search,4028.93,0.84853,0.160112,7.797,2.90696 +1179,delete,74.17,-1.0,-1.0,0.0,2.889 +1180,search,3982.12,0.84731,0.160971,7.883,2.889 +1181,delete,58.276,-1.0,-1.0,0.0,2.87272 +1182,search,3961.72,0.84658,0.161702,7.862,2.87272 +1183,delete,62.292,-1.0,-1.0,0.0,2.85506 +1184,search,3948.22,0.84469,0.163316,7.85,2.85506 +1185,delete,28.765,-1.0,-1.0,0.0,2.84884 +1186,search,3972.6,0.84443,0.163476,8.963,2.84884 +1187,delete,196.841,-1.0,-1.0,0.0,2.81111 +1188,search,3922.23,0.84326,0.164508,8.321,2.81111 +1189,delete,23.199,-1.0,-1.0,0.0,2.80537 +1190,search,3896.89,0.84238,0.164976,8.657,2.80537 +1191,delete,22.773,-1.0,-1.0,0.0,2.80077 +1192,search,3892.02,0.84227,0.165017,8.451,2.80077 +1193,delete,321.021,-1.0,-1.0,0.0,2.70979 +1194,search,3864.4,0.84167,0.165523,8.035,2.70979 +1195,delete,28.838,-1.0,-1.0,0.0,2.70548 +1196,search,3820.93,0.84064,0.166421,8.315,2.70548 +1197,delete,87.099,-1.0,-1.0,0.0,2.68358 +1198,search,3796.31,0.83932,0.167531,7.984,2.68358 +1199,delete,18.117,-1.0,-1.0,0.0,2.67841 +1200,search,3753.97,0.83835,0.168165,8.23,2.67841 +1201,delete,104.701,-1.0,-1.0,0.0,2.64836 +1202,search,3750.17,0.83764,0.167974,7.936,2.64836 +1203,delete,42.191,-1.0,-1.0,0.0,2.63965 +1204,search,3732.79,0.83701,0.168803,7.695,2.63965 +1205,delete,32.237,-1.0,-1.0,0.0,2.62925 +1206,search,3728.64,0.83598,0.169301,7.864,2.62925 +1207,delete,31.942,-1.0,-1.0,0.0,2.61917 +1208,search,3712.29,0.83554,0.169323,7.793,2.61917 +1209,delete,355.419,-1.0,-1.0,0.0,2.50672 +1210,search,3616.35,0.83331,0.17004,7.944,2.50672 +1211,delete,8.816,-1.0,-1.0,0.0,2.50441 +1212,search,3611.67,0.83332,0.170126,7.711,2.50441 +1213,delete,152.668,-1.0,-1.0,0.0,2.45867 +1214,search,3606.39,0.83198,0.170747,7.859,2.45867 +1215,delete,29.287,-1.0,-1.0,0.0,2.45003 +1216,search,3580.23,0.8309,0.171563,7.747,2.45003 +1217,delete,469.425,-1.0,-1.0,0.0,2.28733 +1218,search,3505.1,0.82607,0.174743,8.718,2.28733 +1219,delete,153.687,-1.0,-1.0,0.0,2.24073 +1220,search,3453.08,0.82209,0.177582,7.998,2.24073 +1221,delete,58.594,-1.0,-1.0,0.0,2.2248 +1222,search,3446.45,0.82154,0.178002,7.999,2.2248 +1223,delete,314.14,-1.0,-1.0,0.0,2.11667 +1224,search,3674.33,0.81822,0.179098,7.861,2.11667 +1225,delete,60.23,-1.0,-1.0,0.0,2.09987 +1226,search,3672.06,0.81683,0.17977,7.667,2.09987 +1227,delete,114.421,-1.0,-1.0,0.0,2.06556 +1228,search,3646.77,0.8165,0.179851,7.567,2.06556 +1229,delete,133.483,-1.0,-1.0,0.0,2.02945 +1230,search,3567.58,0.8147,0.180757,8.264,2.02945 +1231,delete,117.354,-1.0,-1.0,0.0,1.99008 +1232,search,3572.55,0.81033,0.183863,7.966,1.99008 +1233,delete,24.795,-1.0,-1.0,0.0,1.98462 +1234,search,3550.76,0.81021,0.183862,8.425,1.98462 +1235,delete,235.574,-1.0,-1.0,0.0,1.90479 +1236,search,3492.4,0.80746,0.185277,8.022,1.90479 +1237,delete,24.324,-1.0,-1.0,0.0,1.89726 +1238,search,3486.61,0.80732,0.185258,8.057,1.89726 +1239,delete,85.468,-1.0,-1.0,0.0,1.87582 +1240,search,3475.77,0.80619,0.185781,8.277,1.87582 +1241,delete,57.912,-1.0,-1.0,0.0,1.85745 +1242,search,3468.89,0.80444,0.18703,7.91,1.85745 +1243,delete,71.36,-1.0,-1.0,0.0,1.83406 +1244,search,3447,0.80337,0.187796,8.087,1.83406 +1245,delete,21.338,-1.0,-1.0,0.0,1.82723 +1246,search,3424.46,0.80209,0.188571,7.633,1.82723 +1247,delete,82.508,-1.0,-1.0,0.0,1.79386 +1248,search,3389.62,0.81257,0.177003,7.581,1.79386 +1249,delete,58.914,-1.0,-1.0,0.0,1.77573 +1250,search,3452.09,0.81238,0.17668,7.619,1.77573 +1251,delete,46.519,-1.0,-1.0,0.0,1.76002 +1252,search,3358.75,0.81045,0.177932,8.465,1.76002 +1253,delete,35.073,-1.0,-1.0,0.0,1.75171 +1254,search,3430.4,0.81051,0.17789,8.015,1.75171 +1255,delete,166.057,-1.0,-1.0,0.0,1.69047 +1256,search,3346.6,0.80753,0.17906,7.75,1.69047 +1257,delete,55.996,-1.0,-1.0,0.0,1.67352 +1258,search,3263.9,0.80705,0.179135,7.776,1.67352 +1259,delete,88.586,-1.0,-1.0,0.0,1.65014 +1260,search,3339.92,0.80672,0.179259,7.649,1.65014 +1261,delete,36.982,-1.0,-1.0,0.0,1.64069 +1262,search,3334.46,0.80516,0.179967,8.082,1.64069 +1263,delete,163.63,-1.0,-1.0,0.0,1.58904 +1264,search,3229.1,0.80486,0.179714,9.626,1.58904 +1265,delete,407.377,-1.0,-1.0,0.0,1.46964 +1266,search,3290.99,0.80132,0.18179,8.041,1.46964 +1267,delete,133.411,-1.0,-1.0,0.0,1.4352 +1268,search,3266.01,0.79971,0.182844,7.885,1.4352 +1269,delete,226.382,-1.0,-1.0,0.0,1.36775 +1270,search,3244.92,0.79943,0.182201,8.015,1.36775 +1271,delete,125.233,-1.0,-1.0,0.0,1.33322 +1272,search,3217.08,0.79724,0.182956,7.731,1.33322 +1273,delete,189.962,-1.0,-1.0,0.0,1.27725 +1274,search,3124.91,0.79615,0.18324,8.008,1.27725 +1275,delete,34.567,-1.0,-1.0,0.0,1.26662 +1276,search,3187.09,0.79548,0.183411,7.486,1.26662 +1277,delete,53.252,-1.0,-1.0,0.0,1.25059 +1278,search,3185.27,0.79393,0.18395,7.772,1.25059 +1279,delete,45.265,-1.0,-1.0,0.0,1.23757 +1280,search,3161.14,0.79329,0.18385,7.587,1.23757 diff --git a/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.1_0.01.csv b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.1_0.01.csv new file mode 100644 index 00000000..5b36e1d9 --- /dev/null +++ b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.1_0.01.csv @@ -0,0 +1,20 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,mainteance_ms,index_mem_gb,num_partitions +2,search,3046.98,0.84693,0.134675,2.186,0.0183474,512 +3,insert,14.156,-1.0,-1.0,3.004,0.0223861,512 +4,search,2186.43,0.85904,0.131452,2.242,0.0223861,512 +5,insert,15.789,-1.0,-1.0,258.383,0.0328126,512 +6,search,2233.5,0.9098,0.11532,443.456,0.0328126,512 +7,insert,21.474,-1.0,-1.0,363.255,0.0423292,512 +8,search,2288.37,0.89642,0.124485,385.37,0.0423292,512 +9,insert,2.728,-1.0,-1.0,273.156,0.044075,512 +10,search,2309.24,0.90145,0.121325,56.822,0.0441819,513 +11,insert,8.477,-1.0,-1.0,2.087,0.0495871,513 +12,search,2336.46,0.91057,0.11373,1227.26,0.0495871,513 +13,insert,16.719,-1.0,-1.0,102.781,0.0604693,515 +14,search,2275.27,0.90893,0.115698,105.085,0.0605239,517 +15,insert,18.723,-1.0,-1.0,2.137,0.0687174,517 +16,search,2379.33,0.91358,0.111333,94.063,0.0688859,518 +17,insert,3.892,-1.0,-1.0,84.857,0.0712152,519 +18,search,2447.64,0.9167,0.109808,92.406,0.0710169,520 +19,insert,32.769,-1.0,-1.0,135.536,0.0784119,521 +20,search,2493.74,0.92503,0.102272,109.37,0.0780337,522 diff --git a/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.2_0.1.csv b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.2_0.1.csv new file mode 100644 index 00000000..398664eb --- /dev/null +++ b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.2_0.1.csv @@ -0,0 +1,469 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,mainteance_ms,index_mem_gb,num_partitions +2,search,3052.35,0.84693,0.134675,2.217,0.0183474,512 +3,insert,14.282,-1.0,-1.0,3.021,0.0223861,512 +4,search,2283.72,0.85904,0.131452,2.216,0.0223861,512 +5,insert,14.95,-1.0,-1.0,258.384,0.0328126,512 +6,search,2255.33,0.9098,0.11532,348.912,0.0328126,512 +7,insert,19.231,-1.0,-1.0,256.457,0.0423292,512 +8,search,2350.71,0.89642,0.124485,57.375,0.0427433,513 +9,insert,2.791,-1.0,-1.0,2.168,0.0441937,513 +10,search,2494.97,0.90505,0.117919,77.743,0.04443,514 +11,insert,6.965,-1.0,-1.0,96.058,0.0489257,515 +12,search,2473.79,0.91055,0.114017,89.258,0.0489114,517 +13,insert,17.309,-1.0,-1.0,2.193,0.0602538,517 +14,search,2367.95,0.91146,0.113818,2.389,0.0602538,517 +15,insert,16.701,-1.0,-1.0,99.638,0.0690209,518 +16,search,2371.72,0.92203,0.104381,114.965,0.0689206,520 +17,insert,8.23,-1.0,-1.0,114.55,0.0709671,521 +18,search,2407.12,0.93166,0.0964676,113.601,0.0708729,522 +19,insert,36.7,-1.0,-1.0,123.103,0.0777375,524 +20,search,2524.69,0.93605,0.0926929,105.129,0.0774755,525 +21,insert,248.299,-1.0,-1.0,347.905,0.197139,533 +22,search,3098.35,0.93738,0.089284,240.915,0.196603,539 +23,insert,311.149,-1.0,-1.0,701.58,0.320968,550 +24,search,3763.64,0.9407,0.0865359,653.624,0.322969,564 +25,insert,34.842,-1.0,-1.0,226.891,0.340715,565 +26,search,3675.66,0.93489,0.0911314,457.782,0.341604,581 +27,insert,24.565,-1.0,-1.0,320.225,0.35472,583 +28,search,3831.67,0.93092,0.0935963,276.442,0.355065,584 +29,insert,47.836,-1.0,-1.0,409.284,0.380826,588 +30,search,3661.84,0.93314,0.0922327,293.379,0.380524,591 +31,insert,263.561,-1.0,-1.0,983.444,0.543114,623 +32,search,3744.25,0.92264,0.100038,651.387,0.543589,643 +33,insert,22.17,-1.0,-1.0,370.234,0.566268,648 +34,search,3704.13,0.9145,0.106455,424.587,0.565921,654 +35,insert,12.771,-1.0,-1.0,296.185,0.58143,655 +36,search,3756.2,0.9156,0.106257,2.038,0.58143,655 +37,insert,20.379,-1.0,-1.0,379.895,0.60236,658 +38,search,4105.65,0.9135,0.109151,460.824,0.603224,659 +39,insert,6.857,-1.0,-1.0,469.649,0.607435,660 +40,search,4797.99,0.9148,0.108053,491.135,0.607466,661 +41,insert,40.07,-1.0,-1.0,500.578,0.620233,663 +42,search,4873.97,0.91496,0.107617,489.722,0.620301,664 +43,insert,9.937,-1.0,-1.0,472.886,0.624885,665 +44,search,4661.96,0.91463,0.107989,3.232,0.624885,665 +45,insert,269.159,-1.0,-1.0,658.811,0.639031,671 +46,search,4890.71,0.91339,0.108861,3.238,0.639031,671 +47,insert,238.904,-1.0,-1.0,1291.38,0.73905,686 +48,search,5221.34,0.91545,0.107941,851.538,0.741011,696 +49,insert,31.528,-1.0,-1.0,654.285,0.758237,699 +50,search,4529.81,0.91473,0.109044,494.586,0.757864,703 +51,insert,132.057,-1.0,-1.0,1559.14,0.893247,734 +52,search,4427.44,0.91232,0.111366,804.942,0.891891,750 +53,insert,21.26,-1.0,-1.0,577.861,0.910586,753 +54,search,4404.74,0.90858,0.113416,586.523,0.911836,757 +55,insert,125.001,-1.0,-1.0,1776.06,1.05273,789 +56,search,4608.43,0.91628,0.110746,936.412,1.05106,802 +57,insert,13.789,-1.0,-1.0,639.656,1.05947,804 +58,search,5009.62,0.91468,0.111657,888.131,1.05992,805 +59,insert,205.321,-1.0,-1.0,2174.54,1.15633,826 +60,search,5927,0.90928,0.11718,962.293,1.15565,828 +61,insert,11.376,-1.0,-1.0,4.373,1.16078,828 +62,search,5763.05,0.90895,0.117372,5.126,1.16078,828 +63,insert,3.814,-1.0,-1.0,4.649,1.1617,828 +64,search,5581.48,0.90894,0.117334,5.087,1.1617,828 +65,insert,111.66,-1.0,-1.0,1263.49,1.19418,836 +66,search,5499.45,0.90933,0.117046,5.138,1.19418,836 +67,insert,36.671,-1.0,-1.0,1006.52,1.21327,842 +68,search,5791.25,0.91032,0.116578,5.819,1.21327,842 +69,insert,58.083,-1.0,-1.0,1096.69,1.23725,850 +70,search,5703.49,0.91142,0.115672,1248.27,1.23743,852 +71,insert,17.695,-1.0,-1.0,1177.65,1.24374,853 +72,search,5995.16,0.91111,0.11604,5.842,1.24374,853 +73,insert,8.156,-1.0,-1.0,5.4,1.24732,853 +74,search,5964.59,0.9112,0.115993,5.956,1.24732,853 +75,insert,23.489,-1.0,-1.0,5.636,1.25521,853 +76,search,6199.02,0.91155,0.115585,5.899,1.25521,853 +77,insert,30.555,-1.0,-1.0,983.17,1.26545,854 +78,search,5287.02,0.91064,0.116364,4.177,1.26545,854 +79,insert,354.395,-1.0,-1.0,777.317,1.29489,855 +80,search,5350.82,0.91039,0.116305,4.391,1.29489,855 +81,insert,32.461,-1.0,-1.0,777.546,1.30896,857 +82,search,4440.17,0.91125,0.114905,4.217,1.30896,857 +83,insert,7.25,-1.0,-1.0,810.006,1.31255,860 +84,search,4484.46,0.9107,0.115327,4.163,1.31255,860 +85,insert,16.15,-1.0,-1.0,911.038,1.32862,865 +86,search,4481.53,0.91193,0.114554,797.126,1.32849,867 +87,insert,187.904,-1.0,-1.0,3042.79,1.51841,920 +88,search,4912.97,0.91393,0.111865,1409.96,1.521,943 +89,insert,160.052,-1.0,-1.0,1997.85,1.67828,981 +90,search,5286.75,0.91467,0.110616,1337.33,1.67712,995 +91,insert,13.936,-1.0,-1.0,997.995,1.69163,997 +92,search,5455.93,0.91434,0.111025,1125.78,1.69192,1003 +93,insert,124.985,-1.0,-1.0,1963.63,1.79726,1024 +94,search,5895.55,0.91415,0.111497,1199.42,1.7972,1032 +95,insert,5.701,-1.0,-1.0,1044.99,1.80288,1033 +96,search,5896.65,0.9146,0.111024,1057.3,1.80322,1034 +97,insert,4.663,-1.0,-1.0,8.24,1.80683,1034 +98,search,5833.63,0.91477,0.110745,8.447,1.80683,1034 +99,insert,5.847,-1.0,-1.0,1088.66,1.80694,1035 +100,search,5799.42,0.91485,0.110668,8.309,1.80694,1035 +101,insert,22.099,-1.0,-1.0,1123.97,1.81983,1037 +102,search,5855.8,0.91494,0.110497,8.317,1.81983,1037 +103,insert,14.465,-1.0,-1.0,1150.57,1.82976,1040 +104,search,5804.81,0.91475,0.110716,1099.26,1.8295,1041 +105,insert,5.179,-1.0,-1.0,8.17,1.83331,1041 +106,search,5895.48,0.91479,0.110797,8.371,1.83331,1041 +107,insert,11.734,-1.0,-1.0,1092.05,1.83849,1042 +108,search,5975.61,0.91458,0.110844,8.252,1.83849,1042 +109,insert,197.997,-1.0,-1.0,3063.23,1.99576,1079 +110,search,6149.16,0.91548,0.11004,1553.92,1.99567,1097 +111,insert,11.732,-1.0,-1.0,1332.83,2.00801,1102 +112,search,6122.51,0.91542,0.110197,1323.44,2.00872,1106 +113,insert,5.782,-1.0,-1.0,8.88,2.0138,1106 +114,search,6141.48,0.91557,0.109872,8.942,2.0138,1106 +115,insert,19.666,-1.0,-1.0,1229.71,2.02434,1107 +116,search,6104.92,0.91617,0.10951,9.044,2.02434,1107 +117,insert,8.674,-1.0,-1.0,8.446,2.03136,1107 +118,search,6172.57,0.91634,0.108783,8.799,2.03136,1107 +119,insert,1.936,-1.0,-1.0,1244.46,2.03269,1108 +120,search,6162.88,0.91626,0.108889,8.647,2.03269,1108 +121,insert,8.075,-1.0,-1.0,1238.1,2.03781,1109 +122,search,6233.4,0.91638,0.108866,8.607,2.03781,1109 +123,insert,153.082,-1.0,-1.0,2143.24,2.15911,1131 +124,search,6420.27,0.91807,0.108131,1547.03,2.15856,1139 +125,insert,26.847,-1.0,-1.0,1583.11,2.17651,1144 +126,search,6462.16,0.91834,0.108179,1470.62,2.17642,1149 +127,insert,25.268,-1.0,-1.0,1513.76,2.19856,1153 +128,search,6399.86,0.91892,0.107972,3.87,2.19856,1153 +129,delete,18.062,-1.0,-1.0,3.497,2.19773,1153 +130,search,6444.74,0.9189,0.107963,3.876,2.19773,1153 +131,delete,4.565,-1.0,-1.0,3.408,2.19771,1153 +132,search,6383.23,0.91877,0.10812,3.772,2.19771,1153 +133,delete,11.167,-1.0,-1.0,3.341,2.19771,1153 +134,search,6483.82,0.91888,0.108127,4.116,2.19771,1153 +135,delete,14.501,-1.0,-1.0,3.386,2.1966,1153 +136,search,6469.52,0.92072,0.106066,3.73,2.1966,1153 +137,delete,2.634,-1.0,-1.0,3.742,2.1966,1153 +138,search,6475.9,0.92067,0.106148,3.546,2.1966,1153 +139,delete,5.285,-1.0,-1.0,3.357,2.19476,1153 +140,search,6483.8,0.92076,0.106168,3.72,2.19476,1153 +141,delete,12.969,-1.0,-1.0,3.371,2.19209,1153 +142,search,6375.5,0.92063,0.10625,3.822,2.19209,1153 +143,delete,9.447,-1.0,-1.0,3.507,2.19094,1153 +144,search,6353.47,0.92101,0.105893,3.852,2.19094,1153 +145,delete,1.08,-1.0,-1.0,3.494,2.18996,1153 +146,search,6277.37,0.92074,0.105992,3.957,2.18996,1153 +147,delete,4.924,-1.0,-1.0,3.46,2.18996,1153 +148,search,6196.31,0.92095,0.105987,4.169,2.18996,1153 +149,delete,116.567,-1.0,-1.0,4.018,2.15109,1153 +150,search,6173.55,0.92505,0.102518,3.965,2.15109,1153 +151,delete,99.398,-1.0,-1.0,3.947,2.11278,1153 +152,search,6065.52,0.92496,0.102772,4.007,2.11278,1153 +153,delete,25.863,-1.0,-1.0,3.64,2.09877,1153 +154,search,6035.93,0.92453,0.10304,3.883,2.09877,1153 +155,delete,16.222,-1.0,-1.0,3.569,2.09375,1153 +156,search,6064.41,0.92446,0.102959,4.155,2.09375,1153 +157,delete,29.56,-1.0,-1.0,3.696,2.08025,1153 +158,search,5924.03,0.92418,0.103226,4.068,2.08025,1153 +159,delete,172.126,-1.0,-1.0,4.003,1.96092,1153 +160,search,5785.15,0.92396,0.103193,3.961,1.96092,1153 +161,delete,27.567,-1.0,-1.0,3.837,1.94874,1153 +162,search,5698.39,0.92407,0.102953,4.005,1.94874,1153 +163,delete,14.272,-1.0,-1.0,3.92,1.94086,1153 +164,search,5659.06,0.92373,0.103241,3.915,1.94086,1153 +165,delete,18.949,-1.0,-1.0,3.666,1.9336,1153 +166,search,5626.15,0.92353,0.103286,4.16,1.9336,1153 +167,delete,4.409,-1.0,-1.0,3.66,1.93146,1153 +168,search,5551.9,0.92352,0.103261,3.884,1.93146,1153 +169,delete,9.558,-1.0,-1.0,3.691,1.92459,1153 +170,search,5584.82,0.9232,0.103599,4.065,1.92459,1153 +171,delete,11.274,-1.0,-1.0,3.64,1.92018,1153 +172,search,5505.71,0.92309,0.103802,4.301,1.92018,1153 +173,delete,31.303,-1.0,-1.0,3.813,1.9063,1153 +174,search,5489.1,0.92229,0.104104,4.112,1.9063,1153 +175,delete,105.432,-1.0,-1.0,3.971,1.83547,1153 +176,search,5490.7,0.92253,0.104055,4.076,1.83547,1153 +177,delete,15.187,-1.0,-1.0,3.74,1.82813,1153 +178,search,5503.23,0.92231,0.104239,4.169,1.82813,1153 +179,delete,143.969,-1.0,-1.0,4.117,1.72375,1153 +180,search,5271.7,0.92058,0.105481,4.099,1.72375,1153 +181,delete,20.913,-1.0,-1.0,3.865,1.71111,1153 +182,search,5206,0.92036,0.105689,4.261,1.71111,1153 +183,delete,100.301,-1.0,-1.0,3.986,1.63978,1153 +184,search,4974.46,0.91493,0.109643,4.182,1.63978,1153 +185,delete,6.688,-1.0,-1.0,3.826,1.63574,1153 +186,search,5002.32,0.91506,0.109521,4.037,1.63574,1153 +187,delete,109.541,-1.0,-1.0,4.053,1.54781,1153 +188,search,4915.33,0.91745,0.107188,4.023,1.54781,1153 +189,delete,7.404,-1.0,-1.0,3.579,1.54396,1153 +190,search,4974.13,0.91742,0.107249,4.184,1.54396,1153 +191,delete,2.242,-1.0,-1.0,3.624,1.54259,1153 +192,search,4918.8,0.91745,0.1073,4.193,1.54259,1153 +193,delete,36.596,-1.0,-1.0,3.98,1.51623,1153 +194,search,4824.22,0.91538,0.108643,4.066,1.51623,1153 +195,delete,19.516,-1.0,-1.0,3.799,1.50287,1153 +196,search,4744.93,0.91433,0.109036,4.189,1.50287,1153 +197,delete,21.643,-1.0,-1.0,3.958,1.48945,1153 +198,search,4716.06,0.91383,0.10949,4.449,1.48945,1153 +199,delete,17.216,-1.0,-1.0,3.661,1.48184,1153 +200,search,4718.11,0.91339,0.109848,4.002,1.48184,1153 +201,delete,5.336,-1.0,-1.0,3.802,1.47975,1153 +202,search,4777.82,0.91323,0.109906,4.074,1.47975,1153 +203,delete,12.722,-1.0,-1.0,3.541,1.4732,1153 +204,search,4731.87,0.91295,0.110299,4.254,1.4732,1153 +205,delete,20.778,-1.0,-1.0,3.843,1.46154,1153 +206,search,4731.49,0.91351,0.10985,3.961,1.46154,1153 +207,delete,31.372,-1.0,-1.0,4.034,1.44092,1153 +208,search,4707.48,0.91391,0.109435,4.22,1.44092,1153 +209,delete,19.623,-1.0,-1.0,3.562,1.43141,1153 +210,search,4630.42,0.91333,0.109776,4.093,1.43141,1153 +211,delete,6.917,-1.0,-1.0,3.931,1.42706,1153 +212,search,4586.97,0.91312,0.109949,4.245,1.42706,1153 +213,delete,12.798,-1.0,-1.0,3.483,1.41816,1153 +214,search,4676.84,0.91272,0.1102,4.19,1.41816,1153 +215,delete,199.513,-1.0,-1.0,3.981,1.26588,1153 +216,search,4289.06,0.90663,0.114276,4.034,1.26588,1153 +217,delete,139.719,-1.0,-1.0,3.927,1.16612,1153 +218,search,4132.07,0.90328,0.116867,4,1.16612,1153 +219,delete,15.351,-1.0,-1.0,3.696,1.15474,1153 +220,search,4048.89,0.90155,0.118081,3.932,1.15474,1153 +221,delete,123.581,-1.0,-1.0,4.231,1.0544,1153 +222,search,3947.95,0.90218,0.118364,3.88,1.0544,1153 +223,delete,7.825,-1.0,-1.0,3.438,1.04891,1153 +224,search,3919.35,0.90061,0.119375,4.146,1.04891,1153 +225,delete,3.973,-1.0,-1.0,3.474,1.04662,1153 +226,search,3917.33,0.90058,0.119389,3.921,1.04662,1153 +227,delete,2.636,-1.0,-1.0,3.582,1.04476,1153 +228,search,3935.56,0.90036,0.119487,4.563,1.04476,1153 +229,delete,31.558,-1.0,-1.0,3.643,1.03179,1153 +230,search,3951.54,0.89919,0.120048,4.035,1.03179,1153 +231,delete,10.124,-1.0,-1.0,3.516,1.02612,1153 +232,search,3909.12,0.89799,0.120694,4.35,1.02612,1153 +233,delete,5.38,-1.0,-1.0,3.452,1.02388,1153 +234,search,3891.82,0.89777,0.120781,4.309,1.02388,1153 +235,delete,8.405,-1.0,-1.0,3.569,1.02006,1153 +236,search,3883.39,0.89786,0.120734,3.899,1.02006,1153 +237,delete,151.855,-1.0,-1.0,4.116,0.910563,1153 +238,search,3741.91,0.8962,0.121527,4.095,0.910563,1153 +239,delete,15.389,-1.0,-1.0,3.978,0.89924,1153 +240,search,3691.3,0.89493,0.121918,4.134,0.89924,1153 +241,delete,7.364,-1.0,-1.0,3.8,0.895275,1153 +242,search,3620.58,0.89419,0.122376,4.14,0.895275,1153 +243,delete,15.819,-1.0,-1.0,3.844,0.883871,1153 +244,search,3694.01,0.89262,0.12356,4.144,0.883871,1153 +245,delete,8.208,-1.0,-1.0,3.556,0.879081,1153 +246,search,3659.78,0.89229,0.123617,3.943,0.879081,1153 +247,delete,2.513,-1.0,-1.0,3.514,0.8774,1153 +248,search,3628.32,0.89203,0.123873,4.02,0.8774,1153 +249,delete,5.582,-1.0,-1.0,3.748,0.872927,1153 +250,search,3639.07,0.89155,0.124305,4.07,0.872927,1153 +251,delete,150.946,-1.0,-1.0,3.911,0.760005,1153 +252,search,3503.98,0.89232,0.122071,4.142,0.760005,1153 +253,delete,19.236,-1.0,-1.0,3.661,0.744569,1153 +254,search,3434.89,0.89072,0.123222,4.109,0.744569,1153 +255,delete,23.618,-1.0,-1.0,3.785,0.725047,1153 +256,search,3430.3,0.88815,0.124611,4.071,0.725047,1153 +257,insert,5.522,-1.0,-1.0,3.491,0.726133,1153 +258,search,3422.48,0.88804,0.124748,3.863,0.726133,1153 +259,insert,16.065,-1.0,-1.0,3.846,0.732707,1153 +260,search,3439.03,0.88908,0.12451,4.072,0.732707,1153 +261,insert,1.972,-1.0,-1.0,3.489,0.732774,1153 +262,search,3418.14,0.88963,0.124277,3.994,0.732774,1153 +263,insert,83.462,-1.0,-1.0,804.713,0.77496,1154 +264,search,3586.01,0.89422,0.122164,746.743,0.776675,1156 +265,insert,41.376,-1.0,-1.0,664,0.792917,1158 +266,search,3637.9,0.89566,0.121119,580,0.79357,1162 +267,insert,15.512,-1.0,-1.0,511.373,0.804373,1165 +268,search,3625.73,0.89722,0.120483,550.332,0.804867,1166 +269,insert,161.238,-1.0,-1.0,952.831,0.889974,1169 +270,search,3956.57,0.8995,0.119162,938.492,0.890233,1171 +271,insert,3.068,-1.0,-1.0,799.571,0.8918,1172 +272,search,3835.65,0.89919,0.119464,796.361,0.892792,1176 +273,insert,12.437,-1.0,-1.0,633.129,0.900367,1177 +274,search,3843.86,0.89992,0.119355,686.219,0.900117,1182 +275,insert,225.914,-1.0,-1.0,1173.93,1.05151,1195 +276,search,4166.73,0.90099,0.120155,1025.88,1.05295,1204 +277,insert,20.736,-1.0,-1.0,755.166,1.06493,1205 +278,search,4698.92,0.90301,0.118091,828.311,1.06402,1215 +279,insert,14.709,-1.0,-1.0,3.992,1.07443,1215 +280,search,4165.15,0.90408,0.117383,4.084,1.07443,1215 +281,insert,204.643,-1.0,-1.0,1128.67,1.20099,1218 +282,search,4299.96,0.9058,0.117827,4.197,1.20099,1218 +283,insert,253.001,-1.0,-1.0,1266.41,1.38004,1229 +284,search,4756.58,0.90979,0.113743,1189.43,1.38021,1240 +285,insert,7.992,-1.0,-1.0,1230.74,1.38999,1242 +286,search,4763.79,0.90991,0.113537,1157.73,1.38867,1246 +287,insert,22.759,-1.0,-1.0,1175.87,1.40193,1248 +288,search,4746.14,0.9102,0.113198,1170.24,1.40302,1251 +289,insert,163.72,-1.0,-1.0,1658.13,1.53253,1271 +290,search,4991.18,0.91291,0.112043,1511.82,1.53292,1288 +291,insert,12.48,-1.0,-1.0,1132.97,1.54126,1289 +292,search,5074.16,0.91458,0.110165,986.55,1.54166,1291 +293,insert,9.028,-1.0,-1.0,946.037,1.54987,1292 +294,search,4976.03,0.91436,0.1107,3.936,1.54987,1292 +295,insert,135.674,-1.0,-1.0,1281.81,1.63765,1296 +296,search,5154.35,0.91618,0.110061,1228.32,1.63729,1297 +297,insert,2.566,-1.0,-1.0,1224.61,1.63849,1298 +298,search,5128.09,0.91607,0.11029,1237.42,1.63783,1300 +299,insert,40.869,-1.0,-1.0,4.58,1.66244,1300 +300,search,5221.84,0.91928,0.107226,1236.41,1.66334,1301 +301,insert,342.361,-1.0,-1.0,2219.76,1.9198,1330 +302,search,5949.96,0.93459,0.0944572,1963.29,1.92088,1348 +303,insert,30.827,-1.0,-1.0,4.143,1.94289,1348 +304,search,5897.76,0.93404,0.0949337,3.926,1.94289,1348 +305,insert,189.113,-1.0,-1.0,1823.75,2.10613,1357 +306,search,6193.41,0.93516,0.0947539,1749.5,2.10772,1367 +307,insert,15.094,-1.0,-1.0,1561.69,2.11762,1370 +308,search,6145.15,0.93549,0.0941362,1690.04,2.11715,1384 +309,insert,4.601,-1.0,-1.0,1372.15,2.12031,1385 +310,search,6245.96,0.93578,0.0939642,1411.44,2.11805,1388 +311,insert,6.952,-1.0,-1.0,4.377,2.12375,1388 +312,search,6145.32,0.93554,0.0942885,4.247,2.12375,1388 +313,insert,19.971,-1.0,-1.0,1370.89,2.1401,1389 +314,search,6205.67,0.93585,0.0941606,4.252,2.1401,1389 +315,insert,4.039,-1.0,-1.0,3.997,2.14185,1389 +316,search,6419.22,0.93569,0.094371,4.699,2.14185,1389 +317,insert,28.355,-1.0,-1.0,1478.38,2.15897,1394 +318,search,9048.61,0.93606,0.0941628,6.876,2.15897,1394 +319,insert,393.965,-1.0,-1.0,4162.67,2.37779,1429 +320,search,6079.68,0.93679,0.0933401,1922.31,2.37782,1442 +321,insert,26.174,-1.0,-1.0,1705.65,2.39405,1447 +322,search,5993.44,0.93659,0.093449,1691.63,2.39595,1449 +323,insert,56.318,-1.0,-1.0,1752.1,2.44048,1456 +324,search,6670.65,0.93745,0.0928764,1588.98,2.44089,1460 +325,insert,1009.93,-1.0,-1.0,2742.26,2.6558,1478 +326,search,7103.15,0.93947,0.0913953,2145.77,2.65662,1493 +327,insert,37.145,-1.0,-1.0,1918.03,2.68409,1498 +328,search,7141.57,0.93954,0.0911321,2061.45,2.68614,1520 +329,insert,5.497,-1.0,-1.0,1620.05,2.69066,1521 +330,search,7159.38,0.93986,0.0908158,1876.56,2.69134,1531 +331,insert,29.237,-1.0,-1.0,1705.85,2.71106,1534 +332,search,7165.61,0.93955,0.0915084,1662.81,2.71149,1535 +333,insert,2.349,-1.0,-1.0,5.757,2.71295,1535 +334,search,7256.15,0.93952,0.0914037,6.233,2.71295,1535 +335,insert,24.585,-1.0,-1.0,1738.52,2.72987,1538 +336,search,7283.3,0.93955,0.0912567,6.064,2.72987,1538 +337,insert,15.998,-1.0,-1.0,6.016,2.73887,1538 +338,search,7107.19,0.93988,0.0908077,6.136,2.73887,1538 +339,insert,32.76,-1.0,-1.0,1668.07,2.75751,1539 +340,search,7176.39,0.94001,0.0906378,5.981,2.75751,1539 +341,insert,3.654,-1.0,-1.0,1757.29,2.76077,1540 +342,search,7197.71,0.94023,0.0905341,1742.13,2.7606,1543 +343,insert,15.744,-1.0,-1.0,1743.88,2.77506,1545 +344,search,7259.57,0.94017,0.0907631,1699.47,2.77449,1547 +345,insert,20.238,-1.0,-1.0,1742.3,2.78993,1549 +346,search,7189.16,0.94045,0.0906642,6.579,2.78993,1549 +347,insert,255.264,-1.0,-1.0,4444.04,2.99918,1601 +348,search,7619.72,0.94147,0.0891616,2549.61,2.99675,1619 +349,insert,25.204,-1.0,-1.0,1930.91,3.01304,1622 +350,search,7594.81,0.9419,0.0885028,7.97,3.01304,1622 +351,insert,249.628,-1.0,-1.0,3884.1,3.19111,1646 +352,search,7823.16,0.94124,0.0893777,2516.78,3.19011,1657 +353,insert,39.868,-1.0,-1.0,2175.3,3.21129,1662 +354,search,7841.26,0.94155,0.0893639,2356.58,3.21083,1671 +355,insert,255.809,-1.0,-1.0,3238.49,3.36661,1696 +356,search,8157,0.94353,0.0873895,2757.86,3.3669,1709 +357,insert,16.516,-1.0,-1.0,2141.51,3.37405,1710 +358,search,8196.74,0.94368,0.0871058,2172.97,3.37399,1711 +359,insert,6.28,-1.0,-1.0,8.941,3.37673,1711 +360,search,8182.95,0.9436,0.087212,9.329,3.37673,1711 +361,insert,39.374,-1.0,-1.0,2239.98,3.40689,1714 +362,search,8288.73,0.94414,0.0865814,9.074,3.40689,1714 +363,insert,13.072,-1.0,-1.0,2209.23,3.41321,1715 +364,search,8192.05,0.94422,0.0865552,9.536,3.41321,1715 +365,insert,33.758,-1.0,-1.0,2251.66,3.43765,1717 +366,search,8231.47,0.94443,0.0862566,9.19,3.43765,1717 +367,insert,17.864,-1.0,-1.0,2237.19,3.45043,1720 +368,search,8278.38,0.94442,0.0862554,9.113,3.45043,1720 +369,insert,37.219,-1.0,-1.0,2301.8,3.4769,1723 +370,search,8188.26,0.94504,0.0854552,9.068,3.4769,1723 +371,insert,7.427,-1.0,-1.0,8.598,3.48077,1723 +372,search,7991.94,0.94476,0.0857052,9.132,3.48077,1723 +373,insert,33.404,-1.0,-1.0,2369.08,3.49968,1727 +374,search,8239.68,0.94514,0.0849398,8.932,3.49968,1727 +375,insert,22.641,-1.0,-1.0,2266.81,3.51311,1728 +376,search,8128.52,0.94516,0.0849293,8.904,3.51311,1728 +377,insert,38.792,-1.0,-1.0,2496.63,3.54356,1735 +378,search,8106.28,0.94565,0.084695,2304.79,3.54381,1736 +379,insert,4.017,-1.0,-1.0,8.91,3.54592,1736 +380,search,8186.97,0.94542,0.0848749,9.557,3.54592,1736 +381,insert,14.016,-1.0,-1.0,2346.47,3.5531,1738 +382,search,8403.18,0.94568,0.0846151,9.314,3.5531,1738 +383,insert,20.972,-1.0,-1.0,2439.98,3.56668,1743 +384,search,8430.89,0.94555,0.0847045,9.121,3.56668,1743 +385,delete,8.564,-1.0,-1.0,8.754,3.56653,1743 +386,search,8461.71,0.94561,0.0847053,8.891,3.56653,1743 +387,delete,11.953,-1.0,-1.0,8.658,3.56565,1743 +388,search,8441.58,0.94572,0.0847462,8.768,3.56565,1743 +389,delete,11.388,-1.0,-1.0,8.617,3.56441,1743 +390,search,8337.02,0.94547,0.0850283,9,3.56441,1743 +391,delete,48.96,-1.0,-1.0,8.779,3.54212,1743 +392,search,8217.54,0.94531,0.0852318,9.452,3.54212,1743 +393,delete,35.803,-1.0,-1.0,9.08,3.54039,1743 +394,search,8132.38,0.9451,0.0853607,9.538,3.54039,1743 +395,delete,23.113,-1.0,-1.0,8.799,3.53324,1743 +396,search,8170.82,0.94486,0.0856316,9.218,3.53324,1743 +397,delete,84.115,-1.0,-1.0,8.864,3.50095,1743 +398,search,8167.56,0.94499,0.0855088,9.068,3.50095,1743 +399,delete,3.86,-1.0,-1.0,9.159,3.50051,1743 +400,search,8188.63,0.94505,0.0854564,9.004,3.50051,1743 +401,delete,11.824,-1.0,-1.0,8.569,3.49808,1743 +402,search,8171.72,0.94511,0.0854112,9.037,3.49808,1743 +403,delete,185.28,-1.0,-1.0,9.08,3.41189,1743 +404,search,8066.22,0.94477,0.0858122,8.948,3.41189,1743 +405,delete,33.028,-1.0,-1.0,8.793,3.40359,1743 +406,search,7962.33,0.9457,0.0849701,9.02,3.40359,1743 +407,delete,57.118,-1.0,-1.0,9.011,3.38102,1743 +408,search,7736.92,0.94591,0.0847342,9.359,3.38102,1743 +409,delete,129.271,-1.0,-1.0,9.086,3.29733,1743 +410,search,7783.45,0.94611,0.0845069,9.489,3.29733,1743 +411,delete,169.371,-1.0,-1.0,9.425,3.22559,1743 +412,search,7569.65,0.94508,0.0851739,9.053,3.22559,1743 +413,delete,15.871,-1.0,-1.0,8.689,3.22075,1743 +414,search,7633.09,0.94494,0.0852703,9.107,3.22075,1743 +415,delete,50.014,-1.0,-1.0,8.974,3.20189,1743 +416,search,7564.91,0.94486,0.0853977,9.053,3.20189,1743 +417,delete,125.898,-1.0,-1.0,8.928,3.13551,1743 +418,search,7473.87,0.94499,0.0851689,9.02,3.13551,1743 +419,delete,16.642,-1.0,-1.0,8.621,3.13185,1743 +420,search,7446.49,0.94476,0.0854248,8.919,3.13185,1743 +421,delete,13.671,-1.0,-1.0,8.654,3.12316,1743 +422,search,7414.94,0.94473,0.085468,8.847,3.12316,1743 +423,delete,90.278,-1.0,-1.0,8.913,3.05712,1743 +424,search,7308.5,0.94484,0.0849506,9.036,3.05712,1743 +425,delete,16.508,-1.0,-1.0,8.582,3.05061,1743 +426,search,7247.53,0.94486,0.0850221,9.53,3.05061,1743 +427,delete,46.729,-1.0,-1.0,8.757,3.04079,1743 +428,search,7225.04,0.94413,0.0860246,9.381,3.04079,1743 +429,delete,293.791,-1.0,-1.0,9.154,2.88888,1743 +430,search,6833.66,0.938,0.0915924,9.24,2.88888,1743 +431,delete,65.129,-1.0,-1.0,8.971,2.85156,1743 +432,search,6812.9,0.93821,0.0915828,9.11,2.85156,1743 +433,delete,137.132,-1.0,-1.0,8.864,2.77189,1743 +434,search,6720.24,0.93767,0.0916961,9.095,2.77189,1743 +435,delete,77.132,-1.0,-1.0,9.056,2.72036,1743 +436,search,6633.73,0.937,0.0925581,9.081,2.72036,1743 +437,delete,10.144,-1.0,-1.0,8.645,2.71365,1743 +438,search,6637.8,0.93698,0.0926248,9.126,2.71365,1743 +439,delete,70.767,-1.0,-1.0,8.921,2.65888,1743 +440,search,6568.08,0.93669,0.0929047,9.081,2.65888,1743 +441,delete,19.925,-1.0,-1.0,8.767,2.6466,1743 +442,search,6531.15,0.93662,0.0927499,8.931,2.6466,1743 +443,delete,17.557,-1.0,-1.0,8.776,2.63339,1743 +444,search,6494.21,0.93692,0.092521,8.908,2.63339,1743 +445,delete,29.027,-1.0,-1.0,8.56,2.61487,1743 +446,search,6387.8,0.93694,0.0923465,9.129,2.61487,1743 +447,delete,228.642,-1.0,-1.0,8.962,2.45733,1743 +448,search,6272.01,0.93607,0.0926135,9.438,2.45733,1743 +449,delete,42.026,-1.0,-1.0,8.941,2.44289,1743 +450,search,6266.65,0.9356,0.0929919,9.422,2.44289,1743 +451,delete,72.88,-1.0,-1.0,8.977,2.40484,1743 +452,search,6210.45,0.93454,0.0941704,9.185,2.40484,1743 +453,delete,185.323,-1.0,-1.0,8.956,2.28499,1743 +454,search,6034.5,0.9339,0.0942232,9.106,2.28499,1743 +455,delete,32.683,-1.0,-1.0,8.747,2.26945,1743 +456,search,6051.89,0.93322,0.0946883,9.073,2.26945,1743 +457,delete,6.434,-1.0,-1.0,8.665,2.26532,1743 +458,search,6048.9,0.93316,0.0947603,9.432,2.26532,1743 +459,delete,27.906,-1.0,-1.0,8.803,2.24885,1743 +460,search,6003.24,0.93325,0.0947971,9.04,2.24885,1743 +461,delete,4.568,-1.0,-1.0,8.88,2.24742,1743 +462,search,6016.16,0.93298,0.0950661,8.971,2.24742,1743 +463,delete,37.349,-1.0,-1.0,8.815,2.22261,1743 +464,search,5912.53,0.93337,0.0947664,9.061,2.22261,1743 +465,delete,17.706,-1.0,-1.0,8.576,2.21194,1743 +466,search,5922,0.93332,0.0947816,8.997,2.21194,1743 +467,delete,24.231,-1.0,-1.0,8.554,2.1986,1743 +468,search,5873.96,0.9332,0.0950936,9.069,2.1986,1743 +469,delete,12.903,-1.0,-1.0,8.653,2.18832,1743 diff --git a/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.3.csv b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.3.csv new file mode 100644 index 00000000..039394ce --- /dev/null +++ b/scripts/big_ann_perf_numbers/scan_using_batching_250_aps_recall_0.9_search_0.3.csv @@ -0,0 +1,1280 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,mainteance_ms,index_mem_gb +2,search,3748.95,0.91201,0.102646,2.974,0.0183461 +3,insert,13.614,-1.0,-1.0,0.0,0.0223849 +4,search,3057.03,0.91992,0.0979933,23.939,0.0224592 +5,insert,36.075,-1.0,-1.0,0.0,0.0330443 +6,search,3121.34,0.95483,0.0772748,99.381,0.0329754 +7,insert,35.987,-1.0,-1.0,0.0,0.0423855 +8,search,3241.03,0.95122,0.0804345,149.885,0.0422513 +9,insert,5.164,-1.0,-1.0,0.0,0.0442505 +10,search,3346.96,0.95818,0.0725195,164.741,0.0442631 +11,insert,13.129,-1.0,-1.0,0.0,0.0495251 +12,search,3459.44,0.95472,0.0765104,202.715,0.0493472 +13,insert,32.082,-1.0,-1.0,0.0,0.0607594 +14,search,3710.29,0.94835,0.0813495,298.256,0.0605403 +15,insert,35.712,-1.0,-1.0,0.0,0.0692233 +16,search,3973.52,0.946,0.0820631,297.233,0.0692376 +17,insert,4.884,-1.0,-1.0,0.0,0.0713612 +18,search,4212.9,0.94276,0.084701,278.457,0.0712972 +19,insert,48.047,-1.0,-1.0,0.0,0.0780582 +20,search,4424.03,0.94469,0.0831914,250.772,0.0779325 +21,insert,447.638,-1.0,-1.0,0.0,0.19733 +22,search,4847.38,0.94669,0.0799093,1227.74,0.196926 +23,insert,457.186,-1.0,-1.0,0.0,0.321974 +24,search,5262.17,0.93336,0.0925517,1921.82,0.321603 +25,insert,56.324,-1.0,-1.0,0.0,0.340791 +26,search,7364.43,0.92558,0.097945,2473.26,0.340803 +27,insert,49.323,-1.0,-1.0,0.0,0.353285 +28,search,9179.81,0.92288,0.09977,1740.47,0.353604 +29,insert,91.165,-1.0,-1.0,0.0,0.380443 +30,search,9166.82,0.92855,0.096077,652.576,0.380629 +31,insert,619.055,-1.0,-1.0,0.0,0.544489 +32,search,11505.4,0.93309,0.093111,4951.6,0.544117 +33,insert,86.913,-1.0,-1.0,0.0,0.566514 +34,search,13416.8,0.93383,0.0926381,3484.35,0.566909 +35,insert,70.459,-1.0,-1.0,0.0,0.582114 +36,search,12470.2,0.93599,0.0915027,1314.85,0.582573 +37,insert,114.705,-1.0,-1.0,0.0,0.603454 +38,search,15553,0.93783,0.0899942,958.943,0.603449 +39,insert,15.146,-1.0,-1.0,0.0,0.606742 +40,search,15617.8,0.93755,0.090397,404.489,0.606713 +41,insert,65.483,-1.0,-1.0,0.0,0.621102 +42,search,15531.9,0.93842,0.0896568,796.485,0.621644 +43,insert,19.609,-1.0,-1.0,0.0,0.625649 +44,search,15454.8,0.93934,0.0891817,475.391,0.625488 +45,insert,349.031,-1.0,-1.0,0.0,0.640433 +46,search,12608.9,0.9409,0.0880706,776.315,0.640496 +47,insert,472.393,-1.0,-1.0,0.0,0.741606 +48,search,16275.3,0.94461,0.0855662,3697.76,0.740908 +49,insert,58.978,-1.0,-1.0,0.0,0.756258 +50,search,15812.3,0.94578,0.0850588,2373.69,0.756896 +51,insert,650.762,-1.0,-1.0,0.0,0.893122 +52,search,18486.4,0.95167,0.0802847,9590.41,0.893064 +53,insert,105.303,-1.0,-1.0,0.0,0.912425 +54,search,20556.1,0.95131,0.0810455,4045.01,0.912818 +55,insert,659.937,-1.0,-1.0,0.0,1.05328 +56,search,23672.2,0.95728,0.0759152,9180.75,1.05401 +57,insert,59.095,-1.0,-1.0,0.0,1.0632 +58,search,23032.5,0.95659,0.0758089,3113.05,1.06259 +59,insert,481.023,-1.0,-1.0,0.0,1.15957 +60,search,25631.6,0.95784,0.0743588,4058.99,1.15849 +61,insert,18.897,-1.0,-1.0,0.0,1.16262 +62,search,26410.8,0.95395,0.08027,2285.74,1.1637 +63,insert,5.858,-1.0,-1.0,0.0,1.16503 +64,search,26971,0.95453,0.0790186,1485.59,1.16426 +65,insert,173.27,-1.0,-1.0,0.0,1.19352 +66,search,25436.5,0.95582,0.0772086,2311.86,1.19431 +67,insert,118.377,-1.0,-1.0,0.0,1.21381 +68,search,27590.2,0.95656,0.0762358,2547.71,1.21439 +69,insert,139.229,-1.0,-1.0,0.0,1.23671 +70,search,25447,0.95742,0.0753606,1361.89,1.23639 +71,insert,41.005,-1.0,-1.0,0.0,1.24248 +72,search,29140.1,0.95763,0.0752386,1632.31,1.24257 +73,insert,21.114,-1.0,-1.0,0.0,1.24584 +74,search,28418.6,0.95757,0.0753229,1344.99,1.24576 +75,insert,77.12,-1.0,-1.0,0.0,1.25383 +76,search,28175.5,0.95768,0.0750396,1643.4,1.25361 +77,insert,93.658,-1.0,-1.0,0.0,1.2645 +78,search,28797.6,0.9576,0.0751443,989.816,1.26461 +79,insert,845.002,-1.0,-1.0,0.0,1.29217 +80,search,22415.4,0.95776,0.0749976,1373.19,1.29238 +81,insert,91.654,-1.0,-1.0,0.0,1.3053 +82,search,29191.4,0.95858,0.0737777,1392.21,1.30562 +83,insert,38.357,-1.0,-1.0,0.0,1.31189 +84,search,29792,0.95885,0.0733661,1622.57,1.31128 +85,insert,104.426,-1.0,-1.0,0.0,1.3277 +86,search,29759.9,0.9593,0.0727576,1594.69,1.32781 +87,insert,1117.91,-1.0,-1.0,0.0,1.52264 +88,search,30787.1,0.96327,0.0676942,15582.9,1.52422 +89,insert,1019.8,-1.0,-1.0,0.0,1.68107 +90,search,34169.4,0.96667,0.06386,16086.1,1.68166 +91,insert,122.495,-1.0,-1.0,0.0,1.69657 +92,search,32976.8,0.96678,0.0644734,4010.37,1.69663 +93,insert,827.294,-1.0,-1.0,0.0,1.80078 +94,search,37109.2,0.96712,0.0638375,5752.57,1.8006 +95,insert,47.05,-1.0,-1.0,0.0,1.80781 +96,search,37810.4,0.96749,0.0636266,2509.31,1.80742 +97,insert,31.976,-1.0,-1.0,0.0,1.81148 +98,search,39507,0.96762,0.0634205,1425.94,1.81135 +99,insert,27.889,-1.0,-1.0,0.0,1.81418 +100,search,40875.2,0.96765,0.0633626,1271.16,1.81424 +101,insert,132.357,-1.0,-1.0,0.0,1.82705 +102,search,40646.6,0.96791,0.0630382,1493.15,1.82677 +103,insert,43.794,-1.0,-1.0,0.0,1.83254 +104,search,39604,0.96808,0.0628118,1488.1,1.83318 +105,insert,35.447,-1.0,-1.0,0.0,1.83698 +106,search,38631.5,0.96815,0.0628426,1296.73,1.83709 +107,insert,66.249,-1.0,-1.0,0.0,1.84299 +108,search,38677.8,0.96817,0.0628348,1322.95,1.84298 +109,insert,1287.6,-1.0,-1.0,0.0,2.0006 +110,search,41152.5,0.96984,0.0606519,12820,1.99993 +111,insert,91.233,-1.0,-1.0,0.0,2.01235 +112,search,42773,0.97048,0.0598906,4296.84,2.01228 +113,insert,55.928,-1.0,-1.0,0.0,2.01788 +114,search,42800.4,0.9704,0.0599167,1953.75,2.01798 +115,insert,128.33,-1.0,-1.0,0.0,2.03031 +116,search,44993.7,0.97085,0.0594764,1777.98,2.03038 +117,insert,45.136,-1.0,-1.0,0.0,2.0356 +118,search,44865.8,0.97094,0.0592616,1508.97,2.0352 +119,insert,17.356,-1.0,-1.0,0.0,2.0371 +120,search,44654.8,0.97098,0.0593001,1473.83,2.03718 +121,insert,69.592,-1.0,-1.0,0.0,2.04358 +122,search,41579.5,0.97104,0.0591428,1523.79,2.04354 +123,insert,1108.95,-1.0,-1.0,0.0,2.16608 +124,search,41835.3,0.97211,0.0581344,5953.15,2.16666 +125,insert,198.112,-1.0,-1.0,0.0,2.1864 +126,search,44581,0.97284,0.0573126,4490.63,2.18669 +127,insert,182.593,-1.0,-1.0,0.0,2.208 +128,search,45031.5,0.97353,0.0564313,3359.55,2.20838 +129,delete,14.156,-1.0,-1.0,0.0,2.20694 +130,search,47893.3,0.9736,0.0564543,1767.94,2.20686 +131,delete,4.495,-1.0,-1.0,0.0,2.20668 +132,search,47533.1,0.97366,0.0563439,1608.55,2.2067 +133,delete,13.714,-1.0,-1.0,0.0,2.20649 +134,search,42035.7,0.97365,0.0564712,102.593,2.20649 +135,delete,45.208,-1.0,-1.0,0.0,2.20436 +136,search,46171.2,0.97401,0.0559797,99.578,2.20436 +137,delete,11.148,-1.0,-1.0,0.0,2.20434 +138,search,46136.1,0.97397,0.0560673,95.865,2.20434 +139,delete,15.03,-1.0,-1.0,0.0,2.20232 +140,search,46758.5,0.97399,0.056044,1557.99,2.20239 +141,delete,20.302,-1.0,-1.0,0.0,2.19794 +142,search,46753.5,0.97405,0.0559093,1597.17,2.19779 +143,delete,16.636,-1.0,-1.0,0.0,2.19417 +144,search,44369.1,0.97399,0.056044,100.448,2.19417 +145,delete,5.793,-1.0,-1.0,0.0,2.19318 +146,search,48659.8,0.97388,0.0561804,99.627,2.19318 +147,delete,27.4,-1.0,-1.0,0.0,2.19246 +148,search,47121.5,0.97391,0.0561853,95.501,2.19246 +149,delete,230.619,-1.0,-1.0,0.0,2.14778 +150,search,46830.8,0.97451,0.0557837,94.038,2.14778 +151,delete,188.4,-1.0,-1.0,0.0,2.10792 +152,search,44544.9,0.97429,0.0560772,98.653,2.10792 +153,delete,51.397,-1.0,-1.0,0.0,2.09341 +154,search,45344.8,0.97403,0.0565428,100.541,2.09341 +155,delete,49.971,-1.0,-1.0,0.0,2.08959 +156,search,46860.4,0.97399,0.0564352,97.205,2.08959 +157,delete,72.512,-1.0,-1.0,0.0,2.07699 +158,search,42812.6,0.97384,0.0566412,92.849,2.07699 +159,delete,318.108,-1.0,-1.0,0.0,1.95404 +160,search,42129.9,0.97344,0.0567896,98.321,1.95404 +161,delete,82.235,-1.0,-1.0,0.0,1.9416 +162,search,47944.9,0.97343,0.0567033,100.236,1.9416 +163,delete,59.083,-1.0,-1.0,0.0,1.93227 +164,search,47539.9,0.97344,0.0567191,101.423,1.93227 +165,delete,69.325,-1.0,-1.0,0.0,1.92234 +166,search,46899,0.97338,0.0568155,92.307,1.92234 +167,delete,9.188,-1.0,-1.0,0.0,1.92008 +168,search,46893.3,0.97331,0.0569972,96.742,1.92008 +169,delete,34.925,-1.0,-1.0,0.0,1.9131 +170,search,43859.7,0.97323,0.0570165,100.664,1.9131 +171,delete,30.817,-1.0,-1.0,0.0,1.90997 +172,search,45038.7,0.97314,0.0570861,98.881,1.90997 +173,delete,80.717,-1.0,-1.0,0.0,1.89786 +174,search,45515.8,0.97287,0.0574233,93.91,1.89786 +175,delete,191.271,-1.0,-1.0,0.0,1.8273 +176,search,46979.7,0.97284,0.0574695,94.638,1.8273 +177,delete,32.047,-1.0,-1.0,0.0,1.82037 +178,search,47828.9,0.97282,0.0574753,99.528,1.82037 +179,delete,333.683,-1.0,-1.0,0.0,1.71742 +180,search,47552.2,0.97152,0.0592729,100.448,1.71742 +181,delete,66.003,-1.0,-1.0,0.0,1.70467 +182,search,39909.9,0.97141,0.0593485,95.321,1.70467 +183,delete,183.266,-1.0,-1.0,0.0,1.63303 +184,search,46210.6,0.969,0.0620927,93.357,1.63303 +185,delete,21.7,-1.0,-1.0,0.0,1.62927 +186,search,45334.4,0.96895,0.0622347,99.5,1.62927 +187,delete,249.353,-1.0,-1.0,0.0,1.54601 +188,search,45254.6,0.97058,0.060474,100,1.54601 +189,delete,29.596,-1.0,-1.0,0.0,1.54291 +190,search,45044.4,0.97052,0.0605272,95.938,1.54291 +191,delete,8.596,-1.0,-1.0,0.0,1.54152 +192,search,42978.5,0.97054,0.0604697,93.578,1.54152 +193,delete,96.44,-1.0,-1.0,0.0,1.51376 +194,search,45938.5,0.96992,0.0611991,100.14,1.51376 +195,delete,86.478,-1.0,-1.0,0.0,1.49775 +196,search,46168.6,0.96945,0.0614333,99.687,1.49775 +197,delete,81.924,-1.0,-1.0,0.0,1.48648 +198,search,46152.7,0.96933,0.0615037,97.671,1.48648 +199,delete,31.306,-1.0,-1.0,0.0,1.48214 +200,search,44959,0.9692,0.0616903,93.334,1.48214 +201,delete,10.808,-1.0,-1.0,0.0,1.47922 +202,search,42579.1,0.96918,0.0616643,96.173,1.47922 +203,delete,38.911,-1.0,-1.0,0.0,1.47295 +204,search,44968.5,0.96917,0.0618432,99.587,1.47295 +205,delete,63.185,-1.0,-1.0,0.0,1.46429 +206,search,44950.9,0.96918,0.0617454,98.93,1.46429 +207,delete,89.908,-1.0,-1.0,0.0,1.4455 +208,search,40992.5,0.9692,0.0615604,92.393,1.4455 +209,delete,27.98,-1.0,-1.0,0.0,1.43658 +210,search,46925.7,0.96915,0.0616568,98.977,1.43658 +211,delete,27.568,-1.0,-1.0,0.0,1.4326 +212,search,44245.7,0.96899,0.0618995,100.173,1.4326 +213,delete,49.308,-1.0,-1.0,0.0,1.42486 +214,search,45737,0.96883,0.0621111,98.873,1.42486 +215,delete,437.63,-1.0,-1.0,0.0,1.26778 +216,search,46182.6,0.96734,0.0638725,94.82,1.26778 +217,delete,282.346,-1.0,-1.0,0.0,1.16504 +218,search,46499.6,0.96587,0.0657298,94.542,1.16504 +219,delete,47.356,-1.0,-1.0,0.0,1.15448 +220,search,41070.8,0.96551,0.0662127,98.867,1.15448 +221,delete,304.342,-1.0,-1.0,0.0,1.05495 +222,search,44396.1,0.96588,0.0658488,99.76,1.05495 +223,delete,40.401,-1.0,-1.0,0.0,1.04851 +224,search,41418.6,0.96542,0.0664561,92.5,1.04851 +225,delete,10.4,-1.0,-1.0,0.0,1.04571 +226,search,48016.6,0.9653,0.0667129,95.004,1.04571 +227,delete,12.163,-1.0,-1.0,0.0,1.04379 +228,search,43833.9,0.96527,0.0667462,101.667,1.04379 +229,delete,66.79,-1.0,-1.0,0.0,1.03283 +230,search,46768.6,0.9647,0.0669362,100.895,1.03283 +231,delete,37.178,-1.0,-1.0,0.0,1.02685 +232,search,44791.5,0.96453,0.0671244,95.885,1.02685 +233,delete,13.162,-1.0,-1.0,0.0,1.02417 +234,search,42046.8,0.96435,0.0671891,93.687,1.02417 +235,delete,14.466,-1.0,-1.0,0.0,1.02058 +236,search,46180.7,0.96436,0.0671891,101.457,1.02058 +237,delete,352.686,-1.0,-1.0,0.0,0.910679 +238,search,45705.5,0.96379,0.0683987,100.526,0.910679 +239,delete,56.354,-1.0,-1.0,0.0,0.898169 +240,search,44704.5,0.96322,0.0688468,95.675,0.898169 +241,delete,18.879,-1.0,-1.0,0.0,0.894298 +242,search,42921.1,0.96317,0.0689725,93.777,0.894298 +243,delete,40.906,-1.0,-1.0,0.0,0.884516 +244,search,43429.9,0.96248,0.0698042,101.425,0.884516 +245,delete,40.37,-1.0,-1.0,0.0,0.879084 +246,search,45589.7,0.9623,0.0699525,100.17,0.879084 +247,delete,13.3,-1.0,-1.0,0.0,0.877764 +248,search,46097.8,0.9622,0.069999,95.793,0.877764 +249,delete,20.973,-1.0,-1.0,0.0,0.873799 +250,search,45648.7,0.96206,0.0702492,93.506,0.873799 +251,delete,316.283,-1.0,-1.0,0.0,0.758317 +252,search,43881.1,0.962,0.069978,100.371,0.758317 +253,delete,84.264,-1.0,-1.0,0.0,0.743387 +254,search,43982,0.96163,0.070281,99.905,0.743387 +255,delete,88.112,-1.0,-1.0,0.0,0.724221 +256,search,46118.5,0.96034,0.0716744,96.78,0.724221 +257,insert,28.775,-1.0,-1.0,0.0,0.725929 +258,search,45826.2,0.96033,0.0716393,93.585,0.725929 +259,insert,75.955,-1.0,-1.0,0.0,0.732452 +260,search,43702.3,0.96063,0.0715502,556.136,0.732482 +261,insert,9.727,-1.0,-1.0,0.0,0.732971 +262,search,45614.8,0.96076,0.0714364,101.135,0.732971 +263,insert,436.32,-1.0,-1.0,0.0,0.772987 +264,search,44422.3,0.9625,0.0698888,680.705,0.77345 +265,insert,161.7,-1.0,-1.0,0.0,0.792564 +266,search,41151.5,0.96411,0.0680986,612.149,0.792599 +267,insert,97.201,-1.0,-1.0,0.0,0.803289 +268,search,45527.3,0.96446,0.0675497,646.664,0.803533 +269,insert,997.968,-1.0,-1.0,0.0,0.898864 +270,search,46676.1,0.96575,0.0661569,1110.08,0.898462 +271,insert,13.314,-1.0,-1.0,0.0,0.899921 +272,search,45845.7,0.96555,0.0665912,907.718,0.899796 +273,insert,69.348,-1.0,-1.0,0.0,0.907798 +274,search,45874,0.96518,0.0668003,786.922,0.907806 +275,insert,1399.95,-1.0,-1.0,0.0,1.06385 +276,search,42379.1,0.9662,0.0656944,2410.53,1.06356 +277,insert,117.428,-1.0,-1.0,0.0,1.07516 +278,search,48260.9,0.96643,0.0655468,1257.04,1.07514 +279,insert,91.76,-1.0,-1.0,0.0,1.08375 +280,search,47385.9,0.96645,0.065664,895.679,1.08389 +281,insert,1014.7,-1.0,-1.0,0.0,1.20857 +282,search,48567,0.96762,0.0642821,1520.85,1.20871 +283,insert,1448.68,-1.0,-1.0,0.0,1.38951 +284,search,50101.5,0.96954,0.0613204,3415.98,1.38894 +285,insert,39.744,-1.0,-1.0,0.0,1.3944 +286,search,50470.5,0.96983,0.0612793,2166.39,1.39397 +287,insert,157.815,-1.0,-1.0,0.0,1.40931 +288,search,51165.7,0.97002,0.061118,1318.8,1.40939 +289,insert,1245.85,-1.0,-1.0,0.0,1.54123 +290,search,46749.8,0.97101,0.0602915,2214.29,1.54109 +291,insert,91.542,-1.0,-1.0,0.0,1.55107 +292,search,51079.1,0.97107,0.0603361,1655.28,1.55017 +293,insert,64.156,-1.0,-1.0,0.0,1.55836 +294,search,49763.3,0.97116,0.0603533,1349.02,1.55867 +295,insert,802.419,-1.0,-1.0,0.0,1.64895 +296,search,52163.4,0.9718,0.059658,1855.32,1.64769 +297,insert,16.701,-1.0,-1.0,0.0,1.64975 +298,search,51297.5,0.97152,0.0599441,1471.45,1.65015 +299,insert,243.725,-1.0,-1.0,0.0,1.67582 +300,search,52293.7,0.97301,0.0581291,1439.4,1.67535 +301,insert,1958.71,-1.0,-1.0,0.0,1.92926 +302,search,52722.5,0.97937,0.0501553,9517.4,1.92937 +303,insert,175.869,-1.0,-1.0,0.0,1.95048 +304,search,56325.8,0.97915,0.0506427,5587.41,1.95031 +305,insert,1226.01,-1.0,-1.0,0.0,2.11169 +306,search,55919.5,0.97917,0.0506267,5253.28,2.11234 +307,insert,142.412,-1.0,-1.0,0.0,2.12458 +308,search,56281.6,0.97936,0.0502403,2955.17,2.12447 +309,insert,36.869,-1.0,-1.0,0.0,2.12738 +310,search,52357.2,0.97958,0.050034,2019.22,2.12809 +311,insert,94.537,-1.0,-1.0,0.0,2.13575 +312,search,57777,0.97963,0.050025,1751.14,2.13554 +313,insert,226.521,-1.0,-1.0,0.0,2.15331 +314,search,56996.3,0.97973,0.0497817,1780.44,2.15346 +315,insert,24.795,-1.0,-1.0,0.0,2.15499 +316,search,57832.8,0.97967,0.0499607,1642.48,2.1549 +317,insert,223.474,-1.0,-1.0,0.0,2.1722 +318,search,58372.7,0.97985,0.0499936,1716.14,2.1721 +319,insert,2170.75,-1.0,-1.0,0.0,2.38896 +320,search,58311.1,0.98095,0.0482529,13772.1,2.38966 +321,insert,170.587,-1.0,-1.0,0.0,2.40691 +322,search,57174.5,0.98152,0.047651,6119.99,2.40697 +323,insert,415.353,-1.0,-1.0,0.0,2.45297 +324,search,61557.1,0.98174,0.0475498,4148.58,2.45253 +325,insert,3190.13,-1.0,-1.0,0.0,2.66782 +326,search,62837.9,0.98259,0.0467781,8161.84,2.66587 +327,insert,313.08,-1.0,-1.0,0.0,2.69634 +328,search,62670.4,0.98284,0.0465157,5978.04,2.69729 +329,insert,33.606,-1.0,-1.0,0.0,2.70057 +330,search,59183.9,0.98301,0.0460707,3229,2.7012 +331,insert,180.27,-1.0,-1.0,0.0,2.71929 +332,search,59201.7,0.98316,0.0458985,2539.1,2.71942 +333,insert,4.696,-1.0,-1.0,0.0,2.72008 +334,search,60260.8,0.9832,0.0458696,2014.68,2.71992 +335,insert,176.962,-1.0,-1.0,0.0,2.73682 +336,search,66724.5,0.98329,0.0458031,2465.07,2.73709 +337,insert,88.737,-1.0,-1.0,0.0,2.74519 +338,search,65475.8,0.98331,0.0457887,2120.8,2.74517 +339,insert,218.945,-1.0,-1.0,0.0,2.76356 +340,search,67689.1,0.98345,0.0455996,2305.57,2.76398 +341,insert,32.414,-1.0,-1.0,0.0,2.76749 +342,search,60456.6,0.98351,0.0454904,2073.42,2.76748 +343,insert,152.495,-1.0,-1.0,0.0,2.78261 +344,search,65260.8,0.98347,0.0457173,2217.48,2.78256 +345,insert,166.077,-1.0,-1.0,0.0,2.79996 +346,search,65432.6,0.98351,0.0456222,2305.09,2.79996 +347,insert,1696.33,-1.0,-1.0,0.0,3.00093 +348,search,68163.8,0.98394,0.0447218,19666.5,3.00085 +349,insert,184.923,-1.0,-1.0,0.0,3.02128 +350,search,67997.7,0.98399,0.0446416,6377.1,3.02092 +351,insert,1516.47,-1.0,-1.0,0.0,3.20385 +352,search,67522.3,0.98453,0.0439062,12689.3,3.201 +353,insert,238.252,-1.0,-1.0,0.0,3.21996 +354,search,69893.4,0.98481,0.0435027,5597.28,3.22193 +355,insert,1419.99,-1.0,-1.0,0.0,3.3814 +356,search,73342.8,0.98512,0.0429029,15073.5,3.38202 +357,insert,106.146,-1.0,-1.0,0.0,3.39019 +358,search,74980.6,0.98535,0.042762,6562.54,3.39055 +359,insert,69.659,-1.0,-1.0,0.0,3.39587 +360,search,68999.3,0.98545,0.0426787,3534.42,3.39619 +361,insert,378.159,-1.0,-1.0,0.0,3.42428 +362,search,75475.9,0.98567,0.0417345,3340.68,3.42379 +363,insert,88.462,-1.0,-1.0,0.0,3.42924 +364,search,66980.6,0.98575,0.0415224,2878.52,3.42973 +365,insert,351.991,-1.0,-1.0,0.0,3.45594 +366,search,75568.2,0.98586,0.0413081,3056.32,3.45568 +367,insert,161.485,-1.0,-1.0,0.0,3.46878 +368,search,74669.2,0.98589,0.0412574,3099.66,3.46873 +369,insert,396.492,-1.0,-1.0,0.0,3.49719 +370,search,71884.2,0.98599,0.041073,3786.27,3.49689 +371,insert,51.725,-1.0,-1.0,0.0,3.50075 +372,search,70564.8,0.9861,0.0408787,3079.46,3.50026 +373,insert,238.402,-1.0,-1.0,0.0,3.51779 +374,search,77344.9,0.98618,0.0407568,3084.27,3.51781 +375,insert,188.565,-1.0,-1.0,0.0,3.53015 +376,search,73126.3,0.98616,0.0407268,3109.33,3.53013 +377,insert,312.745,-1.0,-1.0,0.0,3.55791 +378,search,76789,0.98627,0.0405063,3879.22,3.55712 +379,insert,27.963,-1.0,-1.0,0.0,3.5593 +380,search,71905.1,0.98627,0.040531,3206.3,3.55976 +381,insert,105.372,-1.0,-1.0,0.0,3.56864 +382,search,72194.8,0.98629,0.040536,2971.52,3.56866 +383,insert,146.172,-1.0,-1.0,0.0,3.57877 +384,search,72722.3,0.9863,0.0404791,2957.77,3.57894 +385,delete,13.684,-1.0,-1.0,0.0,3.57838 +386,search,71487.8,0.98636,0.0404234,2839.78,3.5784 +387,delete,22.945,-1.0,-1.0,0.0,3.57579 +388,search,72022.2,0.98637,0.0403916,188.093,3.57579 +389,delete,51.231,-1.0,-1.0,0.0,3.57468 +390,search,72543.2,0.98635,0.040407,2771.9,3.5747 +391,delete,92.781,-1.0,-1.0,0.0,3.5539 +392,search,69420,0.98626,0.0405388,2748.19,3.5539 +393,delete,40.042,-1.0,-1.0,0.0,3.55111 +394,search,71680.3,0.98625,0.04057,2751.63,3.55114 +395,delete,20.835,-1.0,-1.0,0.0,3.5443 +396,search,71646.4,0.98618,0.0406339,189.189,3.5443 +397,delete,225.515,-1.0,-1.0,0.0,3.50938 +398,search,70229.8,0.98617,0.04067,186.103,3.50938 +399,delete,20.535,-1.0,-1.0,0.0,3.50836 +400,search,67042.2,0.98614,0.0406932,185.884,3.50836 +401,delete,43.768,-1.0,-1.0,0.0,3.50386 +402,search,67994.4,0.98621,0.0405815,183.904,3.50386 +403,delete,433.541,-1.0,-1.0,0.0,3.41821 +404,search,70511.3,0.98619,0.0406015,2568.13,3.41818 +405,delete,78.297,-1.0,-1.0,0.0,3.40782 +406,search,70541.9,0.98639,0.0404451,2556.65,3.4078 +407,delete,109.285,-1.0,-1.0,0.0,3.37944 +408,search,71604.5,0.98643,0.0403852,188.803,3.37944 +409,delete,348.507,-1.0,-1.0,0.0,3.2964 +410,search,70318.8,0.98638,0.0405075,187.179,3.2964 +411,delete,429.384,-1.0,-1.0,0.0,3.22488 +412,search,72133,0.98596,0.0412202,186.282,3.22488 +413,delete,61.846,-1.0,-1.0,0.0,3.21929 +414,search,67592.5,0.98593,0.0411993,186.746,3.21929 +415,delete,157.222,-1.0,-1.0,0.0,3.19775 +416,search,66615.7,0.9859,0.0413176,183.176,3.19775 +417,delete,356.312,-1.0,-1.0,0.0,3.13236 +418,search,70359.1,0.98615,0.0407588,2298.62,3.13234 +419,delete,38.154,-1.0,-1.0,0.0,3.12743 +420,search,70228.1,0.98609,0.0408626,186.187,3.12743 +421,delete,56.083,-1.0,-1.0,0.0,3.11795 +422,search,73165.4,0.98605,0.040994,187.701,3.11795 +423,delete,291.116,-1.0,-1.0,0.0,3.05307 +424,search,70839.4,0.98619,0.0408471,184.896,3.05307 +425,delete,66.929,-1.0,-1.0,0.0,3.04774 +426,search,67613.8,0.98619,0.0408716,185.496,3.04774 +427,delete,158.837,-1.0,-1.0,0.0,3.03348 +428,search,72295.7,0.98603,0.0410867,186.489,3.03348 +429,delete,670.898,-1.0,-1.0,0.0,2.8822 +430,search,72217.2,0.98434,0.0435774,188.16,2.8822 +431,delete,177.021,-1.0,-1.0,0.0,2.84579 +432,search,69119.7,0.9842,0.0438258,184.571,2.84579 +433,delete,362.485,-1.0,-1.0,0.0,2.76759 +434,search,69794.3,0.9842,0.0437573,188.563,2.76759 +435,delete,257.841,-1.0,-1.0,0.0,2.72024 +436,search,68062.6,0.98398,0.0441998,184.862,2.72024 +437,delete,41.78,-1.0,-1.0,0.0,2.71505 +438,search,71986.5,0.98397,0.0442512,187.976,2.71505 +439,delete,228.465,-1.0,-1.0,0.0,2.65955 +440,search,70031.7,0.98388,0.0443185,183.662,2.65955 +441,delete,112.115,-1.0,-1.0,0.0,2.64601 +442,search,70552.9,0.98389,0.0444699,188.682,2.64601 +443,delete,70.882,-1.0,-1.0,0.0,2.63225 +444,search,70215,0.98407,0.0441971,185.005,2.63225 +445,delete,122.116,-1.0,-1.0,0.0,2.61664 +446,search,70095.2,0.98389,0.0444924,188.262,2.61664 +447,delete,631.239,-1.0,-1.0,0.0,2.46555 +448,search,70265.4,0.98335,0.045254,187.21,2.46555 +449,delete,116.992,-1.0,-1.0,0.0,2.4492 +450,search,70560.3,0.98332,0.0452998,189.092,2.4492 +451,delete,229.277,-1.0,-1.0,0.0,2.40589 +452,search,69712.3,0.98302,0.0458232,186.742,2.40589 +453,delete,535.491,-1.0,-1.0,0.0,2.28542 +454,search,69868.4,0.98257,0.046383,187.857,2.28542 +455,delete,131.822,-1.0,-1.0,0.0,2.2673 +456,search,70824.1,0.98255,0.0462691,185.52,2.2673 +457,delete,27.085,-1.0,-1.0,0.0,2.26416 +458,search,70275.7,0.98253,0.0463896,189.185,2.26416 +459,delete,114.657,-1.0,-1.0,0.0,2.24573 +460,search,66448.5,0.98264,0.0462694,184.647,2.24573 +461,delete,23.932,-1.0,-1.0,0.0,2.24374 +462,search,70375.9,0.98267,0.0462248,188.002,2.24374 +463,delete,141.515,-1.0,-1.0,0.0,2.22115 +464,search,70002.9,0.98276,0.0459909,184.5,2.22115 +465,delete,69.314,-1.0,-1.0,0.0,2.20906 +466,search,71015.3,0.98268,0.0460462,188.338,2.20906 +467,delete,96.257,-1.0,-1.0,0.0,2.19706 +468,search,70798.3,0.9826,0.0461671,190.199,2.19706 +469,delete,58.408,-1.0,-1.0,0.0,2.18666 +470,search,76713.2,0.9826,0.0462321,187.945,2.18666 +471,delete,256.303,-1.0,-1.0,0.0,2.13065 +472,search,71970.2,0.98258,0.0464394,188.349,2.13065 +473,delete,287.138,-1.0,-1.0,0.0,2.07942 +474,search,70284.7,0.98246,0.0465461,188.443,2.07942 +475,delete,720.303,-1.0,-1.0,0.0,1.89012 +476,search,69603.7,0.98211,0.0466411,189.918,1.89012 +477,delete,156.425,-1.0,-1.0,0.0,1.86599 +478,search,68697.6,0.98214,0.0465508,190.283,1.86599 +479,delete,421.193,-1.0,-1.0,0.0,1.77478 +480,search,68674.1,0.98142,0.0477183,185.261,1.77478 +481,delete,164.322,-1.0,-1.0,0.0,1.75333 +482,search,68791.1,0.98128,0.0477472,188.763,1.75333 +483,delete,545.387,-1.0,-1.0,0.0,1.62266 +484,search,69585.5,0.98078,0.0486506,185.647,1.62266 +485,delete,91.262,-1.0,-1.0,0.0,1.61317 +486,search,68824.5,0.98068,0.0488985,188.741,1.61317 +487,delete,46.66,-1.0,-1.0,0.0,1.60908 +488,search,69750.1,0.98055,0.0490187,186.789,1.60908 +489,delete,117.796,-1.0,-1.0,0.0,1.59208 +490,search,70353.9,0.98038,0.0492895,188.826,1.59208 +491,delete,58.397,-1.0,-1.0,0.0,1.58447 +492,search,69299.3,0.98024,0.0494759,186.061,1.58447 +493,delete,322.295,-1.0,-1.0,0.0,1.52019 +494,search,66393.2,0.97998,0.0496927,188.297,1.52019 +495,delete,83.928,-1.0,-1.0,0.0,1.50876 +496,search,65869,0.97998,0.0497329,187.277,1.50876 +497,delete,184.799,-1.0,-1.0,0.0,1.47716 +498,search,69095.5,0.97986,0.0500477,190.242,1.47716 +499,delete,68.662,-1.0,-1.0,0.0,1.46991 +500,search,68385.9,0.97973,0.0504602,189.228,1.46991 +501,delete,97.848,-1.0,-1.0,0.0,1.45759 +502,search,68073.6,0.97972,0.0503304,190.629,1.45759 +503,delete,78.015,-1.0,-1.0,0.0,1.44884 +504,search,69894.2,0.97978,0.0501722,190.08,1.44884 +505,delete,165.876,-1.0,-1.0,0.0,1.41803 +506,search,68309.1,0.97954,0.0504519,187.404,1.41803 +507,delete,117.224,-1.0,-1.0,0.0,1.4004 +508,search,68822.9,0.97944,0.0504945,187.409,1.4004 +509,delete,72.16,-1.0,-1.0,0.0,1.39108 +510,search,69612.3,0.9795,0.0503976,187.538,1.39108 +511,delete,87.791,-1.0,-1.0,0.0,1.38042 +512,search,64422.2,0.97932,0.0506398,185.561,1.38042 +513,insert,1310.28,-1.0,-1.0,0.0,1.467 +514,search,76931,0.97965,0.0499694,1481.64,1.46684 +515,insert,20.949,-1.0,-1.0,0.0,1.46792 +516,search,68855.4,0.97955,0.0504287,1188.68,1.46796 +517,insert,74.318,-1.0,-1.0,0.0,1.47247 +518,search,69996.2,0.97959,0.0504034,186.66,1.47247 +519,insert,423.154,-1.0,-1.0,0.0,1.49598 +520,search,74168.9,0.97992,0.0500091,1248.19,1.49594 +521,insert,59.108,-1.0,-1.0,0.0,1.49944 +522,search,69746.6,0.97999,0.0498695,1197.79,1.49948 +523,insert,157.841,-1.0,-1.0,0.0,1.51059 +524,search,73477.6,0.98002,0.0498293,1277.04,1.51043 +525,insert,151.896,-1.0,-1.0,0.0,1.51911 +526,search,71637.1,0.98008,0.0496926,1253.26,1.51914 +527,insert,1950.54,-1.0,-1.0,0.0,1.67787 +528,search,75955.1,0.98118,0.0478962,2279.41,1.67708 +529,insert,1150.04,-1.0,-1.0,0.0,1.78285 +530,search,76470.9,0.98201,0.0470062,2560.08,1.78203 +531,insert,231.737,-1.0,-1.0,0.0,1.7957 +532,search,75011.8,0.98213,0.0471571,2119.79,1.79512 +533,insert,88.78,-1.0,-1.0,0.0,1.8008 +534,search,72527.3,0.98223,0.0471334,1967.09,1.80148 +535,insert,99.705,-1.0,-1.0,0.0,1.80765 +536,search,75569.4,0.98227,0.0471918,1666.74,1.80744 +537,insert,37.327,-1.0,-1.0,0.0,1.81071 +538,search,73864.6,0.9823,0.0471271,1525.4,1.81062 +539,insert,147.251,-1.0,-1.0,0.0,1.82055 +540,search,68805.5,0.98231,0.0470552,1519.61,1.82046 +541,insert,1607.66,-1.0,-1.0,0.0,1.98131 +542,search,77636,0.98273,0.0470409,3118.6,1.98117 +543,insert,190.72,-1.0,-1.0,0.0,1.99492 +544,search,76079.6,0.983,0.0465341,2138.79,1.99462 +545,insert,37.576,-1.0,-1.0,0.0,1.99684 +546,search,70381.9,0.983,0.0466415,1671.81,1.99687 +547,insert,659.949,-1.0,-1.0,0.0,2.05069 +548,search,77805.1,0.98364,0.0457879,1731.48,2.05065 +549,insert,1098.87,-1.0,-1.0,0.0,2.14702 +550,search,78396.4,0.98354,0.046035,2479.3,2.14757 +551,insert,106.644,-1.0,-1.0,0.0,2.15799 +552,search,76746.6,0.9836,0.046013,2010.51,2.15792 +553,insert,55.21,-1.0,-1.0,0.0,2.16255 +554,search,73952.1,0.98379,0.0457462,1835.13,2.16287 +555,insert,2021.3,-1.0,-1.0,0.0,2.32931 +556,search,80433.3,0.98486,0.0438956,3211.81,2.32876 +557,insert,130.505,-1.0,-1.0,0.0,2.33845 +558,search,74929.4,0.98488,0.043951,2257.26,2.33832 +559,insert,32.74,-1.0,-1.0,0.0,2.34121 +560,search,75915.6,0.98485,0.0440416,1885.27,2.34119 +561,insert,18.621,-1.0,-1.0,0.0,2.3423 +562,search,75104.8,0.98487,0.0440255,1854.39,2.34226 +563,insert,263.262,-1.0,-1.0,0.0,2.35925 +564,search,76485.5,0.98497,0.0436491,1940.12,2.35925 +565,insert,964.018,-1.0,-1.0,0.0,2.44338 +566,search,81838.3,0.98505,0.0434225,2582.59,2.44315 +567,insert,67.255,-1.0,-1.0,0.0,2.44792 +568,search,76567.6,0.98517,0.0430266,2221.61,2.44801 +569,insert,73.045,-1.0,-1.0,0.0,2.45406 +570,search,75552.1,0.98526,0.042837,2131.94,2.45419 +571,insert,51.971,-1.0,-1.0,0.0,2.4598 +572,search,77079.1,0.98523,0.0429545,1976.51,2.45973 +573,insert,334.412,-1.0,-1.0,0.0,2.48504 +574,search,79951.6,0.98529,0.043067,2105.71,2.48481 +575,insert,227.112,-1.0,-1.0,0.0,2.50314 +576,search,80740.4,0.98534,0.0430724,2073.74,2.50313 +577,insert,194.965,-1.0,-1.0,0.0,2.51783 +578,search,77300.2,0.98545,0.0429124,2244.92,2.51731 +579,insert,2409.76,-1.0,-1.0,0.0,2.77557 +580,search,82732.4,0.98593,0.0424901,11777.7,2.77497 +581,insert,128.566,-1.0,-1.0,0.0,2.78427 +582,search,80102.4,0.98577,0.0427149,5420.92,2.78444 +583,insert,119.563,-1.0,-1.0,0.0,2.79282 +584,search,79142.7,0.98578,0.0429415,2596.01,2.79283 +585,insert,1265.5,-1.0,-1.0,0.0,2.91259 +586,search,83826.7,0.98597,0.0425931,3104.47,2.91323 +587,insert,172.044,-1.0,-1.0,0.0,2.92644 +588,search,80296.5,0.98608,0.0423132,2559.2,2.92652 +589,insert,255.184,-1.0,-1.0,0.0,2.94595 +590,search,82172.9,0.98623,0.0419714,2479.16,2.9456 +591,insert,2159.83,-1.0,-1.0,0.0,3.15265 +592,search,84756,0.9868,0.0412064,12525.3,3.15265 +593,insert,801.433,-1.0,-1.0,0.0,3.23778 +594,search,84814.8,0.98704,0.0405477,7328.97,3.23867 +595,insert,35.625,-1.0,-1.0,0.0,3.24117 +596,search,85090.7,0.98724,0.0404405,3550,3.24066 +597,insert,1121.51,-1.0,-1.0,0.0,3.34148 +598,search,89129.2,0.98732,0.040069,4339.8,3.3424 +599,insert,636.315,-1.0,-1.0,0.0,3.39114 +600,search,87025.8,0.98737,0.0398441,4958.59,3.39152 +601,insert,324.21,-1.0,-1.0,0.0,3.41826 +602,search,85510.2,0.98752,0.0395008,3577,3.41839 +603,insert,31.397,-1.0,-1.0,0.0,3.42051 +604,search,81580.8,0.98758,0.039395,2954.43,3.42034 +605,insert,268.09,-1.0,-1.0,0.0,3.43925 +606,search,85833.4,0.98768,0.0391185,2988.5,3.43919 +607,insert,223.372,-1.0,-1.0,0.0,3.45248 +608,search,86531.1,0.98778,0.0388458,2909.33,3.45241 +609,insert,1112.32,-1.0,-1.0,0.0,3.54344 +610,search,88473.4,0.98802,0.0383722,5196.93,3.54342 +611,insert,121.153,-1.0,-1.0,0.0,3.55425 +612,search,86764,0.98809,0.0382017,3627.69,3.55389 +613,insert,959.187,-1.0,-1.0,0.0,3.63845 +614,search,91138.1,0.98811,0.0381785,5252.44,3.63902 +615,insert,1406.46,-1.0,-1.0,0.0,3.80029 +616,search,93779.4,0.98847,0.0374708,9396.05,3.79976 +617,insert,1360.03,-1.0,-1.0,0.0,3.94137 +618,search,93944.9,0.98866,0.037465,15036.8,3.94115 +619,insert,1166.09,-1.0,-1.0,0.0,4.04847 +620,search,98506.6,0.98886,0.036905,9206.83,4.04843 +621,insert,185.749,-1.0,-1.0,0.0,4.06371 +622,search,92795.1,0.98898,0.0367505,4941.42,4.0639 +623,insert,96.413,-1.0,-1.0,0.0,4.0726 +624,search,91437,0.98907,0.0366011,3767.77,4.07261 +625,insert,419.572,-1.0,-1.0,0.0,4.10768 +626,search,92587,0.98908,0.0366996,4369.76,4.10825 +627,insert,276.55,-1.0,-1.0,0.0,4.12771 +628,search,94438.6,0.98921,0.0365329,4410.1,4.12719 +629,insert,70.489,-1.0,-1.0,0.0,4.13297 +630,search,94828.7,0.98922,0.0366335,3656.07,4.13278 +631,insert,213.258,-1.0,-1.0,0.0,4.14796 +632,search,92315.3,0.98922,0.0366608,3712.73,4.1479 +633,insert,246.905,-1.0,-1.0,0.0,4.16476 +634,search,94940,0.9892,0.0367888,3731.65,4.16426 +635,insert,128.163,-1.0,-1.0,0.0,4.17222 +636,search,94274,0.98921,0.0367512,3598.87,4.17238 +637,insert,2078.03,-1.0,-1.0,0.0,4.35003 +638,search,101298,0.98933,0.036403,14441.2,4.34991 +639,insert,2005.93,-1.0,-1.0,0.0,4.52705 +640,search,101833,0.98969,0.0357867,26941.4,4.5273 +641,delete,210.72,-1.0,-1.0,0.0,4.48516 +642,search,98298.4,0.98957,0.0362258,8465.4,4.48366 +643,delete,14.216,-1.0,-1.0,0.0,4.48243 +644,search,97544.3,0.9897,0.0359986,4232.12,4.48144 +645,delete,20.985,-1.0,-1.0,0.0,4.48088 +646,search,99178.7,0.98972,0.0359243,3801.91,4.4809 +647,delete,91.117,-1.0,-1.0,0.0,4.4657 +648,search,100538,0.98971,0.0359615,3759.57,4.46577 +649,delete,48.467,-1.0,-1.0,0.0,4.46203 +650,search,100877,0.98967,0.0360068,3700.08,4.46205 +651,delete,27.655,-1.0,-1.0,0.0,4.45782 +652,search,99696.9,0.98963,0.036052,3700.82,4.45774 +653,delete,118.631,-1.0,-1.0,0.0,4.43031 +654,search,100787,0.98951,0.0362629,3656.98,4.43026 +655,delete,488.964,-1.0,-1.0,0.0,4.33045 +656,search,99432,0.98923,0.036705,348.82,4.33045 +657,delete,399.017,-1.0,-1.0,0.0,4.24665 +658,search,96923.5,0.98903,0.0368873,3429.1,4.24667 +659,delete,154.344,-1.0,-1.0,0.0,4.21019 +660,search,96425.7,0.9889,0.0370785,349.738,4.21019 +661,delete,142.862,-1.0,-1.0,0.0,4.20263 +662,search,98227.8,0.98886,0.0371481,3358.6,4.2026 +663,delete,97.21,-1.0,-1.0,0.0,4.18521 +664,search,98111.9,0.98875,0.0373179,348.966,4.18521 +665,delete,145.12,-1.0,-1.0,0.0,4.17005 +666,search,97331.1,0.98875,0.0372643,347.813,4.17005 +667,delete,403.62,-1.0,-1.0,0.0,4.10628 +668,search,96182.4,0.98857,0.0374234,346.966,4.10628 +669,delete,464.63,-1.0,-1.0,0.0,4.0369 +670,search,94555.7,0.98843,0.0378042,346.875,4.0369 +671,delete,133.09,-1.0,-1.0,0.0,4.02612 +672,search,98328.8,0.98841,0.0378536,347.179,4.02612 +673,delete,167.227,-1.0,-1.0,0.0,4.00489 +674,search,96978.2,0.98841,0.0379328,348.029,4.00489 +675,delete,318.15,-1.0,-1.0,0.0,3.97331 +676,search,95479.9,0.98815,0.038296,347.77,3.97331 +677,delete,398.752,-1.0,-1.0,0.0,3.89325 +678,search,95425.6,0.98813,0.0383184,347.415,3.89325 +679,delete,92.856,-1.0,-1.0,0.0,3.88261 +680,search,94372.2,0.98814,0.0383051,349.419,3.88261 +681,delete,42.148,-1.0,-1.0,0.0,3.88089 +682,search,95450,0.98816,0.0382619,346.062,3.88089 +683,delete,627.136,-1.0,-1.0,0.0,3.78738 +684,search,93578.8,0.9875,0.0394461,345.507,3.78738 +685,delete,533.723,-1.0,-1.0,0.0,3.69367 +686,search,97220.9,0.98662,0.0410027,347.373,3.69367 +687,delete,67.677,-1.0,-1.0,0.0,3.68726 +688,search,95868.9,0.98658,0.0410136,345.016,3.68726 +689,delete,259.569,-1.0,-1.0,0.0,3.63629 +690,search,94907.1,0.98654,0.0410977,344.129,3.63629 +691,delete,143.517,-1.0,-1.0,0.0,3.62063 +692,search,94658.7,0.98644,0.0412114,343.934,3.62063 +693,delete,383.003,-1.0,-1.0,0.0,3.55723 +694,search,92091.4,0.9864,0.0413192,341.722,3.55723 +695,delete,83.901,-1.0,-1.0,0.0,3.54635 +696,search,97660.8,0.9864,0.0413192,343.846,3.54635 +697,delete,104.081,-1.0,-1.0,0.0,3.53679 +698,search,94899.1,0.98636,0.0413533,341.337,3.53679 +699,delete,37.289,-1.0,-1.0,0.0,3.53007 +700,search,94160.6,0.98631,0.0414478,341.246,3.53007 +701,delete,194.485,-1.0,-1.0,0.0,3.50812 +702,search,94412.5,0.98626,0.0414898,341.146,3.50812 +703,delete,406.064,-1.0,-1.0,0.0,3.43215 +704,search,96946.3,0.98629,0.0415357,340.515,3.43215 +705,delete,123.625,-1.0,-1.0,0.0,3.41815 +706,search,96058.2,0.98619,0.0417672,342.955,3.41815 +707,delete,1052.49,-1.0,-1.0,0.0,3.168 +708,search,95390.5,0.9857,0.0423971,341.876,3.168 +709,delete,403.307,-1.0,-1.0,0.0,3.08665 +710,search,94746.3,0.98535,0.0431347,340.866,3.08665 +711,delete,164.186,-1.0,-1.0,0.0,3.06769 +712,search,93723.3,0.98521,0.0434094,341.148,3.06769 +713,delete,519.014,-1.0,-1.0,0.0,2.9621 +714,search,93518.1,0.98502,0.0436097,338.061,2.9621 +715,delete,115.766,-1.0,-1.0,0.0,2.94693 +716,search,93864.5,0.98496,0.0436804,338.552,2.94693 +717,delete,114.402,-1.0,-1.0,0.0,2.93417 +718,search,93823,0.98487,0.0437521,338.653,2.93417 +719,delete,730.153,-1.0,-1.0,0.0,2.77723 +720,search,93810.5,0.98412,0.0448757,339.081,2.77723 +721,delete,279.766,-1.0,-1.0,0.0,2.7311 +722,search,94652.6,0.98416,0.0448257,339.371,2.7311 +723,delete,95.27,-1.0,-1.0,0.0,2.72078 +724,search,93266.4,0.98408,0.0449323,337.66,2.72078 +725,delete,292.937,-1.0,-1.0,0.0,2.67201 +726,search,95959.1,0.98412,0.0447641,341.611,2.67201 +727,delete,285.645,-1.0,-1.0,0.0,2.6245 +728,search,96647.3,0.98416,0.0448034,341.533,2.6245 +729,delete,296.36,-1.0,-1.0,0.0,2.57339 +730,search,94037.3,0.98391,0.0453015,342.218,2.57339 +731,delete,225.758,-1.0,-1.0,0.0,2.54174 +732,search,93399.1,0.9837,0.0458317,343.323,2.54174 +733,delete,225.804,-1.0,-1.0,0.0,2.50917 +734,search,95187.2,0.98359,0.0458446,342.326,2.50917 +735,delete,389.501,-1.0,-1.0,0.0,2.43352 +736,search,95432.8,0.98354,0.0458172,343.852,2.43352 +737,delete,358.425,-1.0,-1.0,0.0,2.3771 +738,search,89540.5,0.98351,0.0458627,339.146,2.3771 +739,delete,208.669,-1.0,-1.0,0.0,2.3445 +740,search,93534.2,0.98353,0.0459533,336.937,2.3445 +741,delete,428.817,-1.0,-1.0,0.0,2.26812 +742,search,91509.7,0.98356,0.0461295,337.969,2.26812 +743,delete,471.542,-1.0,-1.0,0.0,2.179 +744,search,92041,0.98359,0.0461706,338.891,2.179 +745,delete,573.656,-1.0,-1.0,0.0,2.07225 +746,search,94048.5,0.98329,0.0465823,344.683,2.07225 +747,delete,487.92,-1.0,-1.0,0.0,1.97187 +748,search,92045.5,0.98312,0.0468745,342.032,1.97187 +749,delete,231.923,-1.0,-1.0,0.0,1.93528 +750,search,92564.4,0.98294,0.0471094,343.226,1.93528 +751,delete,92.83,-1.0,-1.0,0.0,1.92202 +752,search,90915.4,0.98291,0.0471493,340.69,1.92202 +753,delete,203.177,-1.0,-1.0,0.0,1.89213 +754,search,92852.8,0.98283,0.0472711,337.692,1.89213 +755,delete,125.672,-1.0,-1.0,0.0,1.87764 +756,search,90540.7,0.98272,0.0474526,338.001,1.87764 +757,delete,112.018,-1.0,-1.0,0.0,1.86378 +758,search,90731.1,0.98271,0.0475425,339.893,1.86378 +759,delete,106.928,-1.0,-1.0,0.0,1.85039 +760,search,91781.7,0.98266,0.047599,343.947,1.85039 +761,delete,155.563,-1.0,-1.0,0.0,1.83001 +762,search,92641.9,0.98264,0.0476747,341.106,1.83001 +763,delete,88.513,-1.0,-1.0,0.0,1.82066 +764,search,95932.7,0.98263,0.0476375,341.961,1.82066 +765,delete,766.473,-1.0,-1.0,0.0,1.6646 +766,search,92285.6,0.98293,0.046455,338.802,1.6646 +767,delete,869.219,-1.0,-1.0,0.0,1.47582 +768,search,90270.5,0.98213,0.0476213,336.202,1.47582 +769,insert,59.916,-1.0,-1.0,0.0,1.47671 +770,search,92587.3,0.98211,0.0476174,335.802,1.47671 +771,insert,136.609,-1.0,-1.0,0.0,1.48314 +772,search,87602.4,0.98213,0.0475582,341.082,1.48314 +773,insert,299.195,-1.0,-1.0,0.0,1.49496 +774,search,91755.3,0.98226,0.0473236,1380.52,1.49489 +775,insert,181.174,-1.0,-1.0,0.0,1.50415 +776,search,93257.7,0.98237,0.0471851,1380.51,1.50413 +777,insert,135.594,-1.0,-1.0,0.0,1.5107 +778,search,94846.7,0.98247,0.0469658,1407.76,1.51074 +779,insert,56.074,-1.0,-1.0,0.0,1.51359 +780,search,90014.9,0.98254,0.0468995,341.905,1.51359 +781,insert,43.443,-1.0,-1.0,0.0,1.51453 +782,search,90371.8,0.98257,0.0468122,337.058,1.51453 +783,insert,273.498,-1.0,-1.0,0.0,1.52779 +784,search,95321,0.98257,0.0467694,341.165,1.52779 +785,insert,189.789,-1.0,-1.0,0.0,1.54224 +786,search,92947.2,0.98276,0.046553,1464.54,1.54226 +787,insert,87.042,-1.0,-1.0,0.0,1.54605 +788,search,94276.1,0.98281,0.0464522,340.101,1.54605 +789,insert,48.264,-1.0,-1.0,0.0,1.54741 +790,search,93066.9,0.9828,0.0465642,340.558,1.54741 +791,insert,162.718,-1.0,-1.0,0.0,1.55412 +792,search,90189.5,0.98264,0.0467639,338.266,1.55412 +793,insert,20.188,-1.0,-1.0,0.0,1.55455 +794,search,91069.1,0.98264,0.0467639,341.546,1.55455 +795,insert,313.221,-1.0,-1.0,0.0,1.57094 +796,search,93476.9,0.98277,0.046588,344.468,1.57094 +797,insert,33.609,-1.0,-1.0,0.0,1.57161 +798,search,91504.5,0.98277,0.0466094,344.034,1.57161 +799,insert,39.369,-1.0,-1.0,0.0,1.57261 +800,search,94111,0.98272,0.0467089,342.252,1.57261 +801,insert,101.616,-1.0,-1.0,0.0,1.57648 +802,search,94588.5,0.98261,0.0469997,344.028,1.57648 +803,insert,4090.35,-1.0,-1.0,0.0,1.87471 +804,search,102635,0.98378,0.044824,4279.84,1.8742 +805,insert,86.542,-1.0,-1.0,0.0,1.87989 +806,search,95552.6,0.98392,0.0448968,3086.21,1.87984 +807,insert,91.689,-1.0,-1.0,0.0,1.88529 +808,search,97047.6,0.98399,0.0447533,1975.18,1.88553 +809,insert,169.122,-1.0,-1.0,0.0,1.89728 +810,search,96273.7,0.98411,0.0445501,1731.88,1.89734 +811,insert,190.522,-1.0,-1.0,0.0,1.90785 +812,search,93795.2,0.98432,0.0441186,1674.17,1.90791 +813,insert,316.803,-1.0,-1.0,0.0,1.92597 +814,search,96336.3,0.98473,0.0433793,280.298,1.92597 +815,insert,12.503,-1.0,-1.0,0.0,1.92653 +816,search,93614,0.98473,0.0433793,281.479,1.92653 +817,insert,551.852,-1.0,-1.0,0.0,1.95747 +818,search,97781.8,0.98471,0.0433984,1882.39,1.95751 +819,insert,114.34,-1.0,-1.0,0.0,1.96319 +820,search,96523.2,0.98466,0.0434377,1715.07,1.96317 +821,insert,78.669,-1.0,-1.0,0.0,1.96853 +822,search,93374.9,0.98464,0.0434766,1748.11,1.9684 +823,insert,186.521,-1.0,-1.0,0.0,1.98199 +824,search,92798.2,0.9847,0.0434281,1786.73,1.98207 +825,insert,428.319,-1.0,-1.0,0.0,2.01108 +826,search,96829.8,0.98484,0.0431073,1800.94,2.01107 +827,insert,113.301,-1.0,-1.0,0.0,2.01864 +828,search,94850,0.98474,0.0433038,285.116,2.01864 +829,insert,2474.08,-1.0,-1.0,0.0,2.18833 +830,search,100521,0.98479,0.0428478,2841.18,2.18838 +831,insert,183.049,-1.0,-1.0,0.0,2.19999 +832,search,96131.5,0.98489,0.042647,2079.76,2.19991 +833,insert,65.708,-1.0,-1.0,0.0,2.20414 +834,search,93400,0.98486,0.042671,1886.46,2.2041 +835,insert,368.174,-1.0,-1.0,0.0,2.22897 +836,search,97172.4,0.98482,0.0426841,285.776,2.22897 +837,insert,160.569,-1.0,-1.0,0.0,2.23894 +838,search,96328.7,0.98478,0.0427843,1914.51,2.23882 +839,insert,409.963,-1.0,-1.0,0.0,2.26669 +840,search,101012,0.98513,0.0423574,2019.56,2.26658 +841,insert,232.497,-1.0,-1.0,0.0,2.2835 +842,search,97464.6,0.98516,0.0423313,1970.32,2.28361 +843,insert,2101.78,-1.0,-1.0,0.0,2.44709 +844,search,101692,0.98521,0.0423597,3216.87,2.44756 +845,insert,1794.22,-1.0,-1.0,0.0,2.59743 +846,search,102260,0.98594,0.0412119,4589.35,2.59734 +847,insert,24.996,-1.0,-1.0,0.0,2.5986 +848,search,96964.9,0.98606,0.0411338,3017.55,2.59861 +849,insert,21.016,-1.0,-1.0,0.0,2.59997 +850,search,100360,0.98614,0.0410846,2361.15,2.60009 +851,insert,53.718,-1.0,-1.0,0.0,2.60409 +852,search,101523,0.98615,0.0410523,2219.78,2.60413 +853,insert,143.702,-1.0,-1.0,0.0,2.61232 +854,search,100974,0.98619,0.0409938,291.326,2.61232 +855,insert,71.631,-1.0,-1.0,0.0,2.61543 +856,search,97145.1,0.98622,0.0409663,2197.27,2.61541 +857,insert,127.665,-1.0,-1.0,0.0,2.62309 +858,search,100782,0.98622,0.0409175,2267.15,2.62311 +859,insert,204.325,-1.0,-1.0,0.0,2.63699 +860,search,97682.4,0.98614,0.0410603,2316.28,2.63696 +861,insert,38.223,-1.0,-1.0,0.0,2.63954 +862,search,102135,0.98614,0.0411819,2233.7,2.63958 +863,insert,361.824,-1.0,-1.0,0.0,2.66765 +864,search,97720.4,0.98626,0.0409804,2394.4,2.66776 +865,insert,67.092,-1.0,-1.0,0.0,2.67208 +866,search,102320,0.98621,0.0410228,2296.86,2.67218 +867,insert,270.559,-1.0,-1.0,0.0,2.69008 +868,search,98206.9,0.9862,0.0410346,2260.26,2.69007 +869,insert,37.459,-1.0,-1.0,0.0,2.69238 +870,search,98918.7,0.9862,0.0409858,293.047,2.69238 +871,insert,218.535,-1.0,-1.0,0.0,2.70626 +872,search,103705,0.9862,0.0409614,2306.88,2.70621 +873,insert,203.744,-1.0,-1.0,0.0,2.7214 +874,search,100417,0.98626,0.0408582,2372.76,2.72139 +875,insert,83.879,-1.0,-1.0,0.0,2.72834 +876,search,100696,0.98628,0.0407901,2335.71,2.72824 +877,insert,16.867,-1.0,-1.0,0.0,2.72929 +878,search,101350,0.9863,0.0408233,293.031,2.72929 +879,insert,1704.09,-1.0,-1.0,0.0,2.85755 +880,search,106014,0.98642,0.0404675,4347.64,2.85697 +881,insert,2471.8,-1.0,-1.0,0.0,3.08398 +882,search,107355,0.98649,0.0404045,8099.06,3.0844 +883,insert,2080.26,-1.0,-1.0,0.0,3.27966 +884,search,110060,0.98719,0.0392555,9194.07,3.27868 +885,insert,48.94,-1.0,-1.0,0.0,3.28265 +886,search,103310,0.98723,0.0393967,3914.77,3.28293 +887,insert,1815.52,-1.0,-1.0,0.0,3.43846 +888,search,110778,0.9877,0.0386653,5893.51,3.43839 +889,insert,1924,-1.0,-1.0,0.0,3.62355 +890,search,110667,0.98788,0.0384087,10851.5,3.62316 +891,insert,194.283,-1.0,-1.0,0.0,3.63609 +892,search,107850,0.98781,0.0386349,5175.58,3.63556 +893,insert,345.082,-1.0,-1.0,0.0,3.66096 +894,search,109862,0.98786,0.0386127,3802.27,3.66064 +895,insert,280.178,-1.0,-1.0,0.0,3.68238 +896,search,109811,0.98792,0.0384733,3272.46,3.68237 +897,delete,84.484,-1.0,-1.0,0.0,3.66156 +898,search,105680,0.98784,0.0384481,319.663,3.66156 +899,delete,50.253,-1.0,-1.0,0.0,3.66003 +900,search,99283.2,0.98783,0.0385091,320.97,3.66003 +901,delete,151.409,-1.0,-1.0,0.0,3.65245 +902,search,107899,0.98788,0.0384087,2987.17,3.65246 +903,delete,60.856,-1.0,-1.0,0.0,3.63464 +904,search,103494,0.98774,0.038626,325.136,3.63464 +905,delete,86.928,-1.0,-1.0,0.0,3.63188 +906,search,104794,0.98774,0.0386778,324.949,3.63188 +907,delete,72.978,-1.0,-1.0,0.0,3.62877 +908,search,105419,0.98771,0.0387038,325.104,3.62877 +909,delete,115.767,-1.0,-1.0,0.0,3.62456 +910,search,106996,0.9877,0.0387428,324.755,3.62456 +911,delete,198.084,-1.0,-1.0,0.0,3.6155 +912,search,106177,0.98744,0.0391948,2934.86,3.61546 +913,delete,42.619,-1.0,-1.0,0.0,3.60309 +914,search,105507,0.98746,0.0390511,326.289,3.60309 +915,delete,159.287,-1.0,-1.0,0.0,3.5893 +916,search,105462,0.98736,0.0393262,326.231,3.5893 +917,delete,58.175,-1.0,-1.0,0.0,3.58733 +918,search,100356,0.9874,0.0392328,326.644,3.58733 +919,delete,88.89,-1.0,-1.0,0.0,3.58482 +920,search,107938,0.98743,0.0391822,325.255,3.58482 +921,delete,67.785,-1.0,-1.0,0.0,3.57949 +922,search,102391,0.98743,0.0391822,324.141,3.57949 +923,delete,165.477,-1.0,-1.0,0.0,3.56757 +924,search,107149,0.98732,0.039262,322.868,3.56757 +925,delete,299.273,-1.0,-1.0,0.0,3.51723 +926,search,102841,0.98706,0.039508,326.058,3.51723 +927,delete,69.138,-1.0,-1.0,0.0,3.51332 +928,search,102472,0.98703,0.0395321,324.143,3.51332 +929,delete,106.223,-1.0,-1.0,0.0,3.5065 +930,search,104879,0.98703,0.0393801,325.764,3.5065 +931,delete,826.36,-1.0,-1.0,0.0,3.36239 +932,search,103650,0.98678,0.0397912,324.193,3.36239 +933,delete,92.451,-1.0,-1.0,0.0,3.34973 +934,search,100126,0.98679,0.0397574,323.887,3.34973 +935,delete,65.114,-1.0,-1.0,0.0,3.34118 +936,search,102171,0.98676,0.0397866,323.711,3.34118 +937,delete,86.939,-1.0,-1.0,0.0,3.33 +938,search,101929,0.98668,0.0399085,322.95,3.33 +939,delete,369.834,-1.0,-1.0,0.0,3.29307 +940,search,101348,0.9865,0.0401977,324.369,3.29307 +941,delete,149.574,-1.0,-1.0,0.0,3.281 +942,search,103019,0.98628,0.0406428,324.739,3.281 +943,delete,18.333,-1.0,-1.0,0.0,3.27961 +944,search,101668,0.98628,0.0406428,323.002,3.27961 +945,delete,261.485,-1.0,-1.0,0.0,3.23064 +946,search,102654,0.98635,0.04058,325.389,3.23064 +947,delete,96.1,-1.0,-1.0,0.0,3.22381 +948,search,100961,0.9863,0.0407006,320.601,3.22381 +949,delete,146.688,-1.0,-1.0,0.0,3.20365 +950,search,101557,0.98632,0.0406555,323.118,3.20365 +951,delete,87.553,-1.0,-1.0,0.0,3.18367 +952,search,104236,0.98617,0.0407926,323.958,3.18367 +953,delete,152.336,-1.0,-1.0,0.0,3.16935 +954,search,101602,0.98614,0.040767,323.892,3.16935 +955,delete,50.358,-1.0,-1.0,0.0,3.16118 +956,search,104842,0.98618,0.0405354,323.639,3.16118 +957,delete,642.385,-1.0,-1.0,0.0,3.03825 +958,search,104534,0.98581,0.0415428,322.806,3.03825 +959,delete,159.52,-1.0,-1.0,0.0,3.02038 +960,search,106027,0.98555,0.0419563,324.527,3.02038 +961,delete,106.669,-1.0,-1.0,0.0,3.01032 +962,search,98824,0.98542,0.0421142,322.689,3.01032 +963,delete,293.363,-1.0,-1.0,0.0,2.96376 +964,search,100623,0.9855,0.0419014,321.522,2.96376 +965,delete,171.265,-1.0,-1.0,0.0,2.935 +966,search,105663,0.98536,0.0422117,325.372,2.935 +967,delete,158.738,-1.0,-1.0,0.0,2.91399 +968,search,101275,0.98521,0.0425248,321.62,2.91399 +969,delete,170.293,-1.0,-1.0,0.0,2.88302 +970,search,99899.3,0.98514,0.0427265,322.25,2.88302 +971,delete,640.51,-1.0,-1.0,0.0,2.75931 +972,search,105230,0.98518,0.0423371,322.882,2.75931 +973,delete,592.997,-1.0,-1.0,0.0,2.63586 +974,search,104406,0.98437,0.0438496,321.861,2.63586 +975,delete,351.676,-1.0,-1.0,0.0,2.56975 +976,search,103478,0.98371,0.0450086,320.774,2.56975 +977,delete,189.065,-1.0,-1.0,0.0,2.53594 +978,search,104236,0.98372,0.0449584,320.827,2.53594 +979,delete,57.155,-1.0,-1.0,0.0,2.52975 +980,search,104619,0.98369,0.0449784,322.091,2.52975 +981,delete,289.254,-1.0,-1.0,0.0,2.47629 +982,search,99443.9,0.98371,0.0450973,320.004,2.47629 +983,delete,136.728,-1.0,-1.0,0.0,2.46256 +984,search,100287,0.98356,0.0454085,320.407,2.46256 +985,delete,145.89,-1.0,-1.0,0.0,2.44482 +986,search,100006,0.98342,0.0456195,318.526,2.44482 +987,delete,162.089,-1.0,-1.0,0.0,2.42591 +988,search,96661.5,0.98338,0.0456964,321.744,2.42591 +989,delete,66.568,-1.0,-1.0,0.0,2.41947 +990,search,100036,0.98336,0.045729,320.699,2.41947 +991,delete,196.352,-1.0,-1.0,0.0,2.38537 +992,search,99704.6,0.9834,0.0453708,319.959,2.38537 +993,delete,263.76,-1.0,-1.0,0.0,2.34778 +994,search,99951,0.9831,0.0460497,319.591,2.34778 +995,delete,224.793,-1.0,-1.0,0.0,2.3138 +996,search,100587,0.98289,0.0463498,320.413,2.3138 +997,delete,100.877,-1.0,-1.0,0.0,2.30187 +998,search,98549.4,0.98291,0.0462499,320.71,2.30187 +999,delete,315.261,-1.0,-1.0,0.0,2.24546 +1000,search,99193,0.98282,0.0463788,320.967,2.24546 +1001,delete,316.98,-1.0,-1.0,0.0,2.19567 +1002,search,98705.7,0.98284,0.0462785,320.794,2.19567 +1003,delete,157.726,-1.0,-1.0,0.0,2.17266 +1004,search,99651,0.98275,0.0465793,319.499,2.17266 +1005,delete,81.904,-1.0,-1.0,0.0,2.16271 +1006,search,98716.8,0.98274,0.0466273,321.168,2.16271 +1007,delete,456.997,-1.0,-1.0,0.0,2.07405 +1008,search,104141,0.98267,0.0466552,322.348,2.07405 +1009,delete,737.756,-1.0,-1.0,0.0,1.92546 +1010,search,101974,0.98252,0.0469944,322.317,1.92546 +1011,delete,926.326,-1.0,-1.0,0.0,1.7433 +1012,search,100754,0.98176,0.0482636,320.632,1.7433 +1013,delete,42.485,-1.0,-1.0,0.0,1.73768 +1014,search,97824.3,0.98168,0.0483392,324.757,1.73768 +1015,delete,670.086,-1.0,-1.0,0.0,1.60326 +1016,search,99679.1,0.98115,0.0489477,323.345,1.60326 +1017,delete,813.436,-1.0,-1.0,0.0,1.44867 +1018,search,98476.9,0.98139,0.0486934,319.774,1.44867 +1019,delete,91.058,-1.0,-1.0,0.0,1.43753 +1020,search,103717,0.98143,0.0484826,319.761,1.43753 +1021,delete,380.719,-1.0,-1.0,0.0,1.37765 +1022,search,101971,0.98152,0.0488941,319.405,1.37765 +1023,delete,285.695,-1.0,-1.0,0.0,1.33484 +1024,search,97854.2,0.98103,0.0493708,320.376,1.33484 +1025,insert,163.645,-1.0,-1.0,0.0,1.34256 +1026,search,101063,0.98113,0.0491657,318.173,1.34256 +1027,insert,1204.11,-1.0,-1.0,0.0,1.39969 +1028,search,106184,0.9817,0.0483271,1429.09,1.39978 +1029,insert,2208.24,-1.0,-1.0,0.0,1.54248 +1030,search,103621,0.98198,0.0476998,2066.48,1.54234 +1031,insert,3306.98,-1.0,-1.0,0.0,1.79153 +1032,search,105702,0.98292,0.0462448,3971.69,1.79144 +1033,insert,2153.13,-1.0,-1.0,0.0,1.95994 +1034,search,106115,0.98426,0.0440066,3553.68,1.95967 +1035,insert,1478,-1.0,-1.0,0.0,2.06538 +1036,search,105392,0.98423,0.0439831,3673.81,2.06548 +1037,insert,68.786,-1.0,-1.0,0.0,2.06934 +1038,search,102763,0.98419,0.0443301,2112.05,2.06915 +1039,insert,440.054,-1.0,-1.0,0.0,2.09794 +1040,search,104681,0.98425,0.0443514,1943.04,2.09793 +1041,insert,54.979,-1.0,-1.0,0.0,2.10152 +1042,search,102151,0.98428,0.044307,1887.57,2.10147 +1043,insert,114.67,-1.0,-1.0,0.0,2.10806 +1044,search,104855,0.98424,0.044313,1916.86,2.10802 +1045,insert,267.431,-1.0,-1.0,0.0,2.12305 +1046,search,104893,0.98423,0.0444579,1903.67,2.12304 +1047,insert,19.498,-1.0,-1.0,0.0,2.12384 +1048,search,107787,0.9842,0.0445051,334.641,2.12384 +1049,insert,104.487,-1.0,-1.0,0.0,2.12955 +1050,search,106221,0.98421,0.0445179,335.37,2.12955 +1051,insert,654.821,-1.0,-1.0,0.0,2.16696 +1052,search,108114,0.98424,0.044583,1979.59,2.16683 +1053,insert,297.941,-1.0,-1.0,0.0,2.18488 +1054,search,107648,0.98424,0.0444707,1973.09,2.18477 +1055,insert,224.855,-1.0,-1.0,0.0,2.19896 +1056,search,102549,0.98424,0.0445156,337.692,2.19896 +1057,insert,50.989,-1.0,-1.0,0.0,2.20109 +1058,search,102420,0.98424,0.0445381,336.545,2.20109 +1059,insert,164.92,-1.0,-1.0,0.0,2.20796 +1060,search,104464,0.9843,0.0444917,1977.99,2.20795 +1061,insert,53.747,-1.0,-1.0,0.0,2.21115 +1062,search,103756,0.9843,0.0444917,1995.1,2.21113 +1063,insert,61.827,-1.0,-1.0,0.0,2.21489 +1064,search,106666,0.98425,0.044666,2046.19,2.215 +1065,insert,1575.69,-1.0,-1.0,0.0,2.33248 +1066,search,117258,0.98434,0.0445311,2795.44,2.33229 +1067,insert,39.976,-1.0,-1.0,0.0,2.33472 +1068,search,107185,0.98438,0.0444334,2268.38,2.33455 +1069,insert,204.902,-1.0,-1.0,0.0,2.34725 +1070,search,109079,0.98456,0.0438843,2181.33,2.34732 +1071,insert,149.176,-1.0,-1.0,0.0,2.35791 +1072,search,105595,0.98457,0.043878,2111.42,2.35789 +1073,insert,158.499,-1.0,-1.0,0.0,2.36702 +1074,search,107371,0.98456,0.0438615,2108.49,2.36698 +1075,insert,65.888,-1.0,-1.0,0.0,2.37034 +1076,search,108723,0.98458,0.0438455,2109.16,2.37034 +1077,insert,160.146,-1.0,-1.0,0.0,2.38119 +1078,search,107784,0.98459,0.0438162,2122.76,2.38118 +1079,insert,133.188,-1.0,-1.0,0.0,2.38887 +1080,search,108520,0.98469,0.0436684,2150.19,2.38887 +1081,insert,1953.68,-1.0,-1.0,0.0,2.53009 +1082,search,115406,0.98494,0.0430987,2692.91,2.52983 +1083,insert,4.594,-1.0,-1.0,0.0,2.53028 +1084,search,100880,0.98485,0.0431934,2295.57,2.53028 +1085,insert,139.696,-1.0,-1.0,0.0,2.53866 +1086,search,109112,0.98487,0.0432232,2259.8,2.53867 +1087,insert,96.064,-1.0,-1.0,0.0,2.54399 +1088,search,106682,0.98489,0.0431834,2272.68,2.54403 +1089,insert,2174,-1.0,-1.0,0.0,2.7085 +1090,search,109583,0.98528,0.0427725,4053.15,2.70924 +1091,insert,298.384,-1.0,-1.0,0.0,2.7311 +1092,search,110597,0.98546,0.0423422,2872.69,2.73092 +1093,insert,110.185,-1.0,-1.0,0.0,2.73903 +1094,search,110168,0.98546,0.042295,2605.23,2.73898 +1095,insert,2114.09,-1.0,-1.0,0.0,2.90637 +1096,search,116747,0.98574,0.0419849,5192.44,2.90632 +1097,insert,116.723,-1.0,-1.0,0.0,2.91542 +1098,search,111137,0.9858,0.0419574,3378.8,2.91487 +1099,insert,80.532,-1.0,-1.0,0.0,2.92004 +1100,search,112620,0.98585,0.0419401,2737.12,2.91984 +1101,insert,240.155,-1.0,-1.0,0.0,2.93596 +1102,search,112533,0.98589,0.0420256,2622.63,2.93622 +1103,insert,509.749,-1.0,-1.0,0.0,2.97577 +1104,search,112736,0.98642,0.0410812,2655.38,2.97565 +1105,insert,16.196,-1.0,-1.0,0.0,2.97662 +1106,search,107445,0.9864,0.041052,2596.43,2.97658 +1107,insert,1209.8,-1.0,-1.0,0.0,3.07261 +1108,search,111024,0.98651,0.040636,3746.02,3.0724 +1109,insert,38.983,-1.0,-1.0,0.0,3.07573 +1110,search,106314,0.98657,0.0407523,3055.5,3.07595 +1111,insert,241.997,-1.0,-1.0,0.0,3.09192 +1112,search,108370,0.98659,0.0407367,2997.27,3.09212 +1113,insert,317.717,-1.0,-1.0,0.0,3.1147 +1114,search,109427,0.98682,0.0403531,2893.3,3.11508 +1115,insert,364.563,-1.0,-1.0,0.0,3.14201 +1116,search,108392,0.98682,0.0403283,2892.77,3.14153 +1117,insert,90.647,-1.0,-1.0,0.0,3.14829 +1118,search,109624,0.98687,0.0402355,2774.78,3.1485 +1119,insert,64.339,-1.0,-1.0,0.0,3.15217 +1120,search,113625,0.98689,0.0401892,2745.79,3.15217 +1121,insert,166.936,-1.0,-1.0,0.0,3.16335 +1122,search,109044,0.98686,0.0402427,2767.5,3.16312 +1123,insert,118.253,-1.0,-1.0,0.0,3.17088 +1124,search,111046,0.98683,0.0401969,2791.11,3.17108 +1125,insert,48.632,-1.0,-1.0,0.0,3.17415 +1126,search,110351,0.98683,0.0401969,2755.62,3.17425 +1127,insert,250.444,-1.0,-1.0,0.0,3.19346 +1128,search,112383,0.98687,0.0401609,2762.09,3.19344 +1129,insert,81.479,-1.0,-1.0,0.0,3.19872 +1130,search,105214,0.98687,0.0401111,2774.49,3.1987 +1131,insert,206.939,-1.0,-1.0,0.0,3.21214 +1132,search,107238,0.9869,0.0400321,2891.1,3.21206 +1133,insert,104.91,-1.0,-1.0,0.0,3.21896 +1134,search,107595,0.98686,0.0401182,2896.82,3.21892 +1135,insert,42.943,-1.0,-1.0,0.0,3.22199 +1136,search,108144,0.98684,0.040139,354.082,3.22199 +1137,insert,203.857,-1.0,-1.0,0.0,3.23102 +1138,search,113924,0.98685,0.0401265,2853.07,3.2311 +1139,insert,13.769,-1.0,-1.0,0.0,3.23204 +1140,search,110554,0.98684,0.0401639,2817.83,3.23201 +1141,insert,1161.84,-1.0,-1.0,0.0,3.32653 +1142,search,111908,0.9869,0.039907,3976.58,3.32697 +1143,insert,127.658,-1.0,-1.0,0.0,3.33653 +1144,search,109154,0.98701,0.039705,3422.18,3.33627 +1145,insert,56.241,-1.0,-1.0,0.0,3.34077 +1146,search,106838,0.987,0.0397372,3109.11,3.34135 +1147,insert,109.179,-1.0,-1.0,0.0,3.34924 +1148,search,109245,0.9871,0.039668,3165.63,3.34878 +1149,insert,31.206,-1.0,-1.0,0.0,3.35077 +1150,search,112014,0.98711,0.0396344,3028.11,3.35061 +1151,insert,34.893,-1.0,-1.0,0.0,3.35297 +1152,search,111449,0.98711,0.0396344,3049.95,3.35298 +1153,delete,35.676,-1.0,-1.0,0.0,3.34958 +1154,search,109545,0.98711,0.0396848,361.216,3.34958 +1155,delete,289.385,-1.0,-1.0,0.0,3.32125 +1156,search,110487,0.98697,0.0398666,361.781,3.32125 +1157,delete,531.549,-1.0,-1.0,0.0,3.24823 +1158,search,107475,0.98675,0.0402434,361.057,3.24823 +1159,delete,688.452,-1.0,-1.0,0.0,3.08659 +1160,search,105333,0.9866,0.0403799,361.292,3.08659 +1161,delete,519.456,-1.0,-1.0,0.0,3.00337 +1162,search,112884,0.98601,0.0415135,359.543,3.00337 +1163,delete,279.441,-1.0,-1.0,0.0,2.9513 +1164,search,106116,0.98598,0.0415156,361.924,2.9513 +1165,delete,106.776,-1.0,-1.0,0.0,2.94458 +1166,search,110746,0.98591,0.0416238,357.686,2.94458 +1167,delete,212.336,-1.0,-1.0,0.0,2.92576 +1168,search,108257,0.98583,0.0415754,356.6,2.92576 +1169,delete,59.116,-1.0,-1.0,0.0,2.91322 +1170,search,105954,0.98573,0.0416624,358.981,2.91322 +1171,delete,98.116,-1.0,-1.0,0.0,2.90279 +1172,search,108864,0.98569,0.0418617,361.711,2.90279 +1173,delete,131.748,-1.0,-1.0,0.0,2.89235 +1174,search,106908,0.98561,0.0418343,419.1,2.89235 +1175,delete,71.969,-1.0,-1.0,0.0,2.88827 +1176,search,105678,0.98561,0.0419061,358.421,2.88827 +1177,delete,45.818,-1.0,-1.0,0.0,2.88348 +1178,search,105577,0.9856,0.0419131,359.509,2.88348 +1179,delete,179.123,-1.0,-1.0,0.0,2.86394 +1180,search,108635,0.98545,0.0420412,356.096,2.86394 +1181,delete,143.105,-1.0,-1.0,0.0,2.84892 +1182,search,104496,0.98533,0.0421201,355.838,2.84892 +1183,delete,134.227,-1.0,-1.0,0.0,2.83236 +1184,search,106131,0.98516,0.0423075,355.687,2.83236 +1185,delete,86.232,-1.0,-1.0,0.0,2.82578 +1186,search,110657,0.98516,0.0424491,355.124,2.82578 +1187,delete,346.264,-1.0,-1.0,0.0,2.78044 +1188,search,105258,0.98511,0.042512,359.539,2.78044 +1189,delete,61.422,-1.0,-1.0,0.0,2.77514 +1190,search,112898,0.9851,0.04257,361.953,2.77514 +1191,delete,44.878,-1.0,-1.0,0.0,2.77245 +1192,search,111129,0.98507,0.0426186,361.087,2.77245 +1193,delete,461.977,-1.0,-1.0,0.0,2.68065 +1194,search,111509,0.98496,0.0426611,358.309,2.68065 +1195,delete,80.069,-1.0,-1.0,0.0,2.67558 +1196,search,110504,0.98494,0.042632,354.658,2.67558 +1197,delete,183.263,-1.0,-1.0,0.0,2.65672 +1198,search,110124,0.98457,0.0433738,354.752,2.65672 +1199,delete,45.926,-1.0,-1.0,0.0,2.65101 +1200,search,110149,0.98458,0.0433636,362.516,2.65101 +1201,delete,182.847,-1.0,-1.0,0.0,2.62336 +1202,search,112064,0.98448,0.0435822,361.292,2.62336 +1203,delete,104.149,-1.0,-1.0,0.0,2.61453 +1204,search,110531,0.98445,0.0436757,361.563,2.61453 +1205,delete,76.688,-1.0,-1.0,0.0,2.60225 +1206,search,110461,0.98447,0.0436803,360.464,2.60225 +1207,delete,75.647,-1.0,-1.0,0.0,2.5931 +1208,search,110510,0.98432,0.0438685,357.051,2.5931 +1209,delete,554.169,-1.0,-1.0,0.0,2.48229 +1210,search,110069,0.9842,0.0439625,354.636,2.48229 +1211,delete,30.223,-1.0,-1.0,0.0,2.4798 +1212,search,109640,0.98417,0.0440069,363.84,2.4798 +1213,delete,308.343,-1.0,-1.0,0.0,2.43384 +1214,search,112009,0.98394,0.0444752,360.374,2.43384 +1215,delete,69.571,-1.0,-1.0,0.0,2.42642 +1216,search,110220,0.98389,0.0445823,362.829,2.42642 +1217,delete,723.941,-1.0,-1.0,0.0,2.2743 +1218,search,110157,0.98341,0.0454752,358.16,2.2743 +1219,delete,265.313,-1.0,-1.0,0.0,2.22938 +1220,search,109583,0.98321,0.0462328,355.481,2.22938 +1221,delete,124.458,-1.0,-1.0,0.0,2.21359 +1222,search,108420,0.98314,0.0464533,351.626,2.21359 +1223,delete,528.966,-1.0,-1.0,0.0,2.10853 +1224,search,108413,0.98258,0.0471447,353.053,2.10853 +1225,delete,114.733,-1.0,-1.0,0.0,2.09138 +1226,search,109164,0.98241,0.0474301,349.306,2.09138 +1227,delete,226.715,-1.0,-1.0,0.0,2.05962 +1228,search,110346,0.98245,0.0472969,359.03,2.05962 +1229,delete,268.812,-1.0,-1.0,0.0,2.02093 +1230,search,109916,0.98215,0.048026,355.355,2.02093 +1231,delete,223.718,-1.0,-1.0,0.0,1.98225 +1232,search,109052,0.98136,0.0493019,359.646,1.98225 +1233,delete,52.538,-1.0,-1.0,0.0,1.97571 +1234,search,108890,0.98136,0.0492816,356.164,1.97571 +1235,delete,406.318,-1.0,-1.0,0.0,1.90439 +1236,search,108949,0.98117,0.0497674,355.943,1.90439 +1237,delete,63.316,-1.0,-1.0,0.0,1.89609 +1238,search,109007,0.98115,0.0497782,353.237,1.89609 +1239,delete,214.345,-1.0,-1.0,0.0,1.87096 +1240,search,109075,0.98106,0.0499544,351.791,1.87096 +1241,delete,160.158,-1.0,-1.0,0.0,1.84977 +1242,search,108969,0.98094,0.0502271,350.291,1.84977 +1243,delete,177.281,-1.0,-1.0,0.0,1.82578 +1244,search,109257,0.98074,0.0505476,355.685,1.82578 +1245,delete,63.886,-1.0,-1.0,0.0,1.81804 +1246,search,105704,0.98074,0.0505278,356.678,1.81804 +1247,delete,167.854,-1.0,-1.0,0.0,1.78373 +1248,search,110346,0.97958,0.0518213,360.958,1.78373 +1249,delete,141.561,-1.0,-1.0,0.0,1.76624 +1250,search,109231,0.97942,0.0520641,361.239,1.76624 +1251,delete,119.651,-1.0,-1.0,0.0,1.751 +1252,search,108558,0.97938,0.0520694,358.978,1.751 +1253,delete,62.73,-1.0,-1.0,0.0,1.74307 +1254,search,108510,0.97939,0.0521015,359.032,1.74307 +1255,delete,289.498,-1.0,-1.0,0.0,1.6858 +1256,search,108146,0.97926,0.0522155,354.65,1.6858 +1257,delete,118.067,-1.0,-1.0,0.0,1.6675 +1258,search,107686,0.97912,0.0524072,354.24,1.6675 +1259,delete,171.132,-1.0,-1.0,0.0,1.64354 +1260,search,107343,0.97905,0.0525184,355.317,1.64354 +1261,delete,83.405,-1.0,-1.0,0.0,1.63394 +1262,search,108352,0.97888,0.0527668,352.16,1.63394 +1263,delete,291.73,-1.0,-1.0,0.0,1.58435 +1264,search,107159,0.97886,0.05268,357.71,1.58435 +1265,delete,605.21,-1.0,-1.0,0.0,1.46851 +1266,search,108248,0.97891,0.0526718,357.202,1.46851 +1267,delete,261.086,-1.0,-1.0,0.0,1.43332 +1268,search,106245,0.97865,0.0530986,356.4,1.43332 +1269,delete,408.63,-1.0,-1.0,0.0,1.37062 +1270,search,108541,0.97888,0.0527857,358.147,1.37062 +1271,delete,268.904,-1.0,-1.0,0.0,1.33192 +1272,search,108613,0.97856,0.053034,356.555,1.33192 +1273,delete,341.932,-1.0,-1.0,0.0,1.27444 +1274,search,106645,0.97854,0.0530985,354.338,1.27444 +1275,delete,96.76,-1.0,-1.0,0.0,1.26343 +1276,search,106850,0.97849,0.0533184,355.204,1.26343 +1277,delete,130.783,-1.0,-1.0,0.0,1.24783 +1278,search,106375,0.9785,0.0535525,354.31,1.24783 +1279,delete,116.543,-1.0,-1.0,0.0,1.23556 +1280,search,106817,0.97846,0.0536125,354.188,1.23556 diff --git a/scripts/big_ann_perf_numbers/scan_using_batching_optimizations.csv b/scripts/big_ann_perf_numbers/scan_using_batching_optimizations.csv new file mode 100644 index 00000000..ff5bf830 --- /dev/null +++ b/scripts/big_ann_perf_numbers/scan_using_batching_optimizations.csv @@ -0,0 +1,251 @@ +step_num,step_type,latency_ms,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,2632.01,0.78814,0.157167,27.1243,17.7481,5.413,0.0184881,1024,38806 +3,insert,13.386,-1.0,-1.0,-1.0,-1.0,4.861,0.0226668,1024,49334 +4,search,1917.41,0.80378,0.157445,25.2878,17.2311,5.214,0.0226668,1024,49334 +5,insert,19.873,-1.0,-1.0,-1.0,-1.0,144.105,0.0334005,1024,73557 +6,search,1888.79,0.8809,0.134059,18.3656,16.172,94.741,0.033569,1028,73557 +7,insert,35.539,-1.0,-1.0,-1.0,-1.0,82.73,0.0429987,1029,95766 +8,search,1946.28,0.87389,0.135667,18.3448,15.5452,83.028,0.0429987,1029,95766 +9,insert,4.601,-1.0,-1.0,-1.0,-1.0,81.497,0.0447654,1029,100025 +10,search,1850.54,0.87441,0.134003,18.1401,15.2176,82.995,0.0447654,1029,100025 +11,insert,10.982,-1.0,-1.0,-1.0,-1.0,84.771,0.0495206,1029,111417 +12,search,1925.55,0.87669,0.133554,17.9087,15.0639,19.749,0.049652,1030,111417 +13,insert,34.869,-1.0,-1.0,-1.0,-1.0,5.36,0.0608124,1030,137139 +14,search,1943.05,0.87135,0.137981,18.0877,14.8735,13.398,0.0608222,1032,137139 +15,insert,18.482,-1.0,-1.0,-1.0,-1.0,5.164,0.0690103,1032,156710 +16,search,1884.59,0.8661,0.137958,18.2658,14.554,12.344,0.0690157,1035,156710 +17,insert,4.59,-1.0,-1.0,-1.0,-1.0,4.7,0.0713474,1035,161773 +18,search,1982.27,0.86496,0.138886,18.2326,14.5256,5.175,0.0713474,1035,161773 +19,insert,28.835,-1.0,-1.0,-1.0,-1.0,4.086,0.0781561,1035,177161 +20,search,1836.34,0.87166,0.134643,17.8515,14.4067,7.754,0.0781561,1035,177161 +21,insert,403.667,-1.0,-1.0,-1.0,-1.0,117.814,0.199758,1063,448898 +22,search,2081.83,0.86327,0.134857,19.2662,15.0447,27.848,0.199809,1069,448898 +23,insert,409.62,-1.0,-1.0,-1.0,-1.0,311.87,0.328989,1120,733493 +24,search,2541.04,0.85706,0.137099,20.1174,15.5544,46.644,0.329308,1131,733493 +25,insert,38.98,-1.0,-1.0,-1.0,-1.0,14.227,0.346145,1132,777443 +26,search,2296.53,0.85619,0.13842,20.2556,15.629,7.125,0.346145,1132,777443 +27,insert,29.156,-1.0,-1.0,-1.0,-1.0,10.562,0.357722,1133,805394 +28,search,2309.62,0.85729,0.137691,20.2172,15.6079,13.596,0.35797,1134,805394 +29,insert,67.381,-1.0,-1.0,-1.0,-1.0,35.009,0.385411,1141,866391 +30,search,2259.64,0.86042,0.137113,19.9102,15.5618,14.225,0.385411,1143,866391 +31,insert,449.841,-1.0,-1.0,-1.0,-1.0,344.588,0.55332,1199,1239816 +32,search,2566.69,0.85078,0.143067,21.0299,16.3049,57.874,0.553319,1203,1239816 +33,insert,49.195,-1.0,-1.0,-1.0,-1.0,21.03,0.573388,1207,1291211 +34,search,2637.32,0.85248,0.141198,21.0615,16.3669,40.36,0.573391,1210,1291211 +35,insert,48.962,-1.0,-1.0,-1.0,-1.0,28.407,0.585617,1215,1324978 +36,search,2679.79,0.85453,0.13992,20.9283,16.3451,7.076,0.585617,1215,1324978 +37,insert,67.274,-1.0,-1.0,-1.0,-1.0,28.986,0.607368,1219,1371885 +38,search,2691.33,0.85524,0.139471,21.0539,16.5981,11.789,0.607368,1220,1371885 +39,insert,8.003,-1.0,-1.0,-1.0,-1.0,6.592,0.610279,1220,1379335 +40,search,2699.52,0.85643,0.139086,21.0585,16.6344,7.21,0.610279,1220,1379335 +41,insert,36.694,-1.0,-1.0,-1.0,-1.0,26.993,0.623715,1226,1413312 +42,search,2733.34,0.85614,0.139715,21.0612,16.6814,7.142,0.623715,1226,1413312 +43,insert,10.595,-1.0,-1.0,-1.0,-1.0,6.634,0.627081,1226,1423411 +44,search,2717.31,0.85652,0.139367,21.0669,16.7085,13.307,0.62731,1227,1423411 +45,insert,196.158,-1.0,-1.0,-1.0,-1.0,11.277,0.640857,1229,1457559 +46,search,2733.39,0.85693,0.13898,20.9978,16.7575,7.014,0.640857,1229,1457559 +47,insert,322.675,-1.0,-1.0,-1.0,-1.0,232.876,0.746731,1259,1688593 +48,search,2906.63,0.86012,0.140045,20.7731,16.9772,77.789,0.746751,1265,1688593 +49,insert,45.189,-1.0,-1.0,-1.0,-1.0,19.325,0.76124,1268,1723505 +50,search,2911.56,0.86114,0.140008,20.7456,17.0214,23.492,0.761241,1270,1723505 +51,insert,347.994,-1.0,-1.0,-1.0,-1.0,199.574,0.900893,1303,2035040 +52,search,3104.82,0.86564,0.138716,20.5745,17.3909,17.913,0.901045,1307,2035040 +53,insert,22.179,-1.0,-1.0,-1.0,-1.0,17.241,0.919777,1311,2078341 +54,search,2444.74,0.86631,0.138402,20.5859,17.3743,8.543,0.919785,1312,2078341 +55,insert,192.021,-1.0,-1.0,-1.0,-1.0,267.73,1.06766,1355,2399905 +56,search,2621.24,0.86822,0.138735,20.3907,17.5395,64.348,1.06788,1369,2399905 +57,insert,19.433,-1.0,-1.0,-1.0,-1.0,8.015,1.07681,1370,2419836 +58,search,2632.25,0.86868,0.138522,20.4308,17.6446,26.228,1.07692,1375,2419836 +59,insert,126.721,-1.0,-1.0,-1.0,-1.0,80.962,1.17577,1386,2643083 +60,search,2689.86,0.86011,0.146646,21.878,19.1613,15.293,1.17595,1389,2643083 +61,insert,7.176,-1.0,-1.0,-1.0,-1.0,13.366,1.1789,1392,2653064 +62,search,2706.09,0.86032,0.146639,21.9275,19.2063,4.959,1.1789,1392,2653064 +63,insert,2.475,-1.0,-1.0,-1.0,-1.0,4.577,1.18047,1392,2656118 +64,search,3115.77,0.8604,0.146416,21.9225,19.2141,13.038,1.18058,1393,2656118 +65,insert,78.884,-1.0,-1.0,-1.0,-1.0,104.077,1.20899,1411,2723424 +66,search,3501.54,0.86183,0.144898,21.9506,19.2159,7.931,1.20899,1411,2723424 +67,insert,45.836,-1.0,-1.0,-1.0,-1.0,73.427,1.22966,1424,2769021 +68,search,3542.28,0.86232,0.144708,21.8205,19.1541,15.812,1.22966,1425,2769021 +69,insert,59.997,-1.0,-1.0,-1.0,-1.0,52.322,1.25219,1432,2821777 +70,search,3580.57,0.86494,0.142718,21.5631,19.0287,14.669,1.2522,1433,2821777 +71,insert,18.517,-1.0,-1.0,-1.0,-1.0,27.016,1.25833,1436,2834921 +72,search,3610.55,0.86539,0.14234,21.5078,19.0046,8.544,1.25833,1436,2834921 +73,insert,7.505,-1.0,-1.0,-1.0,-1.0,19.883,1.26069,1438,2842044 +74,search,3624.63,0.86524,0.142453,21.4984,18.9971,8.232,1.26069,1438,2842044 +75,insert,24.776,-1.0,-1.0,-1.0,-1.0,26.473,1.26741,1441,2858890 +76,search,3649.68,0.86629,0.141337,21.475,18.9623,21.085,1.26766,1443,2858890 +77,insert,32.532,-1.0,-1.0,-1.0,-1.0,25.674,1.27695,1446,2882818 +78,search,3625.03,0.86492,0.142189,21.579,19.0371,8.484,1.27695,1446,2882818 +79,insert,467.313,-1.0,-1.0,-1.0,-1.0,7.24,1.30434,1446,2947806 +80,search,3636.89,0.86495,0.14229,21.6919,19.1546,8.306,1.30434,1446,2947806 +81,insert,34.646,-1.0,-1.0,-1.0,-1.0,29.468,1.31699,1449,2976895 +82,search,3646.04,0.86584,0.141918,21.5183,19.0851,7.975,1.31699,1449,2976895 +83,insert,15.699,-1.0,-1.0,-1.0,-1.0,19.101,1.3229,1451,2990682 +84,search,3660.72,0.86702,0.140947,21.4726,19.0689,9.047,1.3229,1451,2990682 +85,insert,39.612,-1.0,-1.0,-1.0,-1.0,20.07,1.33627,1453,3025613 +86,search,3624.78,0.86791,0.140661,21.375,19.0761,8.325,1.33627,1453,3025613 +87,insert,671.505,-1.0,-1.0,-1.0,-1.0,560.225,1.5433,1524,3468632 +88,search,4018.18,0.87117,0.138193,20.9937,18.8929,88.589,1.54333,1535,3468632 +89,insert,418.166,-1.0,-1.0,-1.0,-1.0,445.405,1.70092,1595,3831692 +90,search,4258.33,0.87535,0.133437,20.6001,18.5074,44.56,1.7011,1600,3831692 +91,insert,40.406,-1.0,-1.0,-1.0,-1.0,34.17,1.71286,1604,3864067 +92,search,4291.22,0.87541,0.1332,20.6264,18.5244,15.297,1.71291,1605,3864067 +93,insert,270.232,-1.0,-1.0,-1.0,-1.0,187.571,1.81663,1628,4103827 +94,search,4401.52,0.87596,0.13334,20.4969,18.4313,28.677,1.81684,1630,4103827 +95,insert,18.795,-1.0,-1.0,-1.0,-1.0,21.703,1.82315,1632,4119898 +96,search,4338.53,0.8768,0.132725,20.449,18.4249,15.911,1.82329,1633,4119898 +97,insert,11.366,-1.0,-1.0,-1.0,-1.0,22.009,1.82724,1635,4129367 +98,search,4399.33,0.87635,0.132979,20.4711,18.4304,9.554,1.82724,1635,4129367 +99,insert,5.836,-1.0,-1.0,-1.0,-1.0,22.732,1.82884,1637,4134502 +100,search,4416.76,0.87634,0.133101,20.4432,18.4272,10.738,1.82884,1637,4134502 +101,insert,37.583,-1.0,-1.0,-1.0,-1.0,38.519,1.84193,1642,4164482 +102,search,4465.68,0.87692,0.132818,20.4162,18.4205,10.145,1.84193,1642,4164482 +103,insert,15.477,-1.0,-1.0,-1.0,-1.0,22.676,1.84756,1644,4178601 +104,search,4443.6,0.87701,0.13287,20.3856,18.3963,10.762,1.84756,1644,4178601 +105,insert,12.599,-1.0,-1.0,-1.0,-1.0,17.064,1.85088,1645,4187774 +106,search,4469.21,0.87704,0.132927,20.3724,18.3854,10.587,1.85088,1645,4187774 +107,insert,22.857,-1.0,-1.0,-1.0,-1.0,9.367,1.85653,1645,4200109 +108,search,4468.94,0.87714,0.1331,20.3782,18.3998,18.019,1.85681,1646,4200109 +109,insert,556.199,-1.0,-1.0,-1.0,-1.0,535.436,2.02015,1713,4557867 +110,search,4724.51,0.87922,0.13168,20.2081,18.4495,76.833,2.02124,1721,4557867 +111,insert,31.283,-1.0,-1.0,-1.0,-1.0,93.1,2.03272,1736,4587252 +112,search,4777.86,0.87917,0.131589,20.1872,18.4653,20.779,2.03301,1737,4587252 +113,insert,12.125,-1.0,-1.0,-1.0,-1.0,25.675,2.03628,1739,4600056 +114,search,4767.28,0.87921,0.131581,20.1737,18.466,12.165,2.03628,1739,4600056 +115,insert,39.108,-1.0,-1.0,-1.0,-1.0,17.433,2.04741,1740,4627422 +116,search,4799.69,0.88041,0.130799,20.0795,18.4134,18.172,2.04748,1741,4627422 +117,insert,11.424,-1.0,-1.0,-1.0,-1.0,10.782,2.05029,1741,4639292 +118,search,4790.3,0.88052,0.130588,20.0782,18.4176,12.466,2.05029,1741,4639292 +119,insert,4.439,-1.0,-1.0,-1.0,-1.0,10.977,2.0514,1741,4643423 +120,search,4490.36,0.88058,0.130556,20.075,18.4178,13.098,2.0514,1741,4643423 +121,insert,22.447,-1.0,-1.0,-1.0,-1.0,38.926,2.05832,1745,4657252 +122,search,4811.11,0.88046,0.130671,20.0593,18.4214,13.487,2.05832,1745,4657252 +123,insert,387.349,-1.0,-1.0,-1.0,-1.0,477.646,2.18625,1784,4938477 +124,search,4741.85,0.88126,0.131424,19.836,18.3256,149.472,2.18696,1797,4938477 +125,insert,63.723,-1.0,-1.0,-1.0,-1.0,68.576,2.20464,1805,4980492 +126,search,4945.4,0.88175,0.131552,19.7332,18.3052,40.516,2.20493,1807,4980492 +127,insert,71.663,-1.0,-1.0,-1.0,-1.0,77.772,2.22549,1817,5030515 +128,search,5042.96,0.88258,0.130504,19.6553,18.281,26.1,2.22549,1818,5030515 +129,delete,19.093,-1.0,-1.0,-1.0,-1.0,13.843,2.22511,1818,5009391 +130,search,4985.66,0.88254,0.130721,19.6659,18.3152,14.896,2.22511,1818,5009391 +131,delete,5.049,-1.0,-1.0,-1.0,-1.0,14.37,2.22511,1818,5004093 +132,search,4884.29,0.8826,0.130672,19.6524,18.3112,14.966,2.22511,1818,5004093 +133,delete,10.465,-1.0,-1.0,-1.0,-1.0,14.63,2.22511,1818,4991787 +134,search,5172.17,0.88266,0.130641,19.6579,18.3209,14.912,2.22511,1818,4991787 +135,delete,15.141,-1.0,-1.0,-1.0,-1.0,13.527,2.22431,1818,4974857 +136,search,5020.67,0.88305,0.13039,19.6213,18.3125,15.709,2.22431,1818,4974857 +137,delete,3.632,-1.0,-1.0,-1.0,-1.0,13.939,2.22431,1818,4971386 +138,search,5145.8,0.88306,0.130408,19.6283,18.3103,15.398,2.22431,1818,4971386 +139,delete,9.093,-1.0,-1.0,-1.0,-1.0,13.458,2.22431,1818,4963487 +140,search,5152.2,0.88293,0.130377,19.6583,18.3337,15.724,2.22431,1818,4963487 +141,delete,21.212,-1.0,-1.0,-1.0,-1.0,13.46,2.2238,1818,4941830 +142,search,5089.69,0.88303,0.130512,19.6694,18.3454,15.821,2.2238,1818,4941830 +143,delete,19.729,-1.0,-1.0,-1.0,-1.0,13.594,2.22143,1818,4925657 +144,search,4939.51,0.88286,0.130569,19.6885,18.3691,15.475,2.22143,1818,4925657 +145,delete,3.089,-1.0,-1.0,-1.0,-1.0,13.569,2.22143,1818,4922686 +146,search,5117.11,0.8829,0.130595,19.6756,18.3759,15.167,2.22143,1818,4922686 +147,delete,8.829,-1.0,-1.0,-1.0,-1.0,13.581,2.22143,1818,4913636 +148,search,4687.71,0.88296,0.130442,19.7007,18.4006,15.293,2.22143,1818,4913636 +149,delete,214.747,-1.0,-1.0,-1.0,-1.0,14.057,2.19857,1818,4717250 +150,search,4674.98,0.88245,0.130901,19.7853,18.5084,16.034,2.19857,1818,4717250 +151,delete,205.28,-1.0,-1.0,-1.0,-1.0,13.772,2.16451,1818,4560269 +152,search,4702.32,0.88134,0.131449,19.9457,18.5862,16.302,2.16451,1818,4560269 +153,delete,42.694,-1.0,-1.0,-1.0,-1.0,14.264,2.15391,1818,4521936 +154,search,4485.95,0.88054,0.132039,20.0548,18.6561,13.066,2.15391,1818,4521936 +155,delete,13.778,-1.0,-1.0,-1.0,-1.0,11.574,2.15176,1818,4502222 +156,search,4816.93,0.88057,0.131992,20.0546,18.6743,16.323,2.15176,1818,4502222 +157,delete,45.791,-1.0,-1.0,-1.0,-1.0,14.145,2.14629,1818,4456039 +158,search,4813.85,0.88001,0.132422,20.1212,18.7246,16.791,2.14629,1818,4456039 +159,delete,360.096,-1.0,-1.0,-1.0,-1.0,13.851,2.03063,1818,4143922 +160,search,4659.24,0.87823,0.133662,20.3417,18.9219,16.181,2.03063,1818,4143922 +161,delete,54.129,-1.0,-1.0,-1.0,-1.0,14.127,2.02303,1818,4101445 +162,search,4631.16,0.87828,0.133838,20.3345,18.9356,15.453,2.02303,1818,4101445 +163,delete,23.591,-1.0,-1.0,-1.0,-1.0,13.519,2.01367,1818,4080175 +164,search,4594.68,0.87784,0.13412,20.4039,18.9657,16.077,2.01367,1818,4080175 +165,delete,37.81,-1.0,-1.0,-1.0,-1.0,13.654,2.00452,1818,4047313 +166,search,4566.84,0.87786,0.134303,20.4205,18.9917,16.011,2.00452,1818,4047313 +167,delete,11.982,-1.0,-1.0,-1.0,-1.0,13.7,2.00213,1818,4040738 +168,search,4577.01,0.87769,0.134525,20.442,18.9951,16.602,2.00213,1818,4040738 +169,delete,23.242,-1.0,-1.0,-1.0,-1.0,13.448,1.9968,1818,4022674 +170,search,4557.45,0.87728,0.13473,20.4726,19.0108,16.509,1.9968,1818,4022674 +171,delete,13.345,-1.0,-1.0,-1.0,-1.0,13.768,1.99291,1818,4015835 +172,search,4779.47,0.87722,0.134757,20.4886,19.027,16.678,1.99291,1818,4015835 +173,delete,50.949,-1.0,-1.0,-1.0,-1.0,13.775,1.97858,1818,3986424 +174,search,4398.77,0.87638,0.135267,20.5634,19.0825,16.575,1.97858,1818,3986424 +175,delete,221.359,-1.0,-1.0,-1.0,-1.0,13.538,1.90948,1818,3804693 +176,search,4468.46,0.87432,0.135894,20.8358,19.1572,16.399,1.90948,1818,3804693 +177,delete,24.189,-1.0,-1.0,-1.0,-1.0,13.992,1.90402,1818,3786352 +178,search,4470.6,0.87397,0.135945,20.8741,19.165,16.117,1.90402,1818,3786352 +179,delete,252,-1.0,-1.0,-1.0,-1.0,13.345,1.80199,1818,3556994 +180,search,4240.15,0.86966,0.137334,21.4185,19.2499,16.171,1.80199,1818,3556994 +181,delete,44.096,-1.0,-1.0,-1.0,-1.0,13.839,1.78699,1818,3526824 +182,search,4305.69,0.86931,0.137605,21.447,19.2757,17.145,1.78699,1818,3526824 +183,delete,188.078,-1.0,-1.0,-1.0,-1.0,13.408,1.70964,1818,3350245 +184,search,4208.79,0.86244,0.142242,22.4081,19.7594,11.86,1.70964,1818,3350245 +185,delete,9.299,-1.0,-1.0,-1.0,-1.0,9.741,1.70627,1818,3340234 +186,search,4044.88,0.86248,0.14221,22.419,19.7658,13.894,1.70627,1818,3340234 +187,delete,166.412,-1.0,-1.0,-1.0,-1.0,12.763,1.6162,1818,3150857 +188,search,4040.95,0.8646,0.140539,21.9776,19.3985,16.938,1.6162,1818,3150857 +189,delete,11.059,-1.0,-1.0,-1.0,-1.0,14.157,1.61417,1818,3143609 +190,search,4165.7,0.86438,0.140673,21.9963,19.4198,10.843,1.61417,1818,3143609 +191,delete,4.444,-1.0,-1.0,-1.0,-1.0,11.889,1.61165,1818,3141038 +192,search,4094.82,0.86432,0.140676,21.9935,19.412,16.674,1.61165,1818,3141038 +193,delete,79.172,-1.0,-1.0,-1.0,-1.0,14.017,1.57978,1818,3081734 +194,search,3835.92,0.86281,0.141605,22.2438,19.5746,14.223,1.57978,1818,3081734 +195,delete,36.41,-1.0,-1.0,-1.0,-1.0,12.749,1.56014,1818,3043966 +196,search,3957.4,0.86208,0.142669,22.3273,19.6372,16.62,1.56014,1818,3043966 +197,delete,48.541,-1.0,-1.0,-1.0,-1.0,14.48,1.54298,1818,3008145 +198,search,3955.07,0.86155,0.142716,22.4142,19.6992,13.272,1.54298,1818,3008145 +199,delete,10.948,-1.0,-1.0,-1.0,-1.0,11.866,1.53834,1818,2999384 +200,search,3900.97,0.861,0.142993,22.4903,19.7293,11.601,1.53834,1818,2999384 +201,delete,5.192,-1.0,-1.0,-1.0,-1.0,9.332,1.53572,1818,2993528 +202,search,3449.55,0.86068,0.143014,22.5371,19.7501,17.535,1.53572,1818,2993528 +203,delete,21.045,-1.0,-1.0,-1.0,-1.0,14.451,1.53002,1818,2980973 +204,search,3959.78,0.8606,0.143098,22.5167,19.7202,17.482,1.53002,1818,2980973 +205,delete,33.381,-1.0,-1.0,-1.0,-1.0,14.656,1.52116,1818,2960291 +206,search,4035.31,0.86061,0.143017,22.5579,19.7135,10.88,1.52116,1818,2960291 +207,delete,33.21,-1.0,-1.0,-1.0,-1.0,9.695,1.50162,1818,2920620 +208,search,3114.5,0.86016,0.143377,22.5352,19.6886,10.775,1.50162,1818,2920620 +209,delete,16.775,-1.0,-1.0,-1.0,-1.0,9.678,1.49058,1818,2900501 +210,search,3118.7,0.85971,0.142991,22.6386,19.7055,10.7,1.49058,1818,2900501 +211,delete,7.223,-1.0,-1.0,-1.0,-1.0,9.46,1.48588,1818,2891660 +212,search,3853.94,0.85936,0.143372,22.6681,19.7394,13.961,1.48588,1818,2891660 +213,delete,15.712,-1.0,-1.0,-1.0,-1.0,12.248,1.47657,1818,2873257 +214,search,3834.44,0.85812,0.143949,22.8108,19.8141,17.018,1.47657,1818,2873257 +215,delete,372.361,-1.0,-1.0,-1.0,-1.0,13.979,1.3127,1818,2537571 +216,search,3722.23,0.85124,0.147583,23.6987,20.1773,17.155,1.3127,1818,2537571 +217,delete,246.559,-1.0,-1.0,-1.0,-1.0,14.116,1.19323,1818,2308194 +218,search,3642.65,0.8453,0.151171,24.312,20.5364,17.577,1.19323,1818,2308194 +219,delete,33.261,-1.0,-1.0,-1.0,-1.0,14.937,1.18109,1818,2285820 +220,search,3620.99,0.84415,0.152147,24.3499,20.5214,12.435,1.18109,1818,2285820 +221,delete,148.05,-1.0,-1.0,-1.0,-1.0,10.136,1.08192,1818,2082858 +222,search,3462.6,0.84261,0.151629,24.6078,20.5822,17.682,1.08192,1818,2082858 +223,delete,15.465,-1.0,-1.0,-1.0,-1.0,14.671,1.0757,1818,2071207 +224,search,3622.73,0.84146,0.152477,24.7149,20.5876,18.589,1.0757,1818,2071207 +225,delete,11.565,-1.0,-1.0,-1.0,-1.0,14.936,1.07328,1818,2065400 +226,search,3619.2,0.84121,0.152693,24.7684,20.6486,14.044,1.07328,1818,2065400 +227,delete,4.633,-1.0,-1.0,-1.0,-1.0,12.014,1.07066,1818,2062105 +228,search,3695.82,0.8409,0.152874,24.7888,20.6526,11.227,1.07066,1818,2062105 +229,delete,19.284,-1.0,-1.0,-1.0,-1.0,9.736,1.05706,1818,2039610 +230,search,3669.43,0.83892,0.153675,25.1294,20.8892,11.449,1.05706,1818,2039610 +231,delete,9.247,-1.0,-1.0,-1.0,-1.0,9.669,1.05025,1818,2027795 +232,search,3618.02,0.83784,0.154555,25.2542,20.9423,17.72,1.05025,1818,2027795 +233,delete,6.887,-1.0,-1.0,-1.0,-1.0,14.056,1.04807,1818,2022859 +234,search,3168.24,0.83751,0.154199,25.287,20.9366,11.084,1.04807,1818,2022859 +235,delete,5.077,-1.0,-1.0,-1.0,-1.0,9.661,1.04502,1818,2016264 +236,search,3103.44,0.83737,0.154328,25.3089,20.9416,11.13,1.04502,1818,2016264 +237,delete,173.802,-1.0,-1.0,-1.0,-1.0,10.223,0.922922,1818,1782866 +238,search,2950.14,0.83203,0.156784,26.1193,21.3925,10.968,0.922922,1818,1782866 +239,delete,15.404,-1.0,-1.0,-1.0,-1.0,9.793,0.910989,1818,1758931 +240,search,2737.08,0.82973,0.157742,26.3511,21.4413,10.913,0.910989,1818,1758931 +241,delete,5.298,-1.0,-1.0,-1.0,-1.0,9.709,0.907239,1818,1751275 +242,search,2744.69,0.82902,0.158101,26.4445,21.4555,10.906,0.907239,1818,1751275 +243,delete,12.776,-1.0,-1.0,-1.0,-1.0,9.744,0.899058,1818,1733264 +244,search,2843.13,0.82606,0.15926,27.4257,22.483,20.171,0.899058,1818,1733264 +245,delete,17.962,-1.0,-1.0,-1.0,-1.0,14.817,0.893956,1818,1722658 +246,search,3641.97,0.82565,0.159138,27.4898,22.497,18.578,0.893956,1818,1722658 +247,delete,6.054,-1.0,-1.0,-1.0,-1.0,15.18,0.892665,1818,1719601 +248,search,3453.26,0.825,0.159104,27.7058,22.7279,11.463,0.892665,1818,1719601 +249,delete,8.317,-1.0,-1.0,-1.0,-1.0,9.489,0.888585,1818,1711648 +250,search,3057.08,0.82427,0.159272,27.8052,22.7611,11.336,0.888585,1818,1711648 +251,delete,189.372,-1.0,-1.0,-1.0,-1.0,10.081,0.766745,1818,1470624 diff --git a/scripts/big_ann_perf_numbers/scan_vary_using_batching_250_no_aps.csv b/scripts/big_ann_perf_numbers/scan_vary_using_batching_250_no_aps.csv new file mode 100644 index 00000000..813fa8c7 --- /dev/null +++ b/scripts/big_ann_perf_numbers/scan_vary_using_batching_250_no_aps.csv @@ -0,0 +1,506 @@ +nprobe_percentage,step_num,step_type,latency_ms,recall_mean,recall_std_dev,mainteance_ms,index_mem_gb +0.95,2,search,9523.51,0.99971,0.00556076,9.033,0.0183461 +0.95,3,insert,18.725,-1.0,-1.0,0.0,0.0223849 +0.95,4,search,8441.92,0.9998,0.00468646,29.819,0.0224592 +0.95,5,insert,32.874,-1.0,-1.0,0.0,0.0330443 +0.95,6,search,8228.4,0.99993,0.0026449,105.336,0.0329754 +0.95,7,insert,45.215,-1.0,-1.0,0.0,0.0423855 +0.95,8,search,8388.71,0.99993,0.00264495,182.055,0.0422158 +0.95,9,insert,5.194,-1.0,-1.0,0.0,0.0442806 +0.95,10,search,8559.42,0.99992,0.00282753,132.976,0.0443737 +0.95,11,insert,11.98,-1.0,-1.0,0.0,0.0493843 +0.95,12,search,9087.21,0.99992,0.00282753,260.257,0.0494251 +0.95,13,insert,45.997,-1.0,-1.0,0.0,0.0606586 +0.95,14,search,9818.98,0.99994,0.0024489,337.415,0.0607059 +0.95,15,insert,26.07,-1.0,-1.0,0.0,0.0692592 +0.95,16,search,10597.9,0.99995,0.0022356,382.59,0.0695305 +0.95,17,insert,5.274,-1.0,-1.0,0.0,0.0716848 +0.95,18,search,10780.6,0.99996,0.00199968,215.298,0.0714449 +0.95,19,insert,44.155,-1.0,-1.0,0.0,0.0780512 +0.95,20,search,11964.3,0.99993,0.00264493,341.831,0.0780985 +0.95,21,insert,463.795,-1.0,-1.0,0.0,0.19728 +0.95,22,search,13010.3,0.99996,0.00199969,1886.63,0.196769 +0.95,23,insert,480.486,-1.0,-1.0,0.0,0.320353 +0.95,24,search,17137.2,0.99995,0.00223565,3961.6,0.322334 +0.95,25,insert,63.865,-1.0,-1.0,0.0,0.34112 +0.95,26,search,23845.3,0.99992,0.00282754,3079.21,0.341868 +0.95,27,insert,48.103,-1.0,-1.0,0.0,0.353831 +0.95,28,search,30776.5,0.99989,0.00331516,2836.04,0.354129 +0.95,29,insert,94.993,-1.0,-1.0,0.0,0.380489 +0.95,30,search,37498.2,0.9999,0.00316067,2164.36,0.381163 +0.95,31,insert,722.609,-1.0,-1.0,0.0,0.544157 +0.95,32,search,40686.7,0.99991,0.00299873,6087.03,0.544795 +0.95,33,insert,89.047,-1.0,-1.0,0.0,0.566606 +0.95,34,search,49355.1,0.99991,0.00299873,4477.65,0.567682 +0.95,35,insert,77.839,-1.0,-1.0,0.0,0.582318 +0.95,36,search,54964.6,0.99993,0.00264507,2566.38,0.582666 +0.95,37,insert,124.842,-1.0,-1.0,0.0,0.60306 +0.95,38,search,59086.5,0.99993,0.00264507,1764.02,0.603237 +0.95,39,insert,12.458,-1.0,-1.0,0.0,0.606432 +0.95,40,search,62049,0.99993,0.00264507,679.175,0.606314 +0.95,41,insert,83.216,-1.0,-1.0,0.0,0.621281 +0.95,42,search,62419.7,0.99991,0.0029987,855.505,0.621315 +0.95,43,insert,34.744,-1.0,-1.0,0.0,0.625901 +0.95,44,search,63523,0.99992,0.00282754,705.264,0.625923 +0.95,45,insert,391.399,-1.0,-1.0,0.0,0.640966 +0.95,46,search,64143.8,0.99993,0.00264507,1095.75,0.641074 +0.95,47,insert,609.552,-1.0,-1.0,0.0,0.741265 +0.95,48,search,64732.8,0.99996,0.00199959,4215.21,0.742498 +0.95,49,insert,76.004,-1.0,-1.0,0.0,0.75791 +0.95,50,search,69199.4,0.99997,0.00173174,2198.96,0.758022 +0.95,51,insert,697.786,-1.0,-1.0,0.0,0.893925 +0.95,52,search,74107.5,0.99995,0.00223571,11123,0.894839 +0.95,53,insert,122.635,-1.0,-1.0,0.0,0.913612 +0.95,54,search,83606.8,0.99994,0.0024489,5471.53,0.913871 +0.95,55,insert,795.383,-1.0,-1.0,0.0,1.05568 +0.95,56,search,90278.6,0.99997,0.00173174,10166.5,1.05468 +0.95,57,insert,75.956,-1.0,-1.0,0.0,1.06361 +0.95,58,search,94437.6,0.99997,0.00173174,3955.38,1.06383 +0.95,59,insert,594.061,-1.0,-1.0,0.0,1.16262 +0.95,60,search,100894,0.99997,0.00173174,7526.01,1.16071 +0.95,61,insert,42.304,-1.0,-1.0,0.0,1.16506 +0.95,62,search,104385,0.99996,0.00199968,3178.89,1.16621 +0.95,63,insert,10.664,-1.0,-1.0,0.0,1.16759 +0.95,64,search,104833,0.99998,0.00141404,2165.31,1.16709 +0.95,65,insert,316.078,-1.0,-1.0,0.0,1.19644 +0.95,66,search,110058,1,0,4346.5,1.19691 +0.95,67,insert,157.995,-1.0,-1.0,0.0,1.21721 +0.95,68,search,113969,0.99999,0.00099998,2253.5,1.21744 +0.95,69,insert,182.197,-1.0,-1.0,0.0,1.2402 +0.95,70,search,114961,0.99999,0.00099998,2267.82,1.24028 +0.95,71,insert,59.067,-1.0,-1.0,0.0,1.24644 +0.95,72,search,115971,0.99999,0.00099998,1543.38,1.2464 +0.95,73,insert,33.238,-1.0,-1.0,0.0,1.24954 +0.95,74,search,114229,0.99999,0.00099998,1369.61,1.24958 +0.95,75,insert,77.963,-1.0,-1.0,0.0,1.25679 +0.95,76,search,115734,0.99999,0.00099998,1425.66,1.25653 +0.95,77,insert,114.327,-1.0,-1.0,0.0,1.2672 +0.95,78,search,115305,0.99999,0.00099998,1521.6,1.2671 +0.95,79,insert,757.383,-1.0,-1.0,0.0,1.29592 +0.95,80,search,119677,0.99998,0.00141406,2552.79,1.29577 +0.95,81,insert,105.653,-1.0,-1.0,0.0,1.30792 +0.95,82,search,118010,0.99998,0.00141406,1794.7,1.30813 +0.95,83,insert,52.859,-1.0,-1.0,0.0,1.31436 +0.95,84,search,120107,0.99998,0.00141406,1491.52,1.31455 +0.95,85,insert,119.219,-1.0,-1.0,0.0,1.3294 +0.95,86,search,124109,0.99998,0.00141406,1885.07,1.32967 +0.95,87,insert,1438.99,-1.0,-1.0,0.0,1.52354 +0.95,88,search,125005,0.99998,0.00141406,22528.9,1.52442 +0.95,89,insert,1336.56,-1.0,-1.0,0.0,1.68435 +0.95,90,search,139715,0.99998,0.00141406,26890.2,1.68461 +0.95,91,insert,144.186,-1.0,-1.0,0.0,1.69942 +0.95,92,search,150429,0.99998,0.00141406,8889.88,1.69942 +0.95,93,insert,981.427,-1.0,-1.0,0.0,1.80371 +0.95,94,search,161753,0.99998,0.00141406,12834.3,1.8043 +0.95,95,insert,67.921,-1.0,-1.0,0.0,1.8114 +0.95,96,search,163073,0.99998,0.00141406,3668.88,1.81084 +0.95,97,insert,47.185,-1.0,-1.0,0.0,1.81493 +0.95,98,search,165485,0.99998,0.00141406,2184.61,1.81487 +0.95,99,insert,26.804,-1.0,-1.0,0.0,1.81702 +0.95,100,search,168676,0.99998,0.00141406,1852.2,1.81694 +0.95,101,insert,151.763,-1.0,-1.0,0.0,1.8304 +0.95,102,search,168870,0.99998,0.00141406,2254.32,1.83048 +0.75,2,search,5157.84,0.99611,0.0205396,4.12,0.0183461 +0.75,3,insert,10.416,-1.0,-1.0,0.0,0.0223849 +0.75,4,search,5089.6,0.99683,0.0182473,14.831,0.0224592 +0.75,5,insert,28.874,-1.0,-1.0,0.0,0.0330443 +0.75,6,search,6423.46,0.99815,0.0139859,80.202,0.0330488 +0.75,7,insert,38.547,-1.0,-1.0,0.0,0.0423284 +0.75,8,search,6743.25,0.99799,0.0152642,165.47,0.0423373 +0.75,9,insert,4.934,-1.0,-1.0,0.0,0.0440534 +0.75,10,search,6821.04,0.99836,0.0137604,176.435,0.0441281 +0.75,11,insert,14.41,-1.0,-1.0,0.0,0.0493398 +0.75,12,search,7128.61,0.99831,0.0137901,212.535,0.0492244 +0.75,13,insert,32.583,-1.0,-1.0,0.0,0.0603089 +0.75,14,search,7527.01,0.99826,0.0142469,305.35,0.0604427 +0.75,15,insert,26.629,-1.0,-1.0,0.0,0.0689651 +0.75,16,search,8143.18,0.99834,0.0138294,395.425,0.0691254 +0.75,17,insert,5.546,-1.0,-1.0,0.0,0.0712429 +0.75,18,search,8802.48,0.99803,0.0152036,325.553,0.0712246 +0.75,19,insert,49.576,-1.0,-1.0,0.0,0.0779957 +0.75,20,search,9656.65,0.99806,0.0150428,311.335,0.0781646 +0.75,21,insert,439.031,-1.0,-1.0,0.0,0.196456 +0.75,22,search,10409,0.99791,0.0147873,1713.58,0.197361 +0.75,23,insert,483.914,-1.0,-1.0,0.0,0.322766 +0.75,24,search,13379.4,0.99769,0.0158647,3672.3,0.322244 +0.75,25,insert,77.429,-1.0,-1.0,0.0,0.341349 +0.75,26,search,17662.1,0.99646,0.0195844,3312.01,0.341526 +0.75,27,insert,42.537,-1.0,-1.0,0.0,0.353945 +0.75,28,search,18012.2,0.99603,0.0206218,2834.92,0.354809 +0.75,29,insert,102.539,-1.0,-1.0,0.0,0.381047 +0.75,30,search,27281.3,0.9965,0.0191771,2427.6,0.380739 +0.75,31,insert,750.721,-1.0,-1.0,0.0,0.544619 +0.75,32,search,32840.1,0.99652,0.0199991,6297.19,0.544806 +0.75,33,insert,115.726,-1.0,-1.0,0.0,0.567317 +0.75,34,search,37795.2,0.99658,0.0196043,4852.08,0.56763 +0.75,35,insert,95.92,-1.0,-1.0,0.0,0.582627 +0.75,36,search,44984.2,0.99717,0.0173504,2718.8,0.582984 +0.75,37,insert,128.757,-1.0,-1.0,0.0,0.602918 +0.75,38,search,45879.6,0.99748,0.0161148,2016.51,0.60316 +0.75,39,insert,18.148,-1.0,-1.0,0.0,0.60627 +0.75,40,search,47197.9,0.99751,0.0160257,899.167,0.606519 +0.75,41,insert,120.724,-1.0,-1.0,0.0,0.621033 +0.75,42,search,48070.4,0.99759,0.0157864,970.479,0.621073 +0.75,43,insert,35.92,-1.0,-1.0,0.0,0.625271 +0.75,44,search,48729.8,0.99776,0.0149999,652.129,0.625482 +0.75,45,insert,405.013,-1.0,-1.0,0.0,0.64049 +0.75,46,search,48826.9,0.9978,0.0149397,1114.44,0.641175 +0.75,47,insert,665.631,-1.0,-1.0,0.0,0.741888 +0.75,48,search,50664.9,0.99803,0.0141125,3839.29,0.742245 +0.75,49,insert,87.116,-1.0,-1.0,0.0,0.757267 +0.75,50,search,53813.6,0.99807,0.0141165,1918.33,0.757701 +0.75,51,insert,838.734,-1.0,-1.0,0.0,0.894337 +0.75,52,search,56850.5,0.99826,0.0134527,10541.5,0.894381 +0.75,53,insert,132.188,-1.0,-1.0,0.0,0.914297 +0.75,54,search,63808.2,0.99824,0.0135249,5486.98,0.915372 +0.75,55,insert,857.849,-1.0,-1.0,0.0,1.05558 +0.75,56,search,68149.6,0.99845,0.0128298,8718.72,1.05475 +0.75,57,insert,81.533,-1.0,-1.0,0.0,1.06308 +0.75,58,search,73723.8,0.99832,0.0132365,3609.6,1.06424 +0.75,59,insert,607.923,-1.0,-1.0,0.0,1.16312 +0.75,60,search,74026.8,0.99842,0.0127877,7151.63,1.16314 +0.75,61,insert,43.485,-1.0,-1.0,0.0,1.16735 +0.75,62,search,72941.1,0.99835,0.0133531,2662.36,1.16609 +0.75,63,insert,10.599,-1.0,-1.0,0.0,1.16751 +0.75,64,search,76467.6,0.99841,0.013133,1607.62,1.1678 +0.75,65,insert,338.492,-1.0,-1.0,0.0,1.19699 +0.75,66,search,80706.2,0.99861,0.0118774,3614.01,1.19703 +0.75,67,insert,150.313,-1.0,-1.0,0.0,1.21641 +0.75,68,search,83828.1,0.99866,0.011671,2375.81,1.21615 +0.75,69,insert,202.397,-1.0,-1.0,0.0,1.23996 +0.75,70,search,84355.5,0.99871,0.0114609,2483.47,1.24047 +0.75,71,insert,57.209,-1.0,-1.0,0.0,1.24617 +0.75,72,search,83249.4,0.99871,0.0114609,1776.57,1.24603 +0.75,73,insert,32.473,-1.0,-1.0,0.0,1.24922 +0.75,74,search,88491.1,0.99873,0.0113757,1286.27,1.24898 +0.75,75,insert,80.764,-1.0,-1.0,0.0,1.25626 +0.75,76,search,84799.6,0.99874,0.0113325,1329.12,1.25636 +0.75,77,insert,124.679,-1.0,-1.0,0.0,1.26762 +0.75,78,search,84922.9,0.99874,0.0113325,1478.46,1.26782 +0.75,79,insert,761.751,-1.0,-1.0,0.0,1.29621 +0.75,80,search,86215.4,0.99878,0.011248,2322.34,1.29592 +0.75,81,insert,121.402,-1.0,-1.0,0.0,1.30874 +0.75,82,search,87678.7,0.99882,0.0110733,1664.39,1.30874 +0.75,83,insert,54.263,-1.0,-1.0,0.0,1.31517 +0.75,84,search,86665.1,0.99884,0.010985,1343.1,1.31479 +0.75,85,insert,126.908,-1.0,-1.0,0.0,1.33024 +0.75,86,search,87677.9,0.99883,0.011029,1618.81,1.33007 +0.75,87,insert,1442.07,-1.0,-1.0,0.0,1.52355 +0.75,88,search,91504.2,0.99907,0.00970343,20731.1,1.52548 +0.75,89,insert,1152.82,-1.0,-1.0,0.0,1.68253 +0.75,90,search,102276,0.99919,0.0089639,26299.4,1.68473 +0.75,91,insert,142.406,-1.0,-1.0,0.0,1.69899 +0.75,92,search,110683,0.99916,0.00912731,9548.98,1.69933 +0.75,93,insert,1085.72,-1.0,-1.0,0.0,1.80314 +0.75,94,search,118291,0.99909,0.00949663,11761.1,1.8034 +0.75,95,insert,71.56,-1.0,-1.0,0.0,1.81062 +0.75,96,search,120615,0.99908,0.00954754,3874.38,1.81054 +0.75,97,insert,52.145,-1.0,-1.0,0.0,1.81498 +0.75,98,search,121596,0.99912,0.00933995,2183.57,1.81515 +0.75,99,insert,28.321,-1.0,-1.0,0.0,1.8174 +0.75,100,search,122336,0.99914,0.00923371,1759.05,1.81738 +0.75,101,insert,148.033,-1.0,-1.0,0.0,1.83002 +0.75,102,search,123854,0.99918,0.00901842,2099.05,1.83005 +0.5,2,search,3743.63,0.97361,0.0540257,2.884,0.0183457 +0.5,3,insert,11.153,-1.0,-1.0,0.0,0.0223845 +0.5,4,search,3598.73,0.97715,0.0498717,3.635,0.0223845 +0.5,5,insert,30.994,-1.0,-1.0,0.0,0.032811 +0.5,6,search,4540.01,0.9874,0.0375665,89.192,0.032607 +0.5,7,insert,38.563,-1.0,-1.0,0.0,0.0422019 +0.5,8,search,4846.43,0.98484,0.0423822,159.183,0.0425038 +0.5,9,insert,5.569,-1.0,-1.0,0.0,0.0440766 +0.5,10,search,4957.5,0.98835,0.0364879,156.109,0.0444186 +0.5,11,insert,13.488,-1.0,-1.0,0.0,0.0491412 +0.5,12,search,5004.8,0.98773,0.037718,195.998,0.0493855 +0.5,13,insert,34.581,-1.0,-1.0,0.0,0.0603616 +0.5,14,search,5334.28,0.98638,0.0403838,252.709,0.0606627 +0.5,15,insert,27.724,-1.0,-1.0,0.0,0.0691303 +0.5,16,search,5421.35,0.98674,0.0394231,227.815,0.0688255 +0.5,17,insert,5.141,-1.0,-1.0,0.0,0.071198 +0.5,18,search,5687.47,0.98717,0.03814,223.956,0.0711552 +0.5,19,insert,48.413,-1.0,-1.0,0.0,0.0782699 +0.5,20,search,5800.35,0.98726,0.0383109,226.684,0.0782426 +0.5,21,insert,437.962,-1.0,-1.0,0.0,0.197909 +0.5,22,search,6458.81,0.98758,0.0366322,1111.19,0.196871 +0.5,23,insert,497.166,-1.0,-1.0,0.0,0.322367 +0.5,24,search,8118.26,0.98306,0.0436939,2786.47,0.321907 +0.5,25,insert,69.664,-1.0,-1.0,0.0,0.341771 +0.5,26,search,10072.1,0.97182,0.0582404,2577.11,0.341055 +0.5,27,insert,48.109,-1.0,-1.0,0.0,0.353585 +0.5,28,search,11947,0.96974,0.0605032,1338.78,0.35286 +0.5,29,insert,89.058,-1.0,-1.0,0.0,0.379365 +0.5,30,search,13399.3,0.97119,0.0587473,794.499,0.380149 +0.5,31,insert,632.677,-1.0,-1.0,0.0,0.54195 +0.5,32,search,13452.8,0.97269,0.057667,4650.08,0.543491 +0.5,33,insert,89.775,-1.0,-1.0,0.0,0.565608 +0.5,34,search,16581.5,0.97178,0.0584663,2672.46,0.566289 +0.5,35,insert,71.556,-1.0,-1.0,0.0,0.580414 +0.5,36,search,18470.8,0.97253,0.0581289,1532.85,0.580893 +0.5,37,insert,92.37,-1.0,-1.0,0.0,0.600819 +0.5,38,search,18330.6,0.97338,0.0567979,958.911,0.600816 +0.5,39,insert,21.539,-1.0,-1.0,0.0,0.604612 +0.5,40,search,19082.3,0.9732,0.0570819,694.963,0.604504 +0.5,41,insert,102.789,-1.0,-1.0,0.0,0.619391 +0.5,42,search,18711.9,0.97372,0.0567056,787.874,0.619435 +0.5,43,insert,30.477,-1.0,-1.0,0.0,0.623922 +0.5,44,search,19655.6,0.97383,0.0565013,830.736,0.623915 +0.5,45,insert,378.58,-1.0,-1.0,0.0,0.639142 +0.5,46,search,19384.2,0.97451,0.0559091,1157.1,0.639118 +0.5,47,insert,563.585,-1.0,-1.0,0.0,0.741071 +0.5,48,search,20200.9,0.97679,0.0534823,4321.88,0.739977 +0.5,49,insert,74.871,-1.0,-1.0,0.0,0.755689 +0.5,50,search,21395.2,0.97775,0.0525398,2115.88,0.754856 +0.5,51,insert,701.766,-1.0,-1.0,0.0,0.891892 +0.5,52,search,22905,0.98016,0.0498644,7573.03,0.893248 +0.5,53,insert,114.214,-1.0,-1.0,0.0,0.911387 +0.5,54,search,25683.9,0.97861,0.0517873,3388.47,0.911412 +0.5,55,insert,669.195,-1.0,-1.0,0.0,1.05338 +0.5,56,search,27611.4,0.98177,0.0481941,6749.72,1.05378 +0.5,57,insert,79.394,-1.0,-1.0,0.0,1.06247 +0.5,58,search,29610.8,0.98182,0.047809,2920.01,1.06225 +0.5,59,insert,629.259,-1.0,-1.0,0.0,1.16263 +0.5,60,search,30548.6,0.98243,0.0468256,5497.48,1.16091 +0.5,61,insert,32.408,-1.0,-1.0,0.0,1.16516 +0.5,62,search,29070.2,0.98031,0.0504931,1591.47,1.16459 +0.5,63,insert,5.033,-1.0,-1.0,0.0,1.16564 +0.5,64,search,30263.8,0.9807,0.0499765,1051.02,1.16574 +0.5,65,insert,249.18,-1.0,-1.0,0.0,1.1948 +0.5,66,search,30841.4,0.98149,0.0486293,2459.94,1.19433 +0.5,67,insert,120.312,-1.0,-1.0,0.0,1.2146 +0.5,68,search,31718.4,0.98176,0.0481391,1335.3,1.21446 +0.5,69,insert,152.82,-1.0,-1.0,0.0,1.2364 +0.5,70,search,32073.5,0.98193,0.0476919,1254.46,1.2368 +0.5,71,insert,40.187,-1.0,-1.0,0.0,1.24216 +0.5,72,search,32523.1,0.98202,0.0476519,1021.83,1.2423 +0.5,73,insert,22.541,-1.0,-1.0,0.0,1.24529 +0.5,74,search,31813.3,0.982,0.0476479,898.995,1.24531 +0.5,75,insert,73.629,-1.0,-1.0,0.0,1.25279 +0.5,76,search,34187.6,0.98213,0.0473263,991.224,1.25237 +0.5,77,insert,96.793,-1.0,-1.0,0.0,1.26381 +0.5,78,search,32661.1,0.98197,0.0474972,962.885,1.2637 +0.5,79,insert,717.384,-1.0,-1.0,0.0,1.29164 +0.5,80,search,32964.9,0.98178,0.0478975,1322.2,1.29165 +0.5,81,insert,89.447,-1.0,-1.0,0.0,1.30421 +0.5,82,search,33698.6,0.9821,0.0474537,1243.35,1.30482 +0.5,83,insert,36.083,-1.0,-1.0,0.0,1.3108 +0.5,84,search,34538.7,0.98218,0.0473759,1028.86,1.31099 +0.5,85,insert,102.746,-1.0,-1.0,0.0,1.32549 +0.5,86,search,33630.9,0.98235,0.0471124,1132.65,1.3256 +0.5,87,insert,1336.38,-1.0,-1.0,0.0,1.51868 +0.5,88,search,34413.8,0.98478,0.0425028,10220.9,1.52046 +0.5,89,insert,992.01,-1.0,-1.0,0.0,1.67934 +0.5,90,search,38139.1,0.98634,0.0395435,11659.5,1.67947 +0.5,91,insert,118.143,-1.0,-1.0,0.0,1.69359 +0.5,92,search,43373.5,0.98622,0.0400773,4030.16,1.69367 +0.5,93,insert,882.897,-1.0,-1.0,0.0,1.79861 +0.5,94,search,43180.6,0.98654,0.0392047,5516.66,1.7988 +0.5,95,insert,60.793,-1.0,-1.0,0.0,1.80626 +0.5,96,search,43812.8,0.98691,0.0386767,2241.62,1.80678 +0.5,97,insert,37.091,-1.0,-1.0,0.0,1.81065 +0.5,98,search,42798.9,0.98704,0.0386018,1381.24,1.81092 +0.5,99,insert,20.421,-1.0,-1.0,0.0,1.81303 +0.5,100,search,45119.1,0.9871,0.0385689,1295.79,1.81312 +0.5,101,insert,150.529,-1.0,-1.0,0.0,1.82562 +0.5,102,search,44183.1,0.98717,0.0383491,1474.14,1.8257 +0.25,2,search,2282.69,0.88485,0.117342,1.716,0.0183457 +0.25,3,insert,9.707,-1.0,-1.0,0.0,0.0223845 +0.25,4,search,2203.75,0.89452,0.112956,1.774,0.0223845 +0.25,5,insert,31.054,-1.0,-1.0,0.0,0.032811 +0.25,6,search,2604.86,0.93574,0.0952189,72.078,0.032607 +0.25,7,insert,38.421,-1.0,-1.0,0.0,0.0422019 +0.25,8,search,2715.59,0.93128,0.0983071,146.99,0.042614 +0.25,9,insert,5.194,-1.0,-1.0,0.0,0.0444508 +0.25,10,search,2752.27,0.94257,0.0871074,119.664,0.0441481 +0.25,11,insert,12.873,-1.0,-1.0,0.0,0.0491334 +0.25,12,search,2846.73,0.93983,0.0899551,151.276,0.0490987 +0.25,13,insert,30.466,-1.0,-1.0,0.0,0.0602396 +0.25,14,search,2918.6,0.93398,0.0949639,161.71,0.0602281 +0.25,15,insert,27.715,-1.0,-1.0,0.0,0.0689651 +0.25,16,search,2973.34,0.93191,0.0951186,179.394,0.069085 +0.25,17,insert,5.675,-1.0,-1.0,0.0,0.0712829 +0.25,18,search,3117.7,0.93227,0.0947425,150.274,0.0714004 +0.25,19,insert,47.577,-1.0,-1.0,0.0,0.0780039 +0.25,20,search,3189.27,0.93655,0.0904867,181.48,0.0780292 +0.25,21,insert,441.609,-1.0,-1.0,0.0,0.197719 +0.25,22,search,3571.69,0.93834,0.0877201,912.217,0.197477 +0.25,23,insert,450.585,-1.0,-1.0,0.0,0.321868 +0.25,24,search,4341.87,0.91593,0.105138,1822.31,0.321488 +0.25,25,insert,62.233,-1.0,-1.0,0.0,0.341099 +0.25,26,search,4786.43,0.89842,0.116361,1729.48,0.341241 +0.25,27,insert,39.278,-1.0,-1.0,0.0,0.353264 +0.25,28,search,5503.58,0.89316,0.118508,729.246,0.353976 +0.25,29,insert,82.9,-1.0,-1.0,0.0,0.380429 +0.25,30,search,5527.95,0.89845,0.115403,566.173,0.37977 +0.25,31,insert,587.56,-1.0,-1.0,0.0,0.543944 +0.25,32,search,6017.92,0.90654,0.111415,3449.11,0.542931 +0.25,33,insert,91.392,-1.0,-1.0,0.0,0.565799 +0.25,34,search,6862.79,0.90644,0.111933,1878.59,0.565218 +0.25,35,insert,56.485,-1.0,-1.0,0.0,0.579554 +0.25,36,search,7543.78,0.90745,0.111673,904.371,0.579638 +0.25,37,insert,82.46,-1.0,-1.0,0.0,0.601868 +0.25,38,search,7443.85,0.90846,0.110802,774.153,0.601668 +0.25,39,insert,13.019,-1.0,-1.0,0.0,0.60504 +0.25,40,search,7758.55,0.90858,0.110928,583.728,0.604879 +0.25,41,insert,69.935,-1.0,-1.0,0.0,0.6212 +0.25,42,search,7828.81,0.91014,0.109952,865.999,0.621318 +0.25,43,insert,19.907,-1.0,-1.0,0.0,0.625059 +0.25,44,search,7679.94,0.91007,0.110048,691.175,0.624947 +0.25,45,insert,346.416,-1.0,-1.0,0.0,0.640347 +0.25,46,search,7876.18,0.91282,0.108133,984.944,0.639864 +0.25,47,insert,488.511,-1.0,-1.0,0.0,0.73934 +0.25,48,search,7968.13,0.91793,0.104952,2848.47,0.740093 +0.25,49,insert,56.876,-1.0,-1.0,0.0,0.755575 +0.25,50,search,8574.73,0.91923,0.104955,1552.22,0.754557 +0.25,51,insert,614.492,-1.0,-1.0,0.0,0.890585 +0.25,52,search,8898.39,0.92771,0.0990787,4777.14,0.891454 +0.25,53,insert,95.807,-1.0,-1.0,0.0,0.911569 +0.25,54,search,9866.91,0.92524,0.10105,2207.39,0.910524 +0.25,55,insert,496.489,-1.0,-1.0,0.0,1.05029 +0.25,56,search,10756.1,0.9338,0.0947638,5256.67,1.05166 +0.25,57,insert,64.877,-1.0,-1.0,0.0,1.06022 +0.25,58,search,11183.9,0.93294,0.0955876,2247.02,1.06059 +0.25,59,insert,433.539,-1.0,-1.0,0.0,1.15772 +0.25,60,search,11644.4,0.934,0.0944534,3835.39,1.15751 +0.25,61,insert,21.641,-1.0,-1.0,0.0,1.16171 +0.25,62,search,12050.7,0.93065,0.0991768,2092,1.16264 +0.25,63,insert,7.967,-1.0,-1.0,0.0,1.16365 +0.25,64,search,11637.3,0.93061,0.099272,1336.6,1.16468 +0.25,65,insert,190.171,-1.0,-1.0,0.0,1.1951 +0.25,66,search,11656.2,0.93284,0.0971381,2451.94,1.19502 +0.25,67,insert,89.76,-1.0,-1.0,0.0,1.21518 +0.25,68,search,12556.2,0.93315,0.0969631,1630.31,1.21553 +0.25,69,insert,143.727,-1.0,-1.0,0.0,1.23926 +0.25,70,search,12137,0.93399,0.0965082,1742.14,1.23899 +0.25,71,insert,27.424,-1.0,-1.0,0.0,1.24378 +0.25,72,search,12357.8,0.93443,0.0961615,1421.86,1.24376 +0.25,73,insert,21.603,-1.0,-1.0,0.0,1.24711 +0.25,74,search,12355.8,0.93426,0.0962957,1400,1.24741 +0.25,75,insert,61.127,-1.0,-1.0,0.0,1.25443 +0.25,76,search,12115.5,0.93436,0.096282,1475.32,1.25452 +0.25,77,insert,70.904,-1.0,-1.0,0.0,1.26581 +0.25,78,search,12431.3,0.93413,0.0961939,1383.47,1.26591 +0.25,79,insert,808.369,-1.0,-1.0,0.0,1.295 +0.25,80,search,12497.5,0.93401,0.096665,1820.87,1.29529 +0.25,81,insert,80.251,-1.0,-1.0,0.0,1.30662 +0.25,82,search,12813,0.9348,0.0958508,1507.13,1.30637 +0.25,83,insert,26.16,-1.0,-1.0,0.0,1.31236 +0.25,84,search,12749,0.93515,0.0951409,1557.52,1.31294 +0.25,85,insert,104.126,-1.0,-1.0,0.0,1.3273 +0.25,86,search,12794.6,0.93577,0.0946486,1581.83,1.32728 +0.25,87,insert,943.232,-1.0,-1.0,0.0,1.51948 +0.25,88,search,13387.7,0.94254,0.0876102,10737.1,1.51868 +0.25,89,insert,940.714,-1.0,-1.0,0.0,1.6786 +0.25,90,search,14735.9,0.94648,0.0836913,12654,1.67838 +0.25,91,insert,107.711,-1.0,-1.0,0.0,1.69204 +0.25,92,search,16373,0.94667,0.0837738,3753.93,1.69365 +0.25,93,insert,727.292,-1.0,-1.0,0.0,1.79968 +0.25,94,search,16155.6,0.94789,0.0824523,5176.82,1.79895 +0.25,95,insert,49.626,-1.0,-1.0,0.0,1.80558 +0.25,96,search,17036.4,0.94816,0.0818819,2363.86,1.80609 +0.25,97,insert,27.707,-1.0,-1.0,0.0,1.80978 +0.25,98,search,17333.6,0.94819,0.0819383,2091.24,1.80984 +0.25,99,insert,18.379,-1.0,-1.0,0.0,1.81137 +0.25,100,search,17081.3,0.94821,0.0818611,38.394,1.81137 +0.25,101,insert,138.01,-1.0,-1.0,0.0,1.82553 +0.25,102,search,17142.3,0.94846,0.0816462,2220.78,1.82553 +0.05,2,search,873.849,0.57321,0.215169,0.753,0.0183461 +0.05,3,insert,13.361,-1.0,-1.0,0.0,0.0223849 +0.05,4,search,856.665,0.58482,0.214377,0.858,0.0223849 +0.05,5,insert,35.555,-1.0,-1.0,0.0,0.0328114 +0.05,6,search,881.401,0.65013,0.234846,65.997,0.0326074 +0.05,7,insert,34.026,-1.0,-1.0,0.0,0.0422023 +0.05,8,search,907.831,0.71023,0.206475,92.964,0.0426144 +0.05,9,insert,4.286,-1.0,-1.0,0.0,0.0444512 +0.05,10,search,918.404,0.71488,0.197403,88.731,0.0441252 +0.05,11,insert,11.808,-1.0,-1.0,0.0,0.0491742 +0.05,12,search,941.415,0.69284,0.200565,96.829,0.0489449 +0.05,13,insert,29.4,-1.0,-1.0,0.0,0.0601796 +0.05,14,search,955.218,0.67446,0.204164,115.475,0.060322 +0.05,15,insert,27.723,-1.0,-1.0,0.0,0.0691642 +0.05,16,search,968.137,0.65255,0.205064,110.948,0.0689296 +0.05,17,insert,5.081,-1.0,-1.0,0.0,0.0712339 +0.05,18,search,971.999,0.64673,0.209309,95.615,0.0708982 +0.05,19,insert,43.227,-1.0,-1.0,0.0,0.0777236 +0.05,20,search,980.94,0.66183,0.207236,107.451,0.0778476 +0.05,21,insert,436.254,-1.0,-1.0,0.0,0.19668 +0.05,22,search,1111.29,0.66877,0.204296,647.443,0.197373 +0.05,23,insert,465.031,-1.0,-1.0,0.0,0.322176 +0.05,24,search,1292.01,0.66837,0.211075,1741.07,0.321703 +0.05,25,insert,60.136,-1.0,-1.0,0.0,0.340587 +0.05,26,search,1254.09,0.65783,0.212576,1092.91,0.341108 +0.05,27,insert,37.104,-1.0,-1.0,0.0,0.352858 +0.05,28,search,1292.64,0.64663,0.217334,418.027,0.353109 +0.05,29,insert,74.381,-1.0,-1.0,0.0,0.380256 +0.05,30,search,1330.56,0.65813,0.218309,463.928,0.379752 +0.05,31,insert,509.231,-1.0,-1.0,0.0,0.543361 +0.05,32,search,1471.14,0.67277,0.212932,2205.56,0.541713 +0.05,33,insert,80.149,-1.0,-1.0,0.0,0.565231 +0.05,34,search,1506.85,0.67319,0.213557,1278.65,0.564945 +0.05,35,insert,56.656,-1.0,-1.0,0.0,0.579606 +0.05,36,search,1553.2,0.67518,0.212154,677.093,0.58035 +0.05,37,insert,71.806,-1.0,-1.0,0.0,0.600103 +0.05,38,search,1503.15,0.67436,0.212779,569.918,0.60058 +0.05,39,insert,17.578,-1.0,-1.0,0.0,0.604429 +0.05,40,search,1557.93,0.67903,0.211491,580.324,0.604298 +0.05,41,insert,62.367,-1.0,-1.0,0.0,0.618968 +0.05,42,search,1562.6,0.68094,0.211189,591.22,0.619109 +0.05,43,insert,14.838,-1.0,-1.0,0.0,0.623476 +0.05,44,search,1556.9,0.68053,0.21106,554.277,0.623353 +0.05,45,insert,329.092,-1.0,-1.0,0.0,0.638415 +0.05,46,search,1579.89,0.6852,0.208619,771.378,0.638332 +0.05,47,insert,390.783,-1.0,-1.0,0.0,0.738241 +0.05,48,search,1647.03,0.69248,0.210438,2014.49,0.73931 +0.05,49,insert,51.919,-1.0,-1.0,0.0,0.755075 +0.05,50,search,1675.36,0.69756,0.210277,1202.83,0.754351 +0.05,51,insert,439.798,-1.0,-1.0,0.0,0.89089 +0.05,52,search,1802.96,0.71888,0.202166,2914.9,0.891649 +0.05,53,insert,77.632,-1.0,-1.0,0.0,0.910222 +0.05,54,search,1833.91,0.71722,0.203346,1733.88,0.909768 +0.05,55,insert,514.027,-1.0,-1.0,0.0,1.05108 +0.05,56,search,2007.24,0.72905,0.198304,3672.65,1.05279 +0.05,57,insert,46.932,-1.0,-1.0,0.0,1.06139 +0.05,58,search,2029.71,0.727,0.201542,1590.45,1.06143 +0.05,59,insert,396.714,-1.0,-1.0,0.0,1.16002 +0.05,60,search,2092.94,0.72834,0.200396,2874.41,1.15954 +0.05,61,insert,16.747,-1.0,-1.0,0.0,1.16378 +0.05,62,search,2117.1,0.72832,0.202387,1294.25,1.1643 +0.05,63,insert,5.294,-1.0,-1.0,0.0,1.16583 +0.05,64,search,2083.26,0.72744,0.202948,1085.3,1.16577 +0.05,65,insert,121.655,-1.0,-1.0,0.0,1.19376 +0.05,66,search,2098.21,0.73114,0.200981,1733.55,1.19402 +0.05,67,insert,77.98,-1.0,-1.0,0.0,1.21405 +0.05,68,search,2133.03,0.7319,0.201766,1337.2,1.21443 +0.05,69,insert,104.974,-1.0,-1.0,0.0,1.23907 +0.05,70,search,2162.73,0.7349,0.19997,1478.22,1.23874 +0.05,71,insert,30.169,-1.0,-1.0,0.0,1.24509 +0.05,72,search,2176.96,0.73663,0.199247,1336.89,1.24488 +0.05,73,insert,11.745,-1.0,-1.0,0.0,1.24772 +0.05,74,search,2173.75,0.73605,0.199683,1256.81,1.24713 +0.05,75,insert,42.752,-1.0,-1.0,0.0,1.25569 +0.05,76,search,2166.39,0.73624,0.199625,1154.5,1.25562 +0.05,77,insert,65.676,-1.0,-1.0,0.0,1.26691 +0.05,78,search,2193.8,0.73787,0.199184,1334.75,1.26651 +0.05,79,insert,731.339,-1.0,-1.0,0.0,1.29381 +0.05,80,search,2226.37,0.73819,0.199011,1374.44,1.29427 +0.05,81,insert,68.476,-1.0,-1.0,0.0,1.30619 +0.05,82,search,2195.05,0.73885,0.198969,1359.59,1.30595 +0.05,83,insert,24.227,-1.0,-1.0,0.0,1.31218 +0.05,84,search,2232.82,0.74189,0.198012,1483.14,1.31255 +0.05,85,insert,81.791,-1.0,-1.0,0.0,1.32874 +0.05,86,search,2231.11,0.74257,0.198453,1423.25,1.32888 +0.05,87,insert,864.217,-1.0,-1.0,0.0,1.51901 +0.05,88,search,2377.17,0.75453,0.192191,6519.5,1.51873 +0.05,89,insert,742.049,-1.0,-1.0,0.0,1.67991 +0.05,90,search,2562.91,0.76709,0.186772,7774.15,1.68172 +0.05,91,insert,55.429,-1.0,-1.0,0.0,1.69436 +0.05,92,search,2637.16,0.76887,0.187009,2934.89,1.69398 +0.05,93,insert,616.954,-1.0,-1.0,0.0,1.80045 +0.05,94,search,2733.78,0.77007,0.187019,3484.94,1.79943 +0.05,95,insert,32.713,-1.0,-1.0,0.0,1.80629 +0.05,96,search,2617.49,0.77078,0.186384,2182.06,1.80663 +0.05,97,insert,18.348,-1.0,-1.0,0.0,1.81046 +0.05,98,search,2708.34,0.7717,0.185967,1766.49,1.81038 +0.05,99,insert,9.343,-1.0,-1.0,0.0,1.81222 +0.05,100,search,2689.76,0.77161,0.186039,1855.31,1.81215 +0.05,101,insert,71.056,-1.0,-1.0,0.0,1.82496 +0.05,102,search,2711.99,0.77238,0.185916,2003.62,1.82586 diff --git a/scripts/big_ann_perf_numbers/temp_perf_debug_scan_0.14_worker_batch_tuning.csv b/scripts/big_ann_perf_numbers/temp_perf_debug_scan_0.14_worker_batch_tuning.csv new file mode 100644 index 00000000..2e08ecf8 --- /dev/null +++ b/scripts/big_ann_perf_numbers/temp_perf_debug_scan_0.14_worker_batch_tuning.csv @@ -0,0 +1,46 @@ +step_num,step_type,latency_ms,worker_partition_size,worker_scan_time_ms,worker_scan_throughput,worker_result_time_ms,measured_ipc,cache_miss_rate,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,3070.27,37.3917,396.028,0.372447,499.333,0.83053,1.59165,0.83796,0.13696,-1.0,-1.0,7.297,0.0184881,1000,38806 +3,insert,24.883,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,11.592,0.0226489,997,49334 +4,search,2432.37,65.61,267.904,0.596273,173.074,1.39491,23.5485,0.85002,0.13685,-1.0,-1.0,17.203,0.0226571,995,49334 +5,insert,20.717,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,148.071,0.0331908,984,73557 +6,search,2305.81,193.134,226.938,0.869705,101.021,0.873411,18.4945,0.91587,0.110639,-1.0,-1.0,137.382,0.0331908,984,73557 +7,insert,24.865,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,136.613,0.0426695,984,95766 +8,search,2351.84,245.376,331.776,0.719487,106.128,1.37675,17.5143,0.91691,0.110489,-1.0,-1.0,87.988,0.0430701,988,95766 +9,insert,3.129,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,73.748,0.0447788,988,100025 +10,search,2372.69,250.163,320.96,0.775866,106.396,1.37115,19.4719,0.91771,0.107466,-1.0,-1.0,74.929,0.0447788,988,100025 +11,insert,8.53,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,77.981,0.0494549,989,111417 +12,search,2446.48,294.312,405.716,0.627239,116.446,1.48712,17.7425,0.91734,0.108494,-1.0,-1.0,23.414,0.0498645,990,111417 +13,insert,29.631,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,6.249,0.0612351,990,137139 +14,search,2421.19,333.836,331.021,0.756576,119.366,1.40499,12.7426,0.91308,0.111165,-1.0,-1.0,15.033,0.061251,992,137139 +15,insert,15.531,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,6.149,0.0699088,992,156710 +16,search,2450.41,367.914,461.266,0.650374,100.993,1.65992,16.2248,0.91007,0.112083,-1.0,-1.0,6.311,0.0699088,992,156710 +17,insert,4.465,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.042,0.0721524,993,161773 +18,search,2491.08,385.117,469.72,0.625897,114.158,1.62688,15.5473,0.91228,0.111388,-1.0,-1.0,6.296,0.0721524,993,161773 +19,insert,30.904,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,8.647,0.0788329,994,177161 +20,search,2463.67,427.16,434.459,0.68179,108.157,1.762,15.8342,0.91619,0.108129,-1.0,-1.0,6.174,0.0788329,994,177161 +21,insert,325.368,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,106.624,0.199226,1015,448898 +22,search,2572.78,924.323,535.212,0.889352,78.4564,1.9074,6.57268,0.90782,0.109883,-1.0,-1.0,15.309,0.199226,1017,448898 +23,insert,398.183,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,252.467,0.328008,1056,733493 +24,search,2882.84,1302.93,798.997,0.793248,90.673,2.12338,10.1353,0.90258,0.111878,-1.0,-1.0,64.024,0.328007,1065,733493 +25,insert,41.729,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.928,0.344779,1067,777443 +26,search,2844.52,1308.8,740.571,0.867858,85.9908,2.02675,8.83081,0.90327,0.111395,-1.0,-1.0,14.563,0.344787,1068,777443 +27,insert,30.127,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,8.481,0.355871,1068,805394 +28,search,2852.76,1337.72,772.889,0.860388,72.3551,1.99488,7.76554,0.9034,0.111453,-1.0,-1.0,8.841,0.355871,1068,805394 +29,insert,60.028,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,40.577,0.384014,1073,866391 +30,search,2866.05,1410.39,780.85,0.898717,76.9226,1.95992,10.2073,0.90458,0.112188,-1.0,-1.0,23.985,0.384013,1075,866391 +31,insert,436.755,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,364.699,0.553439,1125,1239816 +32,search,3141.84,1721.98,947.831,0.901958,91.0181,2.00413,9.49485,0.89642,0.118214,-1.0,-1.0,74.006,0.553553,1134,1239816 +33,insert,44.754,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.727,0.573565,1137,1291211 +34,search,3187.3,1721.41,962.491,0.931293,84.9232,3.14305,12.2368,0.89645,0.118294,-1.0,-1.0,52.214,0.573577,1141,1291211 +35,insert,28.286,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,36.02,0.585395,1147,1324978 +36,search,3206.84,1749.44,979.044,1.00084,83.5524,1.99927,7.53983,0.89755,0.11705,-1.0,-1.0,21.924,0.585403,1149,1324978 +37,insert,42.638,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.522,0.605791,1153,1371885 +38,search,3226.82,1777.43,979.4,0.938982,85.4242,2.0111,7.19635,0.89721,0.116469,-1.0,-1.0,18.322,0.605791,1154,1371885 +39,insert,13.507,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.482,0.60912,1155,1379335 +40,search,3263.26,1779.22,1006.7,0.891796,89.7042,2.02152,7.86378,0.89768,0.116823,-1.0,-1.0,9.444,0.60912,1155,1379335 +41,insert,61.145,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,18.594,0.624903,1157,1413312 +42,search,3345.96,1809.18,1059.58,0.941386,103.23,2.0098,6.21945,0.89914,0.116026,-1.0,-1.0,15.705,0.625015,1158,1413312 +43,insert,16.008,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.139,0.628672,1159,1423411 +44,search,3308.58,1815.56,1041.3,0.928153,96.6002,2.0169,7.3644,0.89885,0.116296,-1.0,-1.0,8.867,0.628672,1159,1423411 +45,insert,218.704,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,51.078,0.644769,1168,1457559 +46,search,3313.21,1816.28,1034.95,0.948786,93.9339,2.04583,10.1682,0.89879,0.116116,-1.0,-1.0,8.532,0.644769,1168,1457559 diff --git a/scripts/big_ann_perf_numbers/temp_perf_debug_scan_0.14_worker_batch_tuning_with_counters.csv b/scripts/big_ann_perf_numbers/temp_perf_debug_scan_0.14_worker_batch_tuning_with_counters.csv new file mode 100644 index 00000000..c36f9e3e --- /dev/null +++ b/scripts/big_ann_perf_numbers/temp_perf_debug_scan_0.14_worker_batch_tuning_with_counters.csv @@ -0,0 +1,76 @@ +step_num,step_type,latency_ms,worker_partition_size,worker_scan_time_ms,worker_scan_throughput,worker_result_time_ms,measured_ipc,cache_miss_rate,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,3035.6,37.3917,420.231,0.382277,504.338,0.867317,3.24812,0.83796,0.13696,-1.0,-1.0,6.95,0.0184881,1000,38806 +3,insert,25.975,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.301,0.022655,996,49334 +4,search,2370.06,65.5482,237.393,0.592827,151.278,1.13594,20.716,0.85049,0.13652,-1.0,-1.0,11.121,0.0226615,993,49334 +5,insert,25.255,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,154.544,0.0329603,983,73557 +6,search,2352.32,193.02,306.821,0.718086,108.047,1.30878,18.2649,0.91581,0.110963,-1.0,-1.0,129.304,0.0329986,984,73557 +7,insert,23.544,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,126.262,0.0424157,984,95766 +8,search,2362.35,243.663,341.842,0.677268,102.619,1.57255,23.9914,0.91496,0.111011,-1.0,-1.0,80.92,0.042911,988,95766 +9,insert,3.253,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,67.407,0.0446756,988,100025 +10,search,2424.36,248.155,387.108,0.597508,114.716,1.40239,25.3986,0.91674,0.108386,-1.0,-1.0,24.446,0.045102,991,100025 +11,insert,15.075,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,7.724,0.0497923,992,111417 +12,search,2456.76,286.378,340.763,0.620804,127.185,1.43018,18.6114,0.91484,0.110118,-1.0,-1.0,14.452,0.0499502,994,111417 +13,insert,18.65,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,6.06,0.0608532,994,137139 +14,search,2392.17,324.008,279.62,0.789245,99.7453,1.37047,15.1104,0.91101,0.112744,-1.0,-1.0,14.047,0.0608716,997,137139 +15,insert,15.206,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,6.198,0.069633,997,156710 +16,search,2385.43,356.422,287.099,0.770457,98.8011,1.46295,15.4498,0.90762,0.113728,-1.0,-1.0,6.247,0.069633,997,156710 +17,insert,4.057,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,6.283,0.0717643,997,161773 +18,search,2425.92,376.748,368.997,0.750144,98.8857,1.71388,18.4104,0.90924,0.113481,-1.0,-1.0,6.297,0.0717643,997,161773 +19,insert,31.156,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,8.606,0.0786085,998,177161 +20,search,2431.31,417.19,368.641,0.754435,97.1014,1.72491,21.242,0.91318,0.109991,-1.0,-1.0,6.338,0.0786085,998,177161 +21,insert,318.612,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,150.53,0.200787,1038,448898 +22,search,2621.51,843.081,509.473,0.839114,75.1851,1.89609,13.5775,0.90153,0.113162,-1.0,-1.0,46.414,0.200816,1049,448898 +23,insert,389.011,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,236.892,0.328393,1094,733493 +24,search,2917,1173.53,766.962,0.842252,78.3771,2.06638,10.4986,0.89616,0.115777,-1.0,-1.0,81.367,0.328673,1110,733493 +25,insert,38.602,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,26.869,0.344932,1114,777443 +26,search,2942.85,1161.84,778.321,0.855797,79.4583,2.18933,8.76152,0.89656,0.115824,-1.0,-1.0,19.327,0.344942,1116,777443 +27,insert,30.004,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.566,0.355953,1117,805394 +28,search,2879.78,1181.81,703.211,0.914815,67.1411,2.10186,9.3667,0.89704,0.115307,-1.0,-1.0,15.215,0.355974,1119,805394 +29,insert,54.017,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,49.861,0.383044,1128,866391 +30,search,2968.2,1235.9,785.407,0.854108,67.4165,2.12757,10.3829,0.89924,0.115321,-1.0,-1.0,23.059,0.383044,1130,866391 +31,insert,453.273,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,334.618,0.552862,1190,1239816 +32,search,3260.36,1498.72,958.818,0.870608,72.1074,2.05855,8.94397,0.88958,0.121369,-1.0,-1.0,68.148,0.552916,1196,1239816 +33,insert,44.691,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,34.798,0.572702,1203,1291211 +34,search,3252.97,1509.86,909.114,0.899267,71.5206,2.1166,6.89881,0.8888,0.121983,-1.0,-1.0,33.866,0.57271,1205,1291211 +35,insert,28.989,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,32.056,0.584599,1212,1324978 +36,search,3299.42,1533.91,928.471,0.938225,69.982,2.09337,8.251,0.89032,0.120622,-1.0,-1.0,16.006,0.584599,1213,1324978 +37,insert,44.573,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,40.031,0.605742,1219,1371885 +38,search,3590.33,1552.76,964.783,0.889823,84.6972,2.17256,11.2212,0.89006,0.121573,-1.0,-1.0,10.689,0.605742,1219,1371885 +39,insert,13.637,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.011,0.608441,1219,1379335 +40,search,3432.21,1560.85,943.132,0.864369,73.1322,2.26521,8.30382,0.89087,0.121418,-1.0,-1.0,9.351,0.608441,1219,1379335 +41,insert,54.656,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,31.722,0.622128,1224,1413312 +42,search,3410.96,1582.26,1028.72,0.83453,80.1023,2.18424,8.96297,0.892,0.120769,-1.0,-1.0,8.91,0.622128,1224,1413312 +43,insert,16.585,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,8.166,0.625911,1224,1423411 +44,search,3369.78,1596.76,990.591,0.882051,81.4331,2.19832,7.79683,0.89242,0.120319,-1.0,-1.0,8.899,0.625911,1224,1423411 +45,insert,197.6,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,26.795,0.642851,1231,1457559 +46,search,3377.11,1610.03,992.018,0.854689,71.6573,2.07907,8.84385,0.89297,0.118827,-1.0,-1.0,8.621,0.642851,1231,1457559 +47,insert,282.212,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,240.219,0.748077,1264,1688593 +48,search,3514.83,1755.28,1079.77,0.908995,76.4786,2.10633,7.74335,0.89387,0.119424,-1.0,-1.0,62.384,0.748077,1270,1688593 +49,insert,36.032,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.023,0.762801,1272,1723505 +50,search,3612.36,1760.31,1143.97,0.900675,98.3982,2.36903,7.57637,0.89462,0.119044,-1.0,-1.0,36.53,0.762807,1276,1723505 +51,insert,312.706,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,208.19,0.900711,1311,2035040 +52,search,3752.47,1954.7,1206.05,0.926171,86.0285,2.08198,7.43964,0.8975,0.118184,-1.0,-1.0,33.065,0.900711,1314,2035040 +53,insert,45.689,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,45.371,0.917223,1318,2078341 +54,search,3781.85,1979.61,1223.22,0.970751,88.3165,2.07803,6.85166,0.89761,0.118593,-1.0,-1.0,10.41,0.917223,1318,2078341 +55,insert,416.74,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,401.633,1.0665,1367,2399905 +56,search,4053.36,2166.51,1387.2,0.922167,112.087,2.16069,7.72146,0.90056,0.118392,-1.0,-1.0,99.117,1.06653,1380,2399905 +57,insert,30.987,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,33.25,1.07438,1384,2419836 +58,search,4054.25,2119.71,1351.65,0.930816,111.615,2.20247,8.51779,0.89909,0.119262,-1.0,-1.0,65.788,1.07442,1391,2419836 +59,insert,222.737,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,108.618,1.17291,1401,2643083 +60,search,4164.69,2178.58,1411.28,0.912069,112.475,2.11429,6.87599,0.8931,0.125411,-1.0,-1.0,61.905,1.17353,1408,2643083 +61,insert,11.629,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,18.731,1.17703,1410,2653064 +62,search,4244.38,2167.7,1475.38,0.921919,125.286,2.15535,7.6848,0.89305,0.125482,-1.0,-1.0,15.291,1.17704,1411,2653064 +63,insert,2.819,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.405,1.17793,1411,2656118 +64,search,4129.84,2167.31,1389.53,0.942525,109.384,2.14315,9.03602,0.89284,0.125525,-1.0,-1.0,10.916,1.17793,1411,2656118 +65,insert,77.318,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,103.552,1.20793,1432,2723424 +66,search,4221.1,2188.06,1421.74,0.949361,116.094,2.1466,9.45562,0.89358,0.125179,-1.0,-1.0,10.057,1.20793,1432,2723424 +67,insert,44.238,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,79.685,1.22887,1447,2769021 +68,search,4280,2202.74,1435.45,0.946927,124.751,2.18624,8.28047,0.8951,0.124238,-1.0,-1.0,19.65,1.22888,1449,2769021 +69,insert,49.515,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,55.921,1.25096,1457,2821777 +70,search,4314.18,2222.62,1454.55,0.897541,121.701,2.18296,7.10808,0.89674,0.122815,-1.0,-1.0,10.321,1.25096,1457,2821777 +71,insert,15.358,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,25.921,1.25685,1461,2834921 +72,search,4336.86,2226.55,1468.08,0.889896,122.891,2.17387,7.53162,0.89742,0.122313,-1.0,-1.0,13.824,1.25685,1461,2834921 +73,insert,7.146,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,10.889,1.25892,1461,2842044 +74,search,4336.7,2231.56,1474.8,0.950467,120.822,2.2214,7.55198,0.89751,0.12232,-1.0,-1.0,13.304,1.25892,1461,2842044 +75,insert,24.165,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.469,1.2636,1462,2858890 +76,search,4361.67,2241.33,1490.39,0.968308,134.077,2.31167,7.23226,0.89745,0.122324,-1.0,-1.0,19.894,1.2637,1463,2858890 diff --git a/scripts/big_ann_perf_numbers/test_metric_collection.csv b/scripts/big_ann_perf_numbers/test_metric_collection.csv new file mode 100644 index 00000000..2dd482ac --- /dev/null +++ b/scripts/big_ann_perf_numbers/test_metric_collection.csv @@ -0,0 +1,11 @@ +batch_size,step_num,step_type,search_latency_ms,worker_partition_size,worker_scan_time_ms,worker_scan_throughput,worker_result_time_ms,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +512,2,search,3482.2,37.5312,781.697,0.457078,388.613,0.8551,0.130106,-1.0,-1.0,10.871,0.0184881,1000,38806 +512,3,insert,23.284,-1.0,-1.0,-1.0,-1.0,19.612,0.022655,996,49334 +512,4,search,2553.16,63.3236,180.072,0.81906,144.894,0.86686,0.128427,-1.0,-1.0,19.651,0.0226469,994,49334 +512,5,insert,21.527,-1.0,-1.0,-1.0,-1.0,165.849,0.0330329,986,73557 +512,6,search,2558.93,174.009,285.811,0.804123,127.678,0.92469,0.104049,-1.0,-1.0,117.095,0.0331769,988,73557 +512,7,insert,24.61,-1.0,-1.0,-1.0,-1.0,116.018,0.0427286,989,95766 +512,8,search,2557.92,218.416,259.189,0.839563,125.21,0.92476,0.103613,-1.0,-1.0,20.489,0.0432125,992,95766 +512,9,insert,3.176,-1.0,-1.0,-1.0,-1.0,6.262,0.0448665,992,100025 +512,10,search,2584.78,223.689,220.772,0.805342,148.186,0.92573,0.101717,-1.0,-1.0,17.197,0.0449,995,100025 +512,11,insert,19.224,-1.0,-1.0,-1.0,-1.0,15.271,0.0497813,997,111417 diff --git a/scripts/big_ann_perf_numbers/test_perf_debug_scan_0.14_tuning.csv b/scripts/big_ann_perf_numbers/test_perf_debug_scan_0.14_tuning.csv new file mode 100644 index 00000000..ac48fae4 --- /dev/null +++ b/scripts/big_ann_perf_numbers/test_perf_debug_scan_0.14_tuning.csv @@ -0,0 +1,26 @@ +step_num,step_type,latency_ms,worker_partition_size,worker_scan_time_ms,worker_scan_throughput,worker_result_time_ms,measured_ipc,cache_miss_rate,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,3168.19,37.4251,328.899,0.550743,612.621,-nan,-nan,0.844,0.1346,-1.0,-1.0,6.735,0.0184881,1000,38806 +3,insert,27.469,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.719,0.022655,996,49334 +4,search,2508.68,65.1954,177.318,0.963355,191.554,-nan,-nan,0.8558,0.133903,-1.0,-1.0,17.764,0.0226615,993,49334 +5,insert,20.935,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,162.229,0.033017,984,73557 +6,search,2364.79,190.054,178.434,1.62821,98.8412,-nan,-nan,0.91846,0.108847,-1.0,-1.0,144.734,0.0330521,985,73557 +7,insert,23.907,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,144.808,0.0425589,986,95766 +8,search,2348.93,238.607,189.89,1.55421,95.834,-nan,-nan,0.91852,0.108405,-1.0,-1.0,87.19,0.0428008,990,95766 +9,insert,3.485,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,74.225,0.0446156,990,100025 +10,search,2394.72,245.023,194.572,1.41505,103.476,-nan,-nan,0.91978,0.10575,-1.0,-1.0,22.238,0.0450652,995,100025 +11,insert,8.361,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.159,0.0498095,997,111417 +12,search,2386.85,278.189,199.22,1.24952,111.249,-nan,-nan,0.91729,0.108302,-1.0,-1.0,22.702,0.0499412,999,111417 +13,insert,28.184,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.479,0.0612192,1001,137139 +14,search,2412.08,314.268,223.304,1.18219,106.811,-nan,-nan,0.9133,0.110833,-1.0,-1.0,16.574,0.0612314,1005,137139 +15,insert,15.287,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,8.984,0.069788,1006,156710 +16,search,2409.13,343.318,237.888,1.20787,101.79,-nan,-nan,0.90963,0.111169,-1.0,-1.0,8.479,0.0697908,1007,156710 +17,insert,3.765,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,8.428,0.0717758,1008,161773 +18,search,2420.02,356.533,237.722,1.17654,100.786,-nan,-nan,0.91096,0.110666,-1.0,-1.0,8.884,0.0718529,1009,161773 +19,insert,31.652,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.454,0.078631,1010,177161 +20,search,2434.27,395.28,257.442,1.20095,98.4046,-nan,-nan,0.91521,0.107631,-1.0,-1.0,6.351,0.078631,1010,177161 +21,insert,317.946,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,210.497,0.201512,1074,448898 +22,search,2673.03,753.477,413.894,1.25247,65.402,-nan,-nan,0.89884,0.114083,-1.0,-1.0,51.738,0.20151,1085,448898 +23,insert,428.467,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,269.614,0.327669,1144,733493 +24,search,3002.29,1034.83,619.84,1.08352,82.3551,-nan,-nan,0.8948,0.11623,-1.0,-1.0,80.543,0.328234,1166,733493 +25,insert,35.426,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.191,0.345577,1170,777443 +26,search,3078.55,1010.14,610.538,1.01519,83.5195,-nan,-nan,0.89313,0.117757,-1.0,-1.0,17.019,0.345672,1172,777443 diff --git a/scripts/big_ann_perf_numbers/test_perf_debug_scan_0.14_worker_batch_tuning.csv b/scripts/big_ann_perf_numbers/test_perf_debug_scan_0.14_worker_batch_tuning.csv new file mode 100644 index 00000000..1381c902 --- /dev/null +++ b/scripts/big_ann_perf_numbers/test_perf_debug_scan_0.14_worker_batch_tuning.csv @@ -0,0 +1,1280 @@ +step_num,step_type,latency_ms,worker_partition_size,worker_scan_time_ms,worker_scan_throughput,worker_result_time_ms,measured_ipc,cache_miss_rate,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +2,search,3247.47,37.4251,331.689,0.510184,648.614,-nan,-nan,0.844,0.1346,-1.0,-1.0,6.695,0.0184881,1000,38806 +3,insert,25.617,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.338,0.022655,996,49334 +4,search,2540.43,65.1954,195.461,0.99775,207.3,-nan,-nan,0.8558,0.133903,-1.0,-1.0,16.939,0.0226615,993,49334 +5,insert,19.949,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,164.352,0.032935,982,73557 +6,search,2350.58,189.735,175.935,1.65929,109.007,-nan,-nan,0.91916,0.108187,-1.0,-1.0,142.438,0.0329729,983,73557 +7,insert,24.085,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,145.763,0.0426744,984,95766 +8,search,2408.17,239.602,188.188,1.4864,107.986,-nan,-nan,0.91905,0.108333,-1.0,-1.0,83.786,0.0429485,987,95766 +9,insert,3.399,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,74.298,0.0447319,987,100025 +10,search,2401.18,247.607,192.967,1.41723,113.025,-nan,-nan,0.92091,0.10518,-1.0,-1.0,18.688,0.0451509,990,100025 +11,insert,8.32,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.81,0.049849,992,111417 +12,search,2404.25,283.16,203.377,1.28774,125.709,-nan,-nan,0.91837,0.107629,-1.0,-1.0,22.419,0.0499784,994,111417 +13,insert,28.707,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,6.171,0.0610747,994,137139 +14,search,2452.88,321.358,221.168,1.25571,121.735,-nan,-nan,0.91528,0.109521,-1.0,-1.0,12.182,0.0610862,996,137139 +15,insert,15.441,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,6.326,0.0696525,996,156710 +16,search,2446.12,356.024,248.257,1.20872,100.562,-nan,-nan,0.91308,0.109879,-1.0,-1.0,6.719,0.0696525,996,156710 +17,insert,3.853,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,8.758,0.0716342,997,161773 +18,search,2438.8,373.487,246.175,1.19235,108.491,-nan,-nan,0.91385,0.109958,-1.0,-1.0,6.346,0.0716342,997,161773 +19,insert,31.511,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,10.406,0.0785551,999,177161 +20,search,2417.02,411.658,261.014,1.26594,103.356,-nan,-nan,0.91716,0.107192,-1.0,-1.0,6.281,0.0785551,999,177161 +21,insert,322.721,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,154.488,0.200933,1036,448898 +22,search,2708.49,842.95,474.008,1.10849,90.2816,-nan,-nan,0.90531,0.110788,-1.0,-1.0,35.874,0.200931,1041,448898 +23,insert,399.981,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,323.52,0.328769,1091,733493 +24,search,2994.8,1174.21,669.034,1.03349,94.9481,-nan,-nan,0.9004,0.113599,-1.0,-1.0,66.913,0.328944,1102,733493 +25,insert,35.069,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,29.291,0.345738,1107,777443 +26,search,3016.23,1167.16,654.749,1.03724,84.4454,-nan,-nan,0.89929,0.114896,-1.0,-1.0,19.189,0.345741,1108,777443 +27,insert,26.067,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,17.498,0.356229,1109,805394 +28,search,3093.11,1196.28,687.579,1.03815,78.9058,-nan,-nan,0.90114,0.113727,-1.0,-1.0,17.461,0.356231,1110,805394 +29,insert,55.811,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,51.268,0.384265,1118,866391 +30,search,3067.15,1254.79,714.868,1.11675,81.1063,-nan,-nan,0.90334,0.112491,-1.0,-1.0,20.417,0.384265,1120,866391 +31,insert,434.69,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,348.394,0.55227,1174,1239816 +32,search,3302.38,1550.25,887.223,1.027,84.8034,-nan,-nan,0.8961,0.118117,-1.0,-1.0,63.66,0.552278,1178,1239816 +33,insert,46.096,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,26.776,0.573493,1183,1291211 +34,search,3347.85,1565.66,905.327,1.04494,83.7677,-nan,-nan,0.89578,0.118547,-1.0,-1.0,49.769,0.5735,1186,1291211 +35,insert,43.682,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,46.199,0.58576,1194,1324978 +36,search,3444.64,1584.95,935.756,0.99145,87.9751,-nan,-nan,0.89719,0.117346,-1.0,-1.0,44.552,0.58577,1198,1324978 +37,insert,61.434,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.089,0.606756,1204,1371885 +38,search,3437.94,1604.69,956.52,1.02288,88.8825,-nan,-nan,0.89618,0.118174,-1.0,-1.0,16.839,0.606756,1205,1371885 +39,insert,12.461,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,8.353,0.609532,1205,1379335 +40,search,3410.13,1609.6,973.105,0.988351,94.9484,-nan,-nan,0.89607,0.118548,-1.0,-1.0,9.248,0.609532,1205,1379335 +41,insert,44.087,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,36.041,0.625083,1211,1413312 +42,search,3454.85,1626.89,975.67,1.04153,80.2463,-nan,-nan,0.89742,0.117832,-1.0,-1.0,14.88,0.6253,1212,1413312 +43,insert,11.141,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,12.258,0.629047,1213,1423411 +44,search,3398.48,1636.44,969.078,1.04653,74.5874,-nan,-nan,0.89749,0.117805,-1.0,-1.0,8.882,0.629047,1213,1423411 +45,insert,203.117,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,43.051,0.644239,1221,1457559 +46,search,3388.06,1648.35,912.967,1.10212,75.6174,-nan,-nan,0.89793,0.116804,-1.0,-1.0,8.814,0.644239,1221,1457559 +47,insert,273.6,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,267.573,0.749406,1255,1688593 +48,search,3625.34,1789.11,1064.85,1.05902,88.5737,-nan,-nan,0.90016,0.116773,-1.0,-1.0,89.809,0.749877,1266,1688593 +49,insert,52.095,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,49.044,0.764106,1272,1723505 +50,search,3697.28,1775.03,1092.25,1.00747,93.0057,-nan,-nan,0.90113,0.115442,-1.0,-1.0,22.637,0.764106,1273,1723505 +51,insert,363.88,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,269.881,0.904015,1312,2035040 +52,search,3951.5,1969.26,1242.76,1.00906,108.238,-nan,-nan,0.90382,0.114674,-1.0,-1.0,34.094,0.904023,1316,2035040 +53,insert,45.887,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,51.689,0.92232,1321,2078341 +54,search,3972.63,1985.61,1245.76,0.987585,111.149,-nan,-nan,0.9038,0.114677,-1.0,-1.0,9.77,0.92232,1321,2078341 +55,insert,398.482,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,490.548,1.07033,1377,2399905 +56,search,4203.19,2132.33,1364.71,0.995361,120.229,-nan,-nan,0.90236,0.117637,-1.0,-1.0,148.974,1.07141,1394,2399905 +57,insert,23.348,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.984,1.07968,1394,2419836 +58,search,4234.97,2114.51,1370.67,1.03681,105.987,-nan,-nan,0.90361,0.115983,-1.0,-1.0,35.601,1.07972,1396,2419836 +59,insert,245.088,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,152.92,1.17676,1409,2643083 +60,search,4289.87,2168.03,1395.52,1.05625,117.672,-nan,-nan,0.89603,0.124549,-1.0,-1.0,51.073,1.17693,1414,2643083 +61,insert,10.324,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.841,1.18076,1415,2653064 +62,search,4231.39,2164.25,1359.35,1.06643,106.408,-nan,-nan,0.89595,0.124396,-1.0,-1.0,13.863,1.18076,1415,2653064 +63,insert,4.94,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,10.62,1.18206,1415,2656118 +64,search,4286.13,2167.84,1414.58,0.985006,120.159,-nan,-nan,0.89608,0.124257,-1.0,-1.0,12.667,1.18206,1415,2656118 +65,insert,96.44,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,127.683,1.20889,1438,2723424 +66,search,4427.55,2182.25,1444.93,1.00564,125.558,-nan,-nan,0.89726,0.12338,-1.0,-1.0,17.284,1.20891,1439,2723424 +67,insert,50.731,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,81.706,1.2296,1453,2769021 +68,search,4478.5,2194.51,1474.12,0.971752,130.866,-nan,-nan,0.89831,0.122862,-1.0,-1.0,15.807,1.2296,1454,2769021 +69,insert,58.868,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,67.143,1.25116,1462,2821777 +70,search,4524.09,2219.21,1498.16,0.956801,133.288,-nan,-nan,0.90033,0.121045,-1.0,-1.0,11.026,1.25116,1462,2821777 +71,insert,20.243,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.852,1.25617,1465,2834921 +72,search,4492.7,2227.61,1486.48,0.982621,128.681,-nan,-nan,0.90108,0.120799,-1.0,-1.0,10.905,1.25617,1465,2834921 +73,insert,11.74,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,10.319,1.25974,1465,2842044 +74,search,4472.35,2233.44,1432.02,1.08092,111.133,-nan,-nan,0.90122,0.120654,-1.0,-1.0,10.655,1.25974,1465,2842044 +75,insert,22.407,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.118,1.26643,1466,2858890 +76,search,4451.67,2244.69,1499.53,0.980039,126.196,-nan,-nan,0.90168,0.120285,-1.0,-1.0,10.943,1.26643,1466,2858890 +77,insert,31.107,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,10.223,1.27611,1466,2882818 +78,search,4478.69,2257.3,1506,0.998915,131.317,-nan,-nan,0.90142,0.12044,-1.0,-1.0,17.983,1.27632,1467,2882818 +79,insert,451.838,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,12.418,1.30428,1468,2947806 +80,search,4482,2290.68,1526.24,0.992914,127.819,-nan,-nan,0.90098,0.120964,-1.0,-1.0,10.728,1.30428,1468,2947806 +81,insert,31.632,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,31.168,1.31604,1472,2976895 +82,search,4548.47,2297.66,1532.82,0.97736,134.684,-nan,-nan,0.90237,0.120179,-1.0,-1.0,11.935,1.31604,1472,2976895 +83,insert,15.766,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,38.479,1.32143,1476,2990682 +84,search,4539.7,2295.85,1542.23,1.05909,119.832,-nan,-nan,0.9025,0.12003,-1.0,-1.0,12.587,1.32143,1476,2990682 +85,insert,46.906,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,37.972,1.33711,1479,3025613 +86,search,4617.63,2318.14,1563.25,1.01272,138.024,-nan,-nan,0.90394,0.119043,-1.0,-1.0,19.633,1.33732,1480,3025613 +87,insert,637.854,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,724.97,1.54453,1576,3468632 +88,search,4975.69,2442.84,1740.81,0.968766,153.493,-nan,-nan,0.90557,0.116489,-1.0,-1.0,78.526,1.54453,1583,3468632 +89,insert,403.651,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,441.781,1.70389,1651,3831692 +90,search,5205.97,2554.48,1806.34,1.07284,148.549,-nan,-nan,0.91036,0.110757,-1.0,-1.0,68.016,1.70424,1659,3831692 +91,insert,32.911,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,47.06,1.71631,1665,3864067 +92,search,5343.85,2547.49,1885.9,1.0381,166.411,-nan,-nan,0.91036,0.110702,-1.0,-1.0,17.647,1.71631,1666,3864067 +93,insert,262.614,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,230.646,1.82019,1698,4103827 +94,search,5483.83,2615.04,1971.46,1.0212,175.162,-nan,-nan,0.91068,0.110878,-1.0,-1.0,47.913,1.82044,1702,4103827 +95,insert,18.017,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,42.606,1.82655,1707,4119898 +96,search,5499.95,2604.41,1970.76,0.990128,172.939,-nan,-nan,0.91086,0.110906,-1.0,-1.0,14.889,1.82655,1707,4119898 +97,insert,10.122,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.915,1.82916,1707,4129367 +98,search,5484.74,2610.67,1974.87,1.02472,174.773,-nan,-nan,0.91098,0.110875,-1.0,-1.0,14.664,1.82916,1707,4129367 +99,insert,5.622,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.033,1.83055,1708,4134502 +100,search,5513.07,2613.02,1976.83,1.02165,167.954,-nan,-nan,0.91127,0.110706,-1.0,-1.0,14.173,1.83055,1708,4134502 +101,insert,33.103,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,42.466,1.84101,1712,4164482 +102,search,5515.33,2621.75,1983.15,1.01529,167.851,-nan,-nan,0.91148,0.111001,-1.0,-1.0,14.438,1.84101,1712,4164482 +103,insert,16.761,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,39.878,1.84731,1716,4178601 +104,search,5546.34,2623.31,1988.96,1.03176,173.108,-nan,-nan,0.91161,0.110939,-1.0,-1.0,14.579,1.84731,1716,4178601 +105,insert,7.691,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.215,1.84944,1718,4187774 +106,search,5509.85,2621.35,1963.75,1.07406,162.847,-nan,-nan,0.91143,0.111133,-1.0,-1.0,18.186,1.84944,1718,4187774 +107,insert,29.291,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.808,1.85475,1718,4200109 +108,search,5394,2626.41,1997.74,1.03681,169.298,-nan,-nan,0.91136,0.111178,-1.0,-1.0,15.845,1.85475,1718,4200109 +109,insert,531.995,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,448.355,2.01489,1788,4557867 +110,search,5176.37,2703.71,2103.93,1.08588,180.518,-nan,-nan,0.91342,0.109554,-1.0,-1.0,38.956,2.01571,1794,4557867 +111,insert,18.944,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,31.098,2.02934,1800,4587252 +112,search,5318.79,2705.27,2047.75,1.07626,165.316,-nan,-nan,0.91418,0.10914,-1.0,-1.0,19.958,2.02934,1800,4587252 +113,insert,16.39,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,32.827,2.03541,1802,4600056 +114,search,5654.72,2709.14,2158.82,0.987462,185.898,-nan,-nan,0.91422,0.109054,-1.0,-1.0,17.132,2.03541,1802,4600056 +115,insert,24.446,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,35.288,2.04504,1806,4627422 +116,search,5753.82,2717.72,2160.91,0.971458,182.168,-nan,-nan,0.91468,0.108513,-1.0,-1.0,19.945,2.04504,1806,4627422 +117,insert,14.656,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.187,2.05065,1806,4639292 +118,search,5869.07,2723.25,2163.05,0.986817,191.155,-nan,-nan,0.91473,0.108308,-1.0,-1.0,20.244,2.05065,1806,4639292 +119,insert,3.709,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.21,2.05155,1806,4643423 +120,search,5887.64,2725.34,2172.18,0.983043,189.816,-nan,-nan,0.91475,0.108249,-1.0,-1.0,20.132,2.05155,1806,4643423 +121,insert,19.371,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,24.611,2.05839,1807,4657252 +122,search,5929.2,2732.14,2173.14,1.03489,187.585,-nan,-nan,0.9149,0.10813,-1.0,-1.0,20.398,2.05839,1807,4657252 +123,insert,328.218,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,538.567,2.18565,1853,4938477 +124,search,6127.68,2791.56,2266.39,0.960952,197.559,-nan,-nan,0.91586,0.108387,-1.0,-1.0,176.556,2.18689,1868,4938477 +125,insert,57.06,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,53.619,2.20463,1873,4980492 +126,search,6184.17,2774.29,2266.34,1.02246,196.13,-nan,-nan,0.91641,0.107931,-1.0,-1.0,48.789,2.20465,1875,4980492 +127,insert,62.016,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,114.92,2.22747,1889,5030515 +128,search,6238.14,2776.54,2297.54,1.00095,193.508,-nan,-nan,0.91677,0.107671,-1.0,-1.0,49.803,2.22767,1892,5030515 +129,delete,18.336,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,53.23,2.22574,1857,5009391 +130,search,6072.57,2765.29,2233.47,1.01781,186.496,-nan,-nan,0.91494,0.109067,-1.0,-1.0,23.175,2.22574,1857,5009391 +131,delete,5.299,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.546,2.22574,1857,5004093 +132,search,6090.84,2762.62,2239.24,0.99419,194.216,-nan,-nan,0.91493,0.109213,-1.0,-1.0,22.841,2.22574,1857,5004093 +133,delete,11.109,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.864,2.22574,1857,4991787 +134,search,6072.1,2750.92,2226.39,0.978737,190.186,-nan,-nan,0.91479,0.109316,-1.0,-1.0,23.781,2.22574,1857,4991787 +135,delete,24.235,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,44.451,2.22323,1854,4974857 +136,search,6091.63,2743.87,2224.58,0.978153,193.477,-nan,-nan,0.91494,0.108938,-1.0,-1.0,23.862,2.22323,1854,4974857 +137,delete,5.354,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.574,2.22323,1854,4971386 +138,search,6072.16,2741.94,2225.1,0.97082,191.118,-nan,-nan,0.91489,0.109076,-1.0,-1.0,23.751,2.22323,1854,4971386 +139,delete,11.473,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.34,2.22323,1854,4963487 +140,search,6102.51,2736.16,2239.03,0.968781,202.361,-nan,-nan,0.9148,0.109122,-1.0,-1.0,23.712,2.22323,1854,4963487 +141,delete,28.103,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,76.215,2.21797,1848,4941830 +142,search,6033.17,2727.24,2220.73,1.01826,187.918,-nan,-nan,0.91458,0.109245,-1.0,-1.0,23.606,2.21797,1848,4941830 +143,delete,20.979,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,96.793,2.21227,1844,4925657 +144,search,5889.56,2725.51,2076.69,1.11362,170.181,-nan,-nan,0.91473,0.109164,-1.0,-1.0,24.545,2.21227,1844,4925657 +145,delete,3.32,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.343,2.21227,1844,4922686 +146,search,6050.31,2723.77,2219.04,1.00981,187.851,-nan,-nan,0.9147,0.109221,-1.0,-1.0,31.861,2.21231,1845,4922686 +147,delete,9.571,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.434,2.21231,1845,4913636 +148,search,6027.66,2714.93,2202.35,1.00352,181.783,-nan,-nan,0.91451,0.109372,-1.0,-1.0,22.964,2.21231,1845,4913636 +149,delete,208.839,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1268.02,2.1264,1749,4717250 +150,search,5736.04,2745.05,2120.02,1.01321,175.773,-nan,-nan,0.91336,0.110239,-1.0,-1.0,43.806,2.12689,1752,4717250 +151,delete,163.29,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1113.46,2.05587,1668,4560269 +152,search,5515.74,2774.9,2037.62,1.00963,176.203,-nan,-nan,0.91297,0.111296,-1.0,-1.0,130.037,2.05822,1680,4560269 +153,delete,27.404,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,162.594,2.04481,1663,4521936 +154,search,5393.05,2755.35,1953.87,1.02771,159.936,-nan,-nan,0.91192,0.111946,-1.0,-1.0,31.747,2.04498,1664,4521936 +155,delete,18.576,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,74.707,2.04061,1660,4502222 +156,search,5448.78,2745.72,2007.61,1.04,164.01,-nan,-nan,0.91198,0.111675,-1.0,-1.0,28.192,2.04084,1661,4502222 +157,delete,41.277,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,215.74,2.02523,1649,4456039 +158,search,5380.61,2738.92,1953.24,1.01867,158.316,-nan,-nan,0.91144,0.112519,-1.0,-1.0,30.528,2.02533,1650,4456039 +159,delete,284.078,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1757.01,1.87218,1491,4143922 +160,search,4936.35,2846.62,1828.84,1.06371,153.171,-nan,-nan,0.91003,0.11434,-1.0,-1.0,162.795,1.87532,1511,4143922 +161,delete,39.744,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,103.249,1.86806,1504,4101445 +162,search,4893.14,2766.67,1767.1,1.11177,143.763,-nan,-nan,0.91034,0.114005,-1.0,-1.0,52.833,1.86826,1508,4101445 +163,delete,18.011,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,42.722,1.86588,1506,4080175 +164,search,4912.77,2740.1,1769.34,1.03502,148.861,-nan,-nan,0.90974,0.11429,-1.0,-1.0,21.781,1.86588,1506,4080175 +165,delete,51.294,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,146.813,1.85609,1498,4047313 +166,search,4929.91,2731.21,1794.38,1.0211,148.083,-nan,-nan,0.91023,0.114008,-1.0,-1.0,37.188,1.85653,1500,4047313 +167,delete,6.561,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.717,1.85653,1500,4040738 +168,search,4948.08,2721.36,1804.24,1.0239,153.813,-nan,-nan,0.9105,0.113968,-1.0,-1.0,20.064,1.85653,1500,4040738 +169,delete,23.117,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,66.531,1.8506,1496,4022674 +170,search,4857.4,2715.88,1743.82,1.01978,141.626,-nan,-nan,0.91014,0.114269,-1.0,-1.0,20.138,1.8506,1496,4022674 +171,delete,8.109,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.565,1.8506,1496,4015835 +172,search,4932.5,2708.8,1799.37,0.994702,152.857,-nan,-nan,0.91009,0.114145,-1.0,-1.0,19.892,1.8506,1496,4015835 +173,delete,39.526,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,35.628,1.84738,1494,3986424 +174,search,4910.81,2681.13,1782.1,1.00745,152.973,-nan,-nan,0.90961,0.114648,-1.0,-1.0,27.665,1.84756,1495,3986424 +175,delete,197.197,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,793.33,1.75678,1406,3804693 +176,search,4581.61,2733.19,1641.87,1.01745,135.338,-nan,-nan,0.90609,0.116488,-1.0,-1.0,107.862,1.75855,1416,3804693 +177,delete,15.257,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,69.284,1.75324,1412,3786352 +178,search,4543.43,2701.28,1659.4,1.00961,141.104,-nan,-nan,0.906,0.116272,-1.0,-1.0,21.091,1.75324,1413,3786352 +179,delete,155.154,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,920.474,1.63049,1270,3556994 +180,search,4118.67,2806.52,1579.01,0.981462,140.917,-nan,-nan,0.89599,0.121748,-1.0,-1.0,102.605,1.63203,1280,3556994 +181,delete,26.962,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,68.828,1.62388,1276,3526824 +182,search,4057.46,2750.89,1573.36,0.980947,132.212,-nan,-nan,0.89574,0.122095,-1.0,-1.0,24.869,1.62388,1277,3526824 +183,delete,169.837,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,809.687,1.53551,1178,3350245 +184,search,3583.34,2808.65,1414.4,1.04713,118.451,-nan,-nan,0.87949,0.140091,-1.0,-1.0,34.637,1.53597,1184,3350245 +185,delete,5.727,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,12.815,1.53545,1180,3340234 +186,search,3775.45,2775.28,1386.52,1.07418,115.266,-nan,-nan,0.88493,0.131791,-1.0,-1.0,59.729,1.53653,1182,3340234 +187,delete,183.459,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,274.986,1.44471,1142,3150857 +188,search,3343.59,2789.51,1373.92,1.04862,115.242,-nan,-nan,0.8878,0.128698,-1.0,-1.0,36.227,1.44567,1147,3150857 +189,delete,4.459,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.926,1.44489,1146,3143609 +190,search,3678.88,2774.72,1356.67,1.0477,115.858,-nan,-nan,0.88821,0.12855,-1.0,-1.0,15.172,1.44489,1146,3143609 +191,delete,2.837,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.799,1.4448,1146,3141038 +192,search,3858.88,2773.23,1428,0.942163,117.559,-nan,-nan,0.888,0.128691,-1.0,-1.0,15.145,1.4448,1146,3141038 +193,delete,61.223,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,172.393,1.42531,1131,3081734 +194,search,3811.76,2743.25,1399.73,0.98777,123.447,-nan,-nan,0.88786,0.129639,-1.0,-1.0,15.094,1.42531,1131,3081734 +195,delete,39.091,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,133.867,1.40939,1119,3043966 +196,search,3745.81,2733.86,1380.28,0.989301,115.467,-nan,-nan,0.88696,0.12995,-1.0,-1.0,42.936,1.40999,1122,3043966 +197,delete,35.424,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,61.331,1.40061,1118,3008145 +198,search,3732.08,2708.44,1367.77,0.995324,115.846,-nan,-nan,0.88638,0.130237,-1.0,-1.0,14.71,1.40061,1118,3008145 +199,delete,12.137,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.509,1.39788,1117,2999384 +200,search,3695.41,2697.03,1336.02,1.02864,117.203,-nan,-nan,0.88604,0.130371,-1.0,-1.0,14.658,1.39788,1117,2999384 +201,delete,6.088,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.634,1.39584,1116,2993528 +202,search,3628.35,2692.83,1278.2,1.09542,107.519,-nan,-nan,0.88606,0.130361,-1.0,-1.0,14.607,1.39584,1116,2993528 +203,delete,16.886,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.624,1.3946,1116,2980973 +204,search,3709.07,2679.14,1337.17,0.987084,117.211,-nan,-nan,0.88599,0.130442,-1.0,-1.0,14.736,1.3946,1116,2980973 +205,delete,25.124,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,50.667,1.3884,1105,2960291 +206,search,3663.21,2666.94,1324.82,1.00665,118.693,-nan,-nan,0.88446,0.131253,-1.0,-1.0,18.178,1.3884,1105,2960291 +207,delete,49.955,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,170.689,1.36242,1055,2920620 +208,search,3528.85,2702.18,1281.08,0.939916,119.503,-nan,-nan,0.88069,0.133619,-1.0,-1.0,16.715,1.36242,1055,2920620 +209,delete,19.65,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,57.655,1.35549,1048,2900501 +210,search,3484.51,2691.69,1265.25,0.950598,111.161,-nan,-nan,0.8796,0.134079,-1.0,-1.0,16.637,1.35549,1048,2900501 +211,delete,10.091,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.022,1.35141,1047,2891660 +212,search,3484.63,2686.91,1270.74,0.97765,107.114,-nan,-nan,0.87962,0.134035,-1.0,-1.0,26.765,1.35148,1048,2891660 +213,delete,17.202,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,61.161,1.34627,1043,2873257 +214,search,3488.86,2677.57,1271.46,0.942013,106.435,-nan,-nan,0.87936,0.134836,-1.0,-1.0,19.928,1.34627,1043,2873257 +215,delete,369.806,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1479.68,1.14875,847,2537571 +216,search,2959.66,2950.98,1138.03,0.992466,98.0244,-nan,-nan,0.87201,0.141139,-1.0,-1.0,397.327,1.15709,887,2537571 +217,delete,216.625,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,956.183,1.03774,747,2308194 +218,search,2635.5,3033.76,1035.13,0.982548,85.9458,-nan,-nan,0.86662,0.144811,-1.0,-1.0,303.177,1.04421,778,2308194 +219,delete,15.796,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,36.79,1.0401,773,2285820 +220,search,2664.41,2815.48,1007.4,0.937145,84.3674,-nan,-nan,0.86553,0.145866,-1.0,-1.0,114.009,1.04053,782,2285820 +221,delete,191.188,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,702.282,0.931892,643,2082858 +222,search,2296.27,2957.97,888.05,0.970132,81.3842,-nan,-nan,0.8549,0.153183,-1.0,-1.0,121.082,0.934506,656,2082858 +223,delete,8.625,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.831,0.932846,653,2071207 +224,search,2253.15,2869.1,846.08,1.01828,77.2851,-nan,-nan,0.85226,0.154795,-1.0,-1.0,22.638,0.933088,654,2071207 +225,delete,6.719,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,18.903,0.931297,653,2065400 +226,search,2278.39,2859.76,869.168,0.923374,80.6006,-nan,-nan,0.85188,0.154997,-1.0,-1.0,10.064,0.931297,653,2065400 +227,delete,3.099,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.246,0.931297,653,2062105 +228,search,2259.02,2850.32,860.538,0.929616,76.6761,-nan,-nan,0.8516,0.15526,-1.0,-1.0,9.906,0.931297,653,2062105 +229,delete,20.146,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,25.382,0.925426,650,2039610 +230,search,2209.16,2824.57,799.868,1.04772,71.2309,-nan,-nan,0.85086,0.155547,-1.0,-1.0,10.199,0.925426,650,2039610 +231,delete,14.085,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,35.785,0.9204,645,2027795 +232,search,2229.65,2829.53,848.907,0.971051,70.876,-nan,-nan,0.84999,0.156205,-1.0,-1.0,27.882,0.92064,647,2027795 +233,delete,4.949,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.634,0.919074,646,2022859 +234,search,2256.11,2816.09,843.54,0.976747,78.8253,-nan,-nan,0.8495,0.156473,-1.0,-1.0,9.711,0.919074,646,2022859 +235,delete,6.068,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.721,0.918346,645,2016264 +236,search,2237.59,2808.5,841.331,0.983559,78.4392,-nan,-nan,0.84934,0.156535,-1.0,-1.0,10.018,0.918346,645,2016264 +237,delete,286.583,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,751.028,0.793305,517,1782866 +238,search,1903.72,3095.71,760.17,0.887256,69.231,-nan,-nan,0.83998,0.165413,-1.0,-1.0,200.908,0.79686,536,1782866 +239,delete,17.84,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,57.041,0.785481,527,1758931 +240,search,1898.4,2951.29,728.935,0.90852,74.76,-nan,-nan,0.83626,0.166359,-1.0,-1.0,47.148,0.785589,529,1758931 +241,delete,6.019,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.356,0.784131,528,1751275 +242,search,1895.25,2929.55,728.348,1.02523,67.0185,-nan,-nan,0.8357,0.16625,-1.0,-1.0,28.57,0.784224,530,1751275 +243,delete,18.536,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.478,0.781006,527,1733264 +244,search,1883.95,2900.72,724.987,0.980999,71.6968,-nan,-nan,0.83375,0.166934,-1.0,-1.0,9.812,0.781006,527,1733264 +245,delete,10.066,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,25.368,0.77889,526,1722658 +246,search,1870.38,2886.97,716.623,0.926433,66.7042,-nan,-nan,0.83306,0.167051,-1.0,-1.0,9.757,0.77889,526,1722658 +247,delete,2.385,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.415,0.77889,526,1719601 +248,search,1864.42,2880.35,714.634,0.9229,68.1556,-nan,-nan,0.8328,0.167259,-1.0,-1.0,9.656,0.77889,526,1719601 +249,delete,7.163,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.193,0.777426,524,1711648 +250,search,1893.66,2871.84,717.578,0.92695,70.6892,-nan,-nan,0.83294,0.167707,-1.0,-1.0,9.806,0.777426,524,1711648 +251,delete,281.255,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,332.799,0.658239,447,1470624 +252,search,1644.25,2988.37,643.236,0.960522,63.6029,-nan,-nan,0.82919,0.170523,-1.0,-1.0,65.749,0.659347,453,1470624 +253,delete,25.279,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,48.538,0.652117,447,1442070 +254,search,1638.66,2898.31,628.387,0.906523,62.5499,-nan,-nan,0.82636,0.171669,-1.0,-1.0,16.157,0.652123,448,1442070 +255,delete,39.605,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,123.359,0.632354,428,1405549 +256,search,1574.95,2955.47,614.627,0.957842,62.2004,-nan,-nan,0.82052,0.175222,-1.0,-1.0,42.976,0.632926,432,1405549 +257,insert,10.858,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,7.614,0.635618,432,1416288 +258,search,1581.91,2931.15,611.738,0.89751,60.0277,-nan,-nan,0.82148,0.174858,-1.0,-1.0,15.343,0.63568,433,1416288 +259,insert,37.579,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,34.39,0.646862,435,1443010 +260,search,1602.22,2960.01,620.766,0.914859,63.873,-nan,-nan,0.8208,0.176376,-1.0,-1.0,17.147,0.646862,436,1443010 +261,insert,6.734,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.263,0.648677,438,1447529 +262,search,1601.93,2946.01,621.983,0.869806,62.4922,-nan,-nan,0.82267,0.175033,-1.0,-1.0,7.9,0.648677,438,1447529 +263,insert,135.408,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,136.268,0.70138,443,1569110 +264,search,1678.61,3148.28,677.016,0.907297,65.9031,-nan,-nan,0.8246,0.175145,-1.0,-1.0,116.995,0.70138,448,1569110 +265,insert,86.922,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,162.318,0.731197,466,1636119 +266,search,1756.93,3138.28,700.504,0.86015,72.1634,-nan,-nan,0.82909,0.172376,-1.0,-1.0,112.359,0.731423,475,1636119 +267,insert,31.364,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.034,0.7436,478,1665152 +268,search,1769.01,3109.26,700.991,0.915684,71.061,-nan,-nan,0.83025,0.172058,-1.0,-1.0,83.72,0.743604,486,1665152 +269,insert,296.221,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,311.878,0.855934,513,1916952 +270,search,1945.72,3313.95,796.831,0.896126,74.983,-nan,-nan,0.83376,0.169836,-1.0,-1.0,92.175,0.85635,519,1916952 +271,insert,4.883,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,9.069,0.857303,519,1920898 +272,search,1912.6,3260.74,777.718,0.939992,72.1861,-nan,-nan,0.83444,0.168858,-1.0,-1.0,60.351,0.857304,521,1920898 +273,insert,21.209,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,34.317,0.864475,524,1942757 +274,search,1957.44,3265.22,807.787,0.960639,71.9841,-nan,-nan,0.83817,0.165899,-1.0,-1.0,48.809,0.864628,527,1942757 +275,insert,444.783,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,621.101,1.0443,594,2328372 +276,search,2241.75,3456.77,948.805,0.986834,86.1372,-nan,-nan,0.84527,0.16313,-1.0,-1.0,245.527,1.04593,619,2328372 +277,insert,36.67,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,76.231,1.05935,628,2363257 +278,search,2311,3329.98,961.356,0.908148,86.7903,-nan,-nan,0.84759,0.161182,-1.0,-1.0,114.935,1.06036,639,2363257 +279,insert,30.308,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,55.131,1.07157,645,2388933 +280,search,2371.95,3294.7,981.285,0.953042,85.6392,-nan,-nan,0.85003,0.159534,-1.0,-1.0,56.499,1.07228,651,2388933 +281,insert,291.072,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,377.28,1.20686,680,2684315 +282,search,2512.68,3429.78,1060.66,0.974092,91.9451,-nan,-nan,0.85315,0.158808,-1.0,-1.0,211.555,1.20867,697,2684315 +283,insert,464.574,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,707.8,1.40005,787,3109157 +284,search,2893.91,3485.52,1227.31,0.947671,107.377,-nan,-nan,0.86263,0.151379,-1.0,-1.0,292.422,1.40231,823,3109157 +285,insert,11.665,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,42.647,1.40723,828,3122751 +286,search,2949.41,3308.35,1218.58,0.950959,100.249,-nan,-nan,0.8623,0.151363,-1.0,-1.0,59.907,1.40768,834,3122751 +287,insert,39.619,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,80.018,1.42605,843,3163679 +288,search,3018,3288.09,1236.52,0.973086,113.833,-nan,-nan,0.86387,0.15031,-1.0,-1.0,43.721,1.42619,847,3163679 +289,insert,373.037,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,505.693,1.56313,903,3472031 +290,search,3231.67,3393.88,1342.28,1.03478,110.179,-nan,-nan,0.86775,0.147918,-1.0,-1.0,229.587,1.56537,922,3472031 +291,insert,23.679,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,48.936,1.57437,928,3495577 +292,search,3314.4,3331.06,1375.36,0.98187,112.035,-nan,-nan,0.86925,0.146365,-1.0,-1.0,144.314,1.57437,936,3495577 +293,insert,18.821,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,42.419,1.58281,939,3515193 +294,search,3363.03,3310.33,1379.33,1.03202,120.666,-nan,-nan,0.86995,0.145957,-1.0,-1.0,70.877,1.58309,944,3515193 +295,insert,254.202,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,365.033,1.67474,986,3723063 +296,search,3487.07,3361.04,1398.73,1.14297,116.852,-nan,-nan,0.87525,0.142732,-1.0,-1.0,176.858,1.67578,1004,3723063 +297,insert,4.343,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.057,1.67768,1005,3727399 +298,search,3534.97,3293.07,1443.31,0.96542,119.105,-nan,-nan,0.8754,0.14269,-1.0,-1.0,23.489,1.6777,1006,3727399 +299,insert,76.215,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,91.579,1.69952,1019,3788002 +300,search,3611.77,3306.8,1486.21,0.95501,127.677,-nan,-nan,0.87817,0.139905,-1.0,-1.0,19.687,1.69963,1020,3788002 +301,insert,696.228,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1160.91,1.97042,1156,4370866 +302,search,4180.71,3485.73,1768.79,0.969352,157.559,-nan,-nan,0.90213,0.121982,-1.0,-1.0,295.781,1.9708,1199,4370866 +303,insert,57.77,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,55.344,1.99031,1204,4420306 +304,search,4260.3,3290.38,1740.14,1.00478,147.249,-nan,-nan,0.90121,0.122298,-1.0,-1.0,63.879,1.99071,1211,4420306 +305,insert,388.84,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,470.51,2.15531,1254,4789669 +306,search,4484.52,3392.35,1860.36,0.95642,161.303,-nan,-nan,0.90362,0.120022,-1.0,-1.0,312.045,2.15725,1279,4789669 +307,insert,28.628,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,58.2,2.1661,1286,4815550 +308,search,4524.92,3334.42,1856.56,1.01501,154.683,-nan,-nan,0.90512,0.118899,-1.0,-1.0,148.016,2.16656,1296,4815550 +309,insert,7.911,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,29.693,2.16938,1299,4823345 +310,search,4522.76,3302.93,1800.4,1.09329,151.366,-nan,-nan,0.90518,0.118736,-1.0,-1.0,74.357,2.16938,1303,4823345 +311,insert,10.535,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,12.142,2.17346,1303,4843493 +312,search,4620.68,3301.07,1873.56,0.939633,163.139,-nan,-nan,0.90521,0.118762,-1.0,-1.0,24.779,2.17346,1304,4843493 +313,insert,41.166,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,137.331,2.18983,1319,4885489 +314,search,4685.96,3296.28,1893.68,0.968189,162.251,-nan,-nan,0.90576,0.118733,-1.0,-1.0,34.968,2.19018,1322,4885489 +315,insert,9.71,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,11.69,2.19332,1322,4892335 +316,search,4673.09,3290.58,1904.05,1.01368,165.117,-nan,-nan,0.90587,0.118811,-1.0,-1.0,12.43,2.19332,1322,4892335 +317,insert,49.852,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,110.429,2.21089,1335,4932996 +318,search,4601,3287.37,1829.57,1.08734,150.034,-nan,-nan,0.90616,0.118443,-1.0,-1.0,12.433,2.21089,1335,4932996 +319,insert,593.142,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,828.068,2.42319,1437,5430684 +320,search,4989.83,3406.28,2006.06,1.17582,163.318,-nan,-nan,0.91094,0.115162,-1.0,-1.0,287.957,2.42793,1471,5430684 +321,insert,43.606,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,87.349,2.44242,1481,5471239 +322,search,5217.89,3341.25,2155.88,1.00832,180.498,-nan,-nan,0.91137,0.114786,-1.0,-1.0,20.604,2.44242,1482,5471239 +323,insert,115.595,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,203.613,2.48753,1506,5575560 +324,search,5228.63,3353.92,2116.3,1.05867,175.497,-nan,-nan,0.91225,0.113563,-1.0,-1.0,39.69,2.48758,1510,5575560 +325,insert,1480.92,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,743.745,2.71036,1578,6069499 +326,search,5652.46,3480.56,2357.97,1.02024,202.967,-nan,-nan,0.91563,0.11156,-1.0,-1.0,366.034,2.7117,1606,6069499 +327,insert,67.688,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,86.121,2.7379,1616,6139647 +328,search,5759.23,3428.48,2376.59,0.996581,202.357,-nan,-nan,0.91702,0.110853,-1.0,-1.0,211.782,2.7397,1636,6139647 +329,insert,9.469,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.19,2.74252,1637,6146884 +330,search,5817.95,3385.96,2393.35,0.991392,204.802,-nan,-nan,0.91729,0.110487,-1.0,-1.0,88.173,2.74304,1643,6146884 +331,insert,53.955,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,103.752,2.76285,1654,6188816 +332,search,5778.19,3374.13,2334.97,1.01998,193.207,-nan,-nan,0.9172,0.110888,-1.0,-1.0,42.003,2.76299,1657,6188816 +333,insert,2.572,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.403,2.76328,1657,6190810 +334,search,5619.48,3368.53,2410.14,0.97144,206.669,-nan,-nan,0.91697,0.111125,-1.0,-1.0,15.691,2.76328,1657,6190810 +335,insert,49.828,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,101.06,2.77954,1669,6230108 +336,search,5660.87,3369.95,2299.56,1.07461,183.262,-nan,-nan,0.9174,0.11043,-1.0,-1.0,41.24,2.77981,1671,6230108 +337,insert,24.502,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,55.185,2.78834,1675,6249699 +338,search,5424.58,3369.5,2443.05,0.96791,212.547,-nan,-nan,0.91803,0.109772,-1.0,-1.0,10.119,2.78834,1675,6249699 +339,insert,33.1,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,62.42,2.80336,1687,6290226 +340,search,5741.51,3372.34,2341.27,1.02686,191.698,-nan,-nan,0.91898,0.108995,-1.0,-1.0,23.85,2.80353,1688,6290226 +341,insert,8.078,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.208,2.80588,1688,6297556 +342,search,6004.76,3371.89,2477.79,0.950462,208.155,-nan,-nan,0.91909,0.109042,-1.0,-1.0,25.651,2.80604,1689,6297556 +343,insert,39.281,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,90.939,2.81943,1699,6330843 +344,search,5916.25,3371.67,2389.48,1.00682,194.674,-nan,-nan,0.91925,0.108823,-1.0,-1.0,23.391,2.81971,1700,6330843 +345,insert,49.132,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,90.337,2.83753,1710,6370265 +346,search,5961.28,3374.56,2414.93,1.02316,199.591,-nan,-nan,0.91989,0.108362,-1.0,-1.0,15.872,2.83753,1710,6370265 +347,insert,585.072,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1025.58,3.0401,1836,6831086 +348,search,6432.08,3385.23,2601.4,1.01005,214.946,-nan,-nan,0.92269,0.105671,-1.0,-1.0,136.552,3.04157,1851,6831086 +349,insert,51.446,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,60.676,3.05854,1856,6876154 +350,search,6453.14,3373.56,2585.6,1.03137,207.192,-nan,-nan,0.92299,0.105466,-1.0,-1.0,43.467,3.05955,1858,6876154 +351,insert,486.223,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,667.573,3.24806,1923,7288266 +352,search,6872.14,3438.51,2843.76,0.968909,238.398,-nan,-nan,0.92333,0.105702,-1.0,-1.0,260.887,3.24991,1945,7288266 +353,insert,55.839,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,108.882,3.2695,1956,7333132 +354,search,6929.52,3402.63,2842.93,0.980906,240.042,-nan,-nan,0.9236,0.105772,-1.0,-1.0,127.2,3.2696,1967,7333132 +355,insert,430.123,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,747.302,3.43459,2059,7696838 +356,search,7148.56,3379.83,2886.97,1.0903,233.97,-nan,-nan,0.92549,0.103703,-1.0,-1.0,268.111,3.43787,2086,7696838 +357,insert,21.753,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,46.107,3.44417,2089,7714498 +358,search,7277.05,3339.99,2913.39,1.05335,246.119,-nan,-nan,0.92563,0.103592,-1.0,-1.0,37.388,3.44417,2090,7714498 +359,insert,12.429,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,38.695,3.44781,2092,7725755 +360,search,7317.71,3340.24,2956.63,1.00846,250.991,-nan,-nan,0.92563,0.103688,-1.0,-1.0,27.57,3.44781,2092,7725755 +361,insert,72.122,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,153.881,3.47426,2106,7789312 +362,search,7412.22,3343.23,3003,1.00827,258.018,-nan,-nan,0.92623,0.103469,-1.0,-1.0,42.122,3.47426,2107,7789312 +363,insert,26.285,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,34.66,3.47837,2108,7803163 +364,search,7383.54,3347.63,3030.94,1.01094,254.412,-nan,-nan,0.92648,0.103216,-1.0,-1.0,62.501,3.47875,2111,7803163 +365,insert,53.445,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,97.177,3.50686,2127,7863722 +366,search,6749.92,3350.76,2787.36,1.16549,214.683,-nan,-nan,0.9271,0.102796,-1.0,-1.0,28.118,3.50729,2129,7863722 +367,insert,24.784,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,53.797,3.52403,2137,7895271 +368,search,7165.88,3351.44,2983.11,1.05444,245.762,-nan,-nan,0.92767,0.102285,-1.0,-1.0,43.922,3.52445,2139,7895271 +369,insert,73.626,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,158.592,3.54825,2158,7961194 +370,search,7554.07,3339.06,3043.14,1.04121,250.702,-nan,-nan,0.92768,0.102291,-1.0,-1.0,78.69,3.54915,2163,7961194 +371,insert,13.175,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,42.572,3.55231,2165,7970239 +372,search,7320.52,3336.88,2832.92,1.17719,218.984,-nan,-nan,0.92773,0.102332,-1.0,-1.0,45.693,3.5526,2167,7970239 +373,insert,56.07,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,97.324,3.56904,2176,8010868 +374,search,7566.85,3340.48,3018.49,1.04616,249.573,-nan,-nan,0.92835,0.101855,-1.0,-1.0,27.026,3.56904,2176,8010868 +375,insert,45.038,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,86.28,3.58208,2184,8040698 +376,search,6950.24,3338.85,2985.79,1.08163,246.04,-nan,-nan,0.92855,0.101788,-1.0,-1.0,42.014,3.5823,2186,8040698 +377,insert,51.882,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,55.749,3.60813,2195,8104748 +378,search,7513.76,3347.67,3076.08,1.06537,260.384,-nan,-nan,0.92931,0.101258,-1.0,-1.0,63.825,3.60848,2199,8104748 +379,insert,5.772,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,26.469,3.60982,2199,8109755 +380,search,7647.37,3345.55,3085.86,1.05033,259.194,-nan,-nan,0.92929,0.101289,-1.0,-1.0,27.968,3.60982,2199,8109755 +381,insert,24.647,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,58.402,3.61798,2202,8130382 +382,search,7620.59,3348.23,3010.95,1.10353,242.677,-nan,-nan,0.92947,0.101211,-1.0,-1.0,55.905,3.61836,2204,8130382 +383,insert,42.875,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,78.319,3.62674,2209,8153290 +384,search,7668.02,3348.5,3051.13,1.09488,251.094,-nan,-nan,0.92956,0.101079,-1.0,-1.0,28.884,3.62674,2209,8153290 +385,delete,26.557,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,26.866,3.62674,2209,8138873 +386,search,7683.66,3343.6,3056.72,1.10707,250.553,-nan,-nan,0.92959,0.101001,-1.0,-1.0,29.267,3.62674,2209,8138873 +387,delete,34.375,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,170.871,3.61832,2203,8115069 +388,search,7670,3340.96,3081.7,1.03258,257.035,-nan,-nan,0.92969,0.100713,-1.0,-1.0,45.678,3.61869,2205,8115069 +389,delete,21.397,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,114.955,3.61303,2199,8101157 +390,search,7764.62,3342,3156.31,1.01771,270.006,-nan,-nan,0.92941,0.100948,-1.0,-1.0,50.764,3.61339,2201,8101157 +391,delete,117.445,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,416.004,3.57285,2170,8014644 +392,search,7681.97,3351.9,3129.75,0.97196,270.484,-nan,-nan,0.92885,0.100629,-1.0,-1.0,78.729,3.57367,2176,8014644 +393,delete,37.464,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,173.186,3.56598,2168,7978358 +394,search,7515.59,3334.29,2991.2,1.0753,244.624,-nan,-nan,0.92833,0.101446,-1.0,-1.0,40.967,3.56609,2169,7978358 +395,delete,22.665,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,257.149,3.55274,2158,7957094 +396,search,7521.83,3341.16,3004.9,1.0565,248.863,-nan,-nan,0.92801,0.101428,-1.0,-1.0,86.888,3.55363,2164,7957094 +397,delete,119.753,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,887.665,3.49478,2125,7823916 +398,search,7400.8,3346.68,2968.97,1.0543,254.641,-nan,-nan,0.92763,0.101616,-1.0,-1.0,131.63,3.49711,2137,7823916 +399,delete,4.775,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.105,3.49711,2137,7818430 +400,search,7418.57,3324.61,2958.99,1.08798,249.628,-nan,-nan,0.92792,0.101269,-1.0,-1.0,35.329,3.49723,2138,7818430 +401,delete,17.636,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,57.388,3.49321,2135,7800702 +402,search,7349.15,3317.74,2903.45,1.10967,239.598,-nan,-nan,0.92717,0.101939,-1.0,-1.0,40.242,3.49341,2136,7800702 +403,delete,329.934,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1905.38,3.35389,2014,7517858 +404,search,6919.22,3379.46,2751.79,1.07233,223.1,-nan,-nan,0.92549,0.103529,-1.0,-1.0,300.075,3.36127,2048,7517858 +405,delete,52.382,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,346.914,3.34233,2031,7451637 +406,search,6999.3,3322.13,2760.75,1.07002,224.239,-nan,-nan,0.9261,0.102909,-1.0,-1.0,80.801,3.34272,2035,7451637 +407,delete,114.218,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,723.831,3.2948,1994,7360417 +408,search,7060.27,3336.38,2860.48,0.978205,244.048,-nan,-nan,0.92505,0.103547,-1.0,-1.0,100.634,3.29567,2001,7360417 +409,delete,224.726,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,837.032,3.19099,1951,7131620 +410,search,6839.74,3337.05,2740.75,1.0613,233.546,-nan,-nan,0.92459,0.103301,-1.0,-1.0,143.068,3.19395,1962,7131620 +411,delete,262.392,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1739.98,3.07617,1850,6867617 +412,search,6623.28,3384.79,2712.8,0.970391,239.434,-nan,-nan,0.92372,0.104474,-1.0,-1.0,270.406,3.08042,1880,6867617 +413,delete,13.583,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,95.905,3.07706,1876,6850098 +414,search,6532.79,3322.53,2600.9,1.06053,216.379,-nan,-nan,0.92368,0.104534,-1.0,-1.0,74.961,3.07746,1882,6850098 +415,delete,75.412,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,492.002,3.04352,1859,6782921 +416,search,6487.53,3323.34,2586.31,1.07042,218.915,-nan,-nan,0.92344,0.104486,-1.0,-1.0,72.742,3.04495,1864,6782921 +417,delete,238.907,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1165.37,2.95114,1802,6562039 +418,search,6249.95,3315.79,2479.18,1.08095,207.181,-nan,-nan,0.92305,0.105265,-1.0,-1.0,140.724,2.95282,1814,6562039 +419,delete,23.928,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,77.329,2.94849,1810,6536285 +420,search,6396.4,3289.49,2583.19,0.975967,222.669,-nan,-nan,0.92325,0.105325,-1.0,-1.0,49.534,2.94873,1812,6536285 +421,delete,21.618,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,152.24,2.93687,1803,6510820 +422,search,6350.52,3288.5,2558.64,0.972112,218.993,-nan,-nan,0.92366,0.105078,-1.0,-1.0,32.264,2.937,1804,6510820 +423,delete,188.496,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,616.144,2.86399,1747,6334168 +424,search,6065.22,3299.56,2401.94,1.08905,200.763,-nan,-nan,0.92191,0.106386,-1.0,-1.0,89.426,2.86526,1755,6334168 +425,delete,18.13,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,73.386,2.86116,1752,6316443 +426,search,6199.85,3280.49,2493.87,0.978208,214.025,-nan,-nan,0.92209,0.106212,-1.0,-1.0,33.972,2.86122,1753,6316443 +427,delete,52.266,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,99.474,2.85217,1746,6266345 +428,search,6013.16,3258.07,2346.78,1.11085,196.799,-nan,-nan,0.92202,0.106483,-1.0,-1.0,20.487,2.85217,1746,6266345 +429,delete,496.006,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,2796.2,2.63182,1505,5883847 +430,search,5412.66,3617.3,2239.62,1.10402,178.508,-nan,-nan,0.91024,0.115128,-1.0,-1.0,626.866,2.64457,1580,5883847 +431,delete,70.643,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,407.219,2.60665,1554,5801380 +432,search,5437,3393.35,2154.01,1.05494,175.33,-nan,-nan,0.90892,0.115225,-1.0,-1.0,117.471,2.60826,1564,5801380 +433,delete,217.856,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1128.52,2.51483,1488,5607567 +434,search,5273.23,3431.17,2129,1.03503,177.952,-nan,-nan,0.90646,0.117174,-1.0,-1.0,146.318,2.51743,1500,5607567 +435,delete,103.938,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,371.715,2.48177,1470,5512176 +436,search,5317.04,3414.76,2189.13,0.972748,193.297,-nan,-nan,0.90537,0.119299,-1.0,-1.0,205.524,2.48612,1480,5512176 +437,delete,9.952,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,79.813,2.48103,1476,5501141 +438,search,5255.19,3390.65,2130.16,1.01803,183.314,-nan,-nan,0.90564,0.118834,-1.0,-1.0,135.576,2.48147,1483,5501141 +439,delete,108.257,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,321.629,2.42167,1467,5369822 +440,search,5261.21,3387.17,2151.58,0.968078,186.099,-nan,-nan,0.90818,0.116107,-1.0,-1.0,149.9,2.42271,1479,5369822 +441,delete,24.929,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,152.465,2.41266,1471,5338985 +442,search,5193.93,3356.95,2095.06,1.04999,177.903,-nan,-nan,0.90788,0.116243,-1.0,-1.0,26.861,2.41277,1472,5338985 +443,delete,30.28,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,86.567,2.39892,1466,5307279 +444,search,5135.41,3353.86,2025.45,1.17273,172.723,-nan,-nan,0.90864,0.115841,-1.0,-1.0,17.378,2.39892,1466,5307279 +445,delete,41.806,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,123.392,2.38775,1459,5268382 +446,search,5222.81,3345.32,2127.18,1.0067,181.798,-nan,-nan,0.90812,0.116251,-1.0,-1.0,39.124,2.38807,1461,5268382 +447,delete,455.945,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1537.78,2.21707,1309,4922787 +448,search,4611.61,3477.77,1836.58,1.11556,150.736,-nan,-nan,0.90203,0.12055,-1.0,-1.0,354.745,2.22398,1343,4922787 +449,delete,33.205,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,180.815,2.21276,1329,4887227 +450,search,4818.56,3388.34,1983.31,0.962429,176.776,-nan,-nan,0.90308,0.119907,-1.0,-1.0,143.892,2.21366,1337,4887227 +451,delete,91.237,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,394.013,2.16853,1299,4795082 +452,search,4639.3,3396.62,1884.27,1.04861,161.531,-nan,-nan,0.90179,0.120242,-1.0,-1.0,94.836,2.16914,1306,4795082 +453,delete,342.548,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1596.19,2.02039,1169,4500175 +454,search,4261.71,3511.26,1763.05,0.966918,155.413,-nan,-nan,0.89651,0.123812,-1.0,-1.0,410.667,2.02898,1205,4500175 +455,delete,30.706,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,117.701,2.01574,1197,4462648 +456,search,4375.34,3421.38,1812.61,0.93558,160.024,-nan,-nan,0.89628,0.123888,-1.0,-1.0,192.485,2.01761,1207,4462648 +457,delete,5.726,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,25.984,2.01571,1206,4456761 +458,search,4369.68,3393.41,1786.79,0.953508,162.524,-nan,-nan,0.89668,0.123411,-1.0,-1.0,87.153,2.0157,1211,4456761 +459,delete,33.906,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,187.816,1.99548,1196,4419992 +460,search,4231.56,3395.93,1688.31,1.01982,146.591,-nan,-nan,0.89653,0.123811,-1.0,-1.0,29.249,1.99548,1197,4419992 +461,delete,6.09,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.455,1.99548,1197,4415725 +462,search,4212.14,3389.66,1680.41,1.02545,139.037,-nan,-nan,0.89669,0.123569,-1.0,-1.0,14.028,1.99548,1197,4415725 +463,delete,52.676,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,267.322,1.97211,1172,4370609 +464,search,4268.98,3432.14,1746.99,0.978419,154.272,-nan,-nan,0.89581,0.124566,-1.0,-1.0,63.452,1.97282,1176,4370609 +465,delete,31.018,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,58.839,1.96092,1171,4345244 +466,search,4261.23,3407.7,1748.66,0.981658,154.883,-nan,-nan,0.89379,0.1255,-1.0,-1.0,25.879,1.96099,1172,4345244 +467,delete,37.606,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,62.642,1.95496,1169,4319734 +468,search,4246.92,3389.14,1744.41,0.934952,157.291,-nan,-nan,0.89327,0.125855,-1.0,-1.0,22.063,1.9551,1170,4319734 +469,delete,19.02,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,44.612,1.94564,1166,4298361 +470,search,4233.77,3380.06,1738.87,0.952064,154.547,-nan,-nan,0.89229,0.126401,-1.0,-1.0,14.922,1.94564,1166,4298361 +471,delete,149.412,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,421.573,1.89144,1116,4188315 +472,search,4107.49,3438.17,1703.49,0.978601,151.525,-nan,-nan,0.89108,0.127707,-1.0,-1.0,151.111,1.89429,1129,4188315 +473,delete,118.892,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,572.277,1.8342,1081,4074567 +474,search,3837.12,3442.32,1527.88,1.12393,127.968,-nan,-nan,0.8902,0.127283,-1.0,-1.0,175.324,1.83731,1097,4074567 +475,delete,441.401,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1693.51,1.64188,926,3691698 +476,search,3423.94,3659.77,1434.88,1.10616,120.814,-nan,-nan,0.88441,0.131971,-1.0,-1.0,494.535,1.65111,981,3691698 +477,delete,37.518,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,137.935,1.62875,968,3642619 +478,search,3417.72,3401.33,1356.73,1.09451,112.677,-nan,-nan,0.88278,0.132913,-1.0,-1.0,89.355,1.6294,975,3642619 +479,delete,240.813,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1221.91,1.52595,874,3421368 +480,search,3206.62,3754.17,1350.29,1.08946,110.302,-nan,-nan,0.89445,0.124311,-1.0,-1.0,390.057,1.53392,902,3421368 +481,delete,40.233,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,116.407,1.51797,889,3378034 +482,search,3152.39,3447.39,1281.98,1.10857,108.594,-nan,-nan,0.88677,0.131019,-1.0,-1.0,108.075,1.51863,898,3378034 +483,delete,284.832,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1019.19,1.39306,784,3115682 +484,search,2961.14,3597.73,1256.78,0.962054,110.315,-nan,-nan,0.87956,0.136816,-1.0,-1.0,337.789,1.39976,816,3115682 +485,delete,17.763,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,40.214,1.39521,813,3094629 +486,search,2995.11,3448.93,1262.51,0.95303,106.314,-nan,-nan,0.88083,0.136437,-1.0,-1.0,133.899,1.39574,823,3094629 +487,delete,8.003,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.814,1.39498,822,3086025 +488,search,3023.57,3396.71,1265.72,0.969439,114.052,-nan,-nan,0.88087,0.13626,-1.0,-1.0,27.44,1.39499,823,3086025 +489,delete,33.298,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,114.767,1.38129,815,3052105 +490,search,2986.82,3393.65,1241.05,1.01622,112.643,-nan,-nan,0.88056,0.136319,-1.0,-1.0,48.828,1.38216,818,3052105 +491,delete,14.97,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,29.384,1.3782,816,3038974 +492,search,2841.11,3371.91,1122.01,1.20808,93.1773,-nan,-nan,0.88026,0.136371,-1.0,-1.0,12.878,1.3782,816,3038974 +493,delete,148.971,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,429.615,1.30681,762,2911063 +494,search,2779.67,3458.87,1144.45,1.07008,98.178,-nan,-nan,0.87776,0.137785,-1.0,-1.0,121.55,1.30933,774,2911063 +495,delete,22.041,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,82.88,1.30107,765,2888785 +496,search,2836.88,3404.59,1185.91,0.956571,107.336,-nan,-nan,0.87832,0.137471,-1.0,-1.0,58.601,1.30203,770,2888785 +497,delete,56.86,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,150.667,1.28107,753,2829718 +498,search,2720.99,3374.53,1114.19,1.05149,96.3758,-nan,-nan,0.87639,0.138943,-1.0,-1.0,39.393,1.28151,756,2829718 +499,delete,15.464,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,26.272,1.27928,755,2816082 +500,search,2749.93,3345.96,1141.41,0.94681,98.724,-nan,-nan,0.87638,0.138929,-1.0,-1.0,11.628,1.27928,755,2816082 +501,delete,27.613,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,86.91,1.26708,748,2790880 +502,search,2643.03,3351.63,1052.33,1.05869,88.8017,-nan,-nan,0.87583,0.139061,-1.0,-1.0,20.811,1.26732,749,2790880 +503,delete,17.106,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,19.836,1.26593,748,2774459 +504,search,2752.49,3331.67,1134,0.975144,103.723,-nan,-nan,0.87556,0.139164,-1.0,-1.0,21.086,1.2661,749,2774459 +505,delete,61.057,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,172.949,1.23522,731,2714683 +506,search,2709.26,3331.88,1111.62,0.958128,101.529,-nan,-nan,0.87379,0.139823,-1.0,-1.0,37.111,1.23564,733,2714683 +507,delete,41.105,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,104.065,1.2236,722,2680986 +508,search,2538.02,3337.49,996.995,1.17072,83.4553,-nan,-nan,0.87343,0.140261,-1.0,-1.0,37.034,1.22388,724,2680986 +509,delete,19.32,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,87.013,1.21484,718,2662966 +510,search,2644.66,3332.5,1093.4,1.00441,97.8257,-nan,-nan,0.87269,0.140466,-1.0,-1.0,26.739,1.21507,719,2662966 +511,delete,24.279,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,29.404,1.21207,716,2642855 +512,search,2604.65,3310.35,1056.81,1.03112,95.653,-nan,-nan,0.87271,0.140844,-1.0,-1.0,21.934,1.21209,717,2642855 +513,insert,374.322,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,309.825,1.31449,741,2906838 +514,search,2775.36,3467.09,1162.22,0.974625,103.444,-nan,-nan,0.87457,0.139712,-1.0,-1.0,52.392,1.31478,745,2906838 +515,insert,5.921,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,10.434,1.31578,745,2913746 +516,search,2766.15,3461.07,1155.95,1.01444,96.4819,-nan,-nan,0.8753,0.139168,-1.0,-1.0,12.951,1.31578,745,2913746 +517,insert,31.044,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,50.522,1.32232,749,2936862 +518,search,2751.2,3471.08,1139,1.02213,99.3175,-nan,-nan,0.87552,0.138997,-1.0,-1.0,12.051,1.32232,749,2936862 +519,insert,128.684,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,155.996,1.35451,758,3009732 +520,search,2853.95,3519.02,1211.15,0.957387,103.653,-nan,-nan,0.87714,0.13883,-1.0,-1.0,127.899,1.3548,768,3009732 +521,insert,19.452,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,42.575,1.36012,771,3025906 +522,search,2834.22,3482.63,1196.79,0.96537,105.231,-nan,-nan,0.87724,0.138847,-1.0,-1.0,22.876,1.36012,772,3025906 +523,insert,46.738,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,87.702,1.37503,779,3061899 +524,search,2932.6,3487.38,1234.96,0.970915,108.791,-nan,-nan,0.87908,0.138187,-1.0,-1.0,51.017,1.37545,783,3061899 +525,insert,57.491,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,122.935,1.39225,795,3101209 +526,search,2938.43,3464.43,1234.49,0.934769,107.038,-nan,-nan,0.8796,0.137699,-1.0,-1.0,24.089,1.39265,796,3101209 +527,insert,610.658,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,813.572,1.57911,884,3522334 +528,search,3284.47,3581.62,1414.26,0.984524,125.218,-nan,-nan,0.88514,0.133021,-1.0,-1.0,316.516,1.58207,915,3522334 +529,insert,353.761,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,527.357,1.70009,971,3797294 +530,search,3566.82,3554.17,1492.76,1.04171,130.125,-nan,-nan,0.89141,0.13043,-1.0,-1.0,211.737,1.7012,987,3797294 +531,insert,101.397,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,140.612,1.72445,1001,3853781 +532,search,3596.81,3484.31,1461.58,1.1936,121.601,-nan,-nan,0.89208,0.129309,-1.0,-1.0,132.222,1.72495,1011,3853781 +533,insert,25.933,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,47.525,1.7327,1015,3873352 +534,search,3641.16,3450.13,1487.19,1.0733,125.192,-nan,-nan,0.89253,0.128807,-1.0,-1.0,95.518,1.73269,1021,3873352 +535,insert,33.163,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.195,1.7415,1024,3893308 +536,search,3631.82,3440.88,1458.72,1.08743,117.393,-nan,-nan,0.89296,0.128787,-1.0,-1.0,53.883,1.74167,1027,3893308 +537,insert,9.589,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.608,1.74556,1027,3901271 +538,search,3712.43,3435.99,1525.93,1.12763,131.816,-nan,-nan,0.89282,0.12889,-1.0,-1.0,14.752,1.74556,1027,3901271 +539,insert,34.024,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,68.932,1.75371,1033,3925164 +540,search,3713.16,3437.9,1522.81,1.04282,127.99,-nan,-nan,0.89352,0.128358,-1.0,-1.0,37.709,1.75432,1035,3925164 +541,insert,528.809,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,756.997,1.93603,1113,4320915 +542,search,3974.13,3539.51,1630.29,1.06531,132.729,-nan,-nan,0.89827,0.124958,-1.0,-1.0,324.92,1.93871,1143,4320915 +543,insert,52.669,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,94.112,1.95393,1154,4360814 +544,search,4177.95,3436.69,1743.69,0.983725,156.938,-nan,-nan,0.89916,0.124573,-1.0,-1.0,177.154,1.95506,1170,4360814 +545,insert,11.961,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,30.128,1.95879,1172,4370220 +546,search,4220.83,3385.95,1755.88,0.96058,149.853,-nan,-nan,0.89991,0.124139,-1.0,-1.0,90.293,1.95984,1180,4370220 +547,insert,188.321,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,343.467,2.02019,1227,4510675 +548,search,4214.14,3347.68,1680.12,1.13174,138.353,-nan,-nan,0.90169,0.122774,-1.0,-1.0,54.637,2.02063,1232,4510675 +549,insert,284.783,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,294.192,2.11909,1248,4744845 +550,search,4535.75,3401.42,1867.08,0.955623,162.386,-nan,-nan,0.89675,0.127609,-1.0,-1.0,112.71,2.11966,1256,4744845 +551,insert,24.47,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,55.785,2.13132,1259,4771433 +552,search,4546.66,3384.75,1874.44,0.968382,165.7,-nan,-nan,0.89763,0.126545,-1.0,-1.0,70.013,2.13182,1265,4771433 +553,insert,13.991,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,54.555,2.13627,1270,4782909 +554,search,4399.73,3365.39,1763.13,1.03325,144.918,-nan,-nan,0.89824,0.125762,-1.0,-1.0,13.89,2.13627,1270,4782909 +555,insert,530.892,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,864.925,2.3127,1392,5180322 +556,search,4885.71,3369.72,1978.3,1.0582,162.331,-nan,-nan,0.90396,0.12014,-1.0,-1.0,119.335,2.31394,1404,5180322 +557,insert,27.296,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,64.688,2.32177,1410,5203220 +558,search,4938.76,3340.47,1974.15,1.04392,163.959,-nan,-nan,0.90451,0.119584,-1.0,-1.0,16.087,2.32177,1410,5203220 +559,insert,7.586,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.597,2.32372,1410,5210154 +560,search,5033.15,3343.99,2065.66,1.00349,174.422,-nan,-nan,0.90471,0.119465,-1.0,-1.0,15.674,2.32372,1410,5210154 +561,insert,9.164,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.433,2.32645,1410,5215426 +562,search,5071.99,3347.03,2071.5,0.994145,183.904,-nan,-nan,0.90476,0.119458,-1.0,-1.0,16.064,2.32645,1410,5215426 +563,insert,63.48,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,69.304,2.33988,1417,5252726 +564,search,4966.6,3354.63,1987.47,1.0503,164.315,-nan,-nan,0.90526,0.119051,-1.0,-1.0,15.736,2.33988,1417,5252726 +565,insert,263.822,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,279.82,2.42579,1433,5447274 +566,search,5168.4,3425.63,2148.87,0.95942,189.255,-nan,-nan,0.90503,0.119292,-1.0,-1.0,207,2.42667,1442,5447274 +567,insert,18.169,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.735,2.43306,1442,5463135 +568,search,5046.75,3408.54,2031.04,1.06007,166.349,-nan,-nan,0.90449,0.119862,-1.0,-1.0,96.7,2.43326,1447,5463135 +569,insert,22.509,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,52.101,2.44028,1451,5478168 +570,search,5032.79,3403.17,2023.18,1.08877,164.916,-nan,-nan,0.9053,0.119142,-1.0,-1.0,86.248,2.44077,1458,5478168 +571,insert,18.182,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,33.493,2.44751,1459,5492309 +572,search,5021.92,3393.2,1984.26,1.14871,160.152,-nan,-nan,0.90522,0.119365,-1.0,-1.0,33.002,2.44774,1461,5492309 +573,insert,89.757,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,118.407,2.47207,1475,5554599 +574,search,5085.2,3387.13,2012.27,1.09612,159.133,-nan,-nan,0.9058,0.119203,-1.0,-1.0,23.438,2.47208,1476,5554599 +575,insert,66.247,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,124.503,2.4921,1488,5604500 +576,search,5328.42,3384.79,2199.45,0.976286,189.918,-nan,-nan,0.9059,0.119068,-1.0,-1.0,41.303,2.4924,1491,5604500 +577,insert,49.336,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,71.343,2.5056,1499,5638834 +578,search,5283.4,3379.09,2150.29,1.02095,174.755,-nan,-nan,0.90618,0.119181,-1.0,-1.0,16.132,2.5056,1499,5638834 +579,insert,900.846,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1310.81,2.78368,1666,6229778 +580,search,5952.23,3379.39,2460.87,0.971077,212.444,-nan,-nan,0.91028,0.114994,-1.0,-1.0,115.477,2.78475,1677,6229778 +581,insert,37.508,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,124.567,2.79862,1688,6261240 +582,search,5928.85,3360.16,2386.01,1.01233,197.431,-nan,-nan,0.91084,0.114296,-1.0,-1.0,34.41,2.79881,1690,6261240 +583,insert,22.512,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,70.918,2.80726,1697,6283315 +584,search,5830.85,3352.92,2296.07,1.08079,180.087,-nan,-nan,0.9109,0.114436,-1.0,-1.0,17.972,2.80726,1697,6283315 +585,insert,329.787,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,460.616,2.93446,1745,6556101 +586,search,5975.41,3390.03,2345.48,1.15426,186.188,-nan,-nan,0.91283,0.113298,-1.0,-1.0,194.012,2.93545,1760,6556101 +587,insert,43.596,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,84.259,2.94832,1767,6587187 +588,search,6189.66,3369.84,2481.64,1.11263,205.25,-nan,-nan,0.91392,0.112172,-1.0,-1.0,102.453,2.94895,1773,6587187 +589,insert,70.808,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,106.136,2.9642,1784,6631526 +590,search,6235.95,3374.71,2524.44,1.03605,207.71,-nan,-nan,0.91493,0.111587,-1.0,-1.0,39.976,2.96468,1786,6631526 +591,insert,619.54,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,825.27,3.169,1900,7108942 +592,search,6860.39,3432.2,2844.45,0.978448,256.884,-nan,-nan,0.91833,0.109222,-1.0,-1.0,164.665,3.17039,1916,7108942 +593,insert,228.255,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,390.544,3.25949,1963,7303976 +594,search,6944.78,3413.71,2829.1,0.99703,240.36,-nan,-nan,0.91972,0.10805,-1.0,-1.0,143.167,3.26063,1976,7303976 +595,insert,7.101,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,29.081,3.26233,1977,7309244 +596,search,7057.96,3391.68,2893.1,0.987915,250.881,-nan,-nan,0.92012,0.107618,-1.0,-1.0,38.715,3.26233,1978,7309244 +597,insert,304.404,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,416.297,3.36515,2027,7542477 +598,search,7112.32,3425.37,2876.5,1.06623,231.551,-nan,-nan,0.92199,0.106785,-1.0,-1.0,155.955,3.36654,2042,7542477 +599,insert,130.077,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,243.829,3.40933,2074,7654438 +600,search,7368.8,3400.67,3038.21,1.01773,251.933,-nan,-nan,0.9229,0.105691,-1.0,-1.0,88.494,3.41035,2082,7654438 +601,insert,72.207,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,135.976,3.43559,2095,7717936 +602,search,7484.03,3398.54,3082.38,1.02002,262.927,-nan,-nan,0.92345,0.105186,-1.0,-1.0,34.246,3.43574,2096,7717936 +603,insert,6.037,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.969,3.43695,2096,7723521 +604,search,7479.2,3398.94,3072.33,0.994627,267.403,-nan,-nan,0.92339,0.10521,-1.0,-1.0,25.177,3.43695,2096,7723521 +605,insert,57.693,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,116.951,3.45443,2107,7767404 +606,search,7394.06,3401.95,2996.95,1.03639,251.806,-nan,-nan,0.92333,0.105209,-1.0,-1.0,34.744,3.45458,2108,7767404 +607,insert,75.178,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,98.384,3.47741,2116,7826115 +608,search,7405.31,3396.42,2954.42,1.054,243.185,-nan,-nan,0.92321,0.105528,-1.0,-1.0,79.2,3.47769,2120,7826115 +609,insert,271.138,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,446.532,3.56683,2167,8036351 +610,search,7557.89,3406.32,3029.97,1.04163,247.868,-nan,-nan,0.92464,0.104993,-1.0,-1.0,93.147,3.56779,2174,8036351 +611,insert,29.179,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,83.326,3.57947,2181,8060364 +612,search,7629.76,3392.42,3075.56,1.02489,256.806,-nan,-nan,0.92489,0.104812,-1.0,-1.0,27.882,3.57947,2181,8060364 +613,insert,270.252,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,356.913,3.66311,2224,8256083 +614,search,7864.04,3396.81,3187.28,1.03013,278.148,-nan,-nan,0.92604,0.103572,-1.0,-1.0,57.193,3.66347,2227,8256083 +615,insert,437.186,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,747.545,3.83333,2293,8625707 +616,search,8185.18,3431.75,3370.12,0.949369,296.404,-nan,-nan,0.92716,0.103087,-1.0,-1.0,310.996,3.83597,2325,8625707 +617,insert,380.581,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,553.971,3.98193,2378,8949227 +618,search,8440.64,3433.68,3501.36,0.980651,297.404,-nan,-nan,0.92936,0.101058,-1.0,-1.0,277.79,3.9845,2402,8949227 +619,insert,288.027,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,343.682,4.08898,2442,9195551 +620,search,8422.47,3416.82,3401.68,1.05564,275.991,-nan,-nan,0.9308,0.0995509,-1.0,-1.0,212.503,4.09092,2463,9195551 +621,insert,48.835,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,63.312,4.10604,2469,9232372 +622,search,8635.16,3390.19,3510.21,1.03918,300.489,-nan,-nan,0.93137,0.0992954,-1.0,-1.0,45.854,4.10606,2471,9232372 +623,insert,25.883,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,71.2,4.11611,2478,9253561 +624,search,8379.65,3389.62,3274.55,1.15119,254.579,-nan,-nan,0.93156,0.0991336,-1.0,-1.0,28.101,4.11611,2479,9253561 +625,insert,93.518,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,150.526,4.14864,2494,9331230 +626,search,8680.24,3394.48,3511.51,1.03662,290.501,-nan,-nan,0.9322,0.0989088,-1.0,-1.0,73.342,4.14976,2500,9331230 +627,insert,53.835,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,111.593,4.16488,2512,9374063 +628,search,8422.11,3382.24,3258.68,1.16954,254.378,-nan,-nan,0.93231,0.0988375,-1.0,-1.0,23.056,4.16488,2512,9374063 +629,insert,32.255,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,21.268,4.17004,2512,9387598 +630,search,8355.27,3384.62,3194.67,1.19213,248.355,-nan,-nan,0.93234,0.0987269,-1.0,-1.0,33.404,4.17021,2513,9387598 +631,insert,70.921,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,75.504,4.18289,2518,9421724 +632,search,8767.78,3383.05,3557.98,1.03513,298.054,-nan,-nan,0.93234,0.098514,-1.0,-1.0,22.658,4.18289,2518,9421724 +633,insert,72.156,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,185.56,4.19932,2538,9461430 +634,search,8852.32,3368.18,3585.05,1.01169,302.96,-nan,-nan,0.93258,0.0981939,-1.0,-1.0,31.761,4.19956,2539,9461430 +635,insert,34.762,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,94.519,4.20633,2547,9479469 +636,search,8863.05,3366.2,3571.39,1.01465,299.804,-nan,-nan,0.93268,0.0981267,-1.0,-1.0,71.728,4.207,2552,9479469 +637,insert,605.325,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,840.918,4.39152,2648,9885997 +638,search,9251.6,3390.96,3756.86,1.02509,323.25,-nan,-nan,0.93473,0.0964742,-1.0,-1.0,236.327,4.39492,2671,9885997 +639,insert,545.018,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,720.892,4.57457,2763,10292043 +640,search,9643.52,3394.03,3931.56,1.00383,326.756,-nan,-nan,0.93719,0.0943849,-1.0,-1.0,168.365,4.57611,2779,10292043 +641,delete,217.226,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1740.63,4.49003,2718,10096584 +642,search,9377.67,3392.37,3762.58,1.07128,312.828,-nan,-nan,0.93683,0.0944116,-1.0,-1.0,226.354,4.49376,2736,10096584 +643,delete,12.154,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,54.925,4.49178,2735,10084840 +644,search,9401.01,3371.73,3767.73,1.03905,306.244,-nan,-nan,0.93666,0.0945367,-1.0,-1.0,44.383,4.49213,2736,10084840 +645,delete,33.205,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,53.037,4.49054,2734,10069802 +646,search,9489.92,3365.9,3823.84,1.05023,320.287,-nan,-nan,0.93665,0.0945439,-1.0,-1.0,31.497,4.49054,2734,10069802 +647,delete,125.673,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,771.14,4.44745,2697,9990953 +648,search,9417.47,3383.93,3816.01,1.0328,328.74,-nan,-nan,0.93615,0.0945666,-1.0,-1.0,155.409,4.44985,2709,9990953 +649,delete,46.864,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,173.907,4.44212,2703,9951625 +650,search,9397.92,3363.22,3792.73,1.01509,316.06,-nan,-nan,0.9362,0.0947268,-1.0,-1.0,69.379,4.44301,2707,9951625 +651,delete,32.152,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,179.941,4.4345,2700,9927065 +652,search,9498.17,3360.99,3870.14,0.980789,341.515,-nan,-nan,0.93625,0.0945808,-1.0,-1.0,46.234,4.43475,2702,9927065 +653,delete,118.44,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,695.537,4.38775,2667,9831098 +654,search,9304.99,3365.77,3769.85,1.0136,316.621,-nan,-nan,0.93543,0.094733,-1.0,-1.0,69.512,4.38836,2672,9831098 +655,delete,458.387,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,2558.28,4.22336,2523,9462172 +656,search,8839.01,3415.62,3591.75,1.00864,303.996,-nan,-nan,0.93307,0.0976823,-1.0,-1.0,353.088,4.23039,2561,9462172 +657,delete,225.258,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,677.84,4.14442,2471,9218902 +658,search,8593.07,3390.12,3464.31,1.01145,289.925,-nan,-nan,0.92907,0.100434,-1.0,-1.0,269.345,4.14816,2488,9218902 +659,delete,133.714,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,681.177,4.09054,2445,9106546 +660,search,8544.22,3378.65,3451.66,1.04905,280.053,-nan,-nan,0.92917,0.10023,-1.0,-1.0,95.461,4.09148,2451,9106546 +661,delete,68.513,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,296.972,4.07934,2444,9059776 +662,search,8620.21,3359.79,3510.96,0.965679,302.513,-nan,-nan,0.92888,0.100503,-1.0,-1.0,58.081,4.07956,2446,9059776 +663,delete,111.585,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,405.993,4.05122,2426,8990616 +664,search,8468.37,3359.41,3416.48,1.0302,285.986,-nan,-nan,0.92799,0.101306,-1.0,-1.0,58.898,4.05212,2429,8990616 +665,delete,65.014,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,303.738,4.03051,2416,8937437 +666,search,8400.73,3352.02,3375.13,1.02591,286.161,-nan,-nan,0.92798,0.101167,-1.0,-1.0,46.487,4.03082,2418,8937437 +667,delete,228.473,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1263.56,3.9438,2348,8766390 +668,search,8333.67,3375.78,3391.66,1.00725,292.279,-nan,-nan,0.92705,0.102075,-1.0,-1.0,171.055,3.94559,2360,8766390 +669,delete,275.38,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,2206.27,3.83523,2246,8532008 +670,search,7961.73,3435.05,3225.82,1.00984,273.054,-nan,-nan,0.92679,0.102386,-1.0,-1.0,380.893,3.84055,2281,8532008 +671,delete,44.608,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,185.425,3.82601,2272,8489178 +672,search,7937.34,3366.96,3186.29,1.05428,263.067,-nan,-nan,0.92652,0.102867,-1.0,-1.0,86.931,3.82634,2277,8489178 +673,delete,75.685,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,311.636,3.79704,2260,8421866 +674,search,7990.98,3357.43,3254.76,1.03803,278.263,-nan,-nan,0.9261,0.103015,-1.0,-1.0,60.65,3.79743,2262,8421866 +675,delete,192.067,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,662.948,3.75169,2226,8288319 +676,search,7522.12,3345.48,2889.76,1.16857,226.402,-nan,-nan,0.92577,0.10336,-1.0,-1.0,84.953,3.75243,2230,8288319 +677,delete,252.19,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,627.197,3.65635,2190,8072197 +678,search,7410.05,3327.53,2860.25,1.14915,227.501,-nan,-nan,0.9279,0.100827,-1.0,-1.0,71.629,3.65676,2193,8072197 +679,delete,39.528,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,359.218,3.63962,2181,8036947 +680,search,7510.31,3328.89,2959.25,1.06954,240.017,-nan,-nan,0.92787,0.10129,-1.0,-1.0,78.922,3.64033,2185,8036947 +681,delete,13.097,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,29.487,3.64033,2185,8026197 +682,search,7599.25,3319.61,3026.86,1.01762,255.01,-nan,-nan,0.92795,0.101076,-1.0,-1.0,31.792,3.64033,2185,8026197 +683,delete,370.667,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1764.57,3.51909,2077,7744229 +684,search,7321.34,3368.61,2956.8,1.01429,245.505,-nan,-nan,0.9268,0.102899,-1.0,-1.0,326.902,3.52354,2109,7744229 +685,delete,256.437,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1372.58,3.427,2014,7541066 +686,search,7173.83,3372.2,2941.73,0.978156,259.156,-nan,-nan,0.92398,0.105788,-1.0,-1.0,337.2,3.43405,2048,7541066 +687,delete,16.511,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,96.442,3.42806,2045,7522190 +688,search,7177.82,3304.04,2907.25,0.966255,246.539,-nan,-nan,0.92385,0.105495,-1.0,-1.0,54.573,3.42877,2047,7522190 +689,delete,146.113,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,550.021,3.36936,2015,7409400 +690,search,7097.22,3322.05,2875.33,0.977189,247.268,-nan,-nan,0.92286,0.106038,-1.0,-1.0,137.737,3.37085,2025,7409400 +691,delete,39.316,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,176.411,3.36071,2016,7375648 +692,search,7071.05,3299.56,2856.72,1.00735,238.646,-nan,-nan,0.92313,0.106055,-1.0,-1.0,29.557,3.36071,2016,7375648 +693,delete,252.394,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,570.051,3.29276,1984,7222068 +694,search,6892.57,3299.73,2736.12,1.05888,226.313,-nan,-nan,0.92327,0.105381,-1.0,-1.0,94.898,3.29391,1989,7222068 +695,delete,33.613,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,174.347,3.2818,1983,7196331 +696,search,6767.67,3295.3,2621.37,1.08934,213.826,-nan,-nan,0.92389,0.104669,-1.0,-1.0,65.21,3.28208,1986,7196331 +697,delete,41.729,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,159.176,3.27279,1980,7170716 +698,search,6897.05,3287.08,2740.17,1.02796,241.945,-nan,-nan,0.92427,0.104561,-1.0,-1.0,43.665,3.27296,1981,7170716 +699,delete,14.931,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,106.808,3.26309,1977,7155255 +700,search,6980.45,3288.01,2815.74,0.981928,251.196,-nan,-nan,0.92437,0.104175,-1.0,-1.0,28.678,3.26309,1977,7155255 +701,delete,73.129,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,120.246,3.25613,1971,7097733 +702,search,6900.75,3267.08,2779.14,0.967854,236.275,-nan,-nan,0.92396,0.104321,-1.0,-1.0,38.835,3.25628,1972,7097733 +703,delete,262.025,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,863.002,3.16736,1908,6924172 +704,search,6708.59,3293.71,2711.07,0.977221,231.803,-nan,-nan,0.92503,0.103516,-1.0,-1.0,128.838,3.16968,1916,6924172 +705,delete,31.046,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,126.337,3.16092,1909,6896276 +706,search,6597.22,3272.53,2607.84,1.04195,214.951,-nan,-nan,0.92468,0.103947,-1.0,-1.0,63.785,3.16142,1912,6896276 +707,delete,744.621,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,3503.36,2.87542,1634,6385507 +708,search,6001.17,3605.73,2553.05,0.966578,226.748,-nan,-nan,0.92238,0.107083,-1.0,-1.0,947.067,2.89213,1745,6385507 +709,delete,170.467,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,976.428,2.80059,1666,6207658 +710,search,5829.21,3356.24,2379.51,0.981776,200.246,-nan,-nan,0.91891,0.110359,-1.0,-1.0,306.113,2.80658,1700,6207658 +711,delete,44.865,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,295.012,2.78662,1686,6162720 +712,search,5897.94,3271.3,2397.77,0.970162,211.238,-nan,-nan,0.91787,0.111736,-1.0,-1.0,92.265,2.78781,1693,6162720 +713,delete,277.191,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1112.59,2.67537,1620,5927803 +714,search,5660.5,3290.83,2291.38,0.951273,195.629,-nan,-nan,0.91457,0.11458,-1.0,-1.0,158.242,2.67827,1633,5927803 +715,delete,30.077,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,133.673,2.6682,1625,5895666 +716,search,5667.22,3266.18,2249.26,1.02081,186.121,-nan,-nan,0.91497,0.114449,-1.0,-1.0,66.037,2.66854,1628,5895666 +717,delete,39.545,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,111.692,2.66148,1620,5868774 +718,search,5679.01,3258.26,2270.17,0.970395,196.108,-nan,-nan,0.91433,0.114956,-1.0,-1.0,58.179,2.66184,1623,5868774 +719,delete,494.825,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1800.9,2.49643,1455,5533540 +720,search,5054.81,3450.22,1998.89,1.10047,160.222,-nan,-nan,0.9086,0.117956,-1.0,-1.0,464.131,2.50526,1500,5533540 +721,delete,116.021,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,359.278,2.45861,1472,5418387 +722,search,5084.31,3302.46,1986.54,1.05529,163.748,-nan,-nan,0.9073,0.119204,-1.0,-1.0,207.064,2.46055,1486,5418387 +723,delete,23.788,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,81.586,2.4524,1481,5398291 +724,search,5098.18,3255.6,1997.12,1.09117,162.876,-nan,-nan,0.90671,0.119615,-1.0,-1.0,101.736,2.45248,1486,5398291 +725,delete,143.506,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,614.725,2.39444,1441,5273546 +726,search,4986.1,3257.5,2004.26,1.02883,169.734,-nan,-nan,0.90472,0.120844,-1.0,-1.0,201.575,2.39666,1456,5273546 +727,delete,105.531,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,628.509,2.35451,1408,5180937 +728,search,4876.57,3298.82,1939.57,1.01924,159.068,-nan,-nan,0.90558,0.11977,-1.0,-1.0,230.307,2.3578,1431,5180937 +729,delete,125.681,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,361.743,2.31195,1393,5079524 +730,search,4857.58,3237.91,1936.95,0.987534,166.544,-nan,-nan,0.90402,0.120692,-1.0,-1.0,85.552,2.31326,1400,5079524 +731,delete,70.586,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,220.263,2.29488,1383,5019265 +732,search,4834.02,3209.31,1920.48,0.999853,160.468,-nan,-nan,0.90345,0.120935,-1.0,-1.0,17.825,2.29488,1383,5019265 +733,delete,85.395,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,146.687,2.27474,1372,4955565 +734,search,4741.11,3182.31,1849.57,1.06009,154.399,-nan,-nan,0.90306,0.121451,-1.0,-1.0,38.685,2.27495,1374,4955565 +735,delete,217.234,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,556.281,2.19118,1331,4776948 +736,search,4680.31,3248.66,1898.49,0.974024,165.98,-nan,-nan,0.90777,0.116387,-1.0,-1.0,180.024,2.19424,1345,4776948 +737,delete,137.242,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,543.34,2.13766,1309,4654026 +738,search,4573.86,3191.35,1799.35,1.02143,154.393,-nan,-nan,0.90618,0.118516,-1.0,-1.0,73.695,2.13844,1315,4654026 +739,delete,76.221,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,353.704,2.10531,1291,4589888 +740,search,4427.61,3187.62,1745.75,1.01142,150.139,-nan,-nan,0.90471,0.119573,-1.0,-1.0,120.913,2.10662,1301,4589888 +741,delete,201.927,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,625.662,2.03192,1251,4430862 +742,search,4354.65,3178.63,1743.74,0.986245,155.012,-nan,-nan,0.90435,0.119953,-1.0,-1.0,130.574,2.03325,1262,4430862 +743,delete,237.226,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1147.81,1.92896,1182,4237049 +744,search,4075.9,3232.63,1614.86,1.00034,136.135,-nan,-nan,0.90152,0.12188,-1.0,-1.0,315.401,1.93614,1205,4237049 +745,delete,283.834,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1065.39,1.80794,1109,4000195 +746,search,3949.85,3295.59,1610.46,0.994881,145.458,-nan,-nan,0.89784,0.124382,-1.0,-1.0,236.872,1.81028,1129,4000195 +747,delete,196.746,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,905.321,1.71548,1064,3803954 +748,search,3791.67,3292.46,1542.51,0.957701,139.552,-nan,-nan,0.89485,0.126324,-1.0,-1.0,271.976,1.71989,1084,3803954 +749,delete,93.572,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,234.882,1.69194,1063,3733214 +750,search,3762.39,3234.82,1524.33,0.990288,133.326,-nan,-nan,0.89382,0.127438,-1.0,-1.0,20.038,1.69194,1063,3733214 +751,delete,35.552,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,118.328,1.68117,1053,3709324 +752,search,3744.52,3241.97,1518.89,1.00569,134.335,-nan,-nan,0.89291,0.128198,-1.0,-1.0,43.782,1.68157,1055,3709324 +753,delete,89.886,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,288.419,1.65218,1028,3649123 +754,search,3650.7,3257.98,1470.48,0.96927,128.833,-nan,-nan,0.89089,0.130356,-1.0,-1.0,71.324,1.65262,1030,3649123 +755,delete,36.104,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,61.275,1.64663,1026,3621475 +756,search,3650.64,3234.64,1475.6,0.947094,128.201,-nan,-nan,0.89059,0.130666,-1.0,-1.0,64.476,1.64685,1029,3621475 +757,delete,35.611,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,109.448,1.63548,1022,3594479 +758,search,3579.5,3229.44,1407.15,1.04619,118.884,-nan,-nan,0.89108,0.130296,-1.0,-1.0,47.088,1.63576,1024,3594479 +759,delete,30.82,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,122.488,1.62239,1016,3569055 +760,search,3653.78,3233.58,1477.48,0.952223,133.08,-nan,-nan,0.89086,0.130192,-1.0,-1.0,46.17,1.62267,1018,3569055 +761,delete,61.047,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,164.095,1.60692,1005,3527896 +762,search,3583.75,3225.76,1447.02,0.964125,128.226,-nan,-nan,0.88963,0.130938,-1.0,-1.0,17.64,1.60692,1005,3527896 +763,delete,26.763,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,83.98,1.60133,1000,3509970 +764,search,3480.78,3226,1351.92,1.04954,111.043,-nan,-nan,0.88992,0.130729,-1.0,-1.0,17.698,1.60133,1000,3509970 +765,delete,512.698,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1165.59,1.42944,863,3187549 +766,search,3143.45,3453.46,1288,1.01697,117.239,-nan,-nan,0.88742,0.130987,-1.0,-1.0,303.647,1.43375,893,3187549 +767,delete,476.465,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1250.19,1.25714,732,2822906 +768,search,2755.54,3551.63,1177.01,0.958107,106.413,-nan,-nan,0.87715,0.13834,-1.0,-1.0,452.244,1.26527,774,2822906 +769,insert,14.994,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,28.433,1.26839,776,2832899 +770,search,2762.04,3355.3,1113.59,1.03477,95.2523,-nan,-nan,0.88066,0.135018,-1.0,-1.0,25.479,1.26869,777,2832899 +771,insert,44.273,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,70.416,1.27982,782,2863523 +772,search,2862.77,3363.67,1192.91,0.982495,106.549,-nan,-nan,0.88142,0.134414,-1.0,-1.0,37.96,1.28012,784,2863523 +773,insert,102.861,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,116.862,1.30563,794,2925169 +774,search,2920.83,3399.47,1216.71,0.982827,110.738,-nan,-nan,0.88412,0.132438,-1.0,-1.0,51.51,1.30581,798,2925169 +775,insert,64.052,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,84.452,1.32143,803,2961356 +776,search,2951.23,3393.36,1239.86,0.97545,105.567,-nan,-nan,0.88367,0.133157,-1.0,-1.0,13.033,1.32143,803,2961356 +777,insert,65.171,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,119.047,1.3365,815,2998678 +778,search,3031.53,3392.55,1252.87,0.970532,111.003,-nan,-nan,0.88504,0.132724,-1.0,-1.0,22.019,1.33663,816,2998678 +779,insert,25.027,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,30.989,1.34094,818,3011024 +780,search,2914.07,3388.44,1183.1,1.13858,102.139,-nan,-nan,0.88462,0.132934,-1.0,-1.0,12.68,1.34094,818,3011024 +781,insert,11.495,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,34.187,1.34403,820,3019515 +782,search,2987.7,3388.45,1248.84,0.989951,111.058,-nan,-nan,0.88449,0.133066,-1.0,-1.0,12.657,1.34403,820,3019515 +783,insert,113.109,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,180.718,1.3715,838,3082298 +784,search,3087.18,3388.34,1278.74,0.953249,114.33,-nan,-nan,0.88572,0.132061,-1.0,-1.0,12.873,1.3715,838,3082298 +785,insert,62.965,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,99.773,1.38997,846,3122090 +786,search,3114.65,3395.49,1294.22,0.944388,111.683,-nan,-nan,0.88718,0.131237,-1.0,-1.0,30.988,1.39025,848,3122090 +787,insert,41.848,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,72.594,1.40013,855,3140294 +788,search,3108.65,3377.43,1296.19,0.97291,117.7,-nan,-nan,0.88724,0.131215,-1.0,-1.0,13.552,1.40013,855,3140294 +789,insert,11.564,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.869,1.40272,856,3147819 +790,search,3102.24,3379.9,1299.73,0.990994,111.301,-nan,-nan,0.88739,0.131289,-1.0,-1.0,13.562,1.40272,856,3147819 +791,insert,59.229,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,94.329,1.41513,865,3180668 +792,search,3076.55,3372.6,1247.68,1.07031,104.348,-nan,-nan,0.88781,0.130877,-1.0,-1.0,14.645,1.41513,865,3180668 +793,insert,7.752,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.448,1.41739,865,3183235 +794,search,3131.31,3374.42,1300.69,0.984249,112.914,-nan,-nan,0.88775,0.130874,-1.0,-1.0,15.03,1.41739,865,3183235 +795,insert,101.02,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,165.564,1.44216,879,3242089 +796,search,3238.92,3390,1345.46,0.941304,118.391,-nan,-nan,0.88832,0.130257,-1.0,-1.0,36.734,1.44265,880,3242089 +797,insert,7.629,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,12.958,1.4442,880,3247192 +798,search,3247.14,3388.46,1350.94,0.966991,121.667,-nan,-nan,0.88845,0.130133,-1.0,-1.0,14,1.4442,880,3247192 +799,insert,12.595,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,12.694,1.44687,880,3254626 +800,search,3224.2,3390.89,1348.78,0.962939,121.835,-nan,-nan,0.88857,0.129976,-1.0,-1.0,13.994,1.44687,880,3254626 +801,insert,33.907,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,64.225,1.45498,884,3274953 +802,search,3235.25,3402.87,1353.13,0.969955,121.187,-nan,-nan,0.88848,0.13011,-1.0,-1.0,13.812,1.45498,884,3274953 +803,insert,1301.39,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1497.01,1.81395,1024,4032688 +804,search,3851.97,3693.89,1673.95,0.94225,149.975,-nan,-nan,0.89663,0.124818,-1.0,-1.0,702.613,1.81702,1100,4032688 +805,insert,25.084,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,47.819,1.82367,1104,4050220 +806,search,3778.67,3409.83,1480.97,1.14869,119.469,-nan,-nan,0.89775,0.123841,-1.0,-1.0,220.979,1.82521,1125,4050220 +807,insert,26.397,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,48.832,1.83371,1128,4067940 +808,search,3865.95,3348.19,1536.44,1.14628,126.917,-nan,-nan,0.89774,0.123994,-1.0,-1.0,25.029,1.83382,1129,4067940 +809,insert,42.909,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,64.909,1.84309,1134,4099369 +810,search,3909.07,3354.33,1552.24,1.07921,129.112,-nan,-nan,0.89835,0.123446,-1.0,-1.0,28.231,1.84351,1135,4099369 +811,insert,53.494,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,75.892,1.85448,1142,4129509 +812,search,4089.42,3363.61,1702.56,0.997582,147.312,-nan,-nan,0.89884,0.122831,-1.0,-1.0,14.115,1.85448,1142,4129509 +813,insert,83.197,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,159.467,1.873,1159,4178669 +814,search,4080.57,3355.35,1657.88,1.03821,141.544,-nan,-nan,0.901,0.121435,-1.0,-1.0,24.523,1.87302,1160,4178669 +815,insert,2.771,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,35.51,1.87397,1162,4180960 +816,search,4099.57,3348.42,1630.85,1.08772,137.791,-nan,-nan,0.90095,0.121555,-1.0,-1.0,14.183,1.87397,1162,4180960 +817,insert,109.501,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,107.168,1.89742,1171,4255359 +818,search,4059.06,3370.19,1611.7,1.09472,133.549,-nan,-nan,0.90156,0.120824,-1.0,-1.0,53.771,1.8983,1175,4255359 +819,insert,27.665,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,65.399,1.9026,1181,4272706 +820,search,4214.72,3351.31,1744.75,0.956053,149.624,-nan,-nan,0.90216,0.120287,-1.0,-1.0,13.607,1.9026,1181,4272706 +821,insert,22.907,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,48.687,1.90888,1184,4287851 +822,search,4214.58,3353.24,1740.58,0.993616,151.39,-nan,-nan,0.9019,0.120437,-1.0,-1.0,24.556,1.90895,1185,4287851 +823,insert,50.887,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,79.122,1.92322,1191,4322788 +824,search,4248.01,3359.57,1762.8,0.949613,156.664,-nan,-nan,0.90179,0.121005,-1.0,-1.0,50.589,1.92382,1195,4322788 +825,insert,126.938,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,166.836,1.95539,1208,4396408 +826,search,4184.7,3359.16,1662.25,1.10836,137.15,-nan,-nan,0.9025,0.119897,-1.0,-1.0,87.671,1.95638,1216,4396408 +827,insert,30.888,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,31.673,1.96368,1218,4415721 +828,search,4167.71,3340.61,1644.54,1.16812,133.735,-nan,-nan,0.90193,0.120169,-1.0,-1.0,23.285,1.96383,1219,4415721 +829,insert,681.591,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,906.233,2.15372,1314,4832079 +830,search,4557.01,3412.8,1802.09,1.14986,148.798,-nan,-nan,0.90589,0.117338,-1.0,-1.0,236.691,2.15552,1337,4832079 +831,insert,44.897,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,101.157,2.16829,1345,4864525 +832,search,4739.6,3348.57,1897.5,1.03549,158.015,-nan,-nan,0.90691,0.116605,-1.0,-1.0,67.116,2.16906,1348,4864525 +833,insert,12.366,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.712,2.17115,1348,4874443 +834,search,4705.02,3350.54,1905.78,1.04607,161,-nan,-nan,0.90686,0.116542,-1.0,-1.0,16.024,2.17115,1348,4874443 +835,insert,107.815,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,142.237,2.19961,1363,4938267 +836,search,4890.78,3369.09,2008.7,0.98171,182.905,-nan,-nan,0.9076,0.11638,-1.0,-1.0,16.824,2.19961,1363,4938267 +837,insert,29.508,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,36.649,2.20659,1365,4961147 +838,search,4848.95,3379.04,1973.45,1.02603,169.769,-nan,-nan,0.90784,0.116248,-1.0,-1.0,31.362,2.20667,1366,4961147 +839,insert,105.339,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,146.531,2.23694,1378,5033740 +840,search,4832.94,3398.08,1947.85,1.0671,165.432,-nan,-nan,0.90887,0.11563,-1.0,-1.0,55.306,2.2374,1381,5033740 +841,insert,63.179,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,71.071,2.2533,1386,5078736 +842,search,5037.19,3407.83,2076.36,1.01233,192.809,-nan,-nan,0.91024,0.114571,-1.0,-1.0,40.696,2.25366,1388,5078736 +843,insert,614.405,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,853.089,2.43534,1489,5467376 +844,search,5361.72,3413.01,2220.49,1.02256,201.879,-nan,-nan,0.91164,0.113533,-1.0,-1.0,305.638,2.43821,1523,5467376 +845,insert,505.48,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,609.737,2.59055,1593,5815547 +846,search,5636.52,3402.83,2304.51,1.00666,198.181,-nan,-nan,0.91476,0.111799,-1.0,-1.0,263.321,2.593,1620,5815547 +847,insert,8.697,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.822,2.59525,1620,5821782 +848,search,5553.14,3350.75,2224.01,1.06381,187.057,-nan,-nan,0.91497,0.111636,-1.0,-1.0,49.629,2.59525,1623,5821782 +849,insert,3.6,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,38.913,2.59617,1625,5825594 +850,search,5790.86,3345.19,2366.53,0.987491,211.604,-nan,-nan,0.91522,0.111476,-1.0,-1.0,25.402,2.5963,1626,5825594 +851,insert,12.687,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,33.98,2.59909,1628,5834880 +852,search,5763.75,3342.62,2359.72,1.00828,204.171,-nan,-nan,0.91513,0.111691,-1.0,-1.0,17.632,2.59909,1628,5834880 +853,insert,47.29,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,49.027,2.60948,1631,5860274 +854,search,5795.25,3348.5,2377.06,0.978885,207.4,-nan,-nan,0.91548,0.111377,-1.0,-1.0,37.186,2.60952,1633,5860274 +855,insert,8.991,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.176,2.61112,1633,5866064 +856,search,5784.22,3348.34,2363.86,1.01217,202.234,-nan,-nan,0.91548,0.111422,-1.0,-1.0,17.484,2.61112,1633,5866064 +857,insert,22.782,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,45.143,2.61504,1636,5884456 +858,search,5632.75,3352.64,2285.37,1.06533,191.616,-nan,-nan,0.91584,0.11113,-1.0,-1.0,31.936,2.61532,1637,5884456 +859,insert,48.415,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,62.079,2.62761,1642,5916786 +860,search,5711.7,3360.52,2283.61,1.10561,188.713,-nan,-nan,0.91589,0.110928,-1.0,-1.0,20.317,2.62761,1642,5916786 +861,insert,15.436,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,36.216,2.629,1644,5923467 +862,search,5885.79,3358.79,2425.81,1.00398,216.466,-nan,-nan,0.91631,0.110563,-1.0,-1.0,31.596,2.62931,1645,5923467 +863,insert,107.17,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,123.235,2.66107,1653,5997844 +864,search,5760.58,3376.34,2315.75,1.04849,195.928,-nan,-nan,0.91512,0.111984,-1.0,-1.0,41.73,2.66122,1655,5997844 +865,insert,19.353,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,68.491,2.66646,1660,6009561 +866,search,5729.56,3369.5,2282.86,1.12699,190.896,-nan,-nan,0.91526,0.111884,-1.0,-1.0,18.672,2.66646,1660,6009561 +867,insert,70.115,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,77.952,2.68242,1666,6052876 +868,search,5770.7,3378.72,2336.38,1.04632,200.368,-nan,-nan,0.91556,0.112026,-1.0,-1.0,31.575,2.68255,1667,6052876 +869,insert,9.94,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.713,2.68506,1667,6059234 +870,search,5848.81,3378.83,2373.92,1.01026,204.667,-nan,-nan,0.91545,0.112149,-1.0,-1.0,18.653,2.68506,1667,6059234 +871,insert,64.829,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,89.489,2.70083,1673,6098225 +872,search,5977.5,3385.32,2460.77,0.986014,211.158,-nan,-nan,0.91558,0.11202,-1.0,-1.0,30.62,2.70116,1674,6098225 +873,insert,61.181,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,79.4,2.7171,1679,6137308 +874,search,5866.11,3390.33,2355.73,1.08059,205.227,-nan,-nan,0.91595,0.11173,-1.0,-1.0,38.395,2.71756,1681,6137308 +875,insert,33.684,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,42.453,2.72506,1683,6154089 +876,search,5960.84,3392.25,2482.83,0.950553,217.145,-nan,-nan,0.91565,0.111954,-1.0,-1.0,18.075,2.72506,1683,6154089 +877,insert,6.115,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.289,2.72606,1683,6156061 +878,search,5841.68,3393.57,2329.03,1.04639,197.031,-nan,-nan,0.91567,0.111948,-1.0,-1.0,17.967,2.72606,1683,6156061 +879,insert,496.619,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,514.082,2.85682,1732,6446004 +880,search,5999.13,3441.6,2444.75,1.08931,202.516,-nan,-nan,0.91792,0.11058,-1.0,-1.0,104.952,2.85787,1743,6446004 +881,insert,731.566,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1007.87,3.09555,1854,6969764 +882,search,6648.51,3484.7,2752.12,0.983634,243.433,-nan,-nan,0.9211,0.108119,-1.0,-1.0,372.6,3.09866,1890,6969764 +883,insert,612.612,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,993.69,3.30536,2022,7422501 +884,search,7244.54,3407.19,2994.15,0.96313,266.475,-nan,-nan,0.92511,0.104812,-1.0,-1.0,388.749,3.31087,2063,7422501 +885,insert,13.955,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.501,3.314,2063,7432053 +886,search,7079.91,3350.23,2786.97,1.11692,233.839,-nan,-nan,0.92544,0.10472,-1.0,-1.0,36.672,3.314,2064,7432053 +887,insert,484.294,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,608.347,3.46718,2135,7789886 +888,search,7289.14,3382.4,2868.33,1.154,225.454,-nan,-nan,0.92697,0.103943,-1.0,-1.0,147.622,3.46827,2148,7789886 +889,insert,574.447,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,706.879,3.65351,2229,8210883 +890,search,7864.17,3420.94,3198.08,1.01738,271.133,-nan,-nan,0.9293,0.101619,-1.0,-1.0,396.44,3.65927,2270,8210883 +891,insert,45.816,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,69.179,3.66876,2275,8240504 +892,search,7865.11,3361.56,3139.26,1.07211,260.478,-nan,-nan,0.92974,0.101403,-1.0,-1.0,73.332,3.66937,2281,8240504 +893,insert,68.778,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,142.723,3.691,2297,8299693 +894,search,7933.46,3347.5,3161.2,1.078,262.646,-nan,-nan,0.93005,0.101324,-1.0,-1.0,54.744,3.69149,2299,8299693 +895,insert,60.817,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,126.286,3.71035,2311,8349126 +896,search,7986.36,3351.61,3191.68,1.05947,268.406,-nan,-nan,0.93067,0.100872,-1.0,-1.0,44.684,3.71055,2312,8349126 +897,delete,104.642,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,517.191,3.67584,2290,8268842 +898,search,8095.53,3356.62,3310.94,1.0011,294.054,-nan,-nan,0.93048,0.100984,-1.0,-1.0,80.172,3.67654,2294,8268842 +899,delete,23.026,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,97.596,3.67143,2290,8250527 +900,search,8088.95,3348.46,3289.66,1.0033,294.895,-nan,-nan,0.93032,0.101108,-1.0,-1.0,32.822,3.67143,2290,8250527 +901,delete,88.713,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,180.785,3.65555,2277,8185565 +902,search,7926.17,3335.72,3178.11,1.018,273.346,-nan,-nan,0.92996,0.101202,-1.0,-1.0,46.152,3.65583,2278,8185565 +903,delete,72.058,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,188.194,3.62977,2261,8126915 +904,search,7712.9,3338.46,3022.87,1.09108,253.387,-nan,-nan,0.9288,0.101818,-1.0,-1.0,42.788,3.63001,2262,8126915 +905,delete,37.253,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,175.473,3.62157,2254,8098533 +906,search,7752.76,3333.68,3071.07,1.09429,254.746,-nan,-nan,0.92861,0.101869,-1.0,-1.0,42.105,3.62186,2255,8098533 +907,delete,32.896,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,206.434,3.61227,2248,8074833 +908,search,7765.41,3332.28,3080.55,1.07358,253.248,-nan,-nan,0.92835,0.102139,-1.0,-1.0,73.59,3.61299,2251,8074833 +909,delete,45.427,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,172.834,3.60443,2243,8035274 +910,search,7729.69,3322.04,3058.12,1.06137,252.908,-nan,-nan,0.92837,0.102038,-1.0,-1.0,57.098,3.60502,2246,8035274 +911,delete,89.781,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,282.559,3.58393,2231,7957223 +912,search,7650.27,3299.54,2997.2,1.07702,257.137,-nan,-nan,0.92742,0.103086,-1.0,-1.0,37.145,3.58401,2232,7957223 +913,delete,43.288,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,246.033,3.56557,2217,7917554 +914,search,7207.04,3300.17,2645.12,1.22085,205.708,-nan,-nan,0.92611,0.103897,-1.0,-1.0,82.445,3.56633,2219,7917554 +915,delete,90.482,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,308.365,3.54055,2199,7861503 +916,search,7708.56,3301.61,3105.03,0.999377,262.596,-nan,-nan,0.92528,0.105123,-1.0,-1.0,92.611,3.5415,2203,7861503 +917,delete,28.712,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,104.031,3.53645,2201,7844487 +918,search,7766.26,3291.52,3132.04,0.971795,273.223,-nan,-nan,0.92594,0.104644,-1.0,-1.0,41.669,3.5365,2202,7844487 +919,delete,44.641,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,54.683,3.53466,2201,7814993 +920,search,7740.23,3279.08,3117.05,0.988277,271.883,-nan,-nan,0.92581,0.104982,-1.0,-1.0,40.623,3.53478,2202,7814993 +921,delete,35.528,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,152.989,3.5274,2196,7793855 +922,search,7625.66,3276.71,3038.11,1.04452,258.819,-nan,-nan,0.9257,0.105088,-1.0,-1.0,43.151,3.52767,2197,7793855 +923,delete,77.066,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,254.907,3.50978,2184,7736054 +924,search,7468.39,3265.81,2920.17,1.0491,239.957,-nan,-nan,0.92539,0.105242,-1.0,-1.0,28.886,3.50978,2184,7736054 +925,delete,190.463,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,643.224,3.44041,2134,7587942 +926,search,7294.25,3271.24,2867.31,1.07001,237.844,-nan,-nan,0.92432,0.105719,-1.0,-1.0,70.598,3.44096,2138,7587942 +927,delete,28.837,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,26.325,3.44064,2138,7565670 +928,search,7412.21,3254.09,2953.33,0.989416,250.844,-nan,-nan,0.9244,0.10575,-1.0,-1.0,38.048,3.4409,2139,7565670 +929,delete,44.176,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,125.887,3.43227,2133,7529948 +930,search,7380.03,3244.16,2934.71,0.998844,248.037,-nan,-nan,0.92409,0.106138,-1.0,-1.0,27.288,3.43227,2133,7529948 +931,delete,634.248,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,3986.13,3.16993,1879,7016700 +932,search,6751.51,3418.99,2772.22,1.03015,240.057,-nan,-nan,0.9206,0.108363,-1.0,-1.0,738.719,3.18435,1954,7016700 +933,delete,23.358,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,138.755,3.17094,1946,6980979 +934,search,6834.73,3275.21,2754.7,1.01681,237.356,-nan,-nan,0.92102,0.108352,-1.0,-1.0,203.299,3.1729,1965,6980979 +935,delete,22.238,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,204.474,3.15922,1958,6954250 +936,search,6766.17,3244.51,2657.34,1.06059,227.349,-nan,-nan,0.92117,0.108004,-1.0,-1.0,43.777,3.15967,1959,6954250 +937,delete,39.954,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,123.18,3.15056,1953,6924596 +938,search,6700.77,3238.58,2620.25,1.06239,215.641,-nan,-nan,0.92068,0.108268,-1.0,-1.0,40.644,3.15093,1954,6924596 +939,delete,199.728,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,540.79,3.10958,1917,6784831 +940,search,6559.92,3214.75,2551.27,1.08448,221.45,-nan,-nan,0.92012,0.10926,-1.0,-1.0,86.588,3.11107,1923,6784831 +941,delete,46.643,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,37.35,3.10854,1922,6745819 +942,search,6684.95,3176.96,2650.55,0.996822,229.405,-nan,-nan,0.91965,0.110053,-1.0,-1.0,38.115,3.10882,1923,6745819 +943,delete,4.699,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,40.661,3.1074,1922,6740944 +944,search,6611.42,3175.25,2584.83,1.04811,221.732,-nan,-nan,0.91964,0.109868,-1.0,-1.0,24.323,3.1074,1922,6740944 +945,delete,147.936,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,978.343,3.03585,1862,6627202 +946,search,6144.55,3230.58,2291.76,1.23631,177.558,-nan,-nan,0.91788,0.110923,-1.0,-1.0,214.897,3.0389,1882,6627202 +947,delete,23.337,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,61.138,3.03331,1879,6607194 +948,search,6489.63,3180.61,2590.12,1.00727,226.946,-nan,-nan,0.91817,0.110392,-1.0,-1.0,83.475,3.03414,1885,6607194 +949,delete,59.612,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,202.91,3.01304,1874,6551065 +950,search,6471.01,3162.99,2560.31,0.989248,219.218,-nan,-nan,0.91782,0.11092,-1.0,-1.0,22.85,3.01304,1874,6551065 +951,delete,33.579,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,97.366,2.99414,1866,6506798 +952,search,6308.61,3158.91,2441.06,1.05654,198.817,-nan,-nan,0.91884,0.109499,-1.0,-1.0,54.162,2.99458,1868,6506798 +953,delete,62.696,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,214.154,2.98003,1858,6459301 +954,search,6265.85,3151.06,2433.75,1.06874,202.889,-nan,-nan,0.91869,0.109963,-1.0,-1.0,44.701,2.98054,1860,6459301 +955,delete,20.113,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,39.241,2.97218,1858,6440069 +956,search,6390.03,3145.61,2512.65,1.02672,209.887,-nan,-nan,0.9186,0.109901,-1.0,-1.0,27.455,2.97218,1858,6440069 +957,delete,471.5,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1722.47,2.80886,1732,6132470 +958,search,6058.73,3213.07,2428.84,0.987886,219.1,-nan,-nan,0.91577,0.111885,-1.0,-1.0,287.613,2.8138,1755,6132470 +959,delete,57.105,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,131.973,2.80031,1745,6086241 +960,search,6033.7,3162.71,2395.44,0.974807,208.845,-nan,-nan,0.91507,0.1122,-1.0,-1.0,40.211,2.80059,1747,6086241 +961,delete,31.017,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,71.303,2.79491,1742,6064734 +962,search,5953.07,3156.53,2316.76,1.03617,199.885,-nan,-nan,0.91483,0.112263,-1.0,-1.0,37.209,2.79493,1743,6064734 +963,delete,158.957,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,803.718,2.73135,1684,5967543 +964,search,5866.47,3227.48,2337.33,1.05612,199.252,-nan,-nan,0.91392,0.112172,-1.0,-1.0,156.417,2.73352,1698,5967543 +965,delete,71.227,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,284.876,2.70567,1675,5906381 +966,search,5726.36,3198.16,2238.62,1.02678,190.462,-nan,-nan,0.91365,0.112744,-1.0,-1.0,54.424,2.70647,1679,5906381 +967,delete,63.369,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,274.659,2.67995,1664,5851849 +968,search,5767.01,3184.75,2283.18,0.975996,198.299,-nan,-nan,0.91296,0.112752,-1.0,-1.0,33.229,2.68029,1665,5851849 +969,delete,72.565,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,254.658,2.64938,1648,5783023 +970,search,5639.75,3181.04,2199.45,0.993896,183.641,-nan,-nan,0.91173,0.113852,-1.0,-1.0,20.253,2.64938,1648,5783023 +971,delete,363.463,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1478.54,2.50841,1533,5496517 +972,search,5392.85,3245.65,2162.98,0.999485,185.256,-nan,-nan,0.90956,0.115782,-1.0,-1.0,314.529,2.51411,1563,5496517 +973,delete,281.782,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1412.68,2.38647,1449,5237981 +974,search,4988.4,3275.28,1955.26,1.06035,163.835,-nan,-nan,0.90808,0.117182,-1.0,-1.0,393.706,2.39355,1485,5237981 +975,delete,166.726,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,532.583,2.31706,1419,5099794 +976,search,5020.62,3256.58,2013.08,0.980703,173.531,-nan,-nan,0.90567,0.118244,-1.0,-1.0,154.945,2.31898,1435,5099794 +977,delete,60.957,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,251.464,2.28964,1417,5032210 +978,search,4820.34,3207.75,1890.41,1.04606,157.523,-nan,-nan,0.90445,0.119604,-1.0,-1.0,79.405,2.29075,1423,5032210 +979,delete,11.364,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,31.678,2.28823,1421,5020211 +980,search,4830.54,3193.93,1865.24,1.05948,155.354,-nan,-nan,0.90472,0.119337,-1.0,-1.0,17.147,2.28823,1421,5020211 +981,delete,143.889,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,442.455,2.2294,1384,4902291 +982,search,4859.13,3200.45,1936.65,0.947442,166.108,-nan,-nan,0.90189,0.121428,-1.0,-1.0,49.275,2.22977,1386,4902291 +983,delete,51.288,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,54.39,2.22636,1384,4872577 +984,search,4832.63,3172.09,1908.4,0.981904,166.354,-nan,-nan,0.90177,0.121487,-1.0,-1.0,18.703,2.22636,1384,4872577 +985,delete,52.584,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,117.509,2.21711,1377,4836730 +986,search,4761.18,3157.31,1901.78,0.979139,167.171,-nan,-nan,0.90144,0.121503,-1.0,-1.0,54.161,2.21749,1380,4836730 +987,delete,54.231,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,101.425,2.20708,1372,4797164 +988,search,4769.79,3130.89,1882.33,1.00173,163.966,-nan,-nan,0.90107,0.121988,-1.0,-1.0,17.809,2.20708,1372,4797164 +989,delete,20.718,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.263,2.20682,1372,4784789 +990,search,4805.71,3120.6,1892.45,0.979808,163.409,-nan,-nan,0.90102,0.121942,-1.0,-1.0,17.076,2.20682,1372,4784789 +991,delete,107.537,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,451.347,2.16706,1342,4705420 +992,search,4667.93,3181.65,1854.36,0.98716,163.136,-nan,-nan,0.90358,0.120105,-1.0,-1.0,153.76,2.17012,1349,4705420 +993,delete,108.191,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,352.608,2.13383,1326,4621807 +994,search,4457.72,3143.5,1679.52,1.17827,136.231,-nan,-nan,0.90243,0.120421,-1.0,-1.0,113.449,2.13447,1333,4621807 +995,delete,84.258,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,365.114,2.09958,1305,4555849 +996,search,4530.05,3144.65,1803.81,0.993125,164.764,-nan,-nan,0.90129,0.120252,-1.0,-1.0,100.505,2.10018,1313,4555849 +997,delete,34.619,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,174.022,2.0825,1304,4526975 +998,search,4548.43,3121.08,1786.7,0.969827,159.465,-nan,-nan,0.90129,0.120152,-1.0,-1.0,24.721,2.0825,1305,4526975 +999,delete,152.81,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,588.902,2.02119,1260,4408448 +1000,search,4298.02,3140.37,1670.45,1.08036,138.766,-nan,-nan,0.89986,0.121594,-1.0,-1.0,130.056,2.02288,1269,4408448 +1001,delete,147.266,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,574.625,1.96461,1223,4303257 +1002,search,4269.93,3157.01,1699.42,0.96846,148.896,-nan,-nan,0.89822,0.122703,-1.0,-1.0,113.813,1.96626,1230,4303257 +1003,delete,48.4,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,204.819,1.94372,1214,4259551 +1004,search,4215.54,3157.73,1687.02,0.99387,155.443,-nan,-nan,0.89755,0.123073,-1.0,-1.0,45.608,1.94432,1217,4259551 +1005,delete,21.618,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,49.799,1.93998,1213,4241434 +1006,search,4239.33,3142.33,1689.02,0.953466,147.4,-nan,-nan,0.89748,0.122865,-1.0,-1.0,28.943,1.94004,1214,4241434 +1007,delete,237.439,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,852.924,1.8518,1154,4052722 +1008,search,4030.56,3167.05,1606.96,0.951968,148.877,-nan,-nan,0.89389,0.125974,-1.0,-1.0,177.691,1.85428,1167,4052722 +1009,delete,400.833,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1160.72,1.70007,1044,3738432 +1010,search,3730.5,3263.9,1520.88,0.979574,138.27,-nan,-nan,0.89105,0.127954,-1.0,-1.0,299.035,1.70373,1064,3738432 +1011,delete,445.406,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1444.81,1.51,900,3372539 +1012,search,3327.79,3456.31,1394.33,0.994651,129.847,-nan,-nan,0.88827,0.12937,-1.0,-1.0,403.678,1.51753,940,3372539 +1013,delete,8.787,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,23.806,1.51393,937,3361432 +1014,search,3256.11,3274,1288.83,1.09529,109.427,-nan,-nan,0.88819,0.12985,-1.0,-1.0,36.394,1.51405,939,3361432 +1015,delete,322.261,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1038.41,1.38165,829,3091692 +1016,search,2993.93,3436.75,1226.88,0.991675,107.832,-nan,-nan,0.88306,0.133298,-1.0,-1.0,386.46,1.38851,867,3091692 +1017,delete,382.507,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1160.78,1.23177,743,2775418 +1018,search,2708.55,3463.58,1099.85,1.10425,93.5673,-nan,-nan,0.87961,0.136017,-1.0,-1.0,366.919,1.2384,777,2775418 +1019,delete,20.082,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,110.116,1.22983,770,2754073 +1020,search,2802.72,3308.1,1161.96,0.933669,107.287,-nan,-nan,0.87924,0.136131,-1.0,-1.0,88.014,1.23053,776,2754073 +1021,delete,146.755,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,481.99,1.17988,735,2638917 +1022,search,2700.03,3330.63,1121.88,0.977232,105.414,-nan,-nan,0.87781,0.13712,-1.0,-1.0,117.044,1.18209,743,2638917 +1023,delete,110.772,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,277.912,1.14767,706,2556494 +1024,search,2555.4,3358.96,1028.52,1.11012,87.8599,-nan,-nan,0.87566,0.139028,-1.0,-1.0,120.234,1.14931,716,2556494 +1025,insert,48.078,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,107.541,1.16172,725,2594934 +1026,search,2672.78,3334.78,1116.84,0.962735,103.239,-nan,-nan,0.878,0.136862,-1.0,-1.0,46.512,1.16218,727,2594934 +1027,insert,314.266,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,288.17,1.24839,748,2802105 +1028,search,2804.53,3456.45,1179.16,0.935239,110.537,-nan,-nan,0.87726,0.138132,-1.0,-1.0,118.89,1.24842,756,2802105 +1029,insert,644.677,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,855.592,1.43204,842,3203975 +1030,search,3119.26,3570.3,1335.83,0.909046,120.259,-nan,-nan,0.884,0.133913,-1.0,-1.0,256.849,1.43273,859,3203975 +1031,insert,919.918,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1084.81,1.72143,960,3830566 +1032,search,3635.12,3710.88,1585.81,0.953223,138.531,-nan,-nan,0.89287,0.129278,-1.0,-1.0,544.714,1.72385,1000,3830566 +1033,insert,623.51,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,977.758,1.91441,1107,4252669 +1034,search,4098.29,3557.7,1744.19,0.950928,154.621,-nan,-nan,0.89925,0.123747,-1.0,-1.0,567.396,1.91639,1152,4252669 +1035,insert,330.107,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,412.972,2.03014,1190,4512107 +1036,search,4334.61,3460.32,1808.37,0.976492,158.062,-nan,-nan,0.90242,0.121415,-1.0,-1.0,382.037,2.03363,1223,4512107 +1037,insert,14.723,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,54.301,2.03758,1227,4525815 +1038,search,4323.72,3379.88,1770.77,1.03449,153.531,-nan,-nan,0.90372,0.120209,-1.0,-1.0,141.452,2.03785,1237,4525815 +1039,insert,117.15,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,268.671,2.0697,1269,4599215 +1040,search,4467.29,3324.04,1832.81,1.02616,158.706,-nan,-nan,0.90466,0.119572,-1.0,-1.0,103.794,2.07041,1277,4599215 +1041,insert,10.635,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,44.047,2.07423,1280,4609851 +1042,search,4542.56,3306.86,1865.76,0.985717,164.639,-nan,-nan,0.9056,0.118971,-1.0,-1.0,41.524,2.07424,1283,4609851 +1043,insert,26.053,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.398,2.07973,1283,4627411 +1044,search,4521.03,3309.33,1859.76,0.998386,163.375,-nan,-nan,0.90545,0.118883,-1.0,-1.0,14.518,2.07973,1283,4627411 +1045,insert,76.058,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,113.83,2.09771,1294,4670607 +1046,search,4470.41,3322.86,1760.49,1.11386,149.885,-nan,-nan,0.90645,0.118526,-1.0,-1.0,23.632,2.09776,1295,4670607 +1047,insert,5.569,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,13.296,2.09877,1295,4675386 +1048,search,4532.04,3323.11,1842.06,1.03616,154.061,-nan,-nan,0.90645,0.118475,-1.0,-1.0,14.467,2.09877,1295,4675386 +1049,insert,25.409,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,33.061,2.10517,1297,4689998 +1050,search,4579.26,3325.71,1879.13,1.00104,160.24,-nan,-nan,0.90646,0.118473,-1.0,-1.0,35.231,2.10554,1298,4689998 +1051,insert,148.476,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,221.12,2.1459,1325,4783971 +1052,search,4692.87,3326.01,1926.63,1.00946,167.128,-nan,-nan,0.90701,0.117628,-1.0,-1.0,26.734,2.1459,1326,4783971 +1053,insert,52.123,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,116.684,2.16112,1338,4826219 +1054,search,4661.39,3324.66,1864.58,1.07285,160.395,-nan,-nan,0.90748,0.117385,-1.0,-1.0,54.978,2.16135,1341,4826219 +1055,insert,48.768,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,95.967,2.17394,1349,4862355 +1056,search,4618.08,3326.04,1783.22,1.07254,145.838,-nan,-nan,0.90785,0.117274,-1.0,-1.0,16,2.17394,1349,4862355 +1057,insert,7.426,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,52.033,2.17588,1353,4869564 +1058,search,4795.39,3317.6,1968.05,0.97857,175.275,-nan,-nan,0.90805,0.117261,-1.0,-1.0,29.015,2.176,1354,4869564 +1059,insert,46.363,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,57.568,2.18759,1359,4898570 +1060,search,4828.7,3324.9,1988.06,0.990374,175.473,-nan,-nan,0.90829,0.117018,-1.0,-1.0,23.836,2.1876,1360,4898570 +1061,insert,10.147,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,44.294,2.18975,1364,4907501 +1062,search,4773.58,3308.95,1913.87,1.05521,168.738,-nan,-nan,0.90783,0.117346,-1.0,-1.0,16.442,2.18975,1364,4907501 +1063,insert,12.151,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,27.657,2.19289,1365,4916386 +1064,search,4739.68,3311.56,1895.35,1.1427,159.251,-nan,-nan,0.90798,0.117489,-1.0,-1.0,15.971,2.19289,1365,4916386 +1065,insert,444.241,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,577.71,2.31464,1420,5194839 +1066,search,4944.97,3359.49,2001.48,1.05795,170.613,-nan,-nan,0.90932,0.116905,-1.0,-1.0,154.424,2.31579,1430,5194839 +1067,insert,8.921,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.023,2.31791,1433,5200182 +1068,search,4961.17,3331.59,1976.41,1.07842,165.293,-nan,-nan,0.90949,0.116802,-1.0,-1.0,86.149,2.31829,1438,5200182 +1069,insert,61.758,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,94.61,2.33325,1449,5230883 +1070,search,5099.12,3310.45,2093.4,1.0162,192.135,-nan,-nan,0.90989,0.116243,-1.0,-1.0,56.458,2.33363,1452,5230883 +1071,insert,39.564,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,36.282,2.3437,1454,5255659 +1072,search,5049.64,3312.63,2025.16,1.0514,175.539,-nan,-nan,0.91006,0.11625,-1.0,-1.0,16.289,2.3437,1454,5255659 +1073,insert,51.797,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,101.323,2.35512,1463,5281693 +1074,search,5006.06,3309.14,1963.72,1.04027,159.591,-nan,-nan,0.91036,0.116023,-1.0,-1.0,31.482,2.35527,1464,5281693 +1075,insert,13.166,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,48.938,2.35744,1468,5290081 +1076,search,5097.87,3304.06,2025.63,1.05547,176.79,-nan,-nan,0.91085,0.115649,-1.0,-1.0,23.845,2.35759,1469,5290081 +1077,insert,47.467,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,69.261,2.3691,1475,5316166 +1078,search,4906.83,3295.71,1882.89,1.21724,148.633,-nan,-nan,0.91078,0.115674,-1.0,-1.0,16.023,2.3691,1475,5316166 +1079,insert,39.567,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,63.614,2.37707,1481,5335531 +1080,search,4923.31,3295.04,1905.25,1.14123,152.212,-nan,-nan,0.91103,0.115809,-1.0,-1.0,40.06,2.37719,1483,5335531 +1081,insert,608.642,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,611.872,2.52933,1554,5678781 +1082,search,5516.94,3337.33,2250.01,0.997154,189.884,-nan,-nan,0.91258,0.114523,-1.0,-1.0,103.887,2.53052,1562,5678781 +1083,insert,0.769,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.664,2.53052,1562,5679848 +1084,search,5510.45,3326.41,2256.97,0.981054,202.607,-nan,-nan,0.9126,0.114631,-1.0,-1.0,16.511,2.53052,1562,5679848 +1085,insert,30.65,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,76.03,2.53931,1569,5702205 +1086,search,5361.33,3322.77,2125.22,1.07094,176.019,-nan,-nan,0.91299,0.114292,-1.0,-1.0,38.613,2.53976,1570,5702205 +1087,insert,24.131,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,68.953,2.54491,1575,5715514 +1088,search,5595.16,3319.28,2279.8,0.970465,206.337,-nan,-nan,0.91329,0.113853,-1.0,-1.0,17.817,2.54491,1575,5715514 +1089,insert,667.81,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,864.758,2.71515,1687,6103499 +1090,search,5814.67,3328.28,2327.59,1.07773,193.562,-nan,-nan,0.91638,0.111713,-1.0,-1.0,192.262,2.71749,1706,6103499 +1091,insert,80.521,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,116.452,2.73889,1720,6152505 +1092,search,5963.84,3296.27,2376.48,1.09394,200.905,-nan,-nan,0.91722,0.111277,-1.0,-1.0,19.521,2.73889,1720,6152505 +1093,insert,32.638,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,42.474,2.74849,1723,6171104 +1094,search,6024.88,3300.12,2421.68,1.04119,208.539,-nan,-nan,0.91762,0.110904,-1.0,-1.0,40.936,2.74895,1725,6171104 +1095,insert,572.161,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,692.198,2.92012,1804,6557850 +1096,search,6272.3,3346.91,2530.14,1.01769,213.645,-nan,-nan,0.91929,0.109913,-1.0,-1.0,221.078,2.92157,1822,6557850 +1097,insert,28.058,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,33.77,2.92859,1824,6579289 +1098,search,6299.03,3323.67,2509.57,1.06264,203.352,-nan,-nan,0.91982,0.109329,-1.0,-1.0,87.855,2.9292,1831,6579289 +1099,insert,19.301,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,58.169,2.93447,1836,6591442 +1100,search,6593.83,3304.5,2583.93,1.03641,220.575,-nan,-nan,0.92,0.109339,-1.0,-1.0,37.863,2.93477,1838,6591442 +1101,insert,62.733,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,132.143,2.95365,1852,6633739 +1102,search,6684.37,3302.49,2647.84,0.986209,225.982,-nan,-nan,0.92077,0.108636,-1.0,-1.0,46.225,2.95406,1855,6633739 +1103,insert,122.51,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,249.702,2.99537,1884,6724357 +1104,search,6709.13,3298.29,2591.52,1.09112,215.645,-nan,-nan,0.92167,0.10802,-1.0,-1.0,66.194,2.99557,1889,6724357 +1105,insert,2.923,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,20.83,2.99617,1889,6727059 +1106,search,6812.29,3288.67,2679.29,0.968912,225.843,-nan,-nan,0.92176,0.107995,-1.0,-1.0,21.956,2.99617,1889,6727059 +1107,insert,319.029,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,485.575,3.0967,1946,6950869 +1108,search,6878.34,3281.28,2646.22,1.09247,219.76,-nan,-nan,0.92238,0.107373,-1.0,-1.0,81.897,3.09697,1949,6950869 +1109,insert,9.217,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.885,3.09899,1949,6958173 +1110,search,6929.42,3281.37,2661.75,1.0485,223.111,-nan,-nan,0.92229,0.107424,-1.0,-1.0,25.024,3.09899,1949,6958173 +1111,insert,99.069,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,105.408,3.1151,1959,6995273 +1112,search,7120.23,3284.26,2800.23,0.976731,254.32,-nan,-nan,0.92241,0.107396,-1.0,-1.0,58.425,3.11591,1962,6995273 +1113,insert,75.696,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,76.863,3.13403,1969,7047862 +1114,search,7135.95,3295.47,2811.39,1.00545,250.609,-nan,-nan,0.92243,0.107274,-1.0,-1.0,24.235,3.13403,1969,7047862 +1115,insert,95.421,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,149.148,3.15923,1984,7110727 +1116,search,7156.6,3301.29,2824.64,1.0136,247.969,-nan,-nan,0.92281,0.106985,-1.0,-1.0,37.233,3.15941,1985,7110727 +1117,insert,21.503,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,30.346,3.16397,1986,7125591 +1118,search,7224.79,3307.18,2848.59,0.990331,259.919,-nan,-nan,0.92315,0.106694,-1.0,-1.0,48.299,3.16421,1988,7125591 +1119,insert,14.116,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,41.78,3.16724,1990,7134442 +1120,search,7089.19,3305.56,2749.74,1.0436,229.596,-nan,-nan,0.92307,0.106772,-1.0,-1.0,23.52,3.16724,1990,7134442 +1121,insert,37.822,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,59.999,3.17599,1993,7162109 +1122,search,7173.12,3311.51,2806.26,1.00045,239.63,-nan,-nan,0.92324,0.106748,-1.0,-1.0,72.569,3.17645,1998,7162109 +1123,insert,21.951,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,47.038,3.18172,2001,7179509 +1124,search,7185.18,3303.15,2809.41,1.03446,242.301,-nan,-nan,0.92329,0.106823,-1.0,-1.0,34.194,3.18194,2002,7179509 +1125,insert,11.138,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,44.157,3.1845,2003,7186372 +1126,search,7095.2,3303.67,2734.69,1.04877,226.589,-nan,-nan,0.92326,0.106876,-1.0,-1.0,23.988,3.1845,2003,7186372 +1127,insert,67.984,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,117.111,3.20323,2012,7231450 +1128,search,7211.26,3304.55,2810.83,1.01033,237.144,-nan,-nan,0.92328,0.106847,-1.0,-1.0,27.184,3.20323,2012,7231450 +1129,insert,26.685,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,32.604,3.20896,2013,7243215 +1130,search,7281.47,3307.47,2863.29,0.9685,250.935,-nan,-nan,0.92314,0.106861,-1.0,-1.0,27.978,3.20896,2013,7243215 +1131,insert,86.453,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,112.221,3.22169,2021,7276107 +1132,search,7330.66,3306.98,2895.27,0.939648,256.402,-nan,-nan,0.92317,0.106767,-1.0,-1.0,59.215,3.22227,2023,7276107 +1133,insert,42.902,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,42.434,3.23004,2025,7292577 +1134,search,7362.35,3308.85,2910.49,0.971741,264.361,-nan,-nan,0.9235,0.106331,-1.0,-1.0,36.105,3.23005,2026,7292577 +1135,insert,12.942,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,40.981,3.23282,2027,7299867 +1136,search,7174.89,3308.17,2794.28,1.04683,236.803,-nan,-nan,0.92344,0.106411,-1.0,-1.0,25.02,3.23282,2027,7299867 +1137,insert,48.469,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,62.128,3.24199,2031,7323342 +1138,search,7223.88,3312.61,2801.57,1.0075,235.056,-nan,-nan,0.92359,0.106196,-1.0,-1.0,26.051,3.24199,2031,7323342 +1139,insert,6.397,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,22.731,3.24339,2031,7325844 +1140,search,7304.36,3313.38,2845.72,0.990627,241.834,-nan,-nan,0.92357,0.106259,-1.0,-1.0,25.549,3.24339,2031,7325844 +1141,insert,405.015,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,305.944,3.33599,2059,7540761 +1142,search,7432.62,3348.57,2925.98,0.987705,248.693,-nan,-nan,0.92434,0.105681,-1.0,-1.0,106.179,3.33672,2069,7540761 +1143,insert,31.921,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,68.281,3.34432,2074,7561926 +1144,search,7509.61,3332.24,2971.85,0.997533,252.334,-nan,-nan,0.92437,0.105653,-1.0,-1.0,38.166,3.34448,2075,7561926 +1145,insert,13.176,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,55.372,3.34835,2079,7571861 +1146,search,7522.37,3333.67,2965.44,0.988849,253.988,-nan,-nan,0.92447,0.105514,-1.0,-1.0,41.019,3.34881,2080,7571861 +1147,insert,31.151,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,73.58,3.35643,2085,7590881 +1148,search,7483.77,3334.07,2929.86,1.00613,246.526,-nan,-nan,0.92428,0.105764,-1.0,-1.0,30.082,3.35643,2085,7590881 +1149,insert,10,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,34.604,3.35806,2086,7595507 +1150,search,7484.11,3333.97,2913.46,1.01552,247.452,-nan,-nan,0.92456,0.105642,-1.0,-1.0,44.738,3.35816,2087,7595507 +1151,insert,10.512,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,72.307,3.36167,2091,7601664 +1152,search,7464.79,3326.45,2892.66,1.02096,241.524,-nan,-nan,0.92432,0.105841,-1.0,-1.0,28.404,3.36167,2091,7601664 +1153,delete,55.568,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,139.354,3.35418,2086,7566707 +1154,search,7543.34,3322.7,2955.52,1.02292,254.88,-nan,-nan,0.92459,0.105484,-1.0,-1.0,27.77,3.35418,2086,7566707 +1155,delete,216.342,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,967.731,3.28169,2039,7396192 +1156,search,7443.03,3325.01,2936.72,0.938122,259.37,-nan,-nan,0.92456,0.104777,-1.0,-1.0,180.287,3.28379,2049,7396192 +1157,delete,421.191,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1989.07,3.14432,1902,7079041 +1158,search,7023.06,3398.55,2798.03,1.00183,251.059,-nan,-nan,0.92278,0.107307,-1.0,-1.0,366.012,3.15059,1935,7079041 +1159,delete,513.676,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1803.22,2.93743,1740,6597115 +1160,search,6307.21,3449.98,2520.94,1.0244,212.103,-nan,-nan,0.91661,0.11137,-1.0,-1.0,271.96,2.94242,1763,6597115 +1161,delete,293.061,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1728.43,2.81179,1640,6308902 +1162,search,5902.35,3488,2314.52,1.0846,193.108,-nan,-nan,0.91451,0.112873,-1.0,-1.0,328.689,2.81687,1670,6308902 +1163,delete,124.905,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,823.807,2.75451,1629,6164514 +1164,search,5849.07,3461.07,2274.79,1.10434,185.277,-nan,-nan,0.91516,0.112377,-1.0,-1.0,245.802,2.75895,1652,6164514 +1165,delete,37.036,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,142.558,2.74823,1643,6133412 +1166,search,5988.73,3415.07,2385.68,0.988888,199.961,-nan,-nan,0.91542,0.112265,-1.0,-1.0,37.035,2.74847,1644,6133412 +1167,delete,105.748,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,154.211,2.73422,1634,6057218 +1168,search,5854.24,3380.87,2283.02,1.12455,195.573,-nan,-nan,0.9147,0.112655,-1.0,-1.0,41.854,2.73431,1636,6057218 +1169,delete,29.121,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,156.51,2.71739,1623,6028053 +1170,search,5962.01,3382.2,2383.99,0.94943,210.724,-nan,-nan,0.9116,0.114816,-1.0,-1.0,29.377,2.71742,1624,6028053 +1171,delete,38.37,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,94.725,2.71087,1620,6001880 +1172,search,5780.52,3372,2235.28,1.09752,187.052,-nan,-nan,0.91103,0.115221,-1.0,-1.0,44.388,2.71135,1622,6001880 +1173,delete,60.836,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,152.472,2.69911,1615,5957180 +1174,search,5680.12,3352.07,2158.58,1.18141,177.291,-nan,-nan,0.91089,0.115137,-1.0,-1.0,49.412,2.69947,1617,5957180 +1175,delete,25.601,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,67.081,2.6976,1616,5939326 +1176,search,5891.83,3338.95,2350.7,0.975489,206.686,-nan,-nan,0.91075,0.115415,-1.0,-1.0,34.155,2.698,1617,5939326 +1177,delete,15.72,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,60.87,2.69395,1614,5924145 +1178,search,5889.81,3335.09,2343.97,0.975365,208.03,-nan,-nan,0.91051,0.11555,-1.0,-1.0,19.63,2.69395,1614,5924145 +1179,delete,87.103,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,329.514,2.66451,1594,5863334 +1180,search,5794.23,3336.17,2316.06,0.983663,199.886,-nan,-nan,0.90999,0.116168,-1.0,-1.0,82.516,2.66585,1600,5863334 +1181,delete,44.237,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,105.873,2.65789,1593,5822703 +1182,search,5757.27,3311.86,2271.17,1.00826,188.214,-nan,-nan,0.90998,0.116349,-1.0,-1.0,45.175,2.65857,1595,5822703 +1183,delete,54.532,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,164.76,2.64444,1585,5775807 +1184,search,5679.92,3296.93,2225.8,1.05659,189.712,-nan,-nan,0.90855,0.117145,-1.0,-1.0,18.887,2.64444,1585,5775807 +1185,delete,35.461,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,159.434,2.63346,1577,5750627 +1186,search,5643.03,3301.73,2228.47,0.997543,188.596,-nan,-nan,0.90846,0.117283,-1.0,-1.0,18.278,2.63346,1577,5750627 +1187,delete,189.742,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,974.26,2.55761,1508,5601207 +1188,search,5495.55,3346.41,2202.33,0.970026,188.976,-nan,-nan,0.90781,0.118192,-1.0,-1.0,149.702,2.56024,1520,5601207 +1189,delete,14.492,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.971,2.56024,1520,5586425 +1190,search,5433.16,3312.43,2102.03,1.09028,172.598,-nan,-nan,0.90771,0.117859,-1.0,-1.0,28.978,2.56053,1521,5586425 +1191,delete,12.315,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,55.706,2.55766,1519,5574023 +1192,search,5497.99,3311.53,2181.97,1.01434,184.665,-nan,-nan,0.90775,0.117789,-1.0,-1.0,17.224,2.55766,1519,5574023 +1193,delete,260.604,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1156.1,2.44979,1440,5356157 +1194,search,5145.82,3354.86,1990.72,1.11528,162.54,-nan,-nan,0.90507,0.119467,-1.0,-1.0,223.797,2.45338,1459,5356157 +1195,delete,17.15,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,15.555,2.45303,1459,5340404 +1196,search,5276.66,3294.72,2081.21,1.01048,174.121,-nan,-nan,0.90587,0.118693,-1.0,-1.0,64.412,2.45391,1463,5340404 +1197,delete,64.304,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,48.136,2.44567,1460,5293259 +1198,search,5260.62,3253.79,2063.73,1.01772,173.584,-nan,-nan,0.90591,0.118713,-1.0,-1.0,18.179,2.44567,1460,5293259 +1199,delete,19.131,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,75.804,2.43974,1456,5279787 +1200,search,5169.58,3252.39,1968.65,1.06356,163.007,-nan,-nan,0.90551,0.119172,-1.0,-1.0,17.462,2.43974,1456,5279787 +1201,delete,86.541,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,314.478,2.4125,1432,5216567 +1202,search,5239.66,3265,2052.78,0.992116,179.157,-nan,-nan,0.90466,0.119681,-1.0,-1.0,60.524,2.41307,1436,5216567 +1203,delete,27.173,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,16.017,2.41282,1436,5196947 +1204,search,5161.07,3234.12,1996.63,1.05338,165.877,-nan,-nan,0.90412,0.120407,-1.0,-1.0,27.286,2.41287,1437,5196947 +1205,delete,31.736,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,48.712,2.40374,1434,5168447 +1206,search,5055.24,3224.55,1908.59,1.08841,160.052,-nan,-nan,0.90372,0.12055,-1.0,-1.0,17.39,2.40374,1434,5168447 +1207,delete,32.333,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,103.052,2.39468,1426,5146927 +1208,search,5165.93,3226.44,2016.14,0.986529,173.473,-nan,-nan,0.90295,0.120914,-1.0,-1.0,16.626,2.39468,1426,5146927 +1209,delete,361.678,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,984.724,2.25497,1356,4887612 +1210,search,4768.37,3243.08,1807.76,1.16204,148.339,-nan,-nan,0.90146,0.122177,-1.0,-1.0,102.297,2.25621,1364,4887612 +1211,delete,7.281,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.761,2.25543,1364,4882301 +1212,search,4923.42,3223.36,1914.77,0.985325,165.262,-nan,-nan,0.90203,0.121672,-1.0,-1.0,28.408,2.25561,1365,4882301 +1213,delete,143.252,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,551.843,2.19956,1323,4780915 +1214,search,4815.28,3250.38,1892.83,1.07093,157.663,-nan,-nan,0.90026,0.122278,-1.0,-1.0,122.189,2.20147,1332,4780915 +1215,delete,16.27,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,14.75,2.20063,1332,4765751 +1216,search,4780.1,3218.27,1877.75,0.98943,160.969,-nan,-nan,0.90016,0.122775,-1.0,-1.0,15.256,2.20063,1332,4765751 +1217,delete,453.226,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,1478.76,2.03809,1194,4452306 +1218,search,4417.17,3332.31,1755.87,0.993145,152.889,-nan,-nan,0.89603,0.127099,-1.0,-1.0,182.27,2.04044,1209,4452306 +1219,delete,110.441,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,438.763,1.98695,1162,4362045 +1220,search,4265.05,3362.47,1721.96,0.994977,146.241,-nan,-nan,0.8948,0.12708,-1.0,-1.0,210.83,1.9889,1180,4362045 +1221,delete,36.339,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,136.84,1.97403,1170,4326917 +1222,search,4271.98,3317.6,1704.32,1.00206,145.292,-nan,-nan,0.89512,0.126652,-1.0,-1.0,39.537,1.97428,1172,4326917 +1223,delete,242.413,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,784.705,1.87751,1100,4109781 +1224,search,4069.81,3351.54,1637.43,0.989783,136.116,-nan,-nan,0.8924,0.128785,-1.0,-1.0,233.098,1.8813,1120,4109781 +1225,delete,30.97,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,147.159,1.86741,1111,4075879 +1226,search,4065.94,3290.38,1613.99,0.99099,143.662,-nan,-nan,0.89235,0.128833,-1.0,-1.0,38.932,1.86771,1113,4075879 +1227,delete,85.22,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,450.104,1.82742,1079,4006503 +1228,search,3938.54,3326.96,1537.24,1.05298,127.797,-nan,-nan,0.89196,0.128835,-1.0,-1.0,187.96,1.83092,1091,4006503 +1229,delete,104.809,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,552.628,1.78283,1047,3922625 +1230,search,3809.52,3376.01,1483.64,1.03636,123.903,-nan,-nan,0.8911,0.129504,-1.0,-1.0,190.834,1.78542,1063,3922625 +1231,delete,93.065,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,350.858,1.74404,1021,3844860 +1232,search,3707.75,3394.91,1439.33,1.08488,118.389,-nan,-nan,0.88755,0.13071,-1.0,-1.0,85.758,1.74528,1027,3844860 +1233,delete,16.404,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,80.179,1.73967,1023,3831414 +1234,search,3753.76,3373.82,1467.42,1.01453,120.894,-nan,-nan,0.88814,0.130309,-1.0,-1.0,49.116,1.7399,1026,3831414 +1235,delete,183.006,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,546.858,1.6726,982,3684234 +1236,search,3649.04,3360.2,1469.96,0.961133,130.727,-nan,-nan,0.88465,0.133314,-1.0,-1.0,48.839,1.67306,985,3684234 +1237,delete,17.414,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,47.96,1.6698,983,3669074 +1238,search,3550.42,3351.03,1390.91,1.11508,116.535,-nan,-nan,0.88442,0.133329,-1.0,-1.0,16.484,1.6698,983,3669074 +1239,delete,65.993,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,139.728,1.65343,972,3620984 +1240,search,3523.24,3336.1,1375.8,1.07562,114.24,-nan,-nan,0.88439,0.133729,-1.0,-1.0,44.839,1.65366,974,3620984 +1241,delete,42.117,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,39.188,1.65003,972,3580113 +1242,search,3561.15,3287.69,1380.16,1.03778,117.579,-nan,-nan,0.88375,0.134318,-1.0,-1.0,16.264,1.65003,972,3580113 +1243,delete,63.968,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,202.022,1.62402,955,3533909 +1244,search,3414.45,3291.95,1283.86,1.13321,107.624,-nan,-nan,0.88211,0.135469,-1.0,-1.0,41.585,1.62416,957,3533909 +1245,delete,19.441,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,48.1,1.61985,954,3519590 +1246,search,3489.25,3278.98,1377.36,1.00551,122.183,-nan,-nan,0.8814,0.136149,-1.0,-1.0,26.701,1.61994,955,3519590 +1247,delete,78.46,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,104.602,1.58542,934,3452537 +1248,search,3434.4,3294.81,1337.17,1.01963,121.524,-nan,-nan,0.88329,0.133633,-1.0,-1.0,37.563,1.58571,936,3452537 +1249,delete,44.755,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,115.591,1.57145,928,3416711 +1250,search,3369.2,3282.89,1326.02,0.981666,117.11,-nan,-nan,0.88309,0.134278,-1.0,-1.0,54.987,1.57188,931,3416711 +1251,delete,33.386,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,70.685,1.56289,926,3387653 +1252,search,3370.56,3255.25,1324.68,1.00347,113.995,-nan,-nan,0.88292,0.134163,-1.0,-1.0,15.776,1.56289,926,3387653 +1253,delete,17.643,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,34.525,1.5572,924,3372157 +1254,search,3395.95,3252.81,1351.82,0.981625,117.847,-nan,-nan,0.88314,0.134194,-1.0,-1.0,15.688,1.5572,924,3372157 +1255,delete,159.625,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,190.399,1.50097,894,3258215 +1256,search,3303.88,3257.48,1312.19,1.00852,118.017,-nan,-nan,0.88056,0.135672,-1.0,-1.0,73.27,1.50197,899,3258215 +1257,delete,42.455,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,68.465,1.48795,892,3223657 +1258,search,3271.39,3233.26,1298.37,0.993107,114.707,-nan,-nan,0.87874,0.13664,-1.0,-1.0,29.477,1.48809,893,3223657 +1259,delete,59.483,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,248.99,1.46178,873,3174982 +1260,search,3249.97,3252.55,1280.49,0.962714,118.199,-nan,-nan,0.8778,0.137564,-1.0,-1.0,69.719,1.46258,877,3174982 +1261,delete,21.915,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,33.255,1.45887,875,3157038 +1262,search,3158.38,3225.6,1214.98,1.03043,105.064,-nan,-nan,0.87708,0.137953,-1.0,-1.0,26.904,1.45907,876,3157038 +1263,delete,131.984,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,426.831,1.40604,849,3058847 +1264,search,3062.54,3234.2,1187.49,1.04133,102.431,-nan,-nan,0.87528,0.13865,-1.0,-1.0,140.08,1.40805,858,3058847 +1265,delete,377.189,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,447.302,1.28369,769,2833531 +1266,search,2790.7,3293.27,1085.19,1.03725,88.457,-nan,-nan,0.86944,0.141716,-1.0,-1.0,123.728,1.28531,778,2833531 +1267,delete,85.054,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,208.316,1.25558,761,2763793 +1268,search,2826.78,3249.98,1131.6,0.937225,92.8669,-nan,-nan,0.86962,0.141183,-1.0,-1.0,66.865,1.25629,766,2763793 +1269,delete,194.337,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,317.011,1.19293,744,2627384 +1270,search,2792.27,3249.21,1106.81,1.01348,100.432,-nan,-nan,0.87145,0.139948,-1.0,-1.0,121.768,1.19454,754,2627384 +1271,delete,85.394,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,282.523,1.16142,732,2553476 +1272,search,2653.41,3218.66,1025.15,1.07635,87.7101,-nan,-nan,0.87065,0.140114,-1.0,-1.0,70.427,1.16203,737,2553476 +1273,delete,147.218,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,366.064,1.10689,701,2443067 +1274,search,2623.55,3244.78,1041.69,0.924011,87.3063,-nan,-nan,0.8693,0.141142,-1.0,-1.0,116.505,1.10899,709,2443067 +1275,delete,24.205,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,69.853,1.10579,704,2421654 +1276,search,2594.78,3194.67,1026.42,0.972842,88.2316,-nan,-nan,0.86811,0.142092,-1.0,-1.0,11.362,1.10579,704,2421654 +1277,delete,36.116,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,59.86,1.09782,699,2391396 +1278,search,2596.54,3172.71,1013.13,0.991487,90.3313,-nan,-nan,0.86699,0.142353,-1.0,-1.0,28.622,1.09796,700,2391396 +1279,delete,41.563,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,70.54,1.09206,695,2367854 +1280,search,2571.7,3151.31,1006.18,0.909026,91.2504,-nan,-nan,0.86739,0.141983,-1.0,-1.0,12.466,1.09206,695,2367854 diff --git a/scripts/big_ann_perf_numbers/test_perf_debug_scan_0.15_batch_versus_nonbatch.csv b/scripts/big_ann_perf_numbers/test_perf_debug_scan_0.15_batch_versus_nonbatch.csv new file mode 100644 index 00000000..449a240f --- /dev/null +++ b/scripts/big_ann_perf_numbers/test_perf_debug_scan_0.15_batch_versus_nonbatch.csv @@ -0,0 +1,26 @@ +num_search_workers,step_num,step_type,search_latency_ms,worker_partition_size,worker_scan_throughput,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors +1,2,search,4777.65,37.312,5.22065,0.8551,0.130106,-1.0,-1.0,10.209,0.0184881,1000,38806 +1,3,insert,20.279,-1.0,-1.0,-1.0,-1.0,13.566,0.022655,996,49334 +1,4,search,5412.6,66.2949,6.35675,0.86686,0.128427,-1.0,-1.0,14.288,0.0226615,993,49334 +1,5,insert,18.384,-1.0,-1.0,-1.0,-1.0,160.439,0.033017,984,73557 +1,6,search,7588.72,195.098,7.56384,0.92515,0.103785,-1.0,-1.0,131.68,0.0331618,986,73557 +1,7,insert,32.757,-1.0,-1.0,-1.0,-1.0,124.282,0.0426658,987,95766 +1,8,search,8668.57,246.309,8.5135,0.92462,0.103689,-1.0,-1.0,30.552,0.0431782,992,95766 +1,9,insert,5.095,-1.0,-1.0,-1.0,-1.0,7.963,0.0449738,993,100025 +1,10,search,8844.58,249.764,8.77157,0.92358,0.103018,-1.0,-1.0,21.31,0.0450048,997,100025 +1,11,insert,11.355,-1.0,-1.0,-1.0,-1.0,7.957,0.049765,998,111417 +1,12,search,9715.83,289.247,9.18782,0.92349,0.103712,-1.0,-1.0,19.631,0.0500151,1002,111417 +1,13,insert,24.324,-1.0,-1.0,-1.0,-1.0,8.25,0.0610939,1003,137139 +1,14,search,10624.4,327.04,9.65616,0.92001,0.105792,-1.0,-1.0,11.911,0.0610947,1004,137139 +1,15,insert,21.929,-1.0,-1.0,-1.0,-1.0,9.06,0.0697341,1005,156710 +1,16,search,11085.7,363.254,10.1911,0.91823,0.106214,-1.0,-1.0,8.385,0.0697341,1005,156710 +1,17,insert,5.59,-1.0,-1.0,-1.0,-1.0,8.888,0.0715754,1006,161773 +1,18,search,8987.83,378.453,14.0274,0.91928,0.106082,-1.0,-1.0,12.499,0.0715754,1006,161773 +1,19,insert,48.525,-1.0,-1.0,-1.0,-1.0,10.652,0.0786999,1008,177161 +1,20,search,9641.55,418.638,14.3387,0.92342,0.102778,-1.0,-1.0,6.701,0.0786999,1008,177161 +1,21,insert,319.002,-1.0,-1.0,-1.0,-1.0,188.001,0.201592,1064,448898 +1,22,search,15838,823.636,16.2556,0.90938,0.10831,-1.0,-1.0,61.443,0.201591,1075,448898 +1,23,insert,400.325,-1.0,-1.0,-1.0,-1.0,276.861,0.32912,1141,733493 +1,24,search,20866.1,1101.08,16.8979,0.90518,0.110083,-1.0,-1.0,108.586,0.329715,1170,733493 +1,25,insert,40.431,-1.0,-1.0,-1.0,-1.0,24.171,0.346687,1175,777443 +1,26,search,21144.6,1048.89,16.971,0.90193,0.112609,-1.0,-1.0,31.049,0.346928,1181,777443 diff --git a/scripts/big_ann_perf_visualize.py b/scripts/big_ann_perf_visualize.py new file mode 100644 index 00000000..ef188fea --- /dev/null +++ b/scripts/big_ann_perf_visualize.py @@ -0,0 +1,282 @@ +import pandas as pd +import matplotlib.pyplot as plt +import os +import numpy as np +import matplotlib.cm as cm +import matplotlib.ticker as mticker +from matplotlib.ticker import ScalarFormatter, LogLocator, FormatStrFormatter + +SECONDS_TO_MS = 10.0 ** 3 +MIN_TO_SECONDS = 60 +MIN_TO_MS = MIN_TO_SECONDS * SECONDS_TO_MS + +SRC_DIR = "big_ann_perf_numbers" +DST_DIR = "bigann_perf_vis" + +NUM_OPERATIONS_TO_VISUALIZE = 1300 +def visualize_perf(src_dataset, config_details): + # Load CSV + df = pd.read_csv(os.path.join(SRC_DIR, src_dataset)) + df = df.head(NUM_OPERATIONS_TO_VISUALIZE) + df = df.fillna(0) + + # Filter rows for step_type == "search" + search_df = df[df['step_type'] == 'search'] + + # Extract data + x = search_df['step_num'].values + recall_mean = np.round(search_df['recall_mean'].values, 2) + latency_search = search_df['latency_ms'].values + + insert_df = df[df['step_type'] == 'insert'] + latency_insert = insert_df['latency_ms'].values + x_insert = insert_df['step_num'].values + + delete_df = df[df['step_type'] == 'delete'] + latency_delete = delete_df['latency_ms'].values + x_delete = delete_df['step_num'].values + + x_maintenance = df['step_num'].values + latency_maintenance = df['mainteance_ms'].values + + # Convert sums to minutes + total_search_min = latency_search.sum() / MIN_TO_MS + total_delete_min = latency_delete.sum() / MIN_TO_MS + total_insert_min = latency_insert.sum() / MIN_TO_MS + total_maintenance_min = latency_maintenance.sum() / MIN_TO_MS + total_all_min = total_search_min + total_delete_min + total_insert_min + total_maintenance_min + + # Create 3 rows x 2 columns subplots + fig, axs = plt.subplots(4, 2, figsize=(12, 12), sharex=True) + + # Row 0 + axs[0, 0].plot(x, recall_mean, color='green') + axs[0, 0].set_ylabel('Recall@10') + axs[0, 0].set_title('Recall Mean') + print("Average Recall for run", src_dataset, "is", np.mean(recall_mean)) + + axs[0, 1].plot(x, latency_search, color='purple') + axs[0, 1].set_ylabel('Latency (ms)') + axs[0, 1].set_title(f'Search Latency (Total: {total_search_min:.2f} min)') + + # Row 1 + axs[1, 0].plot(x_delete, latency_delete, color='red') + axs[1, 0].set_ylabel('Latency (ms)') + axs[1, 0].set_title(f'Delete Latency (Total: {total_delete_min:.2f} min)') + + axs[1, 1].plot(x_insert, latency_insert, color='orange') + axs[1, 1].set_ylabel('Latency (ms)') + axs[1, 1].set_title(f'Insert Latency (Total: {total_insert_min:.2f} min)') + + # Row 2 + axs[2, 0].plot(x_maintenance, latency_maintenance, color='blue') + axs[2, 0].set_xlabel('Step Num') + axs[2, 0].set_ylabel('Latency (ms)') + axs[2, 0].set_title(f'Maintenance Latency (Total: {total_maintenance_min:.2f} min)') + + axs[2, 0].plot(x_maintenance, latency_maintenance, color='blue') + + # Bottom right subplot: Vectors and Partitions over time + non_search_df = df[df['step_type'] != 'search'] + ax1 = axs[2, 1] + ax1.plot(non_search_df['step_num'].values, non_search_df['num_vectors'].values, color='olive', label='Num Vectors') + ax1.set_xlabel('Step Num') + ax1.set_ylabel('Num Vectors', color='olive') + ax1.tick_params(axis='y', labelcolor='olive') + + # Create second y-axis sharing the same x-axis + ax2 = ax1.twinx() + + # Plot num_partitions on right y-axis + ax2.plot(non_search_df['step_num'].values, non_search_df['num_partitions'].values, color='brown', label='Num Partitions') + ax2.set_ylabel('Num Partitions', color='brown') + ax2.tick_params(axis='y', labelcolor='brown') + + lines_1, labels_1 = ax1.get_legend_handles_labels() + lines_2, labels_2 = ax2.get_legend_handles_labels() + ax1.legend(lines_1 + lines_2, labels_1 + labels_2) + + axs[2, 1].set_title('Vector and Partition Count') + + # Plot the scan percentage needed to cover the ground truth vectors + axs[3, 0].plot(search_df['step_num'].values, search_df['gt_scan_mean'].values, color='green') + axs[3, 0].fill_between(search_df['step_num'].values, search_df['gt_scan_mean'].values - search_df['gt_scan_dev'].values, search_df['gt_scan_mean'].values + search_df['gt_scan_dev'].values, + color='green', alpha=0.3) + axs[3, 0].set_xlabel('Step Num') + axs[3, 0].set_ylabel('Partition Scan Percentage') + axs[3, 0].set_title('Rank of GT partitions as Search Candidates') + + # Plot Scan Throughput + axs[3, 1].plot(search_df['step_num'].values, search_df['worker_scan_throughput'].values, color='gray') + axs[3, 1].set_xlabel('Step Num') + axs[3, 1].set_ylabel('Scan Throughput (GB/s)') + axs[3, 1].set_title('Worker Scan Throughput') + + # Overall title including total latency sum in minutes + fig.suptitle(f'{config_details}\nTotal Latency: {total_all_min:.2f} minutes', fontsize=16) + + fig.tight_layout() + plt.savefig(os.path.join(DST_DIR, src_dataset.replace("csv", "png")), dpi=300) + +def visualize_percentage_variation(src_dataset): + # Load CSV + df = pd.read_csv(os.path.join(SRC_DIR, src_dataset)) + search_df = df[df['step_type'] == 'search'] + + # Get unique nprobe_percentage values + nprobe_values = sorted(search_df['nprobe_percentage'].unique()) + colors = plt.rcParams['axes.prop_cycle'].by_key()['color'] + fig, axes = plt.subplots(2, 1, figsize=(10, 8), sharex=True) + + # Top subplot: latency_ms vs step_num + for i, nprobe in enumerate(nprobe_values): + sub_df = search_df[search_df['nprobe_percentage'] == nprobe] + axes[0].plot(sub_df['step_num'].values, sub_df['latency_ms'].values, label=f'Factor Scanned: {nprobe}', color=colors[i]) + total_time_minutes = sub_df['latency_ms'].values.sum() / MIN_TO_MS + axes[0].set_ylabel('Latency (ms)') + axes[0].set_yscale('log') + axes[0].set_title('Search Latency over Step Number') + axes[0].legend() + + # Bottom left subplot: recall_mean vs step_num + for i, nprobe in enumerate(nprobe_values): + sub_df = search_df[search_df['nprobe_percentage'] == nprobe] + axes[1].plot(sub_df['step_num'].values, sub_df['recall_mean'].values, label=f'Factor Scanned: {nprobe}', color=colors[i]) + axes[1].set_xlabel('Step Number') + axes[1].set_ylabel('Recall Mean') + axes[1].set_title('Recall Mean over Step Number') + + plt.tight_layout() + plt.savefig(os.path.join(DST_DIR, src_dataset.replace("csv", "png")), dpi=300) + +def visualize_vary_worker(src_dataset): + # Read CSV + df = pd.read_csv(os.path.join(SRC_DIR, src_dataset)) + search_df = df[df['step_type'] == 'search'] + + # List of metrics to plot + metrics = [ + ("search_latency_ms", "Batch Search Latency (seconds)"), + ("worker_partition_size", "Avg Scan Partition Size"), + ("worker_scan_throughput", "Partition Scan Throughput (GB/s)"), + ("recall_mean", "Query Recall") + ] + + # Get unique worker counts for consistent colors + worker_counts = sorted(search_df['num_search_workers'].unique()) + colors = plt.get_cmap('tab10').colors # or use any other color map + + fig, axs = plt.subplots(2, 2, sharex=True, figsize=(12, 10)) + axs = axs.flatten() + + for idx, (metric, metric_label) in enumerate(metrics): + scale_factor = 1.0 + if metric == "search_latency_ms": + scale_factor = 1000.0 + + ax = axs[idx] + for i, worker in enumerate(worker_counts): + data = search_df[search_df['num_search_workers'] == worker].sort_values('step_num') + ax.plot(data['step_num'].values, data[metric].values/scale_factor, label=f'Num Search Workers: {worker}', color=colors[i % len(colors)]) + + ax.set_xlabel('Step Num') + ax.set_ylabel(metric_label) + ax.set_title(metric_label) + ax.legend() + + if metric == "search_latency_ms": + ax.set_yscale('log') + minor_locator = mticker.LogLocator(subs=np.arange(2, 10)) + ax.yaxis.set_minor_locator(minor_locator) + minor_formatter = mticker.FormatStrFormatter("%.1f") + ax.yaxis.set_minor_formatter(minor_formatter) + + formatter = ScalarFormatter() + formatter.set_scientific(False) + ax.yaxis.set_major_formatter(formatter) + + fig.suptitle(f'Vary Search Workers Experiemnt (Scan Percentage = 15%, Batch Size = 500)', fontsize=16) + + plt.tight_layout() + plt.savefig(os.path.join(DST_DIR, src_dataset.replace("csv", "png")), dpi=300) + +def visualize_vary_batch_size(src_dataset): + # Read CSV + df = pd.read_csv(os.path.join(SRC_DIR, src_dataset)) + search_df = df[df['step_type'] == 'search'] + search_df = search_df[search_df["step_num"] > 5] + + metric_names = [ + ('search_latency_ms', 'Search Latency (ms)'), + ('worker_partition_size', 'Partition Size'), + ('worker_scan_throughput', 'Worker Scan Throughput'), + ('worker_scan_time_ms', 'Worker Scan Time (ms)'), + ('worker_result_time_ms', 'Worker Result Write Time (ms)') + ] + colors = plt.get_cmap('tab10').colors # or use any other color map + + batch_sizes = list(search_df['batch_size'].unique()) + fig, axes = plt.subplots(2, 3, figsize=(16, 12), sharex=True) + axes = axes.flatten() + + for i, (col, title) in enumerate(metric_names): + + ax = axes[i] + for j, batch in enumerate(batch_sizes): + batch_data = search_df[search_df['batch_size'] == batch] + ax.plot(batch_data['step_num'].values, batch_data[col].values, label=f'Batch Size {batch}', color=colors[j % len(colors)]) + ax.set_title(title) + + ax.set_xlabel('Step Num') + ax.legend() + + fig.tight_layout() + plt.savefig(os.path.join(DST_DIR, src_dataset.replace("csv", "png")), dpi=300) + +def visualize_hardware_metrics(src_dataset): + # Read CSV + df = pd.read_csv(os.path.join(SRC_DIR, src_dataset)) + search_df = df[df['step_type'] == 'search'] + fig, axs = plt.subplots(2, 2, figsize=(12, 8), sharex=True) + + # 1) worker_partition_size versus step_num + axs[0, 0].plot(search_df["step_num"].values, search_df["worker_partition_size"].values) + axs[0, 0].set_title("Partition Size vs Step") + axs[0, 0].set_ylabel("Worker Partition Size") + + # 2) worker_scan_throughput versus step_num + axs[0, 1].plot(search_df["step_num"].values, search_df["worker_scan_throughput"].values, color="orange") + axs[0, 1].set_title("Scan Throughput vs Step") + axs[0, 1].set_ylabel("Scan Throughput") + + # 3) measured_ipc versus step_num + axs[1, 0].plot(search_df["step_num"].values, search_df["measured_ipc"].values, color="green") + axs[1, 0].set_title("Measured IPC vs Step") + axs[1, 0].set_xlabel("Step Num") + axs[1, 0].set_ylabel("Measured IPC") + + # 4) cache_miss_rate versus step_num + axs[1, 1].plot(search_df["step_num"].values, search_df["cache_miss_rate"].values, color="red") + axs[1, 1].set_title("Cache Miss Rate vs Step") + axs[1, 1].set_xlabel("Step Num") + axs[1, 1].set_ylabel("Cache Miss Rate") + + fig.suptitle(f'Batch Query Search with Hardware Counters (# of Workers = 8, Batch Size = 256)', fontsize=16) + + fig.tight_layout() + plt.savefig(os.path.join(DST_DIR, src_dataset.replace("csv", "png")), dpi=300) + +if __name__ == "__main__": + ''' + # visualize_vary_worker("perf_debug_scan_0.15_vary_num_search_workers.csv") + # visualize_vary_batch_size("perf_debug_scan_0.15_vary_batch_size.csv") + visualize_hardware_metrics("perf_debug_scan_0.15_counters.csv") + ''' + + configs_to_visualize = [ + ("test_perf_debug_scan_0.14_worker_batch_tuning.csv", "Scan Percentage = 14%, Query Batch Size = 256, Partition Chunk Size = 256"), + ("perf_debug_scan_0.14_worker_batch_tuning_lower.csv", "Scan Percentage = 14%, Query Batch Size = 256, Partition Chunk Size = 128") + ] + + for dataset, config_details in configs_to_visualize: + visualize_perf(dataset, config_details) diff --git a/scripts/bigann_perf_vis/perf_debug_scan_0.14_tuning.png b/scripts/bigann_perf_vis/perf_debug_scan_0.14_tuning.png new file mode 100644 index 00000000..eae31cea Binary files /dev/null and b/scripts/bigann_perf_vis/perf_debug_scan_0.14_tuning.png differ diff --git a/scripts/bigann_perf_vis/perf_debug_scan_0.14_worker_batch_tuning_lower.png b/scripts/bigann_perf_vis/perf_debug_scan_0.14_worker_batch_tuning_lower.png new file mode 100644 index 00000000..efebbad4 Binary files /dev/null and b/scripts/bigann_perf_vis/perf_debug_scan_0.14_worker_batch_tuning_lower.png differ diff --git a/scripts/bigann_perf_vis/perf_debug_scan_0.15_counters.png b/scripts/bigann_perf_vis/perf_debug_scan_0.15_counters.png new file mode 100644 index 00000000..f3c5c84e Binary files /dev/null and b/scripts/bigann_perf_vis/perf_debug_scan_0.15_counters.png differ diff --git a/scripts/bigann_perf_vis/perf_debug_scan_0.15_vary_batch_size.png b/scripts/bigann_perf_vis/perf_debug_scan_0.15_vary_batch_size.png new file mode 100644 index 00000000..ba0f4ac0 Binary files /dev/null and b/scripts/bigann_perf_vis/perf_debug_scan_0.15_vary_batch_size.png differ diff --git a/scripts/bigann_perf_vis/perf_debug_scan_0.15_vary_num_search_workers.png b/scripts/bigann_perf_vis/perf_debug_scan_0.15_vary_num_search_workers.png new file mode 100644 index 00000000..e352b6ab Binary files /dev/null and b/scripts/bigann_perf_vis/perf_debug_scan_0.15_vary_num_search_workers.png differ diff --git a/scripts/bigann_perf_vis/scan_0.12_no_aps_refinment_wma_delete.png b/scripts/bigann_perf_vis/scan_0.12_no_aps_refinment_wma_delete.png new file mode 100644 index 00000000..fa026363 Binary files /dev/null and b/scripts/bigann_perf_vis/scan_0.12_no_aps_refinment_wma_delete.png differ diff --git a/scripts/bigann_perf_vis/scan_0.15_no_aps_refinment_wma_delete.png b/scripts/bigann_perf_vis/scan_0.15_no_aps_refinment_wma_delete.png new file mode 100644 index 00000000..cac87747 Binary files /dev/null and b/scripts/bigann_perf_vis/scan_0.15_no_aps_refinment_wma_delete.png differ diff --git a/scripts/bigann_perf_vis/scan_0.1_no_aps_refinment_wma_delete.png b/scripts/bigann_perf_vis/scan_0.1_no_aps_refinment_wma_delete.png new file mode 100644 index 00000000..d50cb687 Binary files /dev/null and b/scripts/bigann_perf_vis/scan_0.1_no_aps_refinment_wma_delete.png differ diff --git a/scripts/bigann_perf_vis/scan_using_batching_250_aps_recall_0.9_search_0.15.png b/scripts/bigann_perf_vis/scan_using_batching_250_aps_recall_0.9_search_0.15.png new file mode 100644 index 00000000..c949f476 Binary files /dev/null and b/scripts/bigann_perf_vis/scan_using_batching_250_aps_recall_0.9_search_0.15.png differ diff --git a/scripts/bigann_perf_vis/scan_using_batching_250_aps_recall_0.9_search_0.15_0.05.png b/scripts/bigann_perf_vis/scan_using_batching_250_aps_recall_0.9_search_0.15_0.05.png new file mode 100644 index 00000000..d1dcf2e0 Binary files /dev/null and b/scripts/bigann_perf_vis/scan_using_batching_250_aps_recall_0.9_search_0.15_0.05.png differ diff --git a/scripts/bigann_perf_vis/scan_using_batching_250_aps_recall_0.9_search_0.1_0.01.png b/scripts/bigann_perf_vis/scan_using_batching_250_aps_recall_0.9_search_0.1_0.01.png new file mode 100644 index 00000000..28fdbb07 Binary files /dev/null and b/scripts/bigann_perf_vis/scan_using_batching_250_aps_recall_0.9_search_0.1_0.01.png differ diff --git a/scripts/bigann_perf_vis/scan_using_batching_250_aps_recall_0.9_search_0.3.png b/scripts/bigann_perf_vis/scan_using_batching_250_aps_recall_0.9_search_0.3.png new file mode 100644 index 00000000..05be779e Binary files /dev/null and b/scripts/bigann_perf_vis/scan_using_batching_250_aps_recall_0.9_search_0.3.png differ diff --git a/scripts/bigann_perf_vis/scan_using_batching_optimizations.png b/scripts/bigann_perf_vis/scan_using_batching_optimizations.png new file mode 100644 index 00000000..3028aa14 Binary files /dev/null and b/scripts/bigann_perf_vis/scan_using_batching_optimizations.png differ diff --git a/scripts/bigann_perf_vis/scan_vary_using_batching_250_no_aps.png b/scripts/bigann_perf_vis/scan_vary_using_batching_250_no_aps.png new file mode 100644 index 00000000..d34b020c Binary files /dev/null and b/scripts/bigann_perf_vis/scan_vary_using_batching_250_no_aps.png differ diff --git a/scripts/bigann_perf_vis/test_perf_debug_scan_0.14_worker_batch_tuning.png b/scripts/bigann_perf_vis/test_perf_debug_scan_0.14_worker_batch_tuning.png new file mode 100644 index 00000000..a06b2109 Binary files /dev/null and b/scripts/bigann_perf_vis/test_perf_debug_scan_0.14_worker_batch_tuning.png differ diff --git a/scripts/cpp/benchmark/big_ann_debug.cpp b/scripts/cpp/benchmark/big_ann_debug.cpp new file mode 100644 index 00000000..d6eb28ed --- /dev/null +++ b/scripts/cpp/benchmark/big_ann_debug.cpp @@ -0,0 +1,888 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include // For c10::IValue +#include // For torch::load + +#include "quake_index.h" // Quake API header +#include "partition_manager.h" +#include "dynamic_inverted_list.h" +#include "index_partition.h" +#include "parallel.h" + +using torch::Tensor; +using std::vector; +using std::shared_ptr; + +#define DATASET_PATH "/home/devesh/big-ann-benchmark/big-ann-benchmarks/data/MSTuring-30M-clustered/" +#define DIRECTORY_PATH DATASET_PATH "index_arguments/" +#define GROUND_TRUTH_PATH DATASET_PATH "29998994/final_runbook.yaml/" + +constexpr float DELETE_THRESHOLD = 0.7; +constexpr float CAPACITY_THRESHOLD = 1.15; + +constexpr float MS_TO_US = 1000.0; +constexpr float MS_TO_NS = 1000.0 * 1000.0; + +struct Step { + uint32_t step_number; + std::string type; + std::string vectors_path; + std::string ids_path; + + // Default constructor + Step() : step_number(0), type("UNKNOWN") {} +}; + +bool parse_filename(const std::string& filename, + uint32_t& out_step_num, + std::string& out_type, // Changed to std::string& + std::string& out_target) +{ + // 1. Check prefix and suffix + if (filename.find("step_") != 0 || + filename.rfind(".pth") != (filename.length() - 4)) { + return false; + } + + // 2. Extract the core "1_insert_ids" + std::string core = filename.substr(5, filename.length() - 5 - 4); + + // 3. Split the core by underscores + std::vector parts; + std::stringstream ss(core); + std::string part; + while (std::getline(ss, part, '_')) { + parts.push_back(part); + } + + if (parts.size() != 3) { + return false; // Malformed name + } + + // 4. Extract and convert parts + try { + out_step_num = static_cast(std::stoul(parts[0])); + out_type = parts[1]; // Directly assign the string + out_target = parts[2]; + + // Validate the extracted parts + bool valid_type = (out_type == "insert" || out_type == "delete" || out_type == "search"); + bool valid_target = (out_target == "ids" || out_target == "vectors"); + + if (!valid_type || !valid_target) { + return false; + } + + } catch (const std::invalid_argument&) { + return false; // Step number wasn't a number + } catch (const std::out_of_range&) { + return false; // Step number was too large + } + + return true; +} + +std::vector extract_step_details() { + // Use a map to group files by step number. + // Key: step_number, Value: Step struct + std::map step_map; + + // --- POSIX Directory Reading --- + DIR* dir; + struct dirent* ent; + + if ((dir = opendir(DIRECTORY_PATH)) != NULL) { + // Read all files and directories within directory + while ((ent = readdir(dir)) != NULL) { + std::string filename = ent->d_name; + + // Skip "." and ".." + if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) { + continue; + } + + // Skip non .pth files + if (filename.length() < 3 || + filename.rfind(".pth") != (filename.length() - 4)) { + continue; + } + + uint32_t step_num; + std::string type; // Now a string + std::string target; + + if (parse_filename(filename, step_num, type, target)) { + // Get or create the Step struct for this step number + Step& current_step = step_map[step_num]; + + // Populate its data + current_step.step_number = step_num; + current_step.type = type; // Assign the string directly + + std::string full_path = std::string(DIRECTORY_PATH) + filename; + + if (target == "ids") { + current_step.ids_path = full_path; + } else if (target == "vectors") { + current_step.vectors_path = full_path; + } + } else { + std::cerr << "Failed to parse file name " << filename << std::endl; + exit(1); + } + } + closedir(dir); + } else { + std::cerr << "Error: Could not open directory " << DIRECTORY_PATH << std::endl; + exit(1); + } + // --- End of Directory Reading --- + + + // --- Convert map to final vector --- + std::vector steps; + for (std::map::const_iterator it = step_map.begin(); it != step_map.end(); ++it) { + steps.push_back(it->second); + } + return steps; +} + +Tensor load_tensor(std::string file_path) { + torch::jit::Module loaded_module = torch::jit::load(file_path); + return loaded_module.attr("tensor").toTensor();; +} + +constexpr bool RUN_MAINTEANCE = true; +void perform_mainteance(std::shared_ptr index, std::ofstream& result_writer) { + if(RUN_MAINTEANCE) { + std::shared_ptr result = index->maintenance(); + result_writer << result->total_time_us/MS_TO_US << ","; + + std::cout << "Mainteance Metrics: Total Time ms - " << result->total_time_us/MS_TO_US << ", Num Splits - " << result->n_splits << ", Num Deletes - " << result->n_deletes; + std::cout << ", Num Reclusters - " << result->n_recluster << ", Delete Time ms - " << result->delete_time_us/MS_TO_US << ", Split Time ms - " << result->split_time_us/MS_TO_US; + std::cout << ", Recluster Time ms - " << result->recluster_time_us/MS_TO_US << ", Refinement Time ms - " << result->refinement_time_us/MS_TO_US << std::endl; + } +} + +void print_search_metrics(std::shared_ptr timing_info, int level) { + std::cout << "Search Level " << level << " Timing Breakdown: " << std::endl; + std::cout << "\t[Main] Total Time ms - " << timing_info->total_time_ns/MS_TO_NS << std::endl; + std::cout << "\t[Main] Buffer Init Time ms - " << timing_info->buffer_init_time_ns/MS_TO_NS << std::endl; + std::cout << "\t[Main] Copy Query Time ms - " << timing_info->copy_query_time_ns/MS_TO_NS << std::endl; + std::cout << "\t[Main] Job Enqueue Time ms - " << timing_info->job_enqueue_time_ns/MS_TO_NS << std::endl; + std::cout << "\t[Main] Boundary Distance ms - " << timing_info->boundary_distance_time_ns/MS_TO_NS << std::endl; + std::cout << "\t[Main] APS Time ms - " << timing_info->aps_time_ns/MS_TO_NS << std::endl; + std::cout << "\t[Main] Scan Time ms - " << timing_info->scan_time_ns/MS_TO_NS << std::endl; + std::cout << "\t[Main] Job Wait Time ms - " << timing_info->job_wait_time_ns/MS_TO_NS << std::endl; + std::cout << "\t[Main] Result Aggregate Time ms - " << timing_info->result_aggregate_time_ns/MS_TO_NS << std::endl << std::endl; + + std::cout << "\t[Worker] Total Worker Jobs Executed - " << timing_info->total_worker_jobs << std::endl; + std::cout << "\t[Worker] Job Time ms - " << timing_info->worker_job_time_ns/MS_TO_NS << std::endl; + std::cout << "\t[Worker] Wait Time ms - " << timing_info->worker_wait_time_ns/MS_TO_NS << std::endl; + std::cout << "\t[Worker] Process Preamable Time ms - " << timing_info->worker_process_preamble_time_ns/MS_TO_NS << std::endl; + std::cout << "\t[Worker] Total Scan Time ms - " << timing_info->worker_scan_time_ns/MS_TO_NS << std::endl; + std::cout << "\t[Worker] Partition Size - " << timing_info->worker_partition_size << std::endl; + std::cout << "\t[Worker] Partition Scan Bytes - " << timing_info->worker_partition_size_bytes << std::endl; + std::cout << "\t[Worker] Global Partition Scan Through (GB/s) - " << timing_info->worker_scan_throughput << std::endl; + std::cout << "\t[Worker] Local Partition Scan Through (GB/s) - " << timing_info->local_scan_throughput << std::endl; + std::cout << "\t[Worker] Result Enque Time ms - " << timing_info->worker_enqueue_time_ns/MS_TO_NS << std::endl; + std::cout << "\t[Worker] Process Time ms - " << timing_info->worker_process_time_ns/MS_TO_NS << std::endl; + std::cout << "\t[Worker] Worker Batch Scan IPC - " << timing_info->worker_batch_scan_ipc << std::endl; + std::cout << "\t[Worker] Worker Batch Scan Miss Rate - " << timing_info->worker_batch_scan_miss_rate << std::endl; + std::cout << "\t[Worker] Single Scan Time us - " << timing_info->single_scan_job_time_ns/MS_TO_US << std::endl; + std::cout << "\t[Worker] Worker Batch Scan Norms X us - " << timing_info->faiss_norms_x_time_ns/MS_TO_US << std::endl; + std::cout << "\t[Worker] Worker Batch Scan Norms Y us - " << timing_info->faiss_norms_y_time_ns/MS_TO_US << std::endl; + std::cout << "\t[Worker] Sgemm Matrix Multiply us - " << timing_info->sgemm_time_ns/MS_TO_US << std::endl; + std::cout << "\t[Worker] IP TO L2 us - " << timing_info->ip_to_l2_time_ns/MS_TO_US << std::endl; + std::cout << "\t[Worker] Top K Buffer Add us - " << timing_info->top_k_buffer_add_ns/MS_TO_US << std::endl; +} + +std::pair load_ground_truth_data(uint32_t step_num) { + // Open the input file + std::string gt_path = std::string(GROUND_TRUTH_PATH) + "step" + std::to_string(step_num) + ".gt100"; + std::ifstream input_file(gt_path, std::ios::binary | std::ios::in); + if (!input_file) { + throw std::runtime_error("Failed to open file: " + gt_path); + } + + // Read header (two uint32_t values: n and d) + uint32_t n, d; + input_file.read(reinterpret_cast(&n), sizeof(uint32_t)); + input_file.read(reinterpret_cast(&d), sizeof(uint32_t)); + + // Read the gt ids + std::vector ids_data(n * d); + input_file.read(reinterpret_cast(ids_data.data()), n * d * sizeof(int32_t)); + Tensor gt_ids = torch::from_blob(ids_data.data(), {static_cast(n), static_cast(d)}, torch::kInt32).clone(); + + // Read the gt distances + std::vector distances_data(n * d); + input_file.read(reinterpret_cast(distances_data.data()), n * d * sizeof(float)); + Tensor gt_distances = torch::from_blob(distances_data.data(), {static_cast(n), static_cast(d)}, torch::kFloat32).clone(); + + return std::make_pair(gt_ids, gt_distances); // Return empty tensor for distances +} + +float calculateStandardDeviation(const std::vector& data, float mean) { + if (data.empty()) { + return 0.0; // Or handle error appropriately + } + + // Calculate the variance + float sum_sq_diff = 0.0; + for (double value : data) { + sum_sq_diff += std::pow(value - mean, 2); + } + + // Use that to calculate the std dev + return std::sqrt(sum_sq_diff / (data.size() - 1)); +} + +constexpr bool LOG_RECALL_IDS = false; +constexpr bool PRINT_GT_DISTANCES = false; +std::pair calculate_recall(std::shared_ptr index, Tensor queries, std::shared_ptr search_result, uint32_t step_num) { + // Load the tensors + std::pair gt_data = load_ground_truth_data(step_num); + Tensor gt_ids = gt_data.first.to(torch::kInt64).contiguous(); + Tensor gt_distances = gt_data.second.to(torch::kFloat32).contiguous(); + + Tensor predicted_ids = search_result->ids.to(torch::kInt64).contiguous(); + Tensor predicted_distances = search_result->distances.to(torch::kFloat32).contiguous(); + + assert(gt_ids.sizes()[0] == predicted_ids.sizes()[0]); + + // Calculate the recall for each query + int num_queries = predicted_ids.sizes()[0]; + int k = predicted_ids.sizes()[1]; + Tensor gt_top_k = gt_ids.slice(1, 0, k).contiguous(); + + // Calculate the recall for each query + std::vector per_query_recall; + int64_t* gt_ids_ptr = gt_top_k.data_ptr(); + float* gt_distances_ptr = gt_distances.data_ptr(); + + int64_t* predicted_ids_ptr = predicted_ids.data_ptr(); + float* predicted_distances_ptr = predicted_distances.data_ptr(); + for(int i = 0; i < num_queries; i++) { + // Get the ground truth and predicted sets + int64_t* query_gt = gt_ids_ptr + i * k; float* query_gt_distances = gt_distances_ptr + i * k; + int64_t* query_predicted = predicted_ids_ptr + i * k; float* query_predicted_distances = predicted_distances_ptr + i * k; + std::set gt_ids; std::set predicted_ids; + if constexpr(LOG_RECALL_IDS) std::cout << "Query " << i << " Result: " << std::endl; + for(int j = 0; j < k; j++) { + gt_ids.insert(query_gt[j]); + predicted_ids.insert(query_predicted[j]); + } + + // Get their set intersection + std::set intersection_set; + std::set_intersection(gt_ids.begin(), gt_ids.end(), + predicted_ids.begin(), predicted_ids.end(), + std::inserter(intersection_set, intersection_set.begin())); + + // Use that to predict the recall + float query_recall = (1.0 * intersection_set.size())/k; + per_query_recall.push_back(query_recall); + } + + std::shared_ptr partition_store = index->partition_manager_->partition_store_; + queries = queries.contiguous(); + float* all_queries_vectors_ptr = queries.data_ptr(); + int d = partition_store->d_; + + if constexpr(PRINT_GT_DISTANCES) { + for(int i = 0; i < num_queries; i++) { + std::cout << "-------------" << std::endl; + std::cout << "Query " << i << " Ground Truth Details: " << std::endl; + int64_t* query_gt = gt_ids_ptr + i * k; float* query_gt_distances = gt_distances_ptr + i * k; + int64_t* query_predicted = predicted_ids_ptr + i * k; float* query_predicted_distances = predicted_distances_ptr + i * k; + float* query_vector_ptr = all_queries_vectors_ptr + i * d; + + // First log GT vector results + for(int j = 0; j < k; j++) { + int64_t curr_gt_id = query_gt[j]; + std::cout << "\tGT Result " << j << " - ID: " << curr_gt_id << ", GT Distance: " << query_gt_distances[j]; + + // Also calculate the distance of the gt vector to the query vector + std::pair vector_location = partition_store->id_to_location_[curr_gt_id]; + float* gt_vector_ptr = reinterpret_cast(vector_location.first->codes_) + vector_location.second * d; + float distance = std::sqrt(faiss::fvec_L2sqr(query_vector_ptr, reinterpret_cast(gt_vector_ptr), d)); + + std::cout << ", GT Vector Faiss L2 Distance: " << distance << std::endl; + } + + std::cout << std::endl; + + // Now log the predicted results + std::cout << "Query " << i << " Search Details: " << std::endl; + for(int j = 0; j < k; j++) { + float curr_distance = query_predicted_distances[j]; + float l2_value = std::pow(curr_distance, 2); + std::cout << "\tSearch Result " << j << " - ID: " << query_predicted[j] << ", L2 Distance: " << l2_value << std::endl; + } + + std::cout << "-------------" << std::endl; + } + } + + // Return the average recall across all the queries + float mean = std::accumulate(per_query_recall.begin(), per_query_recall.end(), 0.0) / per_query_recall.size(); + float standard_dev = calculateStandardDeviation(per_query_recall, mean); + return std::make_pair(mean, standard_dev); +} + +std::shared_ptr build_index(Step& build_step, int num_search_workers) { + // Verify step + if(build_step.step_number != 1 || build_step.type != "insert") { + std::cerr << "Build Index called with step num " << build_step.step_number << " and type " << build_step.type << std::endl; + exit(1); + } + + // Create and build the index + std::shared_ptr index = std::make_shared(); + + std::shared_ptr build_params = std::make_shared(); + build_params->dimension = 100; + build_params->nlist = 1000; + build_params->num_workers = num_search_workers; + build_params->metric = "l2"; + build_params->niter = 25; + build_params->use_numa = false; + + build_params->parent_params = std::make_shared(); + build_params->parent_params->dimension = 100; + build_params->parent_params->nlist = 1; + build_params->parent_params->metric = "l2"; + build_params->parent_params->num_workers = 0; + build_params->parent_params->use_numa = false; + + Tensor build_vectors = load_tensor(build_step.vectors_path).to(torch::kFloat32); + Tensor build_ids = load_tensor(build_step.ids_path).to(torch::kInt64); + index->build(build_vectors, build_ids, build_params); + + // Also initialize with the default mainteance policy + std::shared_ptr mainteance_policy = std::make_shared(); + mainteance_policy->window_size = 5000; + mainteance_policy->split_threshold_ns = 3250; + mainteance_policy->split_knn_iterations = 6; + mainteance_policy->delete_threshold_ns = 2000; + mainteance_policy->partition_reduction_threshold = 0.22; + mainteance_policy->refinement_radius = 0; + mainteance_policy->refinement_iterations = 5; + mainteance_policy->min_partition_size = 1500; + mainteance_policy->enable_split_rejection = true; + mainteance_policy->enable_delete_rejection = true; + index->initialize_maintenance_policy(mainteance_policy); + + return index; +} + +constexpr bool CALCULATE_RECALL_GIVEN_GT_PARITITONS = false; +std::pair check_gt_partitions_scanned(std::shared_ptr index, Step& search_step, float partition_search_fraction) { + // Load the search query + Tensor search_queries = load_tensor(search_step.vectors_path).to(torch::kFloat32); + + // Get the ground ids that were scanned + std::pair gt_data = load_ground_truth_data(search_step.step_number); + Tensor gt_ids = gt_data.first.to(torch::kInt64); + int k = 10; + Tensor gt_top_k = gt_ids.slice(1, 0, k).contiguous(); + + // Get the partitions that the ground ids belong to + int64_t* gt_ids_ptr = gt_top_k.data_ptr(); + int num_queries = gt_ids.sizes()[0]; + std::shared_ptr partition_store = index->partition_manager_->partition_store_; + if (partition_store->id_to_location_.empty()) partition_store->build_map(); + + std::vector> gt_partitions(num_queries); + for(int i = 0; i < num_queries; i++) { + int64_t* query_gt = gt_ids_ptr + i * k; + for(int j = 0; j < k; j++) { + // Get the partition id for this gt id + int64_t vector_id = query_gt[j]; + if(partition_store->id_to_location_.find(vector_id) == partition_store->id_to_location_.end()) { + std::string err_message = std::string("[ERROR] Failed to find id ") + std::to_string(vector_id) + " in partition store"; + throw std::runtime_error(err_message); + } + + IndexPartition* partition = partition_store->id_to_location_[vector_id].first; + if(partition->partition_id_ == -1) { + std::string err_message = std::string("[ERROR] Vector id ") + std::to_string(vector_id) + " maps to partition with invalid partition id -1"; + throw std::runtime_error(err_message); + } + + gt_partitions[i].push_back(partition->partition_id_); + } + } + + if constexpr(CALCULATE_RECALL_GIVEN_GT_PARITITONS) { + // Initialize the tensor of the gt partitions + Tensor partitions_to_scan = torch::full({num_queries, k}, -1, torch::kInt64); + auto search_partition_id_accessor = partitions_to_scan.accessor(); + for(int i = 0; i < num_queries; i++) { + std::set unique_partitions; + for(int j = 0; j < k; j++) { + int64_t curr_partition_id = static_cast(gt_partitions[i][j]); + if(unique_partitions.find(curr_partition_id) != unique_partitions.end()) { + curr_partition_id = -1; + } else { + unique_partitions.insert(curr_partition_id); + } + search_partition_id_accessor[i][j] = curr_partition_id; + } + } + + // Create the search params for the partition scan + std::shared_ptr search_params = std::make_shared(); + search_params->nprobe = k; + search_params->k = k; + search_params->recall_target = -1.0; + search_params->batched_scan = true; + search_params->batch_size = 500; + search_params->track_hits = false; + + // Get the resulting vectors and return the recall/mean + std::shared_ptr gt_search_result = index->query_coordinator_->scan_partitions(search_queries, partitions_to_scan, search_params); + + std::pair recall_result = calculate_recall(index, search_queries, gt_search_result, search_step.step_number); + std::cout << "Scanning index with GT partitions Recall: Mean - " << recall_result.first << ", Std Dev - " << recall_result.second << std::endl; + } + + // Now search the parent for the query vectors to get the partitions that were scanned + std::shared_ptr search_params = std::make_shared(); + int num_partitions_to_scan = static_cast(index->nlist()); + search_params->k = num_partitions_to_scan; + search_params->batched_scan = true; + search_params->track_hits = false; + search_params->batch_size = 500; + std::shared_ptr parent_index = index->parent_; + std::shared_ptr search_result = parent_index->search(search_queries, search_params); + + // Verify that the ground truth partitions were scanned + Tensor searched_ids = search_result->ids.to(torch::kInt64).contiguous(); + int64_t* searched_ids_ptr = searched_ids.data_ptr(); + Tensor search_dists = search_result->distances.to(torch::kFloat32).contiguous(); + float* searched_dists_ptr = search_dists.data_ptr(); + + std::shared_ptr parent_partition_store = parent_index->partition_manager_->partition_store_; + int d = parent_partition_store->d_; + + search_queries = search_queries.contiguous(); + float* all_queries_vectors_ptr = search_queries.data_ptr(); + std::vector per_query_scan_fraction(num_queries); + for(int i = 0; i < num_queries; i++) { + // Get the ranking of the partitions for this query + float* query_vec_ptr = all_queries_vectors_ptr + i * d; + int64_t* query_searched = searched_ids_ptr + i * num_partitions_to_scan; + float* query_searched_dists = searched_dists_ptr + i * num_partitions_to_scan; + std::unordered_map> partition_loc_map; + + for(int j = 0; j < num_partitions_to_scan; j++) { + int64_t partition_id = query_searched[j]; + if(partition_id != -1) partition_loc_map[partition_id] = std::make_pair(j, query_searched_dists[j]); + } + + // Now determine the max rank of the gt partitions + bool log_query = false; + if(log_query) std::cout << "Query " << i << " GT Partition Ranks: " << std::endl; + int max_rank = -1; + for(int64_t gt_partition_id : gt_partitions[i]) { + if(partition_loc_map.find(gt_partition_id) == partition_loc_map.end()) { + std::cerr << "[ERROR] Query " << i << " didn't get rank for gt partition id " << gt_partition_id << std::endl; + std::cout << "Result for " << num_partitions_to_scan << " partitions: "; + for(int j = 0; j < num_partitions_to_scan; j++) { + std::cout << "(" << query_searched[j] << "," << query_searched_dists[j] << std::endl; + } + std::cout << std::endl; + exit(1); + } + std::pair partition_details = partition_loc_map[gt_partition_id]; + + max_rank = std::max(max_rank, partition_details.first); + if(log_query) { + std::cout << "\t GT Partition ID: " << gt_partition_id << ", Rank: " << partition_details.first << "/" << num_partitions_to_scan << ", Quake Distance: " << partition_details.second; + + // Get the centroid with the ground truth partition id + if(parent_partition_store->id_to_location_.empty()) parent_partition_store->build_map(); + if(parent_partition_store->id_to_location_.find(gt_partition_id) == parent_partition_store->id_to_location_.end()) { + std::string err_message = std::string("[ERROR] Failed to find partition id ") + std::to_string(gt_partition_id) + " in parent partition store"; + throw std::runtime_error(err_message); + } + + // Calculate the distance to it from the query vector + std::pair vector_location = parent_partition_store->id_to_location_[gt_partition_id]; + float* centorid_vec_ptr = reinterpret_cast(vector_location.first->codes_) + vector_location.second * d; + float faiss_distance = std::sqrt(faiss::fvec_L2sqr(query_vec_ptr, centorid_vec_ptr, d)); + std::cout << ", Faiss Distance: " << faiss_distance << std::endl; + } + } + if(log_query) std::cout << std::endl; + + // Now calculate the percentage of centroids we would need to scan to get all the gt partitions + per_query_scan_fraction[i] = (100.0 * max_rank)/num_partitions_to_scan; + } + + // Calculate the mean and std dev of scan fraction + float mean = std::accumulate(per_query_scan_fraction.begin(), per_query_scan_fraction.end(), 0.0) / per_query_scan_fraction.size(); + float standard_dev = calculateStandardDeviation(per_query_scan_fraction, mean); + return std::make_pair(mean, standard_dev); +} + +static int CURR_BATCH_SIZE = 256; +constexpr bool CHECK_QUERY_RECALL = true; +constexpr bool CHECK_GT_PARITIONS_SCANNED = false; +constexpr float RECALL_TARGET = -1.0; // Use this to enable/disable APS +void perform_search(std::shared_ptr index, Step& search_step, std::ofstream& result_writer, float partition_search_fraction) { + // Load the search query + Tensor search_queries = load_tensor(search_step.vectors_path).to(torch::kFloat32); + + // Create the search parameters + std::shared_ptr search_params = std::make_shared(); + search_params->k = 10; + search_params->nprobe = static_cast(partition_search_fraction * index->nlist()); + search_params->recall_target = RECALL_TARGET; + search_params->initial_search_fraction = partition_search_fraction; + search_params->batched_scan = true; + search_params->batch_size = CURR_BATCH_SIZE; + search_params->track_hits = true; + + // Run the search + std::shared_ptr search_result = index->search(search_queries, search_params); + print_search_metrics(search_result->timing_info->parent_info, 0); + std::cout << std::endl; + print_search_metrics(search_result->timing_info, 1); + std::cout << std::endl; + + // Calculate the recall + auto timing_info = search_result->timing_info; + result_writer << search_step.step_number << ",search," << timing_info->total_time_ns/MS_TO_NS << ","; + result_writer << timing_info->worker_partition_size << "," << timing_info->worker_scan_time_ns/MS_TO_NS << ","; + result_writer << timing_info->local_scan_throughput << "," << timing_info->worker_enqueue_time_ns/MS_TO_NS << ","; + result_writer << timing_info->worker_batch_scan_ipc << "," << timing_info->worker_batch_scan_miss_rate << ","; + if constexpr(CHECK_QUERY_RECALL) { + std::pair recall_result = calculate_recall(index, search_queries, search_result, search_step.step_number); + std::cout << "\nSearch Recall: Mean - " << recall_result.first << ", Std Dev - " << recall_result.second << std::endl; + result_writer << recall_result.first << "," << recall_result.second << ","; + } else { + result_writer << "-1.0,-1.0,"; + } + + // Optionally perform gt checks + if constexpr(CHECK_GT_PARITIONS_SCANNED) { + std::pair gt_partitions_result = check_gt_partitions_scanned(index, search_step, partition_search_fraction); + std::cout << "\nGT Percent of Partitions to Scan: Mean - " << gt_partitions_result.first << ", Std Dev - " << gt_partitions_result.second << std::endl; + result_writer << gt_partitions_result.first << "," << gt_partitions_result.second << ","; + } else { + result_writer << "-1.0,-1.0,"; + } +} + +constexpr size_t INSERT_CHUNK_SIZE = 10000; +void perform_insert(std::shared_ptr index, Step& insert_step, std::ofstream& result_writer) { + // Load the arguments + Tensor insert_vectors = load_tensor(insert_step.vectors_path).to(torch::kFloat32); + Tensor insert_ids = load_tensor(insert_step.ids_path).to(torch::kInt64); + + // Insert in the vectors in chunk + int total_time_us = 0; + size_t num_vectors = insert_vectors.size(0); + size_t num_chunks = (num_vectors + INSERT_CHUNK_SIZE - 1)/INSERT_CHUNK_SIZE; + for(size_t i = 0; i < num_chunks; i++) { + size_t start_idx = i * INSERT_CHUNK_SIZE; + size_t end_idx = std::min(start_idx + INSERT_CHUNK_SIZE, num_vectors); + + auto result = index->add(insert_vectors.slice(0, start_idx, end_idx), insert_ids.slice(0, start_idx, end_idx)); + total_time_us += result->modify_time_us; + } + std::cout << "Finished insertion in " << num_chunks << " chunks in " << total_time_us << " us" << std::endl; + + result_writer << insert_step.step_number << ",insert," << total_time_us/MS_TO_US << ","; + result_writer << "-1.0" << "," << "-1.0" << ","; + result_writer << "-1.0" << "," << "-1.0" << ","; + result_writer << "-1.0" << "," << "-1.0" << ","; + result_writer << "-1.0,-1.0,"; + result_writer << "-1.0,-1.0,"; +} + +constexpr size_t DELETE_CHUNK_SIZE = 10000; +void perform_delete(std::shared_ptr index, Step& delete_step, std::ofstream& result_writer) { + Tensor delete_ids = load_tensor(delete_step.ids_path).to(torch::kInt64); + + // Insert in the vectors in chunk + int total_time_us = 0; + size_t num_vectors = delete_ids.size(0); + size_t num_chunks = (num_vectors + DELETE_CHUNK_SIZE - 1)/DELETE_CHUNK_SIZE; + for(size_t i = 0; i < num_chunks; i++) { + size_t start_idx = i * DELETE_CHUNK_SIZE; + size_t end_idx = std::min(start_idx + DELETE_CHUNK_SIZE, num_vectors); + + auto result = index->remove(delete_ids.slice(0, start_idx, end_idx)); + total_time_us += result->modify_time_us; + } + std::cout << "Finished delete in " << num_chunks << " chunks in " << total_time_us << " us" << std::endl; + + result_writer << delete_step.step_number << ",delete," << total_time_us/MS_TO_US << ","; + result_writer << "-1.0" << "," << "-1.0" << ","; + result_writer << "-1.0" << "," << "-1.0" << ","; + result_writer << "-1.0" << "," << "-1.0" << ","; + result_writer << "-1.0,-1.0,"; + result_writer << "-1.0,-1.0,"; +} + +constexpr float BYTES_TO_GB = 1000.0 * 1000.0 * 1000.0; + +float log_memory_stats(std::shared_ptr index, int level, std::ofstream& result_writer) { + auto partition_map = index->partition_manager_->partition_store_->partitions_; + int64_t total_vectors = 0; + int64_t buffer_capacity = 0; + int64_t total_memory = 0; + + for(const auto& pair : partition_map) { + total_vectors += pair.second->num_vectors_; + buffer_capacity += pair.second->buffer_size_; + total_memory += pair.second->buffer_size_ * (pair.second->code_size_ + sizeof(idx_t)); + } + + // Only log the memory for the first level + float total_memory_gb = total_memory/BYTES_TO_GB; + + // std::cout << "Level " << level << " Memory Consumption: Vectors - " << total_vectors << ", Buffer Capacity - " << buffer_capacity << ", Memory (GB) - " << total_memory_gb << std::endl; + + if(index->parent_ != nullptr) { + total_memory_gb += log_memory_stats(index->parent_, level + 1, result_writer); + } + + return total_memory_gb; +} + +// Code to calculate the Sillehoute Score for the matrix +struct PartitionView { + int64_t id; + std::vector vectors; +}; + +std::pair calculate_silhouette_scores(shared_ptr index, int clustering_num_samples = 0, int num_score_calculate_workers = 4) { + if (!index || !index->partition_manager_ || !index->partition_manager_->partition_store_) { + throw std::runtime_error("Invalid QuakeIndex: PartitionManager or Store is null."); + } + + auto pm = index->partition_manager_; + auto store = pm->partition_store_; + int d = pm->d(); + + // 1. Organize vectors by partition for efficient access + std::vector partitions; + int64_t total_vectors = 0; + + for (const auto& entry : store->partitions_) { + int64_t pid = entry.first; + shared_ptr part = entry.second; + if (part->num_vectors_ == 0) continue; + + PartitionView view; + view.id = pid; + view.vectors.reserve(part->num_vectors_); + + const uint8_t* codes = part->codes_; + size_t code_size = part->code_size_; + + for (int64_t i = 0; i < part->num_vectors_; ++i) { + view.vectors.push_back(reinterpret_cast(codes + i * code_size)); + } + partitions.push_back(view); + total_vectors += part->num_vectors_; + } + + // 2. Flatten for parallel iteration (tuples of: partition_idx, vector_idx_in_partition) + struct Task { + size_t p_idx; // Index in `partitions` vector + size_t v_idx; // Index inside that partition + }; + + std::vector tasks; + tasks.reserve(total_vectors); + + for (size_t p = 0; p < partitions.size(); ++p) { + for (size_t v = 0; v < partitions[p].vectors.size(); ++v) { + tasks.push_back({p, v}); + } + } + + // Apply sampling if requested + if (clustering_num_samples > 0) { + unsigned seed = std::chrono::system_clock::now().time_since_epoch().count(); + std::default_random_engine generator(seed); + std::shuffle(tasks.begin(), tasks.end(), generator); + + tasks.resize(clustering_num_samples); + } + + std::vector scores(tasks.size()); + + // 3. Compute scores in parallel + parallel_for(size_t(0), tasks.size(), [&](size_t t) { + const auto& task = tasks[t]; + const auto& current_partition = partitions[task.p_idx]; + const float* current_vec = current_partition.vectors[task.v_idx]; + + // --- Calculate a(i): Mean dist to same cluster --- + double sum_dist_a = 0.0; + size_t count_a = 0; + + for (size_t other_v = 0; other_v < current_partition.vectors.size(); ++other_v) { + if (task.v_idx == other_v) continue; + + // Use Faiss optimized L2 squared distance + float dist_sq = faiss::fvec_L2sqr(current_vec, current_partition.vectors[other_v], d); + sum_dist_a += std::sqrt(dist_sq); + count_a++; + } + + float a_i = (count_a > 0) ? (sum_dist_a / count_a) : 0.0f; + + // Silhouette is 0 for singleton clusters + if (current_partition.vectors.size() <= 1) { + scores[t] = 0.0f; + return; + } + + // --- Calculate b(i): Min mean dist to other clusters --- + float b_i = std::numeric_limits::max(); + + for (size_t p = 0; p < partitions.size(); ++p) { + if (p == task.p_idx) continue; + + const auto& other_partition = partitions[p]; + if (other_partition.vectors.empty()) continue; + + double sum_dist_b = 0.0; + + for (const float* other_vec : other_partition.vectors) { + float dist_sq = faiss::fvec_L2sqr(current_vec, other_vec, d); + sum_dist_b += std::sqrt(dist_sq); + } + + float mean_dist_b = sum_dist_b / other_partition.vectors.size(); + if (mean_dist_b < b_i) { + b_i = mean_dist_b; + } + } + + // Use a_i and b_i to figure out the score + if (b_i == std::numeric_limits::max()) { + scores[t] = 0.0f; + } else { + scores[t] = (b_i - a_i) / std::max(a_i, b_i); + } + + }, num_score_calculate_workers); // End parallel_for + + // Calculate the mean and std dev of scan fraction + float mean = std::accumulate(scores.begin(), scores.end(), 0.0) / scores.size(); + float standard_dev = calculateStandardDeviation(scores, mean); + return std::make_pair(mean, standard_dev); +} + +constexpr float SCAN_PERCENTAGE_RANGE[2] = {0.13, 0.13}; +constexpr size_t MIN_OPERATIONS_BEFORE_MAINTEANCE = 0; +constexpr size_t NUM_OPERATIONS_BETWEEN_MAINTEANCE = 1; +constexpr size_t NUM_TEST_OPERATIONS = 0; + +#define RESULT_WRITE_PATH "../scripts/big_ann_perf_numbers/perf_debug_scan_0.13_worker_batch_clustering_score.csv" + +int main() { + // Configure global params + IndexPartition::delete_resize_threshold_ = DELETE_THRESHOLD; + IndexPartition::capacity_resize_threshold_ = CAPACITY_THRESHOLD; + + // Extract the steps + std::vector steps_arr = extract_step_details(); + std::cout << "Loaded " << steps_arr.size() << " steps" << std::endl; + assert(!steps_arr.empty()); + + // Create the output csv file + std::ofstream result_writer(RESULT_WRITE_PATH); + result_writer << "step_num,step_type,latency_ms,worker_partition_size,worker_scan_time_ms,worker_scan_throughput,worker_result_time_ms,measured_ipc,cache_miss_rate,recall_mean,recall_std_dev,gt_scan_mean,gt_scan_dev,mainteance_ms,index_mem_gb,num_partitions,num_vectors,silleheoute_mean,silleheoute_std_dev" << std::endl; + + int num_default_workers = 8; + float clustering_num_samples = 8192; + int num_score_calculate_workers = 32; + + auto build_start_time = std::chrono::high_resolution_clock::now(); + std::shared_ptr index = build_index(steps_arr[0], num_default_workers); + auto build_end_time = std::chrono::high_resolution_clock::now(); + int64_t build_time_ns = std::chrono::duration_cast(build_end_time - build_start_time).count(); + std::cout << "Initialized Quake Index with batch size in " << build_time_ns/MS_TO_NS << " ms" << std::endl; + + // Now perform the streaming operations against the index + size_t num_test_operations = NUM_TEST_OPERATIONS; + if(num_test_operations == 0) { + num_test_operations = steps_arr.size(); + } + + float curr_scan_percentage = SCAN_PERCENTAGE_RANGE[1]; + float scan_percentage_step = (SCAN_PERCENTAGE_RANGE[1] - SCAN_PERCENTAGE_RANGE[0])/num_test_operations; + + for(size_t i = 1; i <= num_test_operations; i++) { + // Run the step + Step& curr_step = steps_arr[i]; + std::cout << "\n------ START: Step " << curr_step.step_number << " of type " << curr_step.type << " -----" << std::endl; + if(curr_step.type == "search") { + perform_search(index, curr_step, result_writer, curr_scan_percentage); + } else if(curr_step.type == "insert") { + perform_insert(index, curr_step, result_writer); + } else if(curr_step.type == "delete") { + perform_delete(index, curr_step, result_writer); + } else { + std::cerr << "Unsupport step type of " << curr_step.type << std::endl; + exit(1); + } + + // Run the mainteance + if(curr_step.step_number > MIN_OPERATIONS_BEFORE_MAINTEANCE && curr_step.step_number % NUM_OPERATIONS_BETWEEN_MAINTEANCE == 0) { + std::cout << std::endl; + perform_mainteance(index, result_writer); + } else { + result_writer << "0.0,"; + } + + std::cout << std::endl; + float index_memory_gb = log_memory_stats(index, 0, result_writer); + result_writer << index_memory_gb << "," << index->nlist() << "," << index->ntotal() << ","; + + // Get the silleheoute score + std::pair silleheouet_score = calculate_silhouette_scores(index, clustering_num_samples, num_score_calculate_workers); + std::cout << "Silleheoute Score: Mean - " << silleheouet_score.first << ", Std Dev - " << silleheouet_score.second << std::endl; + result_writer << silleheouet_score.first << "," << silleheouet_score.second << "," << std::endl; + + std::cout << "------ FINISH: Step " << curr_step.step_number << " of type " << curr_step.type << " ------\n" << std::endl; + + curr_scan_percentage -= scan_percentage_step; + } + + result_writer.close(); + return 0; +} \ No newline at end of file diff --git a/scripts/test_mainteance.py b/scripts/test_mainteance.py new file mode 100644 index 00000000..2a335ebe --- /dev/null +++ b/scripts/test_mainteance.py @@ -0,0 +1,72 @@ +import quake +import torch + +def mainteance_test(): + # Build the index + code_size = 128 + build_params = quake.IndexBuildParams() + build_params.metric = "l2" + build_params.nlist = 1024 + build_params.num_workers = 8 + build_params.niter = 25 + + parent_params = quake.IndexBuildParams() + parent_params.nlist = 1 + parent_params.metric = "l2" + parent_params.num_workers = 0 + build_params.parent_params = parent_params + + index = quake.QuakeIndex() + num_initial_vectors = 10000 + insert_vectors = torch.randn(num_initial_vectors, code_size).to(torch.float32) + insert_ids = torch.arange(num_initial_vectors).to(torch.int64) + index.build(insert_vectors, insert_ids, build_params) + print("Built index with", num_initial_vectors, "vectors") + + # Set the mainteance policy + m_params = quake.MaintenancePolicyParams() + m_params.window_size = 5000 + m_params.split_threshold_ns = 2100 + m_params.split_knn_iterations = 10 + m_params.delete_threshold_ns = 2750 + m_params.partition_reduction_threshold = 0.45 + m_params.refinement_radius = 0 + m_params.refinement_iterations = 5 + m_params.min_partition_size = 1024 + m_params.enable_split_rejection = True + m_params.enable_delete_rejection = True + index.initialize_maintenance_policy(m_params) + print("Initialized mainteance policy") + + # Run a batch of queries against the index + search_fraction = 0.1 + + search_params = quake.SearchParams() + search_params.k = 10 + search_params.nprobe = int(search_fraction * index.nlist()) + search_params.recall_target = -1.0 + search_params.batched_scan = True + search_params.batch_size = 500 + search_params.track_hits = True + + query_vectors = torch.randn(2500, code_size).to(torch.float32) + index.search(query_vectors, search_params) + print("Finished a search") + + # Now run mainteance + mainteance_result = index.maintenance() + print("Finished mainteance in", mainteance_result.total_time_us, "us") + + # Now try to add some vectors into the index + num_add_vectors = 5000 + add_vectors = torch.randn(num_add_vectors, code_size).to(torch.float32) + add_ids = torch.arange(num_initial_vectors, num_initial_vectors + num_add_vectors).to(torch.int64) + index.add(add_vectors, add_ids) + print("Added", num_add_vectors, "vectors to index") + +if __name__ == "__main__": + quake.IndexPartition.delete_resize_threshold = 0.8 + quake.IndexPartition.capacity_resize_threshold = 1.2 + quake.QueryCoordinator.partition_chunk_size = 256 + + # mainteance_test() \ No newline at end of file diff --git a/setup.py b/setup.py index 981380d8..77f5e948 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ def build_extension(self, ext): cmake_args += ["-DCMAKE_BUILD_WITH_INSTALL_RPATH=TRUE"] cmake_args += ["-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE"] + cmake_args += ["-DQUAKE_SET_ABI_MODE=ON"] try: import torch diff --git a/src/cpp/bindings/wrap.cpp b/src/cpp/bindings/wrap.cpp index 1ef561bd..c31e7a3d 100644 --- a/src/cpp/bindings/wrap.cpp +++ b/src/cpp/bindings/wrap.cpp @@ -53,6 +53,15 @@ PYBIND11_MODULE(_bindings, m) { and maintain your index. )pbdoc"; + /*********** Index Partition Bindings ***********/ + class_(m, "IndexPartition") + .def_readwrite_static("delete_resize_threshold", &IndexPartition::delete_resize_threshold_) + .def_readwrite_static("capacity_resize_threshold", &IndexPartition::capacity_resize_threshold_); + + class_(m, "QueryCoordinator") + .def_readwrite_static("partition_chunk_size", &QueryCoordinator::batch_scan_partition_chunk_size_) + .def_readwrite_static("query_chunk_size", &QueryCoordinator::batch_scan_query_chunk_size_); + /*********** QuakeIndex Binding ***********/ class_>(m, "QuakeIndex") .def(init(), arg("current_level") = 0, @@ -69,6 +78,10 @@ PYBIND11_MODULE(_bindings, m) { " - num_workers: default = " + std::to_string(DEFAULT_NUM_WORKERS); return doc.c_str(); })()) + .def("add_level", &QuakeIndex::add_level, + "Add a new level to the index.\n\n" + "Args:\n" + " build_params (IndexBuildParams): Parameters for building the new level.") .def("search", &QuakeIndex::search, ([]() -> const char* { static const std::string doc = std::string("Search the index for nearest neighbors.\n\n" @@ -127,6 +140,10 @@ PYBIND11_MODULE(_bindings, m) { return oss.str(); }); + // bool use_gpu = false; + // int gpu_batch_size = DEFAULT_GPU_BATCH_SIZE; + // int gpu_sample_size = DEFAULT_GPU_SAMPLE_SIZE; + /*********** IndexBuildParams Binding ***********/ class_>(m, "IndexBuildParams") .def(init<>()) @@ -138,6 +155,19 @@ PYBIND11_MODULE(_bindings, m) { (std::string("Distance metric. default = ") + DEFAULT_METRIC).c_str()) .def_readwrite("num_workers", &IndexBuildParams::num_workers, (std::string("Number of workers. default = ") + std::to_string(DEFAULT_NUM_WORKERS)).c_str()) + .def_readwrite("parent_params", &IndexBuildParams::parent_params, + "Parameters for the parent index, if any.") + .def_readwrite("num_merge_workers", &IndexBuildParams::num_merge_workers, + (std::string("Number of workers for merging. default = ") + std::to_string(DEFAULT_NUM_MERGE_WORKERS)).c_str()) + .def_readwrite("use_numa", &IndexBuildParams::use_numa, + (std::string("Flag to use NUMA for index building. default = ") + std::to_string(false)).c_str()) + .def_readwrite("use_gpu", &IndexBuildParams::use_gpu, + (std::string("Flag to use GPU for index building. default = ") + std::to_string(false)).c_str()) + .def_readwrite("gpu_batch_size", &IndexBuildParams::gpu_batch_size, + (std::string("Batch size for GPU index building. default = ") + std::to_string(DEFAULT_GPU_BATCH_SIZE)).c_str()) + .def_readwrite("gpu_sample_size", &IndexBuildParams::gpu_sample_size, + (std::string("Sample size for GPU index building. default = ") + std::to_string(DEFAULT_GPU_SAMPLE_SIZE)).c_str()) + .def("__repr__", [](const IndexBuildParams &p) { std::ostringstream oss; oss << "{"; @@ -170,6 +200,29 @@ PYBIND11_MODULE(_bindings, m) { (std::string("Threshold to trigger recomputation of APS. default = ") + std::to_string(DEFAULT_RECOMPUTE_THRESHOLD)).c_str()) .def_readwrite("aps_flush_period_us", &SearchParams::aps_flush_period_us, (std::string("APS flush period in microseconds. default = ") + std::to_string(DEFAULT_APS_FLUSH_PERIOD_US)).c_str()) + .def_readwrite("batch_size", &SearchParams::batch_size, + (std::string("Batch size for batched scan. default = ") + std::to_string(MAX_SUBBATCH)).c_str()) + .def_readwrite("k_factor", &SearchParams::k_factor, + "Factor to adjust the number of neighbors to return.") + .def_readwrite("track_hits", &SearchParams::track_hits, + "Flag to track hits for maintenance policy.") + .def_readwrite("use_auncel", &SearchParams::use_auncel, + "Flag to use Auncel recall estimation for search.") + .def_readwrite("auncel_a", &SearchParams::auncel_a, + "Auncel parameter a for recall estimation.") + .def_readwrite("auncel_b", &SearchParams::auncel_b, + "Auncel parameter b for recall estimation.") + .def_readwrite("use_spann", &SearchParams::use_spann, + "Flag to use SPANN for search.") + .def_readwrite("spann_eps", &SearchParams::spann_eps, + "SPANN parameter epsilon for search.") + .def_readwrite("sample_prefix", &SearchParams::sample_prefix, + "Prefix length for APS sampling.") + .def_readwrite("sample_stride", &SearchParams::sample_stride, + "Stride length for APS sampling.") + + .def_readwrite("parent_params", &SearchParams::parent_params, + "Search parameters for the parent index, if any.") .def("__repr__", [](const SearchParams &s) { std::ostringstream oss; oss << "{"; @@ -198,12 +251,18 @@ PYBIND11_MODULE(_bindings, m) { (std::string("Number of refinement iterations. default = ") + std::to_string(DEFAULT_REFINEMENT_ITERATIONS)).c_str()) .def_readwrite("min_partition_size", &MaintenancePolicyParams::min_partition_size, (std::string("Minimum allowed partition size. default = ") + std::to_string(DEFAULT_MIN_PARTITION_SIZE)).c_str()) + .def_readwrite("max_partition_size", &MaintenancePolicyParams::max_partition_size, + (std::string("Maximum allowed partition size. default = ") + std::to_string(-1)).c_str()) .def_readwrite("alpha", &MaintenancePolicyParams::alpha, (std::string("Alpha parameter. default = ") + std::to_string(DEFAULT_ALPHA)).c_str()) .def_readwrite("enable_split_rejection", &MaintenancePolicyParams::enable_split_rejection, (std::string("Enable split rejection. default = ") + std::to_string(DEFAULT_ENABLE_SPLIT_REJECTION)).c_str()) .def_readwrite("enable_delete_rejection", &MaintenancePolicyParams::enable_delete_rejection, (std::string("Enable delete rejection. default = ") + std::to_string(DEFAULT_ENABLE_DELETE_REJECTION)).c_str()) + .def_readwrite("split_knn_iterations", &MaintenancePolicyParams::split_knn_iterations, + (std::string("Number of clustering iterations to perform during a clustering. default = ") + std::to_string(DEFAULT_NITER)).c_str()) + .def_readwrite("partition_reduction_threshold", &MaintenancePolicyParams::partition_reduction_threshold, + (std::string("Threshold for deleting a partition based on the number of vectors it has lost. default = ") + std::to_string(DEFAULT_PARTITION_REDUCTION_THRESHOLD)).c_str()) .def_readwrite("delete_threshold_ns", &MaintenancePolicyParams::delete_threshold_ns, (std::string("Delete threshold (ns). default = ") + std::to_string(DEFAULT_DELETE_THRESHOLD_NS)).c_str()) .def_readwrite("split_threshold_ns", &MaintenancePolicyParams::split_threshold_ns, @@ -233,10 +292,8 @@ PYBIND11_MODULE(_bindings, m) { "Time taken for split operations in microseconds.") .def_readonly("delete_time_us", &MaintenanceTimingInfo::delete_time_us, "Time taken for delete operations in microseconds.") - .def_readonly("split_refine_time_us", &MaintenanceTimingInfo::split_refine_time_us, + .def_readonly("refinement_time_us", &MaintenanceTimingInfo::refinement_time_us, "Time taken for refinement of split operations in microseconds.") - .def_readonly("delete_refine_time_us", &MaintenanceTimingInfo::delete_refine_time_us, - "Time taken for refinement of delete operations in microseconds.") .def_readonly("n_splits", &MaintenanceTimingInfo::n_splits, "Number of partition split operations performed.") .def_readonly("n_deletes", &MaintenanceTimingInfo::n_deletes, @@ -247,8 +304,7 @@ PYBIND11_MODULE(_bindings, m) { oss << "\"total_time_us\": " << t.total_time_us << ", "; oss << "\"split_time_us\": " << t.split_time_us << ", "; oss << "\"delete_time_us\": " << t.delete_time_us << ", "; - oss << "\"split_refine_time_us\": " << t.split_refine_time_us << ", "; - oss << "\"delete_refine_time_us\": " << t.delete_refine_time_us << ", "; + oss << "\"refinement_time_us\": " << t.refinement_time_us << ", "; oss << "\"n_splits\": " << t.n_splits << ", "; oss << "\"n_deletes\": " << t.n_deletes; oss << "}"; @@ -275,6 +331,14 @@ PYBIND11_MODULE(_bindings, m) { return oss.str(); }); + + // double worker_wait_time_ns = 0; ///< Average worker wait time in nanoseconds. + // double worker_process_time_ns = 0; ///< Average worker process time in nanoseconds. + // double worker_process_preamble_time_ns = 0; ///< Average worker process preamble time in nanoseconds. + // double worker_enqueue_time_ns = 0; ///< Average worker enqueue time in nanoseconds. + // double worker_job_time_ns = 0; ///< Average worker job time in nanoseconds. + // double worker_scan_time_ns = 0; ///< Average worker scan time in nanoseconds. + /*********** SearchTimingInfo Binding ***********/ class_>(m, "SearchTimingInfo") .def(init<>()) @@ -282,6 +346,8 @@ PYBIND11_MODULE(_bindings, m) { "Total time taken for the search operation in nanoseconds.") .def_readwrite("buffer_init_time_ns", &SearchTimingInfo::buffer_init_time_ns, "Time spent on initializing buffers in nanoseconds.") + .def_readwrite("copy_query_time_ns", &SearchTimingInfo::copy_query_time_ns, + "Time spent on copying query vectors to NUMA buffers in nanoseconds.") .def_readwrite("job_enqueue_time_ns", &SearchTimingInfo::job_enqueue_time_ns, "Time spent on creating jobs in nanoseconds.") .def_readwrite("boundary_distance_time_ns", &SearchTimingInfo::boundary_distance_time_ns, @@ -300,6 +366,26 @@ PYBIND11_MODULE(_bindings, m) { "Parameters used for the search operation.") .def_readwrite("parent_info", &SearchTimingInfo::parent_info, "Search info for the parent index.") + .def_readwrite("aps_time_ns", &SearchTimingInfo::aps_time_ns, + "Time spent on APS in nanoseconds.") + .def_readwrite("scan_time_ns", &SearchTimingInfo::scan_time_ns, + "Time spent on scanning in nanoseconds.") + .def_readwrite("worker_wait_time_ns", &SearchTimingInfo::worker_wait_time_ns, + "Average worker wait time in nanoseconds.") + .def_readwrite("worker_process_time_ns", &SearchTimingInfo::worker_process_time_ns, + "Average worker process time in nanoseconds.") + .def_readwrite("worker_process_preamble_time_ns", &SearchTimingInfo::worker_process_preamble_time_ns, + "Average worker process preamble time in nanoseconds.") + .def_readwrite("worker_enqueue_time_ns", &SearchTimingInfo::worker_enqueue_time_ns, + "Average worker enqueue time in nanoseconds.") + .def_readwrite("worker_job_time_ns", &SearchTimingInfo::worker_job_time_ns, + "Average worker job time in nanoseconds.") + .def_readwrite("worker_scan_time_ns", &SearchTimingInfo::worker_scan_time_ns, + "Average worker scan time in nanoseconds.") + .def_readwrite("local_scan_throughput", &SearchTimingInfo::local_scan_throughput, + "Average batch scan throughput") + .def_readwrite("worker_partition_size", &SearchTimingInfo::worker_partition_size, + "Average worker partition size") .def("__repr__", [](const SearchTimingInfo &s) { std::ostringstream oss; oss << "{"; diff --git a/src/cpp/include/blas_dist.h b/src/cpp/include/blas_dist.h new file mode 100644 index 00000000..dc1cd6bb --- /dev/null +++ b/src/cpp/include/blas_dist.h @@ -0,0 +1,137 @@ +/***************************************************************************************** + * faiss_exhaustive_L2sqr_buf.h + * + * Header-only replacement for Faiss’ exhaustive_L2sqr_blas()/knn_L2sqr(). + * ─────────────────────────────────────────────────────────────────────── + * • No dynamic allocations on the critical path. + * • Supersedes Faiss’ originals *without* touching any public headers. + * • Thread-safe, preserves numerical behaviour and ResultHandler contract. + * + * Jason — MIT / Quake project, 2025-05-23 + *****************************************************************************************/ +#ifndef FAISS_EXHAUSTIVE_L2SQR_BUF_H +#define FAISS_EXHAUSTIVE_L2SQR_BUF_H + +#include +#include + +#ifndef FINTEGER +#define FINTEGER long +#endif + +extern "C" { +int sgemm_(const char*, const char*, FINTEGER*, FINTEGER*, FINTEGER*, + const float*, const float*, FINTEGER*, + const float*, FINTEGER*, float*, float*, FINTEGER*); +} + +namespace faiss { + +template +void exhaustive_L2sqr_blas_buf( + const float* __restrict x, + const float* __restrict y, + size_t d, + size_t nx, + size_t ny, + size_t db_blas_bs, // = bs_y + BlockResultHandler& res, + float* __restrict ip_block, // bs_x * bs_y + float* __restrict norms_x, // bs_x + float* __restrict norms_y) // bs_y +{ + if (nx == 0 || ny == 0) return; + + constexpr size_t bs_x = 256; + const size_t bs_y = db_blas_bs; + + for (size_t i0 = 0; i0 < nx; i0 += bs_x) { + const size_t i1 = std::min(i0 + bs_x, nx); + const size_t q_chunk = i1 - i0; + + /* ‖x‖² for this query block */ + fvec_norms_L2sqr(norms_x, x + i0 * d, d, q_chunk); + + res.begin_multiple(i0, i1); + + for (size_t j0 = 0; j0 < ny; j0 += bs_y) { + const size_t j1 = std::min(j0 + bs_y, ny); + const size_t db_chunk = j1 - j0; + + /* ‖y‖² for this database block */ + fvec_norms_L2sqr(norms_y, y + j0 * d, d, db_chunk); + + /* SGEMM */ + { + const float one = 1.f; + float zero = 0.f; + FINTEGER nyi = FINTEGER(db_chunk); + FINTEGER nxi = FINTEGER(q_chunk); + FINTEGER di = FINTEGER(d); + sgemm_("Transpose","Not transpose", + &nyi,&nxi,&di, + &one, + y + j0 * d, &di, + x + i0 * d, &di, + &zero, + ip_block, &nyi); + } + + /* IP → L2² */ + for (int64_t qi = 0; qi < static_cast(q_chunk); ++qi) { + float* line_ptr = ip_block + qi * db_chunk; // Pointer to current column in ip_block + const float current_norm_x = norms_x[qi]; + for (size_t pj = 0; pj < db_chunk; ++pj) { + *line_ptr = std::fma(-2.f, *line_ptr, current_norm_x + norms_y[pj]); + line_ptr++; // Move to the next element in the column + } + } + res.add_results(j0, j1, ip_block); + } + res.end_multiple(); + InterruptCallback::check(); + } +} +/* ---------------------------------------------------------------------- */ +/* 2. Public convenience wrapper (mirrors knn_L2sqr()) */ +/* ---------------------------------------------------------------------- */ +inline void knn_L2sqr_buf( + const float* x, + const float* y, + size_t d, + size_t nx, + size_t ny, + size_t k, + float* vals, // (nx × k) + int64_t* ids, // (nx × k) + float* ip_block, + float* norms_x, + float* norms_y_buf, + size_t db_blas_bs = distance_compute_blas_database_bs, + const float* y_norms = nullptr, + const IDSelector* sel = nullptr) +{ + /* selector path falls back to stock Faiss (rare in hot loop) -------- */ + if (sel) { + knn_L2sqr(x, y, d, nx, ny, k, vals, ids, y_norms, sel); + return; + } + + /* choose the same handlers Faiss uses --------------------------------*/ + if (k == 1) { + Top1BlockResultHandler> rh(nx, vals, ids); + exhaustive_L2sqr_blas_buf(x,y,d,nx,ny,db_blas_bs, + rh,ip_block,norms_x,norms_y_buf); + } else if (k < distance_compute_min_k_reservoir) { + HeapBlockResultHandler> rh(nx, vals, ids, k); + exhaustive_L2sqr_blas_buf(x,y,d,nx,ny,db_blas_bs, + rh,ip_block,norms_x,norms_y_buf); + } else { + ReservoirBlockResultHandler> rh(nx, vals, ids, k); + exhaustive_L2sqr_blas_buf(x,y,d,nx,ny,db_blas_bs, + rh,ip_block,norms_x,norms_y_buf); + } +} + +} // namespace faiss +#endif /* FAISS_EXHAUSTIVE_L2SQR_BUF_H */ \ No newline at end of file diff --git a/src/cpp/include/clustering.h b/src/cpp/include/clustering.h index e4b00229..449c99d5 100644 --- a/src/cpp/include/clustering.h +++ b/src/cpp/include/clustering.h @@ -11,10 +11,41 @@ class IndexPartition; +/** + * @brief Clusters vectors into partitions using faiss::Clustering + * + * + * @param vectors The vectors to cluster. + * @param ids The IDs of the vectors. + * @param n_clusters The number of clusters to create. + * @param metric_type The metric type to use for clustering. + * @param niter The number of iterations to run k-means. + * @param initial_centroids The initial centroids to use for k-means. + */ +shared_ptr kmeans_cpu(Tensor vectors, + Tensor ids, + shared_ptr build_params, + Tensor initial_centroids = Tensor()); + +/** + * @brief Clusters vectors into partitions using CuVS k-means. + * + * + * @param vectors The vectors to cluster. + * @param ids The IDs of the vectors. + * @param n_clusters The number of clusters to create. + * @param metric_type The metric type to use for clustering. + * @param niter The number of iterations to run k-means. + * @param initial_centroids The initial centroids to use for k-means. + */ +#ifdef QUAKE_ENABLE_GPU +shared_ptr kmeans_cuvs_sample_and_predict( + Tensor vectors, Tensor ids, shared_ptr build_params); +#endif + /** * @brief Clusters vectors into partitions using k-means. * - * Uses the faiss::Clustering class to cluster vectors into n_clusters partitions. * * @param vectors The vectors to cluster. * @param ids The IDs of the vectors. @@ -25,10 +56,7 @@ class IndexPartition; */ shared_ptr kmeans(Tensor vectors, Tensor ids, - int n_clusters, - MetricType metric_type, - int niter = 5, - bool use_gpu = false, + shared_ptr build_params, Tensor initial_centroids = Tensor()); @@ -47,8 +75,9 @@ shared_ptr kmeans(Tensor vectors, */ tuple>> kmeans_refine_partitions( Tensor centroids, - vector> index_partitions, + vector> &index_partitions, MetricType metric, - int refinement_iterations = 0); + int refinement_iterations = 0, + int num_threads = -1); #endif //CLUSTERING_H diff --git a/src/cpp/include/common.h b/src/cpp/include/common.h index 0938591c..0f59d7df 100644 --- a/src/cpp/include/common.h +++ b/src/cpp/include/common.h @@ -32,20 +32,13 @@ #include #include #include +#include #ifdef QUAKE_USE_NUMA #include #include #endif -#ifdef FAISS_ENABLE_GPU -#include -#include -#include -#include -#include -#endif - using torch::Tensor; using std::vector; using std::unordered_map; @@ -62,6 +55,18 @@ using std::chrono::milliseconds; using faiss::idx_t; using faiss::MetricType; +struct _EnsureSingleOmp { + _EnsureSingleOmp() { + // Disable OpenMP’s dynamic adjustment and nested teams: + omp_set_dynamic(0); + omp_set_max_active_levels(0); + // Force exactly one thread: + omp_set_num_threads(1); + } +}; + +static _EnsureSingleOmp _ensure_single_omp; + // constants static const uint32_t SerializationMagicNumber = 0x44494E4C; static const uint32_t SerializationVersion = 3; @@ -71,6 +76,9 @@ constexpr int DEFAULT_NLIST = 0; ///< Default number of cluste constexpr int DEFAULT_NITER = 5; ///< Default number of k-means iterations used during clustering. constexpr const char* DEFAULT_METRIC = "l2"; ///< Default distance metric (either "l2" for Euclidean or "ip" for inner product). constexpr int DEFAULT_NUM_WORKERS = 0; ///< Default number of workers (0 means single-threaded). +constexpr int DEFAULT_NUM_MERGE_WORKERS = 1; ///< Default number of merge workers (for worker_scan) +constexpr int DEFAULT_GPU_BATCH_SIZE = 100000; ///< Default batch size for GPU index building. +constexpr int DEFAULT_GPU_SAMPLE_SIZE = 1000000; ///< Default sample size for GPU index building. // Default constants for search parameters constexpr int DEFAULT_K = 1; ///< Default number of neighbors to return. @@ -78,9 +86,13 @@ constexpr int DEFAULT_NPROBE = 1; ///< Default number of constexpr float DEFAULT_RECALL_TARGET = -1.0f; ///< Default recall target (a negative value means no adaptive search). constexpr bool DEFAULT_BATCHED_SCAN = false; ///< Default flag for batched scanning. constexpr bool DEFAULT_PRECOMPUTED = true; ///< Default flag to use precomputed incomplete beta fn for APS. -constexpr float DEFAULT_INITIAL_SEARCH_FRACTION = 0.02f; ///< Default initial fraction of partitions to search. +constexpr float DEFAULT_INITIAL_SEARCH_FRACTION = 0.1f; ///< Default initial fraction of partitions to search. constexpr float DEFAULT_RECOMPUTE_THRESHOLD = 0.001f; ///< Default threshold to trigger recomputation of search parameters. -constexpr int DEFAULT_APS_FLUSH_PERIOD_US = 100; ///< Default period (in microseconds) for flushing the APS buffer. +constexpr int DEFAULT_APS_FLUSH_PERIOD_US = 5; ///< Default period (in microseconds) for flushing the APS buffer. +constexpr int MAX_SUBBATCH = 128; +constexpr int MIN_BATCH_SCAN_SIZE = 4; ///< Minimum batch size for scanning partitions. +constexpr int BLAS_DB_BS = 256; +constexpr int DEFAULT_BLAS_Q_BS = 256; // Default constants for maintenance policy parameters constexpr const char* DEFAULT_MAINTENANCE_POLICY = "query_cost"; ///< Default maintenance policy type. @@ -91,8 +103,10 @@ constexpr int DEFAULT_MIN_PARTITION_SIZE = 32; ///< Default minimum allo constexpr float DEFAULT_ALPHA = 0.9f; ///< Default alpha parameter for maintenance. constexpr bool DEFAULT_ENABLE_SPLIT_REJECTION = true; ///< Default flag to enable rejection of splits. constexpr bool DEFAULT_ENABLE_DELETE_REJECTION = true; ///< Default flag to enable rejection of deletions. -constexpr float DEFAULT_DELETE_THRESHOLD_NS = 10.0f; ///< Default threshold in nanoseconds for deletion decisions. -constexpr float DEFAULT_SPLIT_THRESHOLD_NS = 10.0f; ///< Default threshold in nanoseconds for split decisions. +constexpr float DEFAULT_DELETE_THRESHOLD_NS = 100.0f; ///< Default threshold in nanoseconds for deletion decisions. +constexpr float DEFAULT_SPLIT_THRESHOLD_NS = 100.0f; ///< Default threshold in nanoseconds for split decisions. +constexpr float DEFAULT_PARTITION_REDUCTION_THRESHOLD = 0.3; +constexpr float DEFAULT_CHURN_RECLUSTER_THRESHOLD = 0.4; const vector DEFAULT_LATENCY_ESTIMATOR_RANGE_N = {1, 2, 4, 16, 64, 256, 1024, 4096, 16384, 65536}; ///< Default range of n values for latency estimator. const vector DEFAULT_LATENCY_ESTIMATOR_RANGE_K = {1, 4, 16, 64, 256}; ///< Default range of k values for latency estimator. @@ -110,10 +124,14 @@ struct MaintenancePolicyParams { float alpha = DEFAULT_ALPHA; bool enable_split_rejection = DEFAULT_ENABLE_SPLIT_REJECTION; bool enable_delete_rejection = DEFAULT_ENABLE_DELETE_REJECTION; - + int split_knn_iterations = DEFAULT_NITER; + float partition_reduction_threshold = DEFAULT_PARTITION_REDUCTION_THRESHOLD; float delete_threshold_ns = DEFAULT_DELETE_THRESHOLD_NS; float split_threshold_ns = DEFAULT_SPLIT_THRESHOLD_NS; + // SPFresh Param + int max_partition_size = -1; // -1 means default to standard cost-based maintenance, if set then we use size-based thresholding + MaintenancePolicyParams() = default; }; @@ -125,6 +143,7 @@ struct IndexBuildParams { int dimension = 0; int nlist = DEFAULT_NLIST; int num_workers = DEFAULT_NUM_WORKERS; + int num_merge_workers = DEFAULT_NUM_MERGE_WORKERS; int code_size = -1; // for PQ int num_codebooks = -1; // for PQ string metric = DEFAULT_METRIC; @@ -132,11 +151,15 @@ struct IndexBuildParams { bool use_adaptive_nprobe = false; bool use_numa = false; - bool use_gpu = false; bool verify_numa = false; bool same_core = true; bool verbose = false; + // gpu index build params + bool use_gpu = false; + int gpu_batch_size = DEFAULT_GPU_BATCH_SIZE; + int gpu_sample_size = DEFAULT_GPU_SAMPLE_SIZE; + shared_ptr parent_params = nullptr; IndexBuildParams() = default; @@ -174,11 +197,30 @@ struct SearchParams { float recall_target = DEFAULT_RECALL_TARGET; int num_threads = 1; // number of threads to use for search within a single worker float k_factor = 1.0f; - bool use_precomputed = DEFAULT_PRECOMPUTED; bool batched_scan = DEFAULT_BATCHED_SCAN; + int batch_size = MAX_SUBBATCH; + + bool track_hits = true; + bool scan_all = false; + + // APS params + bool use_precomputed = DEFAULT_PRECOMPUTED; float recompute_threshold = DEFAULT_RECOMPUTE_THRESHOLD; float initial_search_fraction = DEFAULT_INITIAL_SEARCH_FRACTION; int aps_flush_period_us = DEFAULT_APS_FLUSH_PERIOD_US; + int sample_prefix = 0; + int sample_stride = 10; + + // Auncel params + bool use_auncel = false; + float auncel_a = 1.0f; + float auncel_b = 1.0f; + + // Spann params + bool use_spann = false; + float spann_eps = 1.25; + + shared_ptr parent_params = nullptr; ///< Search parameters for the parent index, if any. SearchParams() = default; }; @@ -220,11 +262,36 @@ struct SearchTimingInfo { // main thread counters for worker scan int64_t buffer_init_time_ns; ///< Time spent on initializing buffers in nanoseconds. + int64_t copy_query_time_ns; int64_t job_enqueue_time_ns; ///< Time spent on creating jobs in nanoseconds. int64_t boundary_distance_time_ns; ///< Time spent on computing boundary distances in nanoseconds. + int64_t aps_time_ns; ///< Time spent on APS in nanoseconds. + int64_t scan_time_ns; ///< Time spent on scanning in nanoseconds. int64_t job_wait_time_ns; ///< Time spent waiting for jobs to complete in nanoseconds. int64_t result_aggregate_time_ns; ///< Time spent on aggregating results in nanoseconds. int64_t total_time_ns; ///< Total time spent in nanoseconds. + double worker_wait_time_ns = 0; ///< Average worker wait time in nanoseconds. + double worker_process_time_ns = 0; ///< Average worker process time in nanoseconds. + double worker_process_preamble_time_ns = 0; ///< Average worker process preamble time in nanoseconds. + double worker_enqueue_time_ns = 0; ///< Average worker enqueue time in nanoseconds. + double worker_job_time_ns = 0; ///< Average worker job time in nanoseconds. + double worker_scan_time_ns = 0; ///< Average worker scan time in nanoseconds. + + int64_t total_worker_jobs = 0; ///< The number of worker jobs + double worker_partition_size_bytes = 0; ///< Average partition size scanned by worker (in bytes) + double worker_scan_throughput = 0; ///< Average worker scan throughput (bytes/ns = GB/s). + double local_scan_throughput = 0; ///< Scan throughput per job rather than averaged across all workers + double worker_partition_size = 0; ///< Average worker partition size + + double worker_batch_scan_ipc = 0; + double worker_batch_scan_miss_rate = 0; + + double single_scan_job_time_ns = 0; + double faiss_norms_x_time_ns = 0; + double faiss_norms_y_time_ns = 0; + double sgemm_time_ns = 0; + double ip_to_l2_time_ns = 0; + double top_k_buffer_add_ns = 0; }; /** @@ -233,10 +300,12 @@ struct SearchTimingInfo { struct MaintenanceTimingInfo { int64_t n_splits; ///< Number of splits. int64_t n_deletes; ///< Number of merges. + int64_t n_recluster; ///< Number of reclusters + int64_t delete_time_us; ///< Time spent on deletions in microseconds. - int64_t delete_refine_time_us; ///< Time spent on deletions with refinement in microseconds. int64_t split_time_us; ///< Time spent on splits in microseconds. - int64_t split_refine_time_us; ///< Time spent on splits with refinement in microseconds. + int64_t refinement_time_us; ///< Time spent on refinement in microseconds. + int64_t recluster_time_us; ///< Time spent on reclustering int64_t total_time_us; ///< Total time spent in microseconds. }; diff --git a/src/cpp/include/dynamic_inverted_list.h b/src/cpp/include/dynamic_inverted_list.h index e940127e..6ac89364 100644 --- a/src/cpp/include/dynamic_inverted_list.h +++ b/src/cpp/include/dynamic_inverted_list.h @@ -22,8 +22,10 @@ namespace faiss { * It supports dynamic operations (addition, update, removal) across multiple partitions and includes * NUMA-aware functionality. */ - class DynamicInvertedLists : public InvertedLists { + class DynamicInvertedLists { public: + size_t nlist; ///< number of possible key values + size_t code_size; ///< code size per vector in bytes int curr_list_id_ = 0; ///< Next available partition ID. int total_numa_nodes_ = 0; ///< Total NUMA nodes available. @@ -31,6 +33,8 @@ namespace faiss { int d_; ///< Dimensionality of the vectors (derived from code_size). int code_size_; ///< Size in bytes of each vector code. unordered_map> partitions_; ///< Map of partition ID to IndexPartition. + unordered_map> id_to_location_; + unordered_map tombstones_; /** * @brief Constructor for DynamicInvertedLists. @@ -47,7 +51,7 @@ namespace faiss { * * Frees memory by relying on each IndexPartition’s destructor. */ - ~DynamicInvertedLists() override; + ~DynamicInvertedLists(); /** * @brief Return the total number of vectors stored across all partitions. @@ -63,7 +67,7 @@ namespace faiss { * @return Count of vectors in the partition. * @throws std::runtime_error if the partition does not exist. */ - size_t list_size(size_t list_no) const override; + size_t list_size(size_t list_no) const; /** * @brief Get the pointer to the encoded vectors for a partition. @@ -72,7 +76,7 @@ namespace faiss { * @return Pointer to codes. * @throws std::runtime_error if the partition does not exist. */ - const uint8_t* get_codes(size_t list_no) const override; + const uint8_t* get_codes(size_t list_no) const; /** * @brief Get the pointer to the vector IDs for a partition. @@ -81,7 +85,9 @@ namespace faiss { * @return Pointer to IDs. * @throws std::runtime_error if the partition does not exist. */ - const idx_t* get_ids(size_t list_no) const override; + const idx_t* get_ids(size_t list_no) const; + + void build_map(); /** * @brief Release the codes pointer. @@ -91,7 +97,7 @@ namespace faiss { * @param list_no Partition number. * @param codes Unused. */ - void release_codes(size_t list_no, const uint8_t *codes) const override; + void release_codes(size_t list_no, const uint8_t *codes) const; /** * @brief Release the IDs pointer. @@ -101,7 +107,7 @@ namespace faiss { * @param list_no Partition number. * @param ids Unused. */ - void release_ids(size_t list_no, const idx_t *ids) const override; + void release_ids(size_t list_no, const idx_t *ids) const; /** * @brief Remove an entry with the given ID from a specified partition. @@ -124,9 +130,15 @@ namespace faiss { /** * @brief Remove specified vectors from all partitions. * - * @param vectors_to_remove A set of vector IDs to remove. + * @param vectors_to_remove A ptr to the vectors to remove + * @param num_vectors The vectors to remove + */ + void remove_vectors(int64_t* vectors_to_remove, size_t num_vectors, bool update_delta = false); + + /** + * @brief Returns the underlying partition */ - void remove_vectors(std::set vectors_to_remove); + shared_ptr get_partition(size_t list_no); /** * @brief Append new entries (codes and IDs) to a partition. @@ -142,7 +154,8 @@ namespace faiss { size_t list_no, size_t n_entry, const idx_t *ids, - const uint8_t *codes) override; + const uint8_t *codes, + bool update_delta = false); /** * @brief Update existing entries in a partition. @@ -161,7 +174,13 @@ namespace faiss { size_t offset, size_t n_entry, const idx_t *ids, - const uint8_t *codes) override; + const uint8_t *codes); + + /** + * @brief Updates the vector with the provided id to the specified value + * + */ + void write_vector_by_id(idx_t id, float* vector_values); /** * @brief Batch update: move vectors from one partition to new partitions. @@ -236,7 +255,7 @@ namespace faiss { * * Clears all partitions and resets counters. */ - void reset() override; + void reset(); /** * @brief Resize the inverted lists. @@ -246,7 +265,7 @@ namespace faiss { * @param nlist New number of partitions. * @param code_size New code size. */ - void resize(size_t nlist, size_t code_size) override; + void resize(size_t nlist, size_t code_size); /** * @brief Set NUMA configuration for the inverted lists. @@ -325,6 +344,20 @@ namespace faiss { * @return A 1D tensor containing all partition IDs. */ Tensor get_partition_ids(); + + template + inline void map_add(IndexPartition* p, int64_t off, IdT id) noexcept { + id_to_location_[id] = {p, off}; + } + + template + inline void map_erase(IdT id) noexcept { + id_to_location_.erase(id); + } + template + inline void map_swap(IndexPartition* p, int64_t off, IdT id) noexcept { + id_to_location_[id] = {p, off}; + } }; /** diff --git a/src/cpp/include/geometry.h b/src/cpp/include/geometry.h index e16eafc6..4e3922d7 100644 --- a/src/cpp/include/geometry.h +++ b/src/cpp/include/geometry.h @@ -8,6 +8,7 @@ #define STOP 1.0e-8 #define TINY 1.0e-30 +#define CRP_CHECK(cond, msg) ((void)0) using torch::Tensor; using std::vector; @@ -54,64 +55,6 @@ inline void print_array(const float *array, int dimension) { std::cout << std::endl << std::endl; } -inline vector compute_boundary_distances(const Tensor &query, vector centroids, bool euclidean = true) { - - auto start = std::chrono::high_resolution_clock::now(); - int dimension = query.size(0); - - std::vector boundary_distances(centroids.size(), -1.0f); - - const float *query_ptr = query.data_ptr(); - const float *nearest_centroid_ptr = centroids[0]; - - vector line_vector(dimension); - vector midpoint(dimension); - vector residual(dimension); - - auto end = std::chrono::high_resolution_clock::now(); - - // used for euclidean distance - if (euclidean) { - // Compute residual: r = q - c0. - faiss::fvec_sub(dimension, query_ptr, nearest_centroid_ptr, residual.data()); - - // For each centroid j (starting at index 1). - for (int j = 1; j < centroids.size(); j++) { - // Compute v = c_j - c0. - const float* c_j = centroids[j]; - faiss::fvec_sub(dimension, c_j, nearest_centroid_ptr, line_vector.data()); - - // Compute squared norm: A2 = ||v||^2. - float A2 = faiss::fvec_inner_product(line_vector.data(), line_vector.data(), dimension); - float A = std::sqrt(A2); // Guaranteed nonzero. - - // Compute dot product: dot = . - float dot_val = faiss::fvec_inner_product(residual.data(), line_vector.data(), dimension); - - // Instead of computing dot_val/A and 0.5*A separately, - // we compute: d = |dot_val - 0.5 * A2| / A. - float d = std::fabs(dot_val - 0.5f * A2) / A; - boundary_distances[j] = d; - } - } else { - // for dot product distance - float residual_angle = faiss::fvec_inner_product(query_ptr, nearest_centroid_ptr, dimension); - for (int j = 1; j < centroids.size(); j++) { - // get angle of the bisector using dot product - subtract_arrays(centroids[j], nearest_centroid_ptr, line_vector.data(), dimension); - divide_array_by_constant(line_vector.data(), 2.0f, midpoint.data(), dimension); - add_arrays(nearest_centroid_ptr, midpoint.data(), midpoint.data(), dimension); - float norm = faiss::fvec_inner_product(midpoint.data(), midpoint.data(), dimension); - norm = std::sqrt(norm); - divide_array_by_constant(midpoint.data(), norm, midpoint.data(), dimension); - float boundary_angle = faiss::fvec_inner_product(query_ptr, midpoint.data(), dimension); - boundary_distances[j] = std::acos(boundary_angle); - } - } - - return boundary_distances; -} - inline double incomplete_beta(double a, double b, double x) { if (x < 0.0 || x > 1.0) return 1.0 / 0.0; @@ -218,256 +161,332 @@ inline double log_hypersphere_volume(double radius, int dimension) { return log_volume; } -inline double hypersphere_surface_area(double radius, int d) { - return 2 * std::pow(M_PI, d / 2.0) * std::pow(radius, d - 1) / std::tgamma(d / 2.0); -} -inline double compute_partial_cap_volume(double radius, double theta_min, double theta_max, int d) { - // Compute the volume of a hyperspherical cap between angles theta_min and theta_max - if (theta_min < 0 || theta_max > M_PI) { - throw std::invalid_argument("Theta values out of bounds."); - } +inline std::vector +compute_boundary_distances(const Tensor& query, + std::vector& centroids, + bool euclidean) +{ + const int dim = query.size(0); + const float* q = query.data_ptr(); - double cap_volume = 0.0; + const float* c0 = centroids[0]; + std::vector dist(centroids.size(), 0.0f); - // Using the regularized incomplete beta function - double sin2_theta_min = std::sin(theta_min) * std::sin(theta_min); - double sin2_theta_max = std::sin(theta_max) * std::sin(theta_max); + std::vector v(dim); // c_j - c0 + std::vector m(dim); // midpoint for IP - double I_min = incomplete_beta((d - 1.0) / 2.0, 0.5, sin2_theta_min); - double I_max = incomplete_beta((d - 1.0) / 2.0, 0.5, sin2_theta_max); + for (std::size_t j = 1; j < centroids.size(); ++j) { + const float* cj = centroids[j]; - double surface_area = hypersphere_surface_area(radius, d); - cap_volume = surface_area * (I_max - I_min); + if (centroids[j] == nullptr) { + if (euclidean) { + dist[j] = std::numeric_limits::infinity(); + } else { + dist[j] = M_PI_2; // max distance on unit sphere + } + continue; + } - return cap_volume; + if (euclidean) { + /* plane distance d = |q·v − b| / ||v|| , b = ½(||cj||²−||c0||²) */ + faiss::fvec_sub(dim, cj, c0, v.data()); + float v_norm = std::sqrt(faiss::fvec_inner_product(v.data(), v.data(), dim)); + float b = 0.5f * (faiss::fvec_inner_product(cj, cj, dim) - + faiss::fvec_inner_product(c0, c0, dim)); + float dot_qv = faiss::fvec_inner_product(q, v.data(), dim); + dist[j] = std::fabs(dot_qv - b) / (v_norm + 1e-12f); + } else { + faiss::fvec_sub(dim, cj, c0, v.data()); // v = cj - c0 + float v_norm = std::sqrt(faiss::fvec_inner_product(v.data(), v.data(), dim)); + divide_array_by_constant(v.data(), v_norm, v.data(), dim); // v̂ + float s = std::fabs(faiss::fvec_inner_product(q, v.data(), dim)); + s = std::clamp(s, 0.0f, 1.0f); + dist[j] = std::asin(s); // 0–π/2 + } + } + return dist; // dist[0] = 0 by construction } +inline double hyperspherical_cap_volume(double radius, double boundary_distance, int d, bool use_precomputed = true, bool euclidean = true) { -inline double log_hyperspherical_cap_volume(double radius, double boundary_distance, int d, bool ratio = true, - bool use_precomputed = true, bool euclidean = true) { - double h = radius - boundary_distance; - - // Ensure h is within valid range - h = std::max(0.0, std::min(2 * radius, h)); if (euclidean) { - double x = std::sqrt((2 * radius * h - h * h) / (radius * radius)); - // use precomputed incomplete beta function - double inc_beta; - if (use_precomputed) { - inc_beta = incomplete_beta_lookup(x, d); - } else { - inc_beta = incomplete_beta((d + 1.0) / 2.0, 0.5, x); - } - if (inc_beta <= 0.0 || std::isnan(inc_beta) || std::isinf(inc_beta)) { - std::cerr << "Invalid incomplete beta value: " << inc_beta << std::endl; - return -std::numeric_limits::infinity(); - } + // Ensure boundary distance is non-negative double + boundary_distance = std::max(0.0, boundary_distance); - double log_inc_beta = std::log(inc_beta); - double log_cap_volume; - if (!ratio) { - double log_sphere_volume = log_hypersphere_volume(radius, d); - log_cap_volume = std::log(0.5) + log_inc_beta + log_sphere_volume; - } else { - log_cap_volume = std::log(0.5) + log_inc_beta; - } + // If boundary is outside or on the query radius, the cap volume is 0 + if (boundary_distance >= radius) return 0.0; - return log_cap_volume; - } else { - // use intersection of spherical caps - if (ratio != true) { - throw std::invalid_argument("Ratio must be true for dot product distance"); - } + // Calculate x for incomplete beta function + double x = sqrt(1.0 - (boundary_distance / radius) * (boundary_distance / radius)); + x = std::clamp(x, 0.0, 1.0); // Clamp x to [0, 1] - // v_i = (1/2) * [ I( sin^2(phi/2); d/2, 1/2 ) - I( sin^2(theta_i/2); d/2, 1/2 ) ] + // Incomplete Beta parameters + double a = 0.5 * (d + 1.0); + double b = 0.5; + double I = use_precomputed + ? incomplete_beta_lookup(x, d) + : incomplete_beta(a, b, x); - double log_inc_beta = std::log(incomplete_beta((d - 1) / 2.0, 0.5, std::sin(radius / 2.0) * std::sin(radius / 2.0))); - double log_inc_beta_boundary = std::log(incomplete_beta((d - 1) / 2.0, 0.5, std::sin(boundary_distance / 2.0) * std::sin(boundary_distance / 2.0))); - double log_cap_volume = std::log(0.5) + log_inc_beta - log_inc_beta_boundary; - return log_cap_volume; - // compute volume of the intersection of two spherical caps (from the paper: concise formulas for the volume of hyperspherical caps) + return std::clamp(0.5 * I, 0.0, 0.5); + } else { + // spherical / IP ----------------------------------------------------------------- + double theta_q = radius; // query cap angle (rad) + double delta = boundary_distance; // distance to bisector (rad) + /* 1. trivial cases ------------------------------------------------------------ */ + if (delta >= theta_q) return 0.0; // cap entirely in c0 + if (theta_q >= M_PI_2 - delta) return 1.0; // cap entirely in cj - } -} + /* 2. general case: Lee & Kim (2014) cases 9–10 -------------------------------- */ + double t = std::tan(delta) / std::tan(theta_q); // 0 ≤ t < 1 + t = std::clamp(t, 0.0, 1.0); // numerical safety -inline vector compute_intersection_volume(const Tensor &boundary_distances, float query_radius, int dimension, - bool use_precomputed = true) { - auto boundary_distances_ptr = boundary_distances.data_ptr(); - int num_partitions = boundary_distances.size(0); - std::vector partition_volumes(num_partitions, 0.0f); + double alpha = std::acos(t); // 0 < α < π/2 + double x = std::sin(alpha) * std::sin(alpha); // 0 < x < 1 - for (int j = 0; j < num_partitions; j++) { - float boundary_distance = boundary_distances_ptr[j]; + double a = 0.5 * (d - 1); + double b = 0.5; - if (boundary_distance >= query_radius) { - partition_volumes[j] = -1e8; - continue; - } + double Ix = incomplete_beta(a, b, x); // regularised + return 0.5 * Ix; // leakage fraction - double volume_ratio = log_hyperspherical_cap_volume(query_radius, boundary_distance, dimension, false, - use_precomputed); - partition_volumes[j] = volume_ratio; + // double log_inc_beta = std::log(incomplete_beta((d - 1) / 2.0, 0.5, std::sin(radius / 2.0) * std::sin(radius / 2.0))); + // double log_inc_beta_boundary = std::log(incomplete_beta((d - 1) / 2.0, 0.5, std::sin((radius - boundary_distance) / 2.0) * std::sin(radius - boundary_distance) / 2.0)); + // double log_cap_volume = std::log(0.5) + log_inc_beta - log_inc_beta_boundary; + // return std::exp(log_cap_volume); } - - return partition_volumes; } -inline Tensor compute_variance_in_direction_of_query(Tensor query, Tensor centroids, Tensor variance) { - int dimension = query.size(0); - int num_partitions = centroids.size(0); - auto query_ptr = query.data_ptr(); - auto centroids_ptr = centroids.data_ptr(); - auto variance_ptr = variance.data_ptr(); +inline std::vector +compute_recall_profile(const std::vector& boundary_distances, + float query_radius, + int dimension, + std::vector partition_sizes = {}, // Unused in these models + bool use_precomputed = true, + bool euclidean = true) // Unused in these models +{ + const int m = static_cast(boundary_distances.size()); + const float eps = 1e-9f; + + if (!euclidean) { + query_radius = std::acos(query_radius); // Convert to angle in radians for spherical model + } - std::vector variances(num_partitions, 0.0f); + // --- Edge Cases --- + if (m <= 1) { + if (m == 1) return {1.0f}; // Only the central partition exists + return {}; // No partitions defined + } + if (query_radius <= eps) { + std::vector p(m, 0.0f); + p[0] = 1.0f; // Query point is exactly at origin, must be in partition 0 + return p; + } - for (int j = 0; j < num_partitions; j++) { - float *centroid = centroids_ptr + j * dimension; - float *variance = variance_ptr + j * dimension; + // Compute raw cap volumes + std::vector raw_vols(m, 0.0f); + for (int j = 1; j < m; ++j) { + raw_vols[j] = hyperspherical_cap_volume(query_radius, boundary_distances[j], dimension, use_precomputed, euclidean); + } - float *query_minus_centroid = new float[dimension]; - subtract_arrays(query_ptr, centroid, query_minus_centroid, dimension); + float P0 = 0.0f; // Root cell probability + std::vector P_prime(m, 0.0f); // Intermediate neighbor probabilities (k>=1) + float P_prime_sum = 0.0f; // Sum of P_prime[k] for k>=1 - float dot_product = faiss::fvec_inner_product(query_minus_centroid, query_minus_centroid, dimension); + std::vector norm_vols = raw_vols; // Copy raw vols + float S1_for_norm = 0.0f; + for (int j = 1; j < m; ++j) S1_for_norm += norm_vols[j]; - variances[j] = dot_product; - delete[] query_minus_centroid; - } + // S1_for_norm = 1.0; - return torch::tensor(variances).clone(); -} + if (S1_for_norm > eps) { + for (int j = 1; j < m; ++j) norm_vols[j] /= S1_for_norm; + } else { + for (int j = 1; j < m; ++j) norm_vols[j] = 0.0f; + } -inline vector compute_recall_profile(vector boundary_distances, float query_radius, int dimension, - vector partition_sizes = {}, bool use_precomputed = true, - bool euclidean = true) { + P0 = 1.0f; + for (int j = 1; j < m; ++j) P0 *= (1.0f - norm_vols[j]); // Survival probability of the root cell + P0 = std::clamp(P0, 0.0f, 1.0f); - // boundary_distances shape is (num_partitions,) and num_partitions must be greater than 1 - if (boundary_distances.size() < 2) { - throw std::runtime_error("Boundary distances must have at least 2 partitions to create an estimate."); + for (int k = 1; k < m; ++k) { + P_prime[k] = norm_vols[k]; + P_prime_sum += P_prime[k]; } + // Ensure P_prime is non-negative + for (int k = 1; k < m; ++k) P_prime[k] = std::max(0.0f, P_prime[k]); + + // normalize probs + std::vector probs(m, 0.0f); + probs[0] = P0; + + // for (int k = 1; k < m; ++k) { + // if (P_prime_sum > eps) { + // probs[k] = norm_vols[k]; + // } else { + // probs[k] = 0.0f; // If sum is negligible, set to zero + // } + // } + // + // // normalize probabilities + // float S = 0.0f; + // for (int k = 0; k < m; ++k) S += probs[k]; + // if (S > eps) { + // for (int k = 0; k < m; ++k) { + // probs[k] /= S; + // } + // } else { + // // If S is zero, all probabilities remain zero + // for (int k = 0; k < m; ++k) { + // probs[k] = 0.0f; + // } + // } - int num_partitions = boundary_distances.size(); - vector partition_probabilities(num_partitions, 0.0f); + float target = 1.0f - P0; + // Ensure target probability for neighbors is valid [0, 1] + target = std::clamp(target, 0.0f, 1.0f); - double total_volume = 0.0; - bool weigh_using_partition_sizes = partition_sizes.size() == num_partitions; + if (target > eps && P_prime_sum > eps) { + float scale = target / P_prime_sum; + for (int k = 1; k < m; ++k) { + // Ensure final probability is non-negative + probs[k] = std::max(0.0f, P_prime[k] * scale); + } - for (int j = 1; j < num_partitions; j++) { - float boundary_distance = boundary_distances[j]; - if (boundary_distance >= query_radius) { - partition_probabilities[j] = 0.0; - continue; + // Strict renormalization to ensure sum is exactly 1 + float current_sum_k = 0.0f; + for (int k = 1; k < m; ++k) current_sum_k += probs[k]; + + if (current_sum_k > eps) { // Avoid division by zero if sum is negligible + float final_scale = target / current_sum_k; + // Check if scale is finite (handles target=0 case correctly) + if (std::isfinite(final_scale)) { + for (int k = 1; k < m; ++k) { + probs[k] *= final_scale; + // Final clamp for safety + probs[k] = std::max(0.0f, probs[k]); + } + } else if (target <= eps) { + // If target is zero, all neighbor probs should be zero + for (int k = 1; k < m; ++k) probs[k] = 0.0f; + } + } else if (target <= eps) { + // If target is zero and calculated sum is zero, ensure all are zero + for (int k = 1; k < m; ++k) probs[k] = 0.0f; } - - double volume_ratio = std::exp( - log_hyperspherical_cap_volume(query_radius, - boundary_distance, - dimension, - true, - use_precomputed, - euclidean)); - partition_probabilities[j] = (volume_ratio > 0.0) ? volume_ratio : 0.0; } - // TODO: Implement a better way to compute the probabilities for the first partition. This heuristic works well on tested datasets. - partition_probabilities[0] = 2.0 * partition_probabilities[1]; - // partition_probabilities[0] = 1 - partition_probabilities[1]; - - // if (weigh_using_partition_sizes) { - // for (int j = 0; j < num_partitions; j++) { - // partition_probabilities[j] *= partition_sizes[j]; - // } - // } - - // Ensure the probabilities sum to 1 - double sum_probabilities = 0.0; - for (int j = 0; j < num_partitions; j++) { - sum_probabilities += partition_probabilities[j]; + // if the cluster_sizes are given, scale probs by the size of the cluster and renormalize. this is a rudimentary density estimation + if (partition_sizes.size() > 0) { + for (int k = 0; k < m; ++k) { + if (partition_sizes[k] > 0) { + probs[k] *= static_cast(partition_sizes[k]); + } + } } - if (sum_probabilities > 0.0f) { - for (int j = 0; j < num_partitions; j++) { - partition_probabilities[j] /= sum_probabilities; + + // renormalize the probabilities to sum to 1 + float S = 0.0f; + for (int k = 0; k < m; ++k) S += probs[k]; + if (S > eps) { + for (int k = 0; k < m; ++k) { + probs[k] /= S; } } else { - for (int j = 0; j < num_partitions; j++) { - partition_probabilities[j] = 1.0 / num_partitions; + // If S is zero, all probabilities remain zero + for (int k = 0; k < m; ++k) { + probs[k] = 0.0f; } } - // Compute the recall profile - // Tensor recall_profile = torch::cumsum(probabilities_tensor, 0); - - return partition_probabilities; + return probs; } -inline float compute_intersection_volume_one(float boundary_distance, float query_radius, int dimension) { - if (boundary_distance >= query_radius) { - return -1e8; +inline std::vector +compute_recall_profile_auncel( + const std::vector& boundary_distances, + float query_radius, + int K_neighbors, + float a, + float b +) { + const size_t L = boundary_distances.size(); + + // 1) Spherical-cap "angle" terms + std::vector cap_terms(L, 0.0f); + if (query_radius > 1e-9f) { + for (size_t j = 0; j < L; ++j) { + float r = boundary_distances[j] / query_radius; + r = std::clamp(r, -1.0f, 1.0f); + cap_terms[j] = (r >= 1.0f ? 0.0f : std::acos(r)); + } } - double volume_ratio = log_hyperspherical_cap_volume(query_radius, boundary_distance, dimension, true); + // 2) Calculate phi_values for each stage + std::vector phi_values(L + 1); + float running_U_sum = std::accumulate(cap_terms.begin(), cap_terms.end(), 0.0f); - return volume_ratio; -} + for (size_t i = 0; i <= L; ++i) { + float denominator = b - a * running_U_sum; + if (denominator <= 1e-9f) { + phi_values[i] = std::numeric_limits::max(); + } else { + phi_values[i] = 1.0f / denominator; + } + phi_values[i] = std::max(1.0f, phi_values[i]); // Ensure phi >= 1 -inline Tensor estimate_overlap(const Tensor &new_centroid, const Tensor &old_centroid, const Tensor &nbr_centroids) { - Tensor residual = new_centroid - old_centroid; - int dimension = new_centroid.size(0); - - vector old_boundary_distance(nbr_centroids.size(0), -1.0f); - vector new_boundary_distance(nbr_centroids.size(0), -1.0f); - - const float *residual_ptr = residual.data_ptr(); - const float *new_centroid_ptr = new_centroid.data_ptr(); - const float *old_centroid_ptr = old_centroid.data_ptr(); - const float *nbr_centroids_ptr = nbr_centroids.data_ptr(); - - std::vector line_vector(dimension); - std::vector midpoint(dimension); - std::vector projection(dimension); - - // compute distance to old boundary - for (int j = 0; j < nbr_centroids.size(0); j++) { - subtract_arrays(nbr_centroids_ptr + (dimension * j), old_centroid_ptr, line_vector.data(), dimension); - divide_array_by_constant(line_vector.data(), 2.0f, midpoint.data(), dimension); - float norm = faiss::fvec_inner_product(midpoint.data(), midpoint.data(), dimension); - norm = std::sqrt(norm); - old_boundary_distance[j] = norm; + if (i < L) { + running_U_sum -= cap_terms[i]; + } } - // compute distance to new boundary - for (int j = 0; j < nbr_centroids.size(0); j++) { - subtract_arrays(nbr_centroids_ptr + (dimension * j), new_centroid_ptr, line_vector.data(), dimension); - divide_array_by_constant(line_vector.data(), 2.0f, midpoint.data(), dimension); - float norm = faiss::fvec_inner_product(midpoint.data(), midpoint.data(), dimension); - norm = std::sqrt(norm); - new_boundary_distance[j] = norm; - } + // 3) Calculate cumulative recall based on phi_values + std::vector cumulative_recalls(L + 1, 0.0f); + for (size_t i = 0; i <= L; ++i) { + float current_phi = phi_values[i]; + float j_star = 0.0f; + + if (K_neighbors > 0) { + if (current_phi == std::numeric_limits::max()) { + j_star = 0.0f; + } else if (current_phi > 1e-9f) { + j_star = std::floor((float)K_neighbors / current_phi); + j_star = std::min(std::max(0.0f, j_star), (float)K_neighbors); + } + } - Tensor overlap_ratio = torch::empty({nbr_centroids.size(0)}, torch::kFloat32); + float error_i = 1.0f; // Default error if K_neighbors is 0 or no items found + if (K_neighbors > 0) { + error_i = 1.0f - j_star / (float)K_neighbors; + } + cumulative_recalls[i] = std::clamp(1.0f - error_i, 0.0f, 1.0f); + } - // for each neighbor, compute the hyperspherical cap volume, where the radius of the sphere is the distance to new boundary - // and the old boundary distance gives the height of the cap - float mean_new_boundary_distance = 0.0f; - float mean_old_boundary_distance = 0.0f; - for (int j = 0; j < nbr_centroids.size(0); j++) { - mean_new_boundary_distance += new_boundary_distance[j]; - mean_old_boundary_distance += old_boundary_distance[j]; + // 4) Calculate per-stage incremental recall + std::vector recall_profile(L + 1, 0.0f); + if (!cumulative_recalls.empty()) { + recall_profile[0] = cumulative_recalls[0]; + for (size_t i = 1; i <= L; ++i) { + recall_profile[i] = cumulative_recalls[i] - cumulative_recalls[i-1]; + recall_profile[i] = std::max(0.0f, recall_profile[i]); + } } - mean_new_boundary_distance /= nbr_centroids.size(0); - mean_old_boundary_distance /= nbr_centroids.size(0); - for (int j = 0; j < nbr_centroids.size(0); j++) { - overlap_ratio[j] = abs(new_boundary_distance[j] - old_boundary_distance[j]) / mean_old_boundary_distance; + // 5) Normalize the profile to sum to 1 + float S = std::accumulate(recall_profile.begin(), recall_profile.end(), 0.0f); + if (S > 1e-9f) { + for (auto &v : recall_profile) { + v /= S; + } } + // If S is 0 (e.g., K_neighbors=0 or zero predicted recall), profile remains all zeros. - return overlap_ratio; + return recall_profile; } -#endif // CPP_UTILS_GEOMETRY_H +#endif // CPP_UTILS_GEOMETRY_H \ No newline at end of file diff --git a/src/cpp/include/index_partition.h b/src/cpp/include/index_partition.h index 63b022c6..0529a6bf 100644 --- a/src/cpp/include/index_partition.h +++ b/src/cpp/include/index_partition.h @@ -18,16 +18,26 @@ */ class IndexPartition { public: + // Static field for all index partitions + static float delete_resize_threshold_; + static float capacity_resize_threshold_; + int numa_node_ = -1; ///< Assigned NUMA node (-1 if not set) int core_id_ = -1; ///< Mapped thread ID for processing int64_t buffer_size_ = 0; ///< Allocated capacity (in number of vectors) int64_t num_vectors_ = 0; ///< Current number of stored vectors int64_t code_size_ = 0; ///< Size of each code in bytes (must be set before adding vectors) + int64_t partition_id_ = -1; uint8_t* codes_ = nullptr; ///< Pointer to the encoded vectors (raw memory block) idx_t* ids_ = nullptr; ///< Pointer to the vector IDs + int64_t last_snapshot_size_ = 0; ///< The size since the last snapshot + int64_t churn_count_ = 0; ///< Counter for the churn on this index + int64_t delta_count_ = 0; ///< Counter of the delta that has happened since the last snapshot + uint8_t* delta_vec_ = nullptr; ///< Vector keeping track of the delta for this index + std::unordered_map id_to_index_; ///< Map of vector ID to index /// Default constructor. @@ -70,6 +80,10 @@ class IndexPartition { /// Destructor. Frees all allocated memory. ~IndexPartition(); + void allocate_delta_buffer(); + + void reset_delta(); + /** * @brief Set the code size. * @@ -88,7 +102,7 @@ class IndexPartition { * @param new_ids Pointer to the new vector IDs. * @param new_codes Pointer to the new encoded vectors. */ - void append(int64_t n_entry, const idx_t* new_ids, const uint8_t* new_codes); + void append(int64_t n_entry, const idx_t* new_ids, const uint8_t* new_codes, bool update_delta = false); /** * @brief Update existing entries in place. @@ -109,7 +123,7 @@ class IndexPartition { * * @param index Index of the vector to remove. */ - void remove(int64_t index); + int64_t remove(int64_t index, bool update_delta = false); /** * @brief Resize the partition. @@ -128,6 +142,14 @@ class IndexPartition { */ void clear(); + /** + * @brief Checks if the index buffer should be downsized + * + * If the partition determines that its partition is too large, it downsizes the buffer. Note that this + * is a best effort method so the partition could decide not to downsize + */ + void check_buffer_size(); + /** * @brief Find the index of a vector by its ID. * @@ -161,6 +183,7 @@ class IndexPartition { #endif private: + /** * @brief Move data from another partition. * diff --git a/src/cpp/include/list_scanning.h b/src/cpp/include/list_scanning.h index d33e540a..332c6278 100644 --- a/src/cpp/include/list_scanning.h +++ b/src/cpp/include/list_scanning.h @@ -8,8 +8,22 @@ #define LIST_SCANNING_H #include +#include +#include + #include "faiss/utils/Heap.h" #include "faiss/utils/distances.h" +#include "sorting/pdqsort.h" +#include "sorting/floyd_rivest_select.h" +#include "sorting/heap_select.h" +#include "parallel.h" +#include "blas_dist.h" +#include + +using IP_Handler = faiss::HeapBlockResultHandler>; +using L2_Handler = faiss::HeapBlockResultHandler>; +using IP_Single = IP_Handler::SingleResultHandler; +using L2_Single = L2_Handler::SingleResultHandler; inline Tensor calculate_recall(Tensor ids, Tensor gt_ids) { Tensor num_correct = torch::zeros(ids.size(0), torch::kInt64); @@ -36,216 +50,307 @@ inline Tensor calculate_recall(Tensor ids, Tensor gt_ids) { return recall; } -#define TOP_K_BUFFER_CAPACITY (8 * 1024) +template +inline bool better(bool desc, T a, T b) noexcept +{ return desc ? (a > b) : (a < b); } -template +//====================================================================== +// 1. Fast specialised buffer for k == 1 +//====================================================================== +template class TypedTopKBuffer { public: - int k_; // Number of top elements to keep - int curr_offset_ = 0; // Current offset in the buffer - std::vector > topk_; // Buffer to store top-k elements - bool is_descending_; // Flag to indicate sorting order - std::recursive_mutex buffer_mutex_; - std::atomic processing_query_; - std::atomic jobs_left_; - std::atomic partitions_scanned_; - - TypedTopKBuffer(int k, bool is_descending, int buffer_capacity = TOP_K_BUFFER_CAPACITY) - : k_(k), is_descending_(is_descending), topk_(buffer_capacity), processing_query_(true), partitions_scanned_(0) { - assert(k <= buffer_capacity); // Ensure k is smaller than or equal to buffer size - - for (int i = 0; i < topk_.size(); i++) { - if (is_descending_) { - topk_[i] = {-std::numeric_limits::infinity(), -1}; - } else { - topk_[i] = {std::numeric_limits::max(), -1}; + T *vals_; // size = capacity + I *ids_; + int capacity_, head_, k_; + bool is_desc_; + int *ord_; + bool owns_memory_ = true; + int node_; + TypedTopKBuffer(int k, bool desc, int cap, int node) + : capacity_(cap), head_(0), k_(k) { + node_ = node; + + alloc(); + + if (capacity_ < k_) { + string err_msg = "capacity= " + std::to_string(capacity_) + + " must be greater than k= " + std::to_string(k_); + throw std::invalid_argument(err_msg); + } + + if (desc) { + is_desc_ = true; + for (int i = 0; i < capacity_; i++) { + vals_[i] = -std::numeric_limits::infinity(); + ids_[i] = -1; + } + } else { + is_desc_ = false; + for (int i = 0; i < capacity_; i++) { + vals_[i] = std::numeric_limits::infinity(); + ids_[i] = -1; } } } - ~TypedTopKBuffer() = default; + // Create buffer but do not allocate memory (except for ord_) + TypedTopKBuffer(T *vals, I* ids, int cap, int k, bool desc, int node) { + capacity_ = cap; + head_ = 0; + k_ = k; + is_desc_ = desc; + vals_ = vals; + ids_ = ids; + ord_ = static_cast(quake_alloc(sizeof(int) * cap, node)); + owns_memory_ = false; + + if (capacity_ < k_) { + throw std::invalid_argument("capacity must be greater than k"); + } + + if (desc) { + for (int i = 0; i < capacity_; i++) { + vals_[i] = -std::numeric_limits::infinity(); + ids_[i] = -1; + } + } else { + for (int i = 0; i < capacity_; i++) { + vals_[i] = std::numeric_limits::infinity(); + ids_[i] = -1; + } + } + } + + ~TypedTopKBuffer() { + clear(); + } void set_k(int new_k) { - std::lock_guard buffer_lock(buffer_mutex_); - assert(new_k <= topk_.size()); + if (new_k > capacity_) { + clear(); + capacity_ = std::min(new_k * 100, 10000); + alloc(); + } k_ = new_k; reset(); } - void set_processing_query(bool new_value) { - processing_query_.store(new_value, std::memory_order_relaxed); + int k() const { + return k_; } - inline bool currently_processing_query() { - return processing_query_.load(std::memory_order_relaxed); + int capacity() const { + return capacity_; } - void set_jobs_left(int total_jobs) { - jobs_left_.store(total_jobs, std::memory_order_relaxed); + void alloc() { + vals_ = static_cast(quake_alloc(sizeof(T) * capacity_, node_)); + ids_ = static_cast(quake_alloc(sizeof(I) * capacity_, node_)); + ord_ = static_cast(quake_alloc(sizeof(int) * capacity_, node_)); } - void record_skipped_jobs(int skipped_jobs) { - jobs_left_.fetch_sub(skipped_jobs, std::memory_order_relaxed); - } + void clear() { + if (owns_memory_) { + if (vals_) { + quake_free(vals_, sizeof(T) * capacity_); + } - void record_empty_job() { - jobs_left_.fetch_sub(1, std::memory_order_relaxed); - } + if (ids_) { + quake_free(ids_, sizeof(I) * capacity_); + } + } - inline bool finished_all_jobs() { - int curr_jobs_left = jobs_left_.load(std::memory_order_relaxed); - return jobs_left_.load(std::memory_order_relaxed) <= 0; - } + if (ord_) { + quake_free(ord_, sizeof(int) * capacity_); + } - inline int get_num_partitions_scanned() { - return partitions_scanned_.load(std::memory_order_relaxed); + vals_ = nullptr; + ids_ = nullptr; + ord_ = nullptr; } void reset() { - std::lock_guard buffer_lock(buffer_mutex_); - curr_offset_ = 0; - for (int i = 0; i < k_; i++) { - if (is_descending_) { - topk_[i] = { -std::numeric_limits::infinity(), -1 }; - } else { - topk_[i] = { std::numeric_limits::max(), -1 }; - } - } - partitions_scanned_.store(0, std::memory_order_relaxed); + head_ = 0; + // for (int i = 0; i < k_; i++) { + // if (is_desc_) { + // vals_[i] = -std::numeric_limits::infinity(); + // ids_[i] = -1; + // } else { + // vals_[i] = std::numeric_limits::infinity(); + // ids_[i] = -1; + // } + // } } - void add(DistanceType distance, IdType index) { - if (curr_offset_ >= topk_.size()) { - flush(); // Flush the buffer if it is full - } - topk_[curr_offset_++] = {distance, index}; + inline void add(T dist, I idx) { + vals_[head_] = dist; + ids_[head_] = idx; + if (__builtin_expect(++head_ == capacity_, 0)) flush(); } - void batch_add(DistanceType *distances, const IdType *indices, int num_values) { - if (num_values == 0) { - jobs_left_.fetch_sub(1, std::memory_order_relaxed); - return; - } - if (!currently_processing_query()) { - jobs_left_.fetch_sub(1, std::memory_order_relaxed); - return; - } - std::lock_guard lock(buffer_mutex_); + void batch_add(T *distances, I *indices, int num_values) { int pos = 0; while (pos < num_values) { - int available = static_cast(topk_.size()) - curr_offset_; + int available = capacity_ - head_; if (available <= 0) { flush(); - available = static_cast(topk_.size()) - curr_offset_; + available = capacity_ - head_; } int to_copy = std::min(num_values - pos, available); for (int i = 0; i < to_copy; i++) { - topk_[curr_offset_++] = { distances[pos + i], indices[pos + i] }; + vals_[head_] = distances[pos + i]; + ids_[head_] = indices[pos + i]; + head_++; } pos += to_copy; } - jobs_left_.fetch_sub(1, std::memory_order_relaxed); - partitions_scanned_.fetch_add(1, std::memory_order_relaxed); } - DistanceType flush() { - std::lock_guard buffer_lock(buffer_mutex_); - if (curr_offset_ > k_) { - if (is_descending_) { - std::partial_sort(topk_.begin(), topk_.begin() + k_, topk_.begin() + curr_offset_, - [](const auto &a, const auto &b) { return a.first > b.first; }); + T flush() { + + + int n = head_; + int m = std::min(n, k_); + + // 1) build identity permutation + for (int i = 0; i < n; ++i) { + ord_[i] = i; + } + + auto cmp = [&](int a, int b) { + return is_desc_ ? vals_[a] > vals_[b] : vals_[a] < vals_[b]; + }; + + + // 2) select top‐m indices into ord_[0..m) + if (n > m) { + if (m < 10) { + miniselect::heap_select(ord_, ord_ + m, ord_ + n, cmp); + } else if (m < n * 0.001) { + miniselect::floyd_rivest_select(ord_, ord_ + m, ord_ + n, cmp); } else { - std::partial_sort(topk_.begin(), topk_.begin() + k_, topk_.begin() + curr_offset_, - [](const auto &a, const auto &b) { return a.first < b.first; }); + miniselect::pdqpartial_sort_branchless(ord_, ord_ + m, ord_ + n, cmp); } - curr_offset_ = k_; // After flush, retain only the top-k elements + } else { - // sort the curr_offset_ elements - if (is_descending_) { - std::sort(topk_.begin(), topk_.begin() + curr_offset_, - [](const auto &a, const auto &b) { return a.first > b.first; }); - } else { - std::sort(topk_.begin(), topk_.begin() + curr_offset_, - [](const auto &a, const auto &b) { return a.first < b.first; }); - } + miniselect::pdqsort_branchless(ord_, ord_ + n, cmp); } - return topk_[std::min(curr_offset_, k_ - 1)].first; - } - std::vector get_topk() { - std::lock_guard buffer_lock(buffer_mutex_); - flush(); // Ensure the buffer is properly flushed - std::vector topk_distances(std::min(curr_offset_, k_)); - for (int i = 0; i < std::min(curr_offset_, k_); i++) { - topk_distances[i] = topk_[i].first; + // 5) copy the winners back to vals_/ids_ and clamp head_ + std::vector temp_v(m); + std::vector temp_i(m); + + for (int i = 0; i < m; ++i) { + int original_slot_idx = ord_[i]; // ord_[i] is the original index of the i-th best item + // (e.g. ord_[0] is index of best, ord_[1] of 2nd best) + temp_v[i] = vals_[original_slot_idx]; + temp_i[i] = ids_[original_slot_idx]; + } + + // Now copy from temporary buffers to the main buffers + for (int i = 0; i < m; ++i) { + vals_[i] = temp_v[i]; + ids_[i] = temp_i[i]; } - return topk_distances; + // 5) clamp head_ and return the k-th value (or extreme if too few) + head_ = m; + if (head_ == 0) { + return is_desc_ + ? -std::numeric_limits::infinity() + : std::numeric_limits::infinity(); + } + int ret_i = std::min(k_ - 1, head_ - 1); + return vals_[ret_i]; } - DistanceType get_kth_distance() { - std::lock_guard buffer_lock(buffer_mutex_); - flush(); // Ensure the buffer is properly flushed - return topk_[std::min(curr_offset_, k_ - 1)].first; + + std::vector get_topk(bool sort = true) { + if (sort || head_ > k_) flush(); + int n = head_; + std::vector out; + out.reserve(n); + for (int i = 0; i < n; ++i) { + out.push_back(vals_[i]); + } + return out; } - // Get the current top-k indices (after final flush) - std::vector get_topk_indices() { - std::lock_guard buffer_lock(buffer_mutex_); - flush(); // Ensure the buffer is properly flushed + T get_kth_distance() { + flush(); + if (head_ < k_) { + // not enough elements: return the sentinel extreme + return is_desc_ + ? -std::numeric_limits::infinity() + : std::numeric_limits::infinity(); + } + return vals_[k_ - 1]; + } - std::vector topk_indices(std::min(curr_offset_, k_)); - for (int i = 0; i < std::min(curr_offset_, k_); i++) { - topk_indices[i] = topk_[i].second; + std::vector get_topk_indices(bool sort = true) { + if (sort || head_ > k_) flush(); + int n = head_; + std::vector out; + out.reserve(n); + for (int i = 0; i < n; ++i) { + out.push_back(ids_[i]); } - return topk_indices; + return out; } }; // Type alias for convenience using TopkBuffer = TypedTopKBuffer; -inline std::tuple buffers_to_tensor(vector> buffers) { - int n = buffers.size(); - int k = buffers[0]->k_; - Tensor topk_distances = torch::empty({n, k}, torch::kFloat32); - Tensor topk_indices = torch::empty({n, k}, torch::kInt64); +inline vector> create_buffers(int batch_size, int k, bool is_desc, int cap=10000) { + vector> buffers; + for (int i = 0; i < batch_size; i++) { + buffers.push_back(make_shared(k, is_desc, cap, 0)); + } + return buffers; +} - auto topk_distances_accessor = topk_distances.accessor(); - auto topk_indices_accessor = topk_indices.accessor(); +// vector> local_buffers = create_buffers(batch_size, k, (metric_ == faiss::METRIC_INNER_PRODUCT)); - for (int i = 0; i < n; i++) { - vector distances = buffers[i]->get_topk(); - vector indices = buffers[i]->get_topk_indices(); + inline std::tuple buffers_to_tensor(vector> buffers) { + int n = buffers.size(); + int k = buffers[0]->k(); + Tensor topk_distances = torch::empty({n, k}, torch::kFloat32); + Tensor topk_indices = torch::empty({n, k}, torch::kInt64); - int curr_k = std::min(k, (int) distances.size()); + auto topk_distances_accessor = topk_distances.accessor(); + auto topk_indices_accessor = topk_indices.accessor(); - for (int j = 0; j < curr_k; j++) { - topk_distances_accessor[i][j] = distances[j]; - topk_indices_accessor[i][j] = indices[j]; - } - } + for (int i = 0; i < n; i++) { + vector distances = buffers[i]->get_topk(); + vector indices = buffers[i]->get_topk_indices(); - return std::make_tuple(topk_indices, topk_distances); -} + int curr_k = std::min(k, (int) distances.size()); -inline vector> create_buffers(int n, int k, bool is_descending) { - vector> buffers(n); - for (int i = 0; i < n; i++) { - buffers[i] = make_shared(k, is_descending, 10 * k); - } - return buffers; -} + for (int j = 0; j < curr_k; j++) { + topk_distances_accessor[i][j] = distances[j]; + topk_indices_accessor[i][j] = indices[j]; + } + } + + return std::make_tuple(topk_indices, topk_distances); + } inline void scan_list_no_ids_inner_product(const float *query_vec, const float *list_vecs, int list_size, int d, - TopkBuffer &buffer) { + TopkBuffer &buffer, + float pivot) { const float *vec = list_vecs; + float dist; for (int l = 0; l < list_size; l++) { - buffer.add(faiss::fvec_inner_product(query_vec, vec, d), l); + dist = faiss::fvec_inner_product(query_vec, vec, d); + if (dist > pivot) { + buffer.add(dist, l); + } vec += d; // move pointer to next vector } } @@ -254,10 +359,14 @@ inline void scan_list_no_ids_l2(const float *query_vec, const float *list_vecs, int list_size, int d, - TopkBuffer &buffer) { + TopkBuffer &buffer, + float pivot) { const float *vec = list_vecs; for (int l = 0; l < list_size; l++) { - buffer.add(sqrt(faiss::fvec_L2sqr(query_vec, vec, d)), l); + float dist = sqrt(faiss::fvec_L2sqr(query_vec, vec, d)); + if (dist < pivot) { + buffer.add(dist, l); + } vec += d; } } @@ -267,10 +376,14 @@ inline void scan_list_with_ids_inner_product(const float *query_vec, const int64_t *list_ids, int list_size, int d, - TopkBuffer &buffer) { + TopkBuffer &buffer, + float pivot) { const float *vec = list_vecs; for (int l = 0; l < list_size; l++) { - buffer.add(faiss::fvec_inner_product(query_vec, vec, d), list_ids[l]); + float dist = faiss::fvec_inner_product(query_vec, vec, d); + if (dist > pivot) { + buffer.add(dist, list_ids[l]); + } vec += d; } } @@ -280,10 +393,14 @@ inline void scan_list_with_ids_l2(const float *query_vec, const int64_t *list_ids, int list_size, int d, - TopkBuffer &buffer) { + TopkBuffer &buffer, + float pivot) { const float *vec = list_vecs; for (int l = 0; l < list_size; l++) { - buffer.add(sqrt(faiss::fvec_L2sqr(query_vec, vec, d)), list_ids[l]); + float dist = sqrt(faiss::fvec_L2sqr(query_vec, vec, d)); + if (dist < pivot) { + buffer.add(dist, list_ids[l]); + } vec += d; } } @@ -295,75 +412,553 @@ inline void scan_list(const float *query_vec, int list_size, int d, TopkBuffer &buffer, - faiss::MetricType metric = faiss::METRIC_L2) { + faiss::MetricType metric, + float pivot = NULL) { // Dispatch based on metric type and whether list_ids is provided. + + if (pivot == NULL) { + pivot = metric == faiss::METRIC_INNER_PRODUCT + ? -std::numeric_limits::infinity() + : std::numeric_limits::infinity(); + } + if (metric == faiss::METRIC_INNER_PRODUCT) { if (list_ids == nullptr) - scan_list_no_ids_inner_product(query_vec, list_vecs, list_size, d, buffer); + scan_list_no_ids_inner_product(query_vec, list_vecs, list_size, d, buffer, pivot); else - scan_list_with_ids_inner_product(query_vec, list_vecs, list_ids, list_size, d, buffer); + scan_list_with_ids_inner_product(query_vec, list_vecs, list_ids, list_size, d, buffer, pivot); } else { // Assume L2 (or similar) if (list_ids == nullptr) - scan_list_no_ids_l2(query_vec, list_vecs, list_size, d, buffer); + scan_list_no_ids_l2(query_vec, list_vecs, list_size, d, buffer, pivot); else - scan_list_with_ids_l2(query_vec, list_vecs, list_ids, list_size, d, buffer); + scan_list_with_ids_l2(query_vec, list_vecs, list_ids, list_size, d, buffer, pivot); + } +} + +inline void ip_blas( + const float* __restrict x, + const float* __restrict y, + const int64_t *list_ids, + size_t d, + size_t nx, + size_t ny, + size_t db_blas_bs, // = bs_y + size_t k, + vector> &topk_buffers, + float* __restrict ip_block, // nx * bs_y + vector*> pivot = {}) // db_blas_bs +{ + if (nx == 0 || ny == 0) return; + + const size_t bs_x = nx; + const size_t bs_y = db_blas_bs; + int64_t *list_ids_ptr = (int64_t *) list_ids; + + for (size_t i0 = 0; i0 < nx; i0 += bs_x) { + const size_t i1 = std::min(i0 + bs_x, nx); + const size_t q_chunk = i1 - i0; + + for (size_t j0 = 0; j0 < ny; j0 += bs_y) { + const size_t j1 = std::min(j0 + bs_y, ny); + const size_t db_chunk = j1 - j0; + + // use torch matmul +# ifdef __APPLE__ // use torch on macOS + Tensor x_tensor = torch::from_blob((void*) (x + i0 * d), {(int64_t) q_chunk, (int64_t) d}, torch::kFloat32); + Tensor y_tensor = torch::from_blob((void*) (y + j0 * d), {(int64_t) db_chunk, (int64_t) d}, torch::kFloat32); + Tensor ip_tensor = torch::from_blob(ip_block, {(int64_t) q_chunk, (int64_t) db_chunk}, torch::kFloat32); + torch::matmul_out(ip_tensor, x_tensor, y_tensor.transpose(0, 1)); +#else // use BLAS on Linux + { + const float one = 1.f; + float zero = 0.f; + FINTEGER nyi = FINTEGER(db_chunk); + FINTEGER nxi = FINTEGER(q_chunk); + FINTEGER di = FINTEGER(d); + sgemm_("Transpose","Not transpose", + &nyi,&nxi,&di, + &one, + y + j0 * d, &di, + x + i0 * d, &di, + &zero, + ip_block, &nyi); + } +#endif + + + /* IP → L2² */ + if (k > 1) { + for (int64_t qi = 0; qi < static_cast(q_chunk); ++qi) { + float* line_ptr = ip_block + qi * db_chunk; // Pointer to current column in ip_block + + // collect distances closer than pivot + if (pivot.size() > 0) { + float curr_pivot = pivot[qi]->load(std::memory_order_relaxed); + line_ptr = ip_block + qi * db_chunk; // Reset line_ptr to the start of the current column + for (size_t pj = 0; pj < db_chunk; ++pj) { + if (*line_ptr > curr_pivot) { + topk_buffers[qi]->add(*line_ptr, list_ids_ptr[j0 + pj]); + } + line_ptr++; // Move to the next element in the column + } + } else { + topk_buffers[qi]->batch_add(ip_block + qi * db_chunk, list_ids_ptr + j0, db_chunk); + } + } + } else if (k == 1) { + for (int64_t qi = 0; qi < static_cast(q_chunk); ++qi) { + float* line_ptr = ip_block + qi * db_chunk; // Pointer to current column in ip_block + float best_dist = -std::numeric_limits::infinity(); + int64_t best_id = -1; + + for (size_t pj = 0; pj < db_chunk; ++pj) { + if (*line_ptr > best_dist) { + best_dist = *line_ptr; + best_id = list_ids_ptr[j0 + pj]; + } + line_ptr++; // Move to the next element in the column + } + topk_buffers[qi]->add(best_dist, best_id); + } + } + } + } +} + +inline void l2_blas( + const float* __restrict x, + const float* __restrict y, + const int64_t *list_ids, + size_t d, + size_t nx, + size_t ny, + size_t db_blas_bs, // = bs_y + size_t k, + vector> &topk_buffers, + float* __restrict ip_block, // nx * bs_y + float* __restrict norms_x, // bs_x + float* __restrict norms_y, // db_blas_bs + vector*> pivot) +{ + if (nx == 0 || ny == 0) return; + + const size_t bs_x = nx; + const size_t bs_y = db_blas_bs; + int64_t *list_ids_ptr = (int64_t *) list_ids; + + for (size_t i0 = 0; i0 < nx; i0 += bs_x) { + const size_t i1 = std::min(i0 + bs_x, nx); + const size_t q_chunk = i1 - i0; + + /* ‖x‖² for this query block */ + faiss::fvec_norms_L2sqr(norms_x, x + i0 * d, d, q_chunk); + + for (size_t j0 = 0; j0 < ny; j0 += bs_y) { + const size_t j1 = std::min(j0 + bs_y, ny); + const size_t db_chunk = j1 - j0; + + /* ‖y‖² for this database block */ + faiss::fvec_norms_L2sqr(norms_y, y + j0 * d, d, db_chunk); + + // use torch matmul + // Tensor x_tensor = torch::from_blob((void*) (x + i0 * d), {(int64_t) q_chunk, (int64_t) d}, torch::kFloat32); + // Tensor y_tensor = torch::from_blob((void*) (y + j0 * d), {(int64_t) db_chunk, (int64_t) d}, torch::kFloat32); + // Tensor ip_tensor = torch::from_blob(ip_block, {(int64_t) q_chunk, (int64_t) db_chunk}, torch::kFloat32); + // torch::matmul_out(ip_tensor, x_tensor, y_tensor.transpose(0, 1)); + + /* SGEMM */ + { + const float one = 1.f; + float zero = 0.f; + FINTEGER nyi = FINTEGER(db_chunk); + FINTEGER nxi = FINTEGER(q_chunk); + FINTEGER di = FINTEGER(d); + sgemm_("Transpose","Not transpose", + &nyi,&nxi,&di, + &one, + y + j0 * d, &di, + x + i0 * d, &di, + &zero, + ip_block, &nyi); + } + + // for (int64_t qi = 0; qi < (int64_t)q_chunk; ++qi) { + // float* line = ip_block + qi * db_chunk; + // const float xn = norms_x[qi]; + // for (size_t pj = 0; pj < db_chunk; ++pj, ++line) { + // float d2 = xn + norms_y[pj] - 2.f * (*line); + // *line = (d2 < 0.f || !std::isfinite(d2)) ? 0.f : d2; + // } + // } + + /* IP → L2² */ + int num_flushes = 0; + int num_top_k_add_all = 0; + int buffer_not_added = 0; + + float* line_ptr = ip_block; + for (int64_t qi = 0; qi < static_cast(q_chunk); ++qi) { + const float current_norm_x = norms_x[qi]; + + #pragma unroll + for (size_t pj = 0; pj < db_chunk; ++pj) { + *line_ptr = current_norm_x + norms_y[pj] - 2.f * (*line_ptr); + line_ptr++; + } + } + + line_ptr = ip_block; + if (__builtin_expect(pivot.size() > 0, 1)) { + for (int64_t qi = 0; qi < static_cast(q_chunk); ++qi) { + const float curr_pivot = pivot[qi]->load(std::memory_order_relaxed); + const float curr_pivot_sq = curr_pivot * curr_pivot; // Compare squared distances + + #pragma unroll + for (size_t pj = 0; pj < db_chunk; ++pj) { + // Check if distance is within the pivot radius + if (__builtin_expect(*line_ptr < curr_pivot_sq, 0)) { + topk_buffers[qi]->add(std::sqrt(*line_ptr), list_ids_ptr[j0 + pj]); + } + line_ptr++; + } + } + } else { + for (int64_t qi = 0; qi < static_cast(q_chunk); ++qi) { + topk_buffers[qi]->batch_add(line_ptr, list_ids_ptr + j0, db_chunk); + line_ptr += db_chunk; + } + } + } } } inline void batched_scan_list(const float *query_vecs, const float *list_vecs, const int64_t *list_ids, - int num_queries, - int list_size, - int dim, - vector> &topk_buffers, - MetricType metric = faiss::METRIC_L2) { - if (list_size == 0 || list_vecs == nullptr) { - // No list vectors to process; + int num_queries, + int list_size, + int dim, + std::vector> &topk_buffers, + MetricType metric, + /* optional scratch supplied by caller: may be nullptr */ + float *ip_block /* = nullptr */, + float *norms_x /* = nullptr */, + float *norms_y_buf /* = nullptr */, + int blas_db_bs /* = BLAS_DB_BS */, + int blas_q_bs /* = 128 */, + std::vector*> pivots /* = {} */) +{ + + if (list_size == 0 || num_queries == 0 || list_vecs == nullptr) { return; } - // Ensure k does not exceed list_size - int k = topk_buffers[0]->k_; - int k_max = std::min(k, list_size); + const bool need_norm = (metric == faiss::METRIC_L2); + const int k = std::min(topk_buffers[0]->k(), list_size); + + // --------------------------------------------------------------- + // thread‐local scratch buffers: reused across calls, resized only if + // current block exceeds previous capacity + // --------------------------------------------------------------- + thread_local std::vector TLS_ip; // holds up to (blas_q_bs × blas_db_bs) + thread_local std::vector TLS_nx; // holds up to blas_q_bs + thread_local std::vector TLS_ny; // holds up to blas_db_bs + + // --------------------------------------------------------------- + // If L2 and caller gave no norms_x, ensure TLS_nx can hold blas_q_bs + // --------------------------------------------------------------- + if (need_norm && norms_x == nullptr) { + if ((int)TLS_nx.size() < blas_q_bs) { + TLS_nx.resize(blas_q_bs); + } + } - int64_t *labels = (int64_t *) malloc(num_queries * k_max * sizeof(int64_t)); - float *distances = (float *) malloc(num_queries * k_max * sizeof(float)); + // --------------------------------------------------------------- + // Loop over query‐blocks of size ≤ blas_q_bs + // --------------------------------------------------------------- + for (int q_off = 0; q_off < num_queries; q_off += blas_q_bs) { + int q_blk = std::min(blas_q_bs, num_queries - q_off); + const float *q_ptr = query_vecs + size_t(q_off) * dim; + float *q_norms; + + // If caller supplied norms_x, point at offset; else use TLS_nx[0..q_blk-1] + if (need_norm) { + if (norms_x != nullptr) { + q_norms = norms_x + q_off; + } else { + q_norms = TLS_nx.data(); + // compute ‖x‖² for this q‐block + for (int i = 0; i < q_blk; ++i) { + const float *xptr = q_ptr + size_t(i) * dim; + float sumsq = 0.f; + + for (int d = 0; d < dim; ++d) { + sumsq += xptr[d] * xptr[d]; + } + q_norms[i] = sumsq; + } + } + } else { + q_norms = nullptr; + } - if (metric == faiss::METRIC_INNER_PRODUCT) { - faiss::float_minheap_array_t res = {size_t(num_queries), size_t(k_max), labels, distances}; - faiss::knn_inner_product(query_vecs, list_vecs, dim, num_queries, list_size, &res, nullptr); - } else if (metric == faiss::METRIC_L2) { - faiss::float_maxheap_array_t res = {size_t(num_queries), size_t(k_max), labels, distances}; - faiss::knn_L2sqr(query_vecs, list_vecs, dim, num_queries, list_size, &res, nullptr, nullptr); - } else { - throw std::runtime_error("Metric type not supported"); - } + // build a small vector of the q_blk TopkBuffers + std::vector> sub_buffers; + sub_buffers.reserve(q_blk); + for (int i = 0; i < q_blk; ++i) { + sub_buffers.push_back(topk_buffers[q_off + i]); + } - // map the labels to the actual list_ids - if (list_ids != nullptr) { - for (int i = 0; i < num_queries; i++) { - for (int j = 0; j < k_max; j++) { - labels[i * k_max + j] = list_ids[labels[i * k_max + j]]; + // --------------------------------------------------------------- + // Ensure TLS_ip can hold q_blk × blas_db_bs distances if needed + // --------------------------------------------------------------- + if (ip_block == nullptr) { + size_t need_ip = size_t(q_blk) * size_t(blas_db_bs); + if (TLS_ip.size() < need_ip) { + std::cout << "Resizing TLS_ip from " << TLS_ip.size() + << " to " << need_ip << std::endl; + TLS_ip.resize(need_ip); } } - } - // if the metric is l2, convert the distances to sqrt - if (metric == faiss::METRIC_L2) { - for (int i = 0; i < num_queries * k_max; i++) { - distances[i] = sqrt(distances[i]); + // --------------------------------------------------------------- + // Loop over database blocks of size ≤ blas_db_bs + // --------------------------------------------------------------- + for (int db_off = 0; db_off < list_size; db_off += blas_db_bs) { + int blk = std::min(blas_db_bs, list_size - db_off); + const float *d_ptr = list_vecs + size_t(db_off) * dim; + const int64_t *d_ids = list_ids + db_off; + + // Decide where to write "ip_block": either caller‐owned or TLS_ip + float *blk_ip = (ip_block != nullptr ? ip_block + : TLS_ip.data()); + + // If L2 and caller gave no norms_y_buf, ensure TLS_ny fits blk + + float *blk_norm_y = nullptr; + if (need_norm) { + if (norms_y_buf != nullptr) { + blk_norm_y = norms_y_buf; + } else { + if ((int)TLS_ny.size() < blk) { + TLS_ny.resize(blk); + } + blk_norm_y = TLS_ny.data(); + } + } + + // Dispatch to the appropriate BLAS kernel + if (metric == faiss::METRIC_INNER_PRODUCT) { + ip_blas(q_ptr, + d_ptr, + d_ids, + dim, + q_blk, + blk, + blk, + k, + sub_buffers, + blk_ip, + pivots); + } else { // faiss::METRIC_L2 + l2_blas(q_ptr, + d_ptr, + d_ids, + dim, + q_blk, + blk, + blk, + k, + sub_buffers, + blk_ip, + q_norms, + blk_norm_y, + pivots); + } } } +} - // add distances to the topk buffers - for (int i = 0; i < num_queries; i++) { - topk_buffers[i]->batch_add(distances + i * k_max, labels + i * k_max, k_max); - } +// inline void batched_scan_list(const float *query_vecs, +// const float *list_vecs, +// const int64_t *list_ids, +// int num_queries, +// int list_size, +// int dim, +// std::vector> &topk_buffers, +// MetricType metric, +// /* optional scratch supplied by caller: may be nullptr */ +// float *ip_block /* = nullptr */, +// float *norms_x /* = nullptr */, +// float *norms_y_buf /* = nullptr */, +// int blas_db_bs /* = BLAS_DB_BS */, +// int blas_q_bs /* = 128 */, +// std::vector*> pivots /* = {} */) +// { +// if (list_size == 0 || num_queries == 0 || list_vecs == nullptr) { +// return; +// } +// +// const bool need_norm = (metric == faiss::METRIC_L2); +// // each query has a TopkBuffer in topk_buffers[0..num_queries-1] +// const int k = std::min(topk_buffers[0]->k(), list_size); +// +// /* ------------------------------------------------------------------ */ +// /* TEMPORARY BUFFERS (only if caller did NOT supply) */ +// /* ------------------------------------------------------------------ */ +// std::vector tmp_ip; // for caller‐absent ip_block +// std::vector tmp_nx; // for norms_x if caller did not supply +// std::vector tmp_ny; // for norms_y_buf if caller did not supply +// +// /* ------------------------------------------------------------------ */ +// /* PRE‐COMPUTE ‖x‖² FOR ALL QUERIES IF L2 AND NO norms_x GIVEN */ +// /* ------------------------------------------------------------------ */ +// if (need_norm && norms_x == nullptr) { +// tmp_nx.resize(blas_q_bs); +// norms_x = tmp_nx.data(); +// } +// +// /* ------------------------------------------------------------------ */ +// /* LOOP OVER QUERY BLOCKS OF SIZE ≤ blas_q_bs */ +// /* ------------------------------------------------------------------ */ +// for (int q_off = 0; q_off < num_queries; q_off += blas_q_bs) { +// int q_blk = std::min(blas_q_bs, num_queries - q_off); +// const float *q_ptr = query_vecs + size_t(q_off) * dim; +// float *q_norms = need_norm ? (norms_x + q_off) : nullptr; +// +// // build a subvector of TopkBuffers for these q_blk queries +// std::vector> sub_buffers; +// sub_buffers.reserve(q_blk); +// for (int i = 0; i < q_blk; ++i) { +// sub_buffers.push_back(topk_buffers[q_off + i]); +// } +// +// /* ------------------------------------------------------------------ */ +// /* OPTIONALLY GROW tmp_ip TO HOLD q_blk × blas_db_bs DISTANCES */ +// /* (only if caller did not supply ip_block) */ +// /* ------------------------------------------------------------------ */ +// if (ip_block == nullptr) { +// size_t needed = size_t(q_blk) * size_t(blas_db_bs); +// if (tmp_ip.size() < needed) { +// tmp_ip.resize(needed); +// } +// } +// +// /* ------------------------------------------------------------------ */ +// /* LOOP OVER DATABASE BLOCKS OF SIZE ≤ blas_db_bs */ +// /* ------------------------------------------------------------------ */ +// for (int db_off = 0; db_off < list_size; db_off += blas_db_bs) { +// int blk = std::min(blas_db_bs, list_size - db_off); +// const float *d_ptr = list_vecs + size_t(db_off) * dim; +// const int64_t *d_ids = list_ids + db_off; +// float *blk_ip = ip_block ? ip_block : tmp_ip.data(); +// float *blk_norm_y = nullptr; +// +// /* if L2 and no caller norms_y_buf given, grow tmp_ny to size blk */ +// if (need_norm) { +// if (norms_y_buf != nullptr) { +// blk_norm_y = norms_y_buf; +// } else { +// if ((int)tmp_ny.size() < blk) { +// tmp_ny.resize(blk); +// } +// blk_norm_y = tmp_ny.data(); +// } +// } +// +// if (metric == faiss::METRIC_INNER_PRODUCT) { +// // q_blk queries, blk database rows → distances in blk_ip +// ip_blas(q_ptr, +// d_ptr, +// d_ids, +// dim, +// q_blk, +// blk, +// blk, +// k, +// sub_buffers, +// blk_ip, +// pivots); +// } else if (metric == faiss::METRIC_L2) { +// l2_blas(q_ptr, +// d_ptr, +// d_ids, +// dim, +// q_blk, +// blk, +// blk, +// k, +// sub_buffers, +// blk_ip, +// q_norms, +// blk_norm_y, +// pivots); +// } else { +// throw std::runtime_error("batched_scan_list: unsupported metric"); +// } +// } +// } +// } + +// +// +// inline void batched_scan_list(const float *query_vecs, +// const float *list_vecs, +// const int64_t *list_ids, +// int num_queries, +// int list_size, +// int dim, +// vector> &topk_buffers, +// MetricType metric, +// float *ip_block = nullptr, +// float *norms_x = nullptr, +// float *norms_y_buf = nullptr, +// int blas_db_bs = BLAS_DB_BS, +// vector*> pivots = {}) { +// if (list_size == 0 || list_vecs == nullptr) { +// // No list vectors to process; +// return; +// } +// +// // Ensure k does not exceed list_size +// int k = topk_buffers[0]->k(); +// int k_max = std::min(k, list_size); +// +// if (metric == faiss::METRIC_INNER_PRODUCT) { +// ip_blas( +// query_vecs, +// list_vecs, +// list_ids, +// dim, +// num_queries, +// list_size, +// blas_db_bs, +// k_max, +// topk_buffers, +// ip_block, +// pivots +// ); +// } else if (metric == faiss::METRIC_L2) { +// l2_blas( +// query_vecs, +// list_vecs, +// list_ids, +// dim, +// num_queries, +// list_size, +// blas_db_bs, +// k_max, +// topk_buffers, +// ip_block, +// norms_x, +// norms_y_buf, +// pivots +// ); +// } else { +// throw std::runtime_error("Metric type not supported"); +// } +// } - free(labels); - free(distances); -} // } -#endif //LIST_SCANNING_H +#endif //LIST_SCANNING_H \ No newline at end of file diff --git a/src/cpp/include/maintenance_policies.h b/src/cpp/include/maintenance_policies.h index 6ee68075..84c329ec 100644 --- a/src/cpp/include/maintenance_policies.h +++ b/src/cpp/include/maintenance_policies.h @@ -46,11 +46,11 @@ class MaintenancePolicy { */ void reset(); - private: shared_ptr partition_manager_; ///< Manages partition state. shared_ptr params_; ///< Maintenance parameters. shared_ptr cost_estimator_; ///< Cost estimator for maintenance actions. shared_ptr hit_count_tracker_; ///< Hit count tracker for partition hit rates. + std::unordered_map snapshot_sizes_; /// Map to keep track of index sizes /** * @brief Perform local refinement on a set of partition IDs. @@ -58,6 +58,8 @@ class MaintenancePolicy { * @param partition_ids Tensor of partition IDs. */ void local_refinement(const Tensor& partition_ids); +private: + static constexpr bool debug_ = false; }; #endif // MAINTENANCE_POLICY_REFACTORED_H \ No newline at end of file diff --git a/src/cpp/include/parallel.h b/src/cpp/include/parallel.h index 15f9c30d..21841126 100644 --- a/src/cpp/include/parallel.h +++ b/src/cpp/include/parallel.h @@ -38,27 +38,183 @@ inline bool set_thread_affinity(int core_id) { #endif } + +#ifdef QUAKE_USE_NUMA +#include + #include + #include + + inline void* quake_alloc(size_t sz, int node) { + void* ptr = numa_alloc_onnode(sz, node); + if (ptr == nullptr || ptr == MAP_FAILED) { + throw std::bad_alloc(); + } + memset(ptr, 0, sz); // Initialize memory to zero + return ptr; + } + + inline void quake_free(void* ptr, size_t sz) noexcept { + if (ptr) { + numa_free(ptr, sz); + } + } + + inline int cpu_numa_node(int cpu) { + return numa_node_of_cpu(cpu); + } + + inline int get_num_numa_nodes() { + return numa_num_configured_nodes(); + } + + inline int get_current_cpu_numa_node() { + if (numa_available() < 0) return 0; // NUMA not available or error + int current_cpu = sched_getcpu(); + if (current_cpu < 0) { + // perror("sched_getcpu failed"); // User prefers less output + return -1; // Error + } + return numa_node_of_cpu(current_cpu); + } + + // Helper to get the NUMA node of a memory address + inline int get_memory_numa_node(const void* ptr) { + if (numa_available() < 0) return 0; // NUMA not available or error + if (!ptr) { + return -1; // Invalid pointer (e.g. NULL) + } + + // Align pointer to page boundary for numa_move_pages query + void* page_address = (void*)((uintptr_t)ptr & ~((uintptr_t)getpagesize() - 1)); + + void *pages_to_query[] = { page_address }; + int status_array[] = { -1 }; // To store the node of the page + + errno = 0; // Clear errno before system call + // pid = 0 for current process's address space + // nodes = NULL to query current node + // flags = 0 for query + if (numa_move_pages(0, 1, pages_to_query, NULL, status_array, 0) == 0) { + // Success. status_array[0] contains the node. + // It can be -1 if the page has a policy like MPOL_INTERLEAVE or is not mapped to a specific node. + return status_array[0]; + } else { + // numa_move_pages failed, return negative errno. + // errno is set by numa_move_pages. + // Common errors: EFAULT (-14), EINVAL (-22), EPERM (-1), ESRCH (-3), ENOMEM (-12) + return -errno; + } +} + + // Verifies if the memory at memory_address is on the same NUMA node as the current CPU. + // variable_name is for logging/debugging by the caller if verification fails. + inline bool verify_numa_locality(const void* memory_address, const char* variable_name) { + if (numa_available() < 0) return true; // NUMA not available, assume locality + if (get_num_numa_nodes() <= 1) return true; // Single NUMA node system + + if (!memory_address) { + std::cout << "Warning: Cannot verify NUMA locality for null pointer: " << variable_name << std::endl; + return true; // Or false, based on desired strictness for nullptrs. True avoids false positives. + } + + int current_cpu_node = get_current_cpu_numa_node(); + if (current_cpu_node < 0) { + std::cout << "Error: Could not determine current CPU NUMA node for " << variable_name << std::endl; + return false; // Cannot verify + } + + int memory_page_node = get_memory_numa_node(memory_address); + if (memory_page_node < -1) { // -2 indicates query error + std::cout << "memory_page_node = " << memory_page_node << std::endl; + std::cout << "Error: Could not determine NUMA node for memory of " << variable_name << std::endl; + return false; // Cannot verify due to error + } + if (memory_page_node == -1) { + // This typically means the page is not mapped or has a default/interleaved policy + // that doesn't map to a single specific node in a way numa_move_pages can report. + // For strict checking, this could be considered non-local if cpu_node is specific. + // Caller can log: "Warning: Memory for " << variable_name << " has undetermined/interleaved policy (node -1). CPU node: " << current_cpu_node + // A common case for MPOL_DEFAULT is allocation on the node of first touch. + // If it's truly interleaved, it's not "local" to any single node. + // If it's default and first touched by current_cpu_node, it would be local. + // This check is tricky. For now, if we can't determine a specific node, assume it might not be local. + std::cout << "Warning: Memory for " << variable_name << " has undetermined/interleaved policy (node -1). CPU node: " << current_cpu_node << std::endl; + return false; + } + + bool is_local = (current_cpu_node == memory_page_node); + + if (is_local) { + // std::cout << "NUMA locality verified for " << variable_name << std::endl; + } else { + std::cout << "Warning: NUMA locality mismatch for " << variable_name + << ": CPU is on node " << current_cpu_node + << ", but memory is on node " << memory_page_node << std::endl; + } + + return is_local; + } +#else +#include +#include + +inline void* quake_alloc(size_t sz, int /*node*/) { + void* ptr = std::malloc(sz); + if (!ptr) { + throw std::bad_alloc(); + } + memset(ptr, 0, sz); // Initialize memory to zero + return ptr; +} + +inline void quake_free(void* ptr, size_t /*sz*/) noexcept { + std::free(ptr); +} + +inline int cpu_numa_node(int /*cpu*/) { + return 0; // Not applicable +} + +inline int get_num_numa_nodes() { + return 1; // Not applicable +} + +inline bool verify_numa_locality(const void* /*memory_address*/, const char* /*variable_name*/) { + return true; +} +inline int get_current_cpu_numa_node() { return 0; } +inline int get_memory_numa_node(const void* /*ptr*/) { return 0; } + +#endif + + template void parallel_for(IndexType start, IndexType end, Function func, int num_threads = -1) { if (num_threads <= 0) { num_threads = std::thread::hardware_concurrency(); - } - IndexType total = end - start; - IndexType chunk = (total + num_threads - 1) / num_threads; - std::vector> futures; - for (IndexType t = 0; t < num_threads; ++t) { - IndexType chunk_start = start + t * chunk; - IndexType chunk_end = std::min(end, chunk_start + chunk); - if (chunk_start >= chunk_end) - break; - futures.push_back(std::async(std::launch::async, [=]() { - for (IndexType i = chunk_start; i < chunk_end; i++) { - func(i); - } - })); - } - for (auto &f : futures) { - f.get(); + } else if (num_threads == 1) { + for (IndexType i = start; i < end; i++) { + func(i); + } + } else { + IndexType total = end - start; + IndexType chunk = (total + num_threads - 1) / num_threads; + std::vector> futures; + for (IndexType t = 0; t < num_threads; ++t) { + IndexType chunk_start = start + t * chunk; + IndexType chunk_end = std::min(end, chunk_start + chunk); + if (chunk_start >= chunk_end) + break; + futures.push_back(std::async(std::launch::async, [=]() { + for (IndexType i = chunk_start; i < chunk_end; i++) { + func(i); + } + })); + } + + for (auto &f : futures) { + f.get(); + } } } diff --git a/src/cpp/include/partition_manager.h b/src/cpp/include/partition_manager.h index 9a5018a2..03655a2d 100644 --- a/src/cpp/include/partition_manager.h +++ b/src/cpp/include/partition_manager.h @@ -26,6 +26,7 @@ class PartitionManager { shared_ptr parent_ = nullptr; ///< Pointer to a higher-level parent index. std::shared_ptr partition_store_ = nullptr; ///< Pointer to the inverted lists. int64_t curr_partition_id_ = 0; ///< Current partition ID. + int num_workers_ = 0; ///< Number of workers for parallel processing. bool debug_ = false; ///< If true, print debug information. bool check_uniques_ = false; ///< If true, check that vector IDs are unique and don't already exist in the index. @@ -56,14 +57,14 @@ class PartitionManager { * @param assignments Tensor of shape [num_vectors] containing partition IDs. If not provided, vectors are assigned using the parent index. * @return Timing information for the operation. */ - shared_ptr add(const Tensor &vectors, const Tensor &vector_ids, const Tensor &assignments = Tensor(), bool check_uniques = true); + shared_ptr add(const Tensor &vectors, const Tensor &vector_ids, const Tensor &assignments = Tensor(), bool check_uniques = true, bool record_delta = false); /** * @brief Remove vectors by ID from the index. * @param ids Tensor of shape [num_to_remove]. * @return Timing information for the operation. */ - shared_ptr remove(const Tensor &ids); + shared_ptr remove(const Tensor &ids, bool record_delta = false); /** * @brief Get vectors by ID. @@ -80,7 +81,7 @@ class PartitionManager { * @brief Split a given partition into multiple smaller ones. * @param partition_ids The partition IDs to split. */ - shared_ptr split_partitions(const Tensor &partition_ids); + shared_ptr split_partitions(const Tensor &partition_ids, int knn_iteration = DEFAULT_NITER); /** * @brief Refine selected partitions using k-means @@ -102,6 +103,22 @@ class PartitionManager { */ void add_partitions(shared_ptr partitions); + /** + * @brief Calculate the churn factor of the index + */ + float get_churn_factor(int64_t churn_factor); + + /** + * @brief Returns the percentage of the cluster that has been deleted since last mainteance + */ + float get_delete_factor(int64_t partition_id); + + /** + * @brief Updates the centroid based on its delta + * + */ + int64_t update_centroid(int64_t partition_id, float* centroid_buffer); + /** * @brief Select partitions and their centroids. * @param partition_ids Tensor of shape [num_partitions] containing partition IDs. @@ -113,13 +130,13 @@ class PartitionManager { * @brief Distribute the partitions across multiple workers. * @param num_workers The number of workers to distribute the partitions across. */ - void distribute_partitions(int num_workers); + void distribute_partitions(int num_workers, bool use_numa = false); /** * @brief Set the core ID for a given partition. * @param partition_id The ID of the partition. */ - void set_partition_core_id(int64_t partition_id, int core_id); + void set_partition_core_id(int64_t partition_id, int core_id, bool use_numa = false); /** * @brief Return the core ID for a given partition. diff --git a/src/cpp/include/quake_index.h b/src/cpp/include/quake_index.h index c0cccc21..69edb3f8 100644 --- a/src/cpp/include/quake_index.h +++ b/src/cpp/include/quake_index.h @@ -49,6 +49,8 @@ class QuakeIndex { */ shared_ptr build(Tensor x, Tensor ids, shared_ptr build_params); + void add_level(shared_ptr params); + /** * @brief Search for vectors in the index. * @param x Tensor of shape [num_queries, dimension]. @@ -120,7 +122,7 @@ class QuakeIndex { * @param path Path to load the index. * @param n_workers Number of workers to use for query processing. */ - void load(const std::string &path, int n_workers = 0); + void load(const std::string &path, shared_ptr params); /** * @brief Get the total number of vectors in the index. diff --git a/src/cpp/include/query_coordinator.h b/src/cpp/include/query_coordinator.h index a3ae0dd5..86e5d032 100644 --- a/src/cpp/include/query_coordinator.h +++ b/src/cpp/include/query_coordinator.h @@ -10,7 +10,9 @@ #include #include #include -#include +#include +#include +#include "blockingconcurrentqueue.h" class QuakeIndex; class PartitionManager; @@ -21,13 +23,19 @@ class PartitionManager; * A ScanJob encapsulates all parameters required to perform a scan on a given index partition. */ struct ScanJob { - int64_t partition_id; ///< The identifier of the partition to be scanned. - int k; ///< The number of neighbors (Top-K) to return. - const float* query_vector; ///< Pointer to the query vector. - vector query_ids; ///< Global query IDs; used in batched mode. - bool is_batched = false; ///< Indicates whether this is a batched query job. - int64_t num_queries = 0; ///< The number of queries in batched mode. - int rank = 0; ///< Rank of the partition + int64_t job_id; ///< Unique identifier for the job. + int64_t partition_id; ///< The identifier of the partition to be scanned. + int k; ///< The number of neighbors (Top-K) to return. + const float* query_vector; ///< Pointer to the query vector. + bool is_batched = false; ///< Indicates whether this is a batched query job. + int64_t num_queries = 0; + + int query_id; + int rank = 0; ///< Rank of the partition + + std::shared_ptr> query_ids; ///< Global query IDs; used in batched mode. + std::shared_ptr> ranks; ///< Rank of the partition for each query + bool scan_all = false; }; /** @@ -38,6 +46,10 @@ struct ScanJob { */ class QueryCoordinator { public: + // Static field for all query coordinators + static int batch_scan_partition_chunk_size_; + static int batch_scan_query_chunk_size_; + // Public member variables (for internal use) shared_ptr partition_manager_; ///< Manager for partition assignments. shared_ptr maintenance_policy_; ///< Policy for index maintenance. @@ -50,27 +62,105 @@ class QueryCoordinator { * Each core maintains its own pool of Top‑K buffers, a local query buffer, and a dedicated job queue. */ struct CoreResources { - int core_id; ///< Logical identifier of the core. - vector> topk_buffer_pool; ///< Preallocated Top‑K buffers. - vector local_query_buffer; ///< Local aggregator for query results. - moodycamel::BlockingConcurrentQueue job_queue; ///< Job queue for scan jobs. + int core_id; + std::vector> topk_buffer_pool; + + // Thread-local buffers for batched queries + float* batch_queries = nullptr; // batched query buffer (NUMA-allocated) + size_t batch_q_capacity = 0; // number of floats in batch_queries + + float* blas_ip_block = nullptr; // capacity: db_blas_bs * max_Q + size_t blas_ip_capacity = 0; // # floats + + float* blas_norms_x = nullptr; // capacity: max_Q + size_t blas_norms_x_cap = 0; + + float* blas_norms_y = nullptr; // capacity: db_blas_bs (re-used per chunk) + size_t blas_norms_y_cap = 0; + + int64_t job_counter = 0; ///< Job counter for this core. + int64_t queries_counter = 0; ///< Number of queries processed by this core. + int64_t wait_time_ns = 0; ///< Time spent waiting for jobs. + int64_t process_time_ns = 0; ///< Time spent processing jobs. + int64_t process_preamble_time_ns = 0; ///< Time spent on job processing (excluding waiting). + int64_t scan_time_ns = 0; ///< Time spent on scanning. + int64_t enqueue_time_ns = 0; ///< Time spent enqueuing. + int64_t job_time_ns = 0; ///< Time spent on job processing (excluding waiting). + + int64_t bytes_scan_total = 0; ///< Total partition bytes scanned for throughput calculation. + int64_t partition_size = 0; + int64_t num_scan_jobs = 0; + float per_job_scan_throughput = 0; + + float per_job_ipc = 0; + float measured_ipc_count = 0; + float per_job_cache_miss_rate = 0; + float measured_cache_count = 0; + + int64_t batch_scan_total_time_ns = 0; + int64_t faiss_norms_x_time_ns = 0; + int64_t faiss_norms_y_time_ns = 0; + int64_t sgemm_time_ns = 0; + int64_t ip_to_l2_time_ns = 0; + int64_t top_k_buffer_add_ns = 0; + }; + + struct NUMAResources { + float* local_query_buffer = nullptr; + void* metric_tracker_ptr; + + size_t buffer_size = 0; + moodycamel::BlockingConcurrentQueue job_queue; + }; + + struct ResultJob { + int query_id; + int rank; + std::vector distances; + std::vector indices; + }; + + struct MergeResources { + moodycamel::BlockingConcurrentQueue queue; + std::vector> handlers; // points to HandlerIP or HandlerL2 }; vector core_resources_; ///< Per‑core resources for worker threads. + vector numa_resources_; + vector merge_res_; // size == num_merge_workers_ + bool workers_initialized_ = false; ///< Flag indicating if worker threads are initialized. int num_workers_; ///< Total number of worker threads. + int num_merge_workers_; ///< Number of merge worker threads. vector worker_threads_; ///< Container for worker threads. + vector merge_threads_; ///< Container for merge threads. vector worker_job_counter_; ///< Job counters for each worker. - vector> global_topk_buffer_pool_; ///< Global aggregator buffers. + + // The underlying buffers holding the data for the global buffers + float* global_heap_vals_buffer_{nullptr}; + int64_t* global_heap_ids_buffer_{nullptr}; + size_t global_heap_buffer_capacity_{0}; + + shared_ptr>> global_min_heaps_; ///< Global aggregator buffers. + shared_ptr>> global_max_heaps_; ///< Global aggregator buffers. + std::mutex global_mutex_; ///< Mutex for global synchronization. std::condition_variable global_cv_; ///< Condition variable for thread coordination. - std::atomic stop_workers_; ///< Flag to signal workers to terminate. + std::atomic stop_workers_; ///< Flag to signal workers to terminate. bool debug_ = false; ///< Debug mode flag. + std::vector job_buffer_; + int next_job_id_ = 0; ///< ID for the next job to be processed. + vector> query_dist_pivots_; ///< Pivots for each query to speed up sorting + vector> query_done_flags_; ///< Flags to indicate if a query is done from APS + vector> max_rank_; ///< Maximum rank for each query to ensure APS doesn't overshoot vector>> job_flags_; ///< Flags to track job completion std::atomic job_pull_time_ns = 0; ///< Time spent pulling jobs from the queue. std::atomic job_process_time_ns = 0; ///< Time spent processing jobs. + std::atomic total_left_; + vector> per_query_total_left_; ///< Total jobs left for each query. + int current_level_; ///< Current level of the coordinator, used for debugging. /** * @brief Constructs a QueryCoordinator. @@ -85,7 +175,10 @@ class QueryCoordinator { shared_ptr partition_manager, shared_ptr maintenance_policy, MetricType metric, - int num_workers=0); + int current_level = 0, + int num_workers=0, + bool use_numa=false, + int num_merge_workers=1); /** * @brief Destructor for QueryCoordinator. @@ -148,7 +241,7 @@ class QueryCoordinator { * * @param num_workers Number of worker threads to initialize. */ - void initialize_workers(int num_workers); + void initialize_workers(int num_workers, int num_merge_workers=1, bool use_numa=false); /** * @brief Shuts down all worker threads. @@ -166,6 +259,9 @@ class QueryCoordinator { */ void partition_scan_worker_fn(int worker_id); + template + void merge_worker_fn(int worker_id); + /** * @brief Worker thread function to perform partition scanning. * @@ -190,6 +286,42 @@ class QueryCoordinator { * @param d Dimensionality of the query vectors. */ void allocate_core_resources(int core_idx, int num_queries, int k, int d); + + void process_scan_job(ScanJob job, CoreResources &res); + + // handles the non‐batched branch + void handle_nonbatched_job(const ScanJob &job, + CoreResources &res, + NUMAResources &nr); + + // handles the batched branch + void handle_batched_job(const ScanJob &job, + CoreResources &res, + NUMAResources &nr); + + // — Main‐thread helpers — + void init_global_buffers(int64_t nQ, int K, + Tensor &partition_ids, + shared_ptr params); + + void copy_query_to_numa(const float *xptr, int64_t nQ, int64_t D); + + void enqueue_scan_jobs(Tensor x, + Tensor partition_ids, + shared_ptr params); + + void enqueue_result_job(ResultJob job); + + void drain_and_apply_aps(Tensor x, + Tensor partition_ids, + shared_ptr params, + shared_ptr timing); + + std::shared_ptr + aggregate_scan_results(int64_t nQ, int K, + shared_ptr timing, + Tensor out_ids, + Tensor out_dists); }; #endif //QUERY_COORDINATOR_H diff --git a/src/cpp/include/sorting/atomicops.h b/src/cpp/include/sorting/atomicops.h new file mode 100644 index 00000000..1832dee8 --- /dev/null +++ b/src/cpp/include/sorting/atomicops.h @@ -0,0 +1,772 @@ +// ©2013-2016 Cameron Desrochers. +// Distributed under the simplified BSD license (see the license file that +// should have come with this header). +// Uses Jeff Preshing's semaphore implementation (under the terms of its +// separate zlib license, embedded below). + +#pragma once + +// Provides portable (VC++2010+, Intel ICC 13, GCC 4.7+, and anything C++11 compliant) implementation +// of low-level memory barriers, plus a few semi-portable utility macros (for inlining and alignment). +// Also has a basic atomic type (limited to hardware-supported atomics with no memory ordering guarantees). +// Uses the AE_* prefix for macros (historical reasons), and the "moodycamel" namespace for symbols. + +#include +#include +#include +#include +#include +#include + +// Platform detection +#if defined(__INTEL_COMPILER) +#define AE_ICC +#elif defined(_MSC_VER) +#define AE_VCPP +#elif defined(__GNUC__) +#define AE_GCC +#endif + +#if defined(_M_IA64) || defined(__ia64__) +#define AE_ARCH_IA64 +#elif defined(_WIN64) || defined(__amd64__) || defined(_M_X64) || defined(__x86_64__) +#define AE_ARCH_X64 +#elif defined(_M_IX86) || defined(__i386__) +#define AE_ARCH_X86 +#elif defined(_M_PPC) || defined(__powerpc__) +#define AE_ARCH_PPC +#else +#define AE_ARCH_UNKNOWN +#endif + + +// AE_UNUSED +#define AE_UNUSED(x) ((void)x) + +// AE_NO_TSAN/AE_TSAN_ANNOTATE_* +// For GCC +#if defined(__SANITIZE_THREAD__) +#define AE_TSAN_IS_ENABLED +#endif +// For clang +#if defined(__has_feature) +#if __has_feature(thread_sanitizer) && !defined(AE_TSAN_IS_ENABLED) +#define AE_TSAN_IS_ENABLED +#endif +#endif + +#ifdef AE_TSAN_IS_ENABLED +#if __cplusplus >= 201703L // inline variables require C++17 +namespace moodycamel { inline int ae_tsan_global; } +#define AE_TSAN_ANNOTATE_RELEASE() AnnotateHappensBefore(__FILE__, __LINE__, (void *)(&::moodycamel::ae_tsan_global)) +#define AE_TSAN_ANNOTATE_ACQUIRE() AnnotateHappensAfter(__FILE__, __LINE__, (void *)(&::moodycamel::ae_tsan_global)) +extern "C" void AnnotateHappensBefore(const char*, int, void*); +extern "C" void AnnotateHappensAfter(const char*, int, void*); +#else // when we can't work with tsan, attempt to disable its warnings +#define AE_NO_TSAN __attribute__((no_sanitize("thread"))) +#endif +#endif + +#ifndef AE_NO_TSAN +#define AE_NO_TSAN +#endif + +#ifndef AE_TSAN_ANNOTATE_RELEASE +#define AE_TSAN_ANNOTATE_RELEASE() +#define AE_TSAN_ANNOTATE_ACQUIRE() +#endif + + +// AE_FORCEINLINE +#if defined(AE_VCPP) || defined(AE_ICC) +#define AE_FORCEINLINE __forceinline +#elif defined(AE_GCC) +//#define AE_FORCEINLINE __attribute__((always_inline)) +#define AE_FORCEINLINE inline +#else +#define AE_FORCEINLINE inline +#endif + + +// AE_ALIGN +#if defined(AE_VCPP) || defined(AE_ICC) +#define AE_ALIGN(x) __declspec(align(x)) +#elif defined(AE_GCC) +#define AE_ALIGN(x) __attribute__((aligned(x))) +#else +// Assume GCC compliant syntax... +#define AE_ALIGN(x) __attribute__((aligned(x))) +#endif + + +// Portable atomic fences implemented below: + +namespace moodycamel { + + enum memory_order { + memory_order_relaxed, + memory_order_acquire, + memory_order_release, + memory_order_acq_rel, + memory_order_seq_cst, + + // memory_order_sync: Forces a full sync: + // #LoadLoad, #LoadStore, #StoreStore, and most significantly, #StoreLoad + memory_order_sync = memory_order_seq_cst + }; + +} // end namespace moodycamel + +#if (defined(AE_VCPP) && (_MSC_VER < 1700 || defined(__cplusplus_cli))) || (defined(AE_ICC) && __INTEL_COMPILER < 1600) +// VS2010 and ICC13 don't support std::atomic_*_fence, implement our own fences + +#include + +#if defined(AE_ARCH_X64) || defined(AE_ARCH_X86) +#define AeFullSync _mm_mfence +#define AeLiteSync _mm_mfence +#elif defined(AE_ARCH_IA64) +#define AeFullSync __mf +#define AeLiteSync __mf +#elif defined(AE_ARCH_PPC) +#include +#define AeFullSync __sync +#define AeLiteSync __lwsync +#endif + + +#ifdef AE_VCPP +#pragma warning(push) +#pragma warning(disable: 4365) // Disable erroneous 'conversion from long to unsigned int, signed/unsigned mismatch' error when using `assert` +#ifdef __cplusplus_cli +#pragma managed(push, off) +#endif +#endif + +namespace moodycamel { + +AE_FORCEINLINE void compiler_fence(memory_order order) AE_NO_TSAN +{ + switch (order) { + case memory_order_relaxed: break; + case memory_order_acquire: _ReadBarrier(); break; + case memory_order_release: _WriteBarrier(); break; + case memory_order_acq_rel: _ReadWriteBarrier(); break; + case memory_order_seq_cst: _ReadWriteBarrier(); break; + default: assert(false); + } +} + +// x86/x64 have a strong memory model -- all loads and stores have +// acquire and release semantics automatically (so only need compiler +// barriers for those). +#if defined(AE_ARCH_X86) || defined(AE_ARCH_X64) +AE_FORCEINLINE void fence(memory_order order) AE_NO_TSAN +{ + switch (order) { + case memory_order_relaxed: break; + case memory_order_acquire: _ReadBarrier(); break; + case memory_order_release: _WriteBarrier(); break; + case memory_order_acq_rel: _ReadWriteBarrier(); break; + case memory_order_seq_cst: + _ReadWriteBarrier(); + AeFullSync(); + _ReadWriteBarrier(); + break; + default: assert(false); + } +} +#else +AE_FORCEINLINE void fence(memory_order order) AE_NO_TSAN +{ + // Non-specialized arch, use heavier memory barriers everywhere just in case :-( + switch (order) { + case memory_order_relaxed: + break; + case memory_order_acquire: + _ReadBarrier(); + AeLiteSync(); + _ReadBarrier(); + break; + case memory_order_release: + _WriteBarrier(); + AeLiteSync(); + _WriteBarrier(); + break; + case memory_order_acq_rel: + _ReadWriteBarrier(); + AeLiteSync(); + _ReadWriteBarrier(); + break; + case memory_order_seq_cst: + _ReadWriteBarrier(); + AeFullSync(); + _ReadWriteBarrier(); + break; + default: assert(false); + } +} +#endif +} // end namespace moodycamel +#else +// Use standard library of atomics +#include + +namespace moodycamel { + + AE_FORCEINLINE void compiler_fence(memory_order order) AE_NO_TSAN + { + switch (order) { + case memory_order_relaxed: break; + case memory_order_acquire: std::atomic_signal_fence(std::memory_order_acquire); break; + case memory_order_release: std::atomic_signal_fence(std::memory_order_release); break; + case memory_order_acq_rel: std::atomic_signal_fence(std::memory_order_acq_rel); break; + case memory_order_seq_cst: std::atomic_signal_fence(std::memory_order_seq_cst); break; + default: assert(false); + } + } + + AE_FORCEINLINE void fence(memory_order order) AE_NO_TSAN + { + switch (order) { + case memory_order_relaxed: break; + case memory_order_acquire: AE_TSAN_ANNOTATE_ACQUIRE(); std::atomic_thread_fence(std::memory_order_acquire); break; + case memory_order_release: AE_TSAN_ANNOTATE_RELEASE(); std::atomic_thread_fence(std::memory_order_release); break; + case memory_order_acq_rel: AE_TSAN_ANNOTATE_ACQUIRE(); AE_TSAN_ANNOTATE_RELEASE(); std::atomic_thread_fence(std::memory_order_acq_rel); break; + case memory_order_seq_cst: AE_TSAN_ANNOTATE_ACQUIRE(); AE_TSAN_ANNOTATE_RELEASE(); std::atomic_thread_fence(std::memory_order_seq_cst); break; + default: assert(false); + } + } + +} // end namespace moodycamel + +#endif + + +#if !defined(AE_VCPP) || (_MSC_VER >= 1700 && !defined(__cplusplus_cli)) +#define AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC +#endif + +#ifdef AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC +#include +#endif +#include + +// WARNING: *NOT* A REPLACEMENT FOR std::atomic. READ CAREFULLY: +// Provides basic support for atomic variables -- no memory ordering guarantees are provided. +// The guarantee of atomicity is only made for types that already have atomic load and store guarantees +// at the hardware level -- on most platforms this generally means aligned pointers and integers (only). +namespace moodycamel { + template + class weak_atomic + { + public: + AE_NO_TSAN weak_atomic() : value() { } +#ifdef AE_VCPP + #pragma warning(push) +#pragma warning(disable: 4100) // Get rid of (erroneous) 'unreferenced formal parameter' warning +#endif + template AE_NO_TSAN weak_atomic(U&& x) : value(std::forward(x)) { } +#ifdef __cplusplus_cli + // Work around bug with universal reference/nullptr combination that only appears when /clr is on + AE_NO_TSAN weak_atomic(nullptr_t) : value(nullptr) { } +#endif + AE_NO_TSAN weak_atomic(weak_atomic const& other) : value(other.load()) { } + AE_NO_TSAN weak_atomic(weak_atomic&& other) : value(std::move(other.load())) { } +#ifdef AE_VCPP +#pragma warning(pop) +#endif + + AE_FORCEINLINE operator T() const AE_NO_TSAN { return load(); } + + +#ifndef AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC + template AE_FORCEINLINE weak_atomic const& operator=(U&& x) AE_NO_TSAN { value = std::forward(x); return *this; } + AE_FORCEINLINE weak_atomic const& operator=(weak_atomic const& other) AE_NO_TSAN { value = other.value; return *this; } + + AE_FORCEINLINE T load() const AE_NO_TSAN { return value; } + + AE_FORCEINLINE T fetch_add_acquire(T increment) AE_NO_TSAN + { +#if defined(AE_ARCH_X64) || defined(AE_ARCH_X86) + if (sizeof(T) == 4) return _InterlockedExchangeAdd((long volatile*)&value, (long)increment); +#if defined(_M_AMD64) + else if (sizeof(T) == 8) return _InterlockedExchangeAdd64((long long volatile*)&value, (long long)increment); +#endif +#else +#error Unsupported platform +#endif + assert(false && "T must be either a 32 or 64 bit type"); + return value; + } + + AE_FORCEINLINE T fetch_add_release(T increment) AE_NO_TSAN + { +#if defined(AE_ARCH_X64) || defined(AE_ARCH_X86) + if (sizeof(T) == 4) return _InterlockedExchangeAdd((long volatile*)&value, (long)increment); +#if defined(_M_AMD64) + else if (sizeof(T) == 8) return _InterlockedExchangeAdd64((long long volatile*)&value, (long long)increment); +#endif +#else +#error Unsupported platform +#endif + assert(false && "T must be either a 32 or 64 bit type"); + return value; + } +#else + template + AE_FORCEINLINE weak_atomic const& operator=(U&& x) AE_NO_TSAN + { + value.store(std::forward(x), std::memory_order_relaxed); + return *this; + } + + AE_FORCEINLINE weak_atomic const& operator=(weak_atomic const& other) AE_NO_TSAN + { + value.store(other.value.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; + } + + AE_FORCEINLINE T load() const AE_NO_TSAN { return value.load(std::memory_order_relaxed); } + + AE_FORCEINLINE T fetch_add_acquire(T increment) AE_NO_TSAN + { + return value.fetch_add(increment, std::memory_order_acquire); + } + + AE_FORCEINLINE T fetch_add_release(T increment) AE_NO_TSAN + { + return value.fetch_add(increment, std::memory_order_release); + } +#endif + + + private: +#ifndef AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC + // No std::atomic support, but still need to circumvent compiler optimizations. + // `volatile` will make memory access slow, but is guaranteed to be reliable. + volatile T value; +#else + std::atomic value; +#endif + }; + +} // end namespace moodycamel + + + +// Portable single-producer, single-consumer semaphore below: + +#if defined(_WIN32) +// Avoid including windows.h in a header; we only need a handful of +// items, so we'll redeclare them here (this is relatively safe since +// the API generally has to remain stable between Windows versions). +// I know this is an ugly hack but it still beats polluting the global +// namespace with thousands of generic names or adding a .cpp for nothing. +extern "C" { + struct _SECURITY_ATTRIBUTES; + __declspec(dllimport) void* __stdcall CreateSemaphoreW(_SECURITY_ATTRIBUTES* lpSemaphoreAttributes, long lInitialCount, long lMaximumCount, const wchar_t* lpName); + __declspec(dllimport) int __stdcall CloseHandle(void* hObject); + __declspec(dllimport) unsigned long __stdcall WaitForSingleObject(void* hHandle, unsigned long dwMilliseconds); + __declspec(dllimport) int __stdcall ReleaseSemaphore(void* hSemaphore, long lReleaseCount, long* lpPreviousCount); +} +#elif defined(__MACH__) +#include +#elif defined(__unix__) +#include +#elif defined(FREERTOS) +#include +#include +#include +#endif + +namespace moodycamel +{ + // Code in the spsc_sema namespace below is an adaptation of Jeff Preshing's + // portable + lightweight semaphore implementations, originally from + // https://github.com/preshing/cpp11-on-multicore/blob/master/common/sema.h + // LICENSE: + // Copyright (c) 2015 Jeff Preshing + // + // This software is provided 'as-is', without any express or implied + // warranty. In no event will the authors be held liable for any damages + // arising from the use of this software. + // + // Permission is granted to anyone to use this software for any purpose, + // including commercial applications, and to alter it and redistribute it + // freely, subject to the following restrictions: + // + // 1. The origin of this software must not be misrepresented; you must not + // claim that you wrote the original software. If you use this software + // in a product, an acknowledgement in the product documentation would be + // appreciated but is not required. + // 2. Altered source versions must be plainly marked as such, and must not be + // misrepresented as being the original software. + // 3. This notice may not be removed or altered from any source distribution. + namespace spsc_sema + { +#if defined(_WIN32) + class Semaphore + { + private: + void* m_hSema; + + Semaphore(const Semaphore& other); + Semaphore& operator=(const Semaphore& other); + + public: + AE_NO_TSAN Semaphore(int initialCount = 0) : m_hSema() + { + assert(initialCount >= 0); + const long maxLong = 0x7fffffff; + m_hSema = CreateSemaphoreW(nullptr, initialCount, maxLong, nullptr); + assert(m_hSema); + } + + AE_NO_TSAN ~Semaphore() + { + CloseHandle(m_hSema); + } + + bool wait() AE_NO_TSAN + { + const unsigned long infinite = 0xffffffff; + return WaitForSingleObject(m_hSema, infinite) == 0; + } + + bool try_wait() AE_NO_TSAN + { + return WaitForSingleObject(m_hSema, 0) == 0; + } + + bool timed_wait(std::uint64_t usecs) AE_NO_TSAN + { + return WaitForSingleObject(m_hSema, (unsigned long)(usecs / 1000)) == 0; + } + + void signal(int count = 1) AE_NO_TSAN + { + while (!ReleaseSemaphore(m_hSema, count, nullptr)); + } + }; +#elif defined(__MACH__) + //--------------------------------------------------------- + // Semaphore (Apple iOS and OSX) + // Can't use POSIX semaphores due to http://lists.apple.com/archives/darwin-kernel/2009/Apr/msg00010.html + //--------------------------------------------------------- + class Semaphore + { + private: + semaphore_t m_sema; + + Semaphore(const Semaphore& other); + Semaphore& operator=(const Semaphore& other); + + public: + AE_NO_TSAN Semaphore(int initialCount = 0) : m_sema() + { + assert(initialCount >= 0); + kern_return_t rc = semaphore_create(mach_task_self(), &m_sema, SYNC_POLICY_FIFO, initialCount); + assert(rc == KERN_SUCCESS); + AE_UNUSED(rc); + } + + AE_NO_TSAN ~Semaphore() + { + semaphore_destroy(mach_task_self(), m_sema); + } + + bool wait() AE_NO_TSAN + { + return semaphore_wait(m_sema) == KERN_SUCCESS; + } + + bool try_wait() AE_NO_TSAN + { + return timed_wait(0); + } + + bool timed_wait(std::uint64_t timeout_usecs) AE_NO_TSAN + { + mach_timespec_t ts; + ts.tv_sec = static_cast(timeout_usecs / 1000000); + ts.tv_nsec = static_cast((timeout_usecs % 1000000) * 1000); + + // added in OSX 10.10: https://developer.apple.com/library/prerelease/mac/documentation/General/Reference/APIDiffsMacOSX10_10SeedDiff/modules/Darwin.html + kern_return_t rc = semaphore_timedwait(m_sema, ts); + return rc == KERN_SUCCESS; + } + + void signal() AE_NO_TSAN + { + while (semaphore_signal(m_sema) != KERN_SUCCESS); + } + + void signal(int count) AE_NO_TSAN + { + while (count-- > 0) + { + while (semaphore_signal(m_sema) != KERN_SUCCESS); + } + } + }; +#elif defined(__unix__) + //--------------------------------------------------------- + // Semaphore (POSIX, Linux) + //--------------------------------------------------------- + class Semaphore + { + private: + sem_t m_sema; + + Semaphore(const Semaphore& other); + Semaphore& operator=(const Semaphore& other); + + public: + AE_NO_TSAN Semaphore(int initialCount = 0) : m_sema() + { + assert(initialCount >= 0); + int rc = sem_init(&m_sema, 0, static_cast(initialCount)); + assert(rc == 0); + AE_UNUSED(rc); + } + + AE_NO_TSAN ~Semaphore() + { + sem_destroy(&m_sema); + } + + bool wait() AE_NO_TSAN + { + // http://stackoverflow.com/questions/2013181/gdb-causes-sem-wait-to-fail-with-eintr-error + int rc; + do + { + rc = sem_wait(&m_sema); + } + while (rc == -1 && errno == EINTR); + return rc == 0; + } + + bool try_wait() AE_NO_TSAN + { + int rc; + do { + rc = sem_trywait(&m_sema); + } while (rc == -1 && errno == EINTR); + return rc == 0; + } + + bool timed_wait(std::uint64_t usecs) AE_NO_TSAN + { + struct timespec ts; + const int usecs_in_1_sec = 1000000; + const int nsecs_in_1_sec = 1000000000; + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += static_cast(usecs / usecs_in_1_sec); + ts.tv_nsec += static_cast(usecs % usecs_in_1_sec) * 1000; + // sem_timedwait bombs if you have more than 1e9 in tv_nsec + // so we have to clean things up before passing it in + if (ts.tv_nsec >= nsecs_in_1_sec) { + ts.tv_nsec -= nsecs_in_1_sec; + ++ts.tv_sec; + } + + int rc; + do { + rc = sem_timedwait(&m_sema, &ts); + } while (rc == -1 && errno == EINTR); + return rc == 0; + } + + void signal() AE_NO_TSAN + { + while (sem_post(&m_sema) == -1); + } + + void signal(int count) AE_NO_TSAN + { + while (count-- > 0) + { + while (sem_post(&m_sema) == -1); + } + } + }; +#elif defined(FREERTOS) + //--------------------------------------------------------- + // Semaphore (FreeRTOS) + //--------------------------------------------------------- + class Semaphore + { + private: + SemaphoreHandle_t m_sema; + + Semaphore(const Semaphore& other); + Semaphore& operator=(const Semaphore& other); + + public: + AE_NO_TSAN Semaphore(int initialCount = 0) : m_sema() + { + assert(initialCount >= 0); + m_sema = xSemaphoreCreateCounting(static_cast(~0ull), static_cast(initialCount)); + assert(m_sema); + } + + AE_NO_TSAN ~Semaphore() + { + vSemaphoreDelete(m_sema); + } + + bool wait() AE_NO_TSAN + { + return xSemaphoreTake(m_sema, portMAX_DELAY) == pdTRUE; + } + + bool try_wait() AE_NO_TSAN + { + // Note: In an ISR context, if this causes a task to unblock, + // the caller won't know about it + if (xPortIsInsideInterrupt()) + return xSemaphoreTakeFromISR(m_sema, NULL) == pdTRUE; + return xSemaphoreTake(m_sema, 0) == pdTRUE; + } + + bool timed_wait(std::uint64_t usecs) AE_NO_TSAN + { + std::uint64_t msecs = usecs / 1000; + TickType_t ticks = static_cast(msecs / portTICK_PERIOD_MS); + if (ticks == 0) + return try_wait(); + return xSemaphoreTake(m_sema, ticks) == pdTRUE; + } + + void signal() AE_NO_TSAN + { + // Note: In an ISR context, if this causes a task to unblock, + // the caller won't know about it + BaseType_t rc; + if (xPortIsInsideInterrupt()) + rc = xSemaphoreGiveFromISR(m_sema, NULL); + else + rc = xSemaphoreGive(m_sema); + assert(rc == pdTRUE); + AE_UNUSED(rc); + } + + void signal(int count) AE_NO_TSAN + { + while (count-- > 0) + signal(); + } + }; +#else +#error Unsupported platform! (No semaphore wrapper available) +#endif + + //--------------------------------------------------------- + // LightweightSemaphore + //--------------------------------------------------------- + class LightweightSemaphore + { + public: + typedef std::make_signed::type ssize_t; + + private: + weak_atomic m_count; + Semaphore m_sema; + + bool waitWithPartialSpinning(std::int64_t timeout_usecs = -1) AE_NO_TSAN + { + ssize_t oldCount; + // Is there a better way to set the initial spin count? + // If we lower it to 1000, testBenaphore becomes 15x slower on my Core i7-5930K Windows PC, + // as threads start hitting the kernel semaphore. + int spin = 1024; + while (--spin >= 0) + { + if (m_count.load() > 0) + { + m_count.fetch_add_acquire(-1); + return true; + } + compiler_fence(memory_order_acquire); // Prevent the compiler from collapsing the loop. + } + oldCount = m_count.fetch_add_acquire(-1); + if (oldCount > 0) + return true; + if (timeout_usecs < 0) + { + if (m_sema.wait()) + return true; + } + if (timeout_usecs > 0 && m_sema.timed_wait(static_cast(timeout_usecs))) + return true; + // At this point, we've timed out waiting for the semaphore, but the + // count is still decremented indicating we may still be waiting on + // it. So we have to re-adjust the count, but only if the semaphore + // wasn't signaled enough times for us too since then. If it was, we + // need to release the semaphore too. + while (true) + { + oldCount = m_count.fetch_add_release(1); + if (oldCount < 0) + return false; // successfully restored things to the way they were + // Oh, the producer thread just signaled the semaphore after all. Try again: + oldCount = m_count.fetch_add_acquire(-1); + if (oldCount > 0 && m_sema.try_wait()) + return true; + } + } + + public: + AE_NO_TSAN LightweightSemaphore(ssize_t initialCount = 0) : m_count(initialCount), m_sema() + { + assert(initialCount >= 0); + } + + bool tryWait() AE_NO_TSAN + { + if (m_count.load() > 0) + { + m_count.fetch_add_acquire(-1); + return true; + } + return false; + } + + bool wait() AE_NO_TSAN + { + return tryWait() || waitWithPartialSpinning(); + } + + bool wait(std::int64_t timeout_usecs) AE_NO_TSAN + { + return tryWait() || waitWithPartialSpinning(timeout_usecs); + } + + void signal(ssize_t count = 1) AE_NO_TSAN + { + assert(count >= 0); + ssize_t oldCount = m_count.fetch_add_release(count); + assert(oldCount >= -1); + if (oldCount < 0) + { + m_sema.signal(1); + } + } + + std::size_t availableApprox() const AE_NO_TSAN + { + ssize_t count = m_count.load(); + return count > 0 ? static_cast(count) : 0; + } + }; + } // end namespace spsc_sema +} // end namespace moodycamel + +#if defined(AE_VCPP) && (_MSC_VER < 1700 || defined(__cplusplus_cli)) +#pragma warning(pop) +#ifdef __cplusplus_cli +#pragma managed(pop) +#endif +#endif \ No newline at end of file diff --git a/src/cpp/include/sorting/floyd_rivest_select.h b/src/cpp/include/sorting/floyd_rivest_select.h new file mode 100644 index 00000000..db213b6c --- /dev/null +++ b/src/cpp/include/sorting/floyd_rivest_select.h @@ -0,0 +1,129 @@ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace miniselect { + namespace floyd_rivest_detail { + + enum floyd_rivest_constants { + kQCap = 600, + }; + + template + struct CompareRefType { + // Pass the comparator by lvalue reference. Or in debug mode, using a + // debugging wrapper that stores a reference. + using type = typename std::add_lvalue_reference::type; + }; + + template ::difference_type> + inline void floyd_rivest_select_loop(Iter begin, DiffType left, DiffType right, + DiffType k, Compare comp) { + while (right > left) { + DiffType size = right - left; + if (size > floyd_rivest_constants::kQCap) { + DiffType n = right - left + 1; + DiffType i = k - left + 1; + + double z = log(n); + double s = 0.5 * exp(2 * z / 3); + double sd = 0.5 * sqrt(z * s * (n - s) / n); + if (i < n / 2) { + sd *= -1.0; + } + DiffType new_left = + std::max(left, static_cast(k - i * s / n + sd)); + DiffType new_right = + std::min(right, static_cast(k + (n - i) * s / n + sd)); + floyd_rivest_select_loop(begin, new_left, + new_right, k, comp); + } + DiffType i = left; + DiffType j = right; + + std::swap(begin[left], begin[k]); + const bool to_swap = comp(begin[left], begin[right]); + if (to_swap) { + std::swap(begin[left], begin[right]); + } + // Make sure that non copyable types compile. + const auto& t = to_swap ? begin[left] : begin[right]; + while (i < j) { + std::swap(begin[i], begin[j]); + i++; + j--; + while (comp(begin[i], t)) { + i++; + } + while (comp(t, begin[j])) { + j--; + } + } + + if (to_swap) { + std::swap(begin[left], begin[j]); + } else { + j++; + std::swap(begin[right], begin[j]); + } + + if (j <= k) { + left = j + 1; + } + if (k <= j) { + right = j - 1; + } + } + } + + } // namespace floyd_rivest_detail + + template + inline void floyd_rivest_partial_sort(Iter begin, Iter mid, Iter end, + Compare comp) { + if (begin == end) return; + if (begin == mid) return; + using CompType = typename floyd_rivest_detail::CompareRefType::type; + using DiffType = typename std::iterator_traits::difference_type; + floyd_rivest_detail::floyd_rivest_select_loop( + begin, DiffType{0}, static_cast(end - begin - 1), + static_cast(mid - begin - 1), comp); + // std::sort proved to be better than other sorts because of pivoting. + std::sort(begin, mid, comp); + } + + template + inline void floyd_rivest_partial_sort(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + floyd_rivest_partial_sort(begin, mid, end, std::less()); + } + + template + inline void floyd_rivest_select(Iter begin, Iter mid, Iter end, Compare comp) { + if (mid == end) return; + using CompType = typename floyd_rivest_detail::CompareRefType::type; + using DiffType = typename std::iterator_traits::difference_type; + floyd_rivest_detail::floyd_rivest_select_loop( + begin, DiffType{0}, static_cast(end - begin - 1), + static_cast(mid - begin), comp); + } + + template + inline void floyd_rivest_select(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + floyd_rivest_select(begin, mid, end, std::less()); + } + +} // namespace miniselect \ No newline at end of file diff --git a/src/cpp/include/sorting/heap_select.h b/src/cpp/include/sorting/heap_select.h new file mode 100644 index 00000000..d51eb9b7 --- /dev/null +++ b/src/cpp/include/sorting/heap_select.h @@ -0,0 +1,106 @@ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +#pragma once + +#include +#include +#include +#include +#include + +namespace miniselect { + namespace heap_select_detail { + + template + struct CompareRefType { + // Pass the comparator by lvalue reference. Or in debug mode, using a + // debugging wrapper that stores a reference. + using type = typename std::add_lvalue_reference::type; + }; + + template + inline void sift_down(Iter first, Compare comp, + typename std::iterator_traits::difference_type len, + Iter start) { + using difference_type = typename std::iterator_traits::difference_type; + using value_type = typename std::iterator_traits::value_type; + difference_type child = start - first; + + if (len < 2 || (len - 2) / 2 < child) return; + + child = 2 * child + 1; + Iter child_i = first + child; + + if ((child + 1) < len && comp(*child_i, *(child_i + 1))) { + ++child_i; + ++child; + } + + if (comp(*child_i, *start)) { + return; + } + + value_type top(std::move(*start)); + do { + *start = std::move(*child_i); + start = child_i; + + if ((len - 2) / 2 < child) { + break; + } + + child = 2 * child + 1; + child_i = first + child; + + if ((child + 1) < len && comp(*child_i, *(child_i + 1))) { + ++child_i; + ++child; + } + } while (!comp(*child_i, top)); + *start = std::move(top); + } + + template + inline void heap_select_loop(Iter first, Iter middle, Iter last, Compare comp) { + std::make_heap(first, middle, comp); + typename std::iterator_traits::difference_type len = middle - first; + for (Iter i = middle; i != last; ++i) { + if (comp(*i, *first)) { + std::swap(*i, *first); + sift_down(first, comp, len, first); + } + } + } + + } // namespace heap_select_detail + + template + inline void heap_select(Iter first, Iter middle, Iter end, Compare comp) { + if (middle == end) return; + heap_select_detail::heap_select_loop(first, middle + 1, end, comp); + std::swap(*first, *middle); + } + + template + inline void heap_select(Iter first, Iter middle, Iter end) { + heap_select(first, middle, end, + std::less::value_type>()); + } + + template + inline void heap_partial_sort(Iter first, Iter middle, Iter end, Compare comp) { + heap_select_detail::heap_select_loop(first, middle, end, comp); + std::sort_heap(first, middle, comp); + } + + template + inline void heap_partial_sort(Iter first, Iter middle, Iter end) { + heap_partial_sort( + first, middle, end, + std::less::value_type>()); + } + +} // namespace miniselect \ No newline at end of file diff --git a/src/cpp/include/sorting/median_common.h b/src/cpp/include/sorting/median_common.h new file mode 100644 index 00000000..4e85cd8e --- /dev/null +++ b/src/cpp/include/sorting/median_common.h @@ -0,0 +1,454 @@ +/* Copyright Andrei Alexandrescu, 2016-, + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +#pragma once + +#include +#include +#include +#include + +namespace miniselect { + namespace median_common_detail { + + template + struct CompareRefType { + // Pass the comparator by lvalue reference. Or in debug mode, using a + // debugging wrapper that stores a reference. + using type = typename std::add_lvalue_reference::type; + }; +/** +Swaps the median of r[a], r[b], and r[c] into r[b]. +*/ + template ::difference_type> + inline void median3(Iter r, DiffType a, DiffType b, DiffType c, + Compare&& comp) { + if (comp(r[b], r[a])) // b < a + { + if (comp(r[b], r[c])) // b < a, b < c + { + if (comp(r[c], r[a])) // b < c < a + std::swap(r[b], r[c]); + else // b < a <= c + std::swap(r[b], r[a]); + } + } else if (comp(r[c], r[b])) // a <= b, c < b + { + if (comp(r[c], r[a])) // c < a <= b + std::swap(r[b], r[a]); + else // a <= c < b + std::swap(r[b], r[c]); + } + } + +/** +Sorts in place r[a], r[b], and r[c]. +*/ + template ::difference_type> + inline void sort3(Iter r, DiffType a, DiffType b, DiffType c, Compare&& comp) { + typedef typename std::iterator_traits::value_type T; + if (comp(r[b], r[a])) // b < a + { + if (comp(r[c], r[b])) // c < b < a + { + std::swap(r[a], r[c]); // a < b < c + } else // b < a, b <= c + { + T t = std::move(r[a]); + r[a] = std::move(r[b]); + if (comp(r[c], t)) // b <= c < a + { + r[b] = std::move(r[c]); + r[c] = std::move(t); + } else // b < a <= c + { + r[b] = std::move(t); + } + } + } else if (comp(r[c], r[b])) // a <= b, c < b + { + T t = std::move(r[c]); + r[c] = std::move(r[b]); + if (comp(t, r[a])) // c < a < b + { + r[b] = std::move(r[a]); + r[a] = std::move(t); + } else // a <= c < b + { + r[b] = std::move(t); + } + } + + assert(!comp(r[b], r[a]) && !comp(r[c], r[b])); + } + +/** +If leanRight == false, swaps the lower median of r[a]...r[d] into r[b] and +the minimum into r[a]. If leanRight == true, swaps the upper median of +r[a]...r[d] into r[c] and the minimum into r[d]. +*/ + template ::difference_type> + inline void partition4(Iter r, DiffType a, DiffType b, DiffType c, DiffType d, + Compare&& comp) { + assert(a != b && a != c && a != d && b != c && b != d && c != d); + /* static */ if (leanRight) { + // In the median of 5 algorithm, consider r[e] infinite + if (comp(r[c], r[a])) { + std::swap(r[a], r[c]); + } // a <= c + if (comp(r[d], r[b])) { + std::swap(r[b], r[d]); + } // a <= c, b <= d + if (comp(r[d], r[c])) { + std::swap(r[c], r[d]); // a <= d, b <= c < d + std::swap(r[a], r[b]); // b <= d, a <= c < d + } // a <= c <= d, b <= d + if (comp(r[c], r[b])) { // a <= c <= d, c < b <= d + std::swap(r[b], r[c]); // a <= b <= c <= d + } // a <= b <= c <= d + } else { + // In the median of 5 algorithm consider r[a] infinitely small, then + // change b->a. c->b, d->c, e->d + if (comp(r[c], r[a])) { + std::swap(r[a], r[c]); + } + if (comp(r[c], r[b])) { + std::swap(r[b], r[c]); + } + if (comp(r[d], r[a])) { + std::swap(r[a], r[d]); + } + if (comp(r[d], r[b])) { + std::swap(r[b], r[d]); + } else { + if (comp(r[b], r[a])) { + std::swap(r[a], r[b]); + } + } + } + } + +/** +Places the median of r[a]...r[e] in r[c] and partitions the other elements +around it. +*/ + template ::difference_type> + inline void partition5(Iter r, DiffType a, DiffType b, DiffType c, DiffType d, + DiffType e, Compare&& comp) { + assert(a != b && a != c && a != d && a != e && b != c && b != d && b != e && + c != d && c != e && d != e); + if (comp(r[c], r[a])) { + std::swap(r[a], r[c]); + } + if (comp(r[d], r[b])) { + std::swap(r[b], r[d]); + } + if (comp(r[d], r[c])) { + std::swap(r[c], r[d]); + std::swap(r[a], r[b]); + } + if (comp(r[e], r[b])) { + std::swap(r[b], r[e]); + } + if (comp(r[e], r[c])) { + std::swap(r[c], r[e]); + if (comp(r[c], r[a])) { + std::swap(r[a], r[c]); + } + } else { + if (comp(r[c], r[b])) { + std::swap(r[b], r[c]); + } + } + } + +/** +Implements Hoare partition. +*/ + template ::difference_type> + inline Iter pivot_partition(Iter r, DiffType k, DiffType length, + Compare&& comp) { + assert(k < length); + std::swap(*r, r[k]); + DiffType lo = 1, hi = length - 1; + for (;; ++lo, --hi) { + for (;; ++lo) { + if (lo > hi) goto loop_done; + if (!comp(r[lo], *r)) break; + } + // found the left bound: r[lo] >= r[0] + assert(lo <= hi); + for (; comp(*r, r[hi]); --hi) { + } + if (lo >= hi) break; + // found the right bound: r[hi] <= r[0], swap & make progress + std::swap(r[lo], r[hi]); + } + loop_done: + --lo; + std::swap(r[lo], *r); + return r + lo; + } + +/** +Implements the quickselect algorithm, parameterized with a partition function. +*/ + template + inline void quickselect(Iter r, Iter mid, Iter end, Compare&& comp) { + if (r == end || mid >= end) return; + assert(r <= mid && mid < end); + for (;;) switch (end - r) { + case 1: + return; + case 2: + if (comp(r[1], *r)) std::swap(*r, r[1]); + return; + case 3: + sort3(r, 0, 1, 2, comp); + return; + case 4: + switch (mid - r) { + case 0: + goto select_min; + case 1: + partition4(r, 0, 1, 2, 3, comp); + break; + case 2: + partition4(r, 0, 1, 2, 3, comp); + break; + case 3: + goto select_max; + default: + assert(false); + } + return; + default: + assert(end - r > 4); + if (r == mid) { + select_min: + auto pivot = r; + for (++mid; mid < end; ++mid) + if (comp(*mid, *pivot)) pivot = mid; + std::swap(*r, *pivot); + return; + } + if (mid + 1 == end) { + select_max: + auto pivot = r; + for (mid = r + 1; mid < end; ++mid) + if (comp(*pivot, *mid)) pivot = mid; + std::swap(*pivot, *(end - 1)); + return; + } + auto pivot = partition(r, end, comp); + if (pivot == mid) return; + if (mid < pivot) { + end = pivot; + } else { + r = pivot + 1; + } + } + } + +/** +Returns the index of the median of r[a], r[b], and r[c] without writing +anything. +*/ + template ::difference_type> + inline DiffType median_index(const Iter r, DiffType a, DiffType b, DiffType c, + Compare&& comp) { + if (comp(r[c], r[a])) std::swap(a, c); + if (comp(r[c], r[b])) return c; + if (comp(r[b], r[a])) return a; + return b; + } + +/** +Returns the index of the median of r[a], r[b], r[c], and r[d] without writing +anything. If leanRight is true, computes the upper median. Otherwise, conputes +the lower median. +*/ + template ::difference_type> + inline DiffType median_index(Iter r, DiffType a, DiffType b, DiffType c, + DiffType d, Compare&& comp) { + if (comp(r[d], r[c])) std::swap(c, d); + if (leanRight) { + if (comp(r[c], r[a])) { + assert(comp(r[c], r[a]) && !comp(r[d], r[c])); // so r[c]) is out + return median_index(r, a, b, d, comp); + } + } else { + if (!comp(r[d], r[a])) { + return median_index(r, a, b, c, comp); + } + } + // Could return median_index(r, b, c, d) but we already know r[c] <= r[d] + if (!comp(r[c], r[b])) return c; + if (comp(r[d], r[b])) return d; + return b; + } + +/** +Tukey's Ninther: compute the median of r[_1], r[_2], r[_3], then the median of +r[_4], r[_5], r[_6], then the median of r[_7], r[_8], r[_9], and then swap the +median of those three medians into r[_5]. +*/ + template ::difference_type> + inline void ninther(Iter r, DiffType _1, DiffType _2, DiffType _3, DiffType _4, + DiffType _5, DiffType _6, DiffType _7, DiffType _8, + DiffType _9, Compare&& comp) { + _2 = median_index(r, _1, _2, _3, comp); + _8 = median_index(r, _7, _8, _9, comp); + if (comp(r[_8], r[_2])) std::swap(_2, _8); + if (comp(r[_6], r[_4])) std::swap(_4, _6); + // Here we know that r[_2] and r[_8] are the other two medians and that + // r[_2] <= r[_8]. We also know that r[_4] <= r[_6] + if (comp(r[_5], r[_4])) { + // r[_4] is the median of r[_4], r[_5], r[_6] + } else if (comp(r[_6], r[_5])) { + // r[_6] is the median of r[_4], r[_5], r[_6] + _4 = _6; + } else { + // Here we know r[_5] is the median of r[_4], r[_5], r[_6] + if (comp(r[_5], r[_2])) return std::swap(r[_5], r[_2]); + if (comp(r[_8], r[_5])) return std::swap(r[_5], r[_8]); + // This is the only path that returns with no swap + return; + } + // Here we know r[_4] is the median of r[_4], r[_5], r[_6] + if (comp(r[_4], r[_2])) + _4 = _2; + else if (comp(r[_8], r[_4])) + _4 = _8; + std::swap(r[_5], r[_4]); + } + +/** +Input assumptions: +(a) hi <= rite +(c) the range r[0 .. hi] contains elements no smaller than r[0] +Output guarantee: same as Hoare partition using r[0] as pivot. Returns the new +position of the pivot. +*/ + template ::difference_type> + inline DiffType expand_partition_right(Iter r, DiffType hi, DiffType rite, + Compare&& comp) { + DiffType pivot = 0; + assert(pivot <= hi); + assert(hi <= rite); + // First loop: spend r[pivot .. hi] + for (; pivot < hi; --rite) { + if (rite == hi) goto done; + if (!comp(r[rite], r[0])) continue; + ++pivot; + std::swap(r[rite], r[pivot]); + } + // Second loop: make left and pivot meet + for (; rite > pivot; --rite) { + if (!comp(r[rite], r[0])) continue; + while (rite > pivot) { + ++pivot; + if (comp(r[0], r[pivot])) { + std::swap(r[rite], r[pivot]); + break; + } + } + } + + done: + std::swap(r[0], r[pivot]); + return pivot; + } + +/** +Input assumptions: +(a) lo > 0, lo <= pivot +(b) the range r[lo .. pivot] already contains elements no greater than r[pivot] +Output guarantee: Same as Hoare partition around r[pivot]. Returns the new +position of the pivot. +*/ + template ::difference_type> + inline DiffType expand_partition_left(Iter r, DiffType lo, DiffType pivot, + Compare&& comp) { + assert(lo > 0 && lo <= pivot); + DiffType left = 0; + const auto oldPivot = pivot; + for (; lo < pivot; ++left) { + if (left == lo) goto done; + if (!comp(r[oldPivot], r[left])) continue; + --pivot; + std::swap(r[left], r[pivot]); + } + // Second loop: make left and pivot meet + for (;; ++left) { + if (left == pivot) break; + if (!comp(r[oldPivot], r[left])) continue; + for (;;) { + if (left == pivot) goto done; + --pivot; + if (comp(r[pivot], r[oldPivot])) { + std::swap(r[left], r[pivot]); + break; + } + } + } + + done: + std::swap(r[oldPivot], r[pivot]); + return pivot; + } + +/** +Input assumptions: +(a) lo <= pivot, pivot < hi, hi <= length +(b) the range r[lo .. pivot] already contains elements no greater than +r[pivot] +(c) the range r[pivot .. hi] already contains elements no smaller than +r[pivot] +Output guarantee: Same as Hoare partition around r[pivot], returning the new +position of the pivot. +*/ + template ::difference_type> + inline DiffType expand_partition(Iter r, DiffType lo, DiffType pivot, + DiffType hi, DiffType length, Compare&& comp) { + assert(lo <= pivot && pivot < hi && hi <= length); + --hi; + --length; + DiffType left = 0; + for (;; ++left, --length) { + for (;; ++left) { + if (left == lo) + return pivot + expand_partition_right(r + pivot, hi - pivot, + length - pivot, comp); + if (comp(r[pivot], r[left])) break; + } + for (;; --length) { + if (length == hi) + return left + + expand_partition_left(r + left, lo - left, pivot - left, comp); + if (!comp(r[pivot], r[length])) break; + } + std::swap(r[left], r[length]); + } + } + + } // namespace median_common_detail +} // namespace miniselect \ No newline at end of file diff --git a/src/cpp/include/sorting/pdqsort.h b/src/cpp/include/sorting/pdqsort.h new file mode 100644 index 00000000..2bd768b0 --- /dev/null +++ b/src/cpp/include/sorting/pdqsort.h @@ -0,0 +1,937 @@ +/* + pdqsort.h - Pattern-defeating quicksort. + + Copyright (c) 2015 Orson Peters + + This software is provided 'as-is', without any express or implied warranty. + In no event will the authors be held liable for any damages arising from the + use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software in a + product, an acknowledgment in the product documentation would be appreciated + but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source distribution. +*/ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +// Adjusted by Danila Kutenin to support pdqselect and pdqpartial_sort. + +#ifndef PDQSORT_H +#define PDQSORT_H + +#include +#include +#include +#include +#include + +#if __cplusplus >= 201103L +#include +#include +#define PDQSORT_PREFER_MOVE(x) std::move(x) +#else +#define PDQSORT_PREFER_MOVE(x) (x) +#endif + +#include "median_common.h" + +namespace miniselect { + namespace pdqsort_detail { + + template + struct CompareRefType { + // Pass the comparator by lvalue reference. Or in debug mode, using a + // debugging wrapper that stores a reference. + using type = typename std::add_lvalue_reference::type; + }; + + enum { + // Partitions below this size are sorted using insertion sort. + insertion_sort_threshold = 24, + + // Partitions above this size use Tukey's ninther to select the pivot. + ninther_threshold = 128, + + // When we detect an already sorted partition, attempt an insertion sort that + // allows this + // amount of element moves before giving up. + partial_insertion_sort_limit = 8, + + // Must be multiple of 8 due to loop unrolling, and < 256 to fit in unsigned + // char. + block_size = 64, + + // Cacheline size, assumes power of two. + cacheline_size = 64 + + }; + +#if __cplusplus >= 201103L + template + struct is_default_compare : std::false_type {}; + template + struct is_default_compare> : std::true_type {}; + template + struct is_default_compare> : std::true_type {}; +#endif + +// Returns floor(log2(n)), assumes n > 0. + template + inline int log2(T n) { + int log = 0; + while (n >>= 1) ++log; + return log; + } + +// Sorts [begin, end) using insertion sort with the given comparison function. + template + inline void insertion_sort(Iter begin, Iter end, Compare& comp) { + typedef typename std::iterator_traits::value_type T; + if (begin == end) return; + + for (Iter cur = begin + 1; cur != end; ++cur) { + Iter sift = cur; + Iter sift_1 = cur - 1; + + // Compare first so we can avoid 2 moves for an element already positioned + // correctly. + if (comp(*sift, *sift_1)) { + T tmp = PDQSORT_PREFER_MOVE(*sift); + + do { + *sift-- = PDQSORT_PREFER_MOVE(*sift_1); + } while (sift != begin && comp(tmp, *--sift_1)); + + *sift = PDQSORT_PREFER_MOVE(tmp); + } + } + } + +// Sorts [begin, end) using insertion sort with the given comparison function. +// Assumes +// *(begin - 1) is an element smaller than or equal to any element in [begin, +// end). + template + inline void unguarded_insertion_sort(Iter begin, Iter end, Compare& comp) { + typedef typename std::iterator_traits::value_type T; + if (begin == end) return; + + for (Iter cur = begin + 1; cur != end; ++cur) { + Iter sift = cur; + Iter sift_1 = cur - 1; + + // Compare first so we can avoid 2 moves for an element already positioned + // correctly. + if (comp(*sift, *sift_1)) { + T tmp = PDQSORT_PREFER_MOVE(*sift); + + do { + *sift-- = PDQSORT_PREFER_MOVE(*sift_1); + } while (comp(tmp, *--sift_1)); + + *sift = PDQSORT_PREFER_MOVE(tmp); + } + } + } + +// Attempts to use insertion sort on [begin, end). Will return false if more +// than partial_insertion_sort_limit elements were moved, and abort sorting. +// Otherwise it will successfully sort and return true. + template + inline bool partial_insertion_sort(Iter begin, Iter end, Compare& comp) { + typedef typename std::iterator_traits::value_type T; + if (begin == end) return true; + + std::size_t limit = 0; + for (Iter cur = begin + 1; cur != end; ++cur) { + Iter sift = cur; + Iter sift_1 = cur - 1; + + // Compare first so we can avoid 2 moves for an element already positioned + // correctly. + if (comp(*sift, *sift_1)) { + T tmp = PDQSORT_PREFER_MOVE(*sift); + + do { + *sift-- = PDQSORT_PREFER_MOVE(*sift_1); + } while (sift != begin && comp(tmp, *--sift_1)); + + *sift = PDQSORT_PREFER_MOVE(tmp); + limit += cur - sift; + } + + if (limit > partial_insertion_sort_limit) return false; + } + + return true; + } + + template + inline void sort2(Iter a, Iter b, Compare& comp) { + if (comp(*b, *a)) std::iter_swap(a, b); + } + +// Sorts the elements *a, *b and *c using comparison function comp. + template + inline void sort3(Iter a, Iter b, Iter c, Compare& comp) { + sort2(a, b, comp); + sort2(b, c, comp); + sort2(a, b, comp); + } + + template + inline T* align_cacheline(T* p) { +#if defined(UINTPTR_MAX) && __cplusplus >= 201103L + std::uintptr_t ip = reinterpret_cast(p); +#else + std::size_t ip = reinterpret_cast(p); +#endif + ip = (ip + cacheline_size - 1) & -cacheline_size; + return reinterpret_cast(ip); + } + + template + inline void swap_offsets(Iter first, Iter last, unsigned char* offsets_l, + unsigned char* offsets_r, int num, bool use_swaps) { + typedef typename std::iterator_traits::value_type T; + if (use_swaps) { + // This case is needed for the descending distribution, where we need + // to have proper swapping for pdqsort to remain O(n). + for (int i = 0; i < num; ++i) { + std::iter_swap(first + offsets_l[i], last - offsets_r[i]); + } + } else if (num > 0) { + Iter l = first + offsets_l[0]; + Iter r = last - offsets_r[0]; + T tmp(PDQSORT_PREFER_MOVE(*l)); + *l = PDQSORT_PREFER_MOVE(*r); + for (int i = 1; i < num; ++i) { + l = first + offsets_l[i]; + *r = PDQSORT_PREFER_MOVE(*l); + r = last - offsets_r[i]; + *l = PDQSORT_PREFER_MOVE(*r); + } + *r = PDQSORT_PREFER_MOVE(tmp); + } + } + +// Partitions [begin, end) around pivot *begin using comparison function comp. +// Elements equal to the pivot are put in the right-hand partition. Returns the +// position of the pivot after partitioning and whether the passed sequence +// already was correctly partitioned. Assumes the pivot is a median of at least +// 3 elements and that [begin, end) is at least insertion_sort_threshold long. +// Uses branchless partitioning. + template + inline std::pair partition_right_branchless(Iter begin, Iter end, + Compare& comp) { + typedef typename std::iterator_traits::value_type T; + + // Move pivot into local for speed. + T pivot(PDQSORT_PREFER_MOVE(*begin)); + Iter first = begin; + Iter last = end; + + // Find the first element greater than or equal than the pivot (the median of + // 3 guarantees this exists). + while (comp(*++first, pivot)) + ; + + // Find the first element strictly smaller than the pivot. We have to guard + // this search if there was no element before *first. + if (first - 1 == begin) + while (first < last && !comp(*--last, pivot)) + ; + else + while (!comp(*--last, pivot)) + ; + + // If the first pair of elements that should be swapped to partition are the + // same element, the passed in sequence already was correctly partitioned. + bool already_partitioned = first >= last; + if (!already_partitioned) { + std::iter_swap(first, last); + ++first; + } + + // The following branchless partitioning is derived from "BlockQuicksort: How + // Branch Mispredictions don’t affect Quicksort" by Stefan Edelkamp and Armin + // Weiss. + unsigned char offsets_l_storage[block_size + cacheline_size]; + unsigned char offsets_r_storage[block_size + cacheline_size]; + unsigned char* offsets_l = align_cacheline(offsets_l_storage); + unsigned char* offsets_r = align_cacheline(offsets_r_storage); + int num_l, num_r, start_l, start_r; + num_l = num_r = start_l = start_r = 0; + + while (last - first > 2 * block_size) { + // Fill up offset blocks with elements that are on the wrong side. + if (num_l == 0) { + start_l = 0; + Iter it = first; + for (unsigned char i = 0; i < block_size;) { + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + } + } + if (num_r == 0) { + start_r = 0; + Iter it = last; + for (unsigned char i = 0; i < block_size;) { + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + } + } + + // Swap elements and update block sizes and first/last boundaries. + int num = std::min(num_l, num_r); + swap_offsets(first, last, offsets_l + start_l, offsets_r + start_r, num, + num_l == num_r); + num_l -= num; + num_r -= num; + start_l += num; + start_r += num; + if (num_l == 0) first += block_size; + if (num_r == 0) last -= block_size; + } + + int l_size = 0, r_size = 0; + int unknown_left = (int)(last - first) - ((num_r || num_l) ? block_size : 0); + if (num_r) { + // Handle leftover block by assigning the unknown elements to the other + // block. + l_size = unknown_left; + r_size = block_size; + } else if (num_l) { + l_size = block_size; + r_size = unknown_left; + } else { + // No leftover block, split the unknown elements in two blocks. + l_size = unknown_left / 2; + r_size = unknown_left - l_size; + } + + // Fill offset buffers if needed. + if (unknown_left && !num_l) { + start_l = 0; + Iter it = first; + for (unsigned char i = 0; i < l_size;) { + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + } + } + if (unknown_left && !num_r) { + start_r = 0; + Iter it = last; + for (unsigned char i = 0; i < r_size;) { + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + } + } + + int num = std::min(num_l, num_r); + swap_offsets(first, last, offsets_l + start_l, offsets_r + start_r, num, + num_l == num_r); + num_l -= num; + num_r -= num; + start_l += num; + start_r += num; + if (num_l == 0) first += l_size; + if (num_r == 0) last -= r_size; + + // We have now fully identified [first, last)'s proper position. Swap the last + // elements. + if (num_l) { + offsets_l += start_l; + while (num_l--) std::iter_swap(first + offsets_l[num_l], --last); + first = last; + } + if (num_r) { + offsets_r += start_r; + while (num_r--) std::iter_swap(last - offsets_r[num_r], first), ++first; + last = first; + } + + // Put the pivot in the right place. + Iter pivot_pos = first - 1; + *begin = PDQSORT_PREFER_MOVE(*pivot_pos); + *pivot_pos = PDQSORT_PREFER_MOVE(pivot); + + return std::make_pair(pivot_pos, already_partitioned); + } + +// Partitions [begin, end) around pivot *begin using comparison function comp. +// Elements equal to the pivot are put in the right-hand partition. Returns the +// position of the pivot after partitioning and whether the passed sequence +// already was correctly partitioned. Assumes the pivot is a median of at least +// 3 elements and that [begin, end) is at least insertion_sort_threshold long. + template + inline std::pair partition_right(Iter begin, Iter end, + Compare& comp) { + typedef typename std::iterator_traits::value_type T; + + // Move pivot into local for speed. + T pivot(PDQSORT_PREFER_MOVE(*begin)); + + Iter first = begin; + Iter last = end; + + // Find the first element greater than or equal than the pivot (the median of + // 3 guarantees this exists). + while (comp(*++first, pivot)) + ; + + // Find the first element strictly smaller than the pivot. We have to guard + // this search if there was no element before *first. + if (first - 1 == begin) + while (first < last && !comp(*--last, pivot)) + ; + else + while (!comp(*--last, pivot)) + ; + + // If the first pair of elements that should be swapped to partition are the + // same element, the passed in sequence already was correctly partitioned. + bool already_partitioned = first >= last; + + // Keep swapping pairs of elements that are on the wrong side of the pivot. + // Previously swapped pairs guard the searches, which is why the first + // iteration is special-cased above. + while (first < last) { + std::iter_swap(first, last); + while (comp(*++first, pivot)) + ; + while (!comp(*--last, pivot)) + ; + } + + // Put the pivot in the right place. + Iter pivot_pos = first - 1; + *begin = PDQSORT_PREFER_MOVE(*pivot_pos); + *pivot_pos = PDQSORT_PREFER_MOVE(pivot); + + return std::make_pair(pivot_pos, already_partitioned); + } + +// Similar function to the one above, except elements equal to the pivot are put +// to the left of the pivot and it doesn't check or return if the passed +// sequence already was partitioned. Since this is rarely used (the many equal +// case), and in that case pdqsort already has O(n) performance, no block +// quicksort is applied here for simplicity. + template + inline Iter partition_left(Iter begin, Iter end, Compare& comp) { + typedef typename std::iterator_traits::value_type T; + + T pivot(PDQSORT_PREFER_MOVE(*begin)); + Iter first = begin; + Iter last = end; + + while (comp(pivot, *--last)) + ; + + if (last + 1 == end) + while (first < last && !comp(pivot, *++first)) + ; + else + while (!comp(pivot, *++first)) + ; + + while (first < last) { + std::iter_swap(first, last); + while (comp(pivot, *--last)) + ; + while (!comp(pivot, *++first)) + ; + } + + Iter pivot_pos = last; + *begin = PDQSORT_PREFER_MOVE(*pivot_pos); + *pivot_pos = PDQSORT_PREFER_MOVE(pivot); + + return pivot_pos; + } + + template + inline void pdqsort_loop(Iter begin, Iter end, Compare& comp, int bad_allowed, + bool leftmost = true) { + typedef typename std::iterator_traits::difference_type diff_t; + + // Use a while loop for tail recursion elimination. + while (true) { + diff_t size = end - begin; + + // Insertion sort is faster for small arrays. + if (size < insertion_sort_threshold) { + if (leftmost) + insertion_sort(begin, end, comp); + else + unguarded_insertion_sort(begin, end, comp); + return; + } + + // Choose pivot as median of 3 or pseudomedian of 9. + diff_t s2 = size / 2; + if (size > ninther_threshold) { + sort3(begin, begin + s2, end - 1, comp); + sort3(begin + 1, begin + (s2 - 1), end - 2, comp); + sort3(begin + 2, begin + (s2 + 1), end - 3, comp); + sort3(begin + (s2 - 1), begin + s2, begin + (s2 + 1), comp); + std::iter_swap(begin, begin + s2); + } else + sort3(begin + s2, begin, end - 1, comp); + + // If *(begin - 1) is the end of the right partition of a previous partition + // operation there is no element in [begin, end) that is smaller than + // *(begin - 1). Then if our pivot compares equal to *(begin - 1) we change + // strategy, putting equal elements in the left partition, greater elements + // in the right partition. We do not have to recurse on the left partition, + // since it's sorted (all equal). + if (!leftmost && !comp(*(begin - 1), *begin)) { + begin = partition_left(begin, end, comp) + 1; + continue; + } + + // Partition and get results. + std::pair part_result = + Branchless ? partition_right_branchless(begin, end, comp) + : partition_right(begin, end, comp); + Iter pivot_pos = part_result.first; + bool already_partitioned = part_result.second; + + // Check for a highly unbalanced partition. + diff_t l_size = pivot_pos - begin; + diff_t r_size = end - (pivot_pos + 1); + bool highly_unbalanced = l_size < size / 8 || r_size < size / 8; + + // If we got a highly unbalanced partition we shuffle elements to break many + // patterns. + if (highly_unbalanced) { + // If we had too many bad partitions, switch to heapsort to guarantee O(n + // log n). + if (--bad_allowed == 0) { + std::make_heap(begin, end, comp); + std::sort_heap(begin, end, comp); + return; + } + + if (l_size >= insertion_sort_threshold) { + std::iter_swap(begin, begin + l_size / 4); + std::iter_swap(pivot_pos - 1, pivot_pos - l_size / 4); + + if (l_size > ninther_threshold) { + std::iter_swap(begin + 1, begin + (l_size / 4 + 1)); + std::iter_swap(begin + 2, begin + (l_size / 4 + 2)); + std::iter_swap(pivot_pos - 2, pivot_pos - (l_size / 4 + 1)); + std::iter_swap(pivot_pos - 3, pivot_pos - (l_size / 4 + 2)); + } + } + + if (r_size >= insertion_sort_threshold) { + std::iter_swap(pivot_pos + 1, pivot_pos + (1 + r_size / 4)); + std::iter_swap(end - 1, end - r_size / 4); + + if (r_size > ninther_threshold) { + std::iter_swap(pivot_pos + 2, pivot_pos + (2 + r_size / 4)); + std::iter_swap(pivot_pos + 3, pivot_pos + (3 + r_size / 4)); + std::iter_swap(end - 2, end - (1 + r_size / 4)); + std::iter_swap(end - 3, end - (2 + r_size / 4)); + } + } + } else { + // If we were decently balanced and we tried to sort an already + // partitioned sequence try to use insertion sort. + if (already_partitioned && + partial_insertion_sort(begin, pivot_pos, comp) && + partial_insertion_sort(pivot_pos + 1, end, comp)) + return; + } + + // Sort the left partition first using recursion and do tail recursion + // elimination for the right-hand partition. + pdqsort_loop(begin, pivot_pos, comp, bad_allowed, + leftmost); + begin = pivot_pos + 1; + leftmost = false; + } + } + + template + inline void pdqpartial_sort_loop(Iter begin, Iter mid, Iter end, Compare& comp, + int bad_allowed, bool leftmost = true) { + typedef typename std::iterator_traits::difference_type diff_t; + + // Use a while loop for tail recursion elimination. + while (true) { + diff_t size = end - begin; + + // Insertion sort is faster for small arrays. + if (size < insertion_sort_threshold) { + if (leftmost) + insertion_sort(begin, end, comp); + else + unguarded_insertion_sort(begin, end, comp); + return; + } + + // Choose pivot as median of 3 or pseudomedian of 9. + diff_t s2 = size / 2; + if (size > ninther_threshold) { + sort3(begin, begin + s2, end - 1, comp); + sort3(begin + 1, begin + (s2 - 1), end - 2, comp); + sort3(begin + 2, begin + (s2 + 1), end - 3, comp); + sort3(begin + (s2 - 1), begin + s2, begin + (s2 + 1), comp); + std::iter_swap(begin, begin + s2); + } else + sort3(begin + s2, begin, end - 1, comp); + + // If *(begin - 1) is the end of the right partition of a previous partition + // operation there is no element in [begin, end) that is smaller than + // *(begin - 1). Then if our pivot compares equal to *(begin - 1) we change + // strategy, putting equal elements in the left partition, greater elements + // in the right partition. We do not have to recurse on the left partition, + // since it's sorted (all equal). + if (!leftmost && !comp(*(begin - 1), *begin)) { + begin = partition_left(begin, end, comp) + 1; + continue; + } + + // Partition and get results. + std::pair part_result = + Branchless ? partition_right_branchless(begin, end, comp) + : partition_right(begin, end, comp); + Iter pivot_pos = part_result.first; + bool already_partitioned = part_result.second; + + // Check for a highly unbalanced partition. + diff_t l_size = pivot_pos - begin; + diff_t r_size = end - (pivot_pos + 1); + bool highly_unbalanced = l_size < size / 8 || r_size < size / 8; + + // If we got a highly unbalanced partition we shuffle elements to break many + // patterns. + if (highly_unbalanced) { + // If we had too many bad partitions, switch to heapsort to guarantee O(n + // log n). + if (--bad_allowed == 0) { + std::make_heap(begin, end, comp); + std::sort_heap(begin, end, comp); + return; + } + + if (l_size >= insertion_sort_threshold) { + std::iter_swap(begin, begin + l_size / 4); + std::iter_swap(pivot_pos - 1, pivot_pos - l_size / 4); + + if (l_size > ninther_threshold) { + std::iter_swap(begin + 1, begin + (l_size / 4 + 1)); + std::iter_swap(begin + 2, begin + (l_size / 4 + 2)); + std::iter_swap(pivot_pos - 2, pivot_pos - (l_size / 4 + 1)); + std::iter_swap(pivot_pos - 3, pivot_pos - (l_size / 4 + 2)); + } + } + + if (r_size >= insertion_sort_threshold) { + std::iter_swap(pivot_pos + 1, pivot_pos + (1 + r_size / 4)); + std::iter_swap(end - 1, end - r_size / 4); + + if (r_size > ninther_threshold) { + std::iter_swap(pivot_pos + 2, pivot_pos + (2 + r_size / 4)); + std::iter_swap(pivot_pos + 3, pivot_pos + (3 + r_size / 4)); + std::iter_swap(end - 2, end - (1 + r_size / 4)); + std::iter_swap(end - 3, end - (2 + r_size / 4)); + } + } + } else { + // If we were decently balanced and we tried to sort an already + // partitioned sequence try to use insertion sort. + if (already_partitioned && + partial_insertion_sort(begin, pivot_pos, comp) && + partial_insertion_sort(pivot_pos + 1, end, comp)) + return; + } + + // Sort the left partition first using recursion and do tail recursion + // elimination for the right-hand partition. + if (pivot_pos < mid) { + pdqsort_loop(begin, pivot_pos, comp, + bad_allowed, leftmost); + begin = pivot_pos + 1; + leftmost = false; + } else { + end = pivot_pos; + } + } + } + + template + inline void pdqselect_loop(Iter begin, Iter mid, Iter end, Compare& comp, + int bad_allowed, bool leftmost = true) { + typedef typename std::iterator_traits::difference_type diff_t; + + // Use a while loop for tail recursion elimination. + while (true) { + diff_t size = end - begin; + + // Insertion sort is faster for small arrays. + if (size < insertion_sort_threshold) { + if (leftmost) + insertion_sort(begin, end, comp); + else + unguarded_insertion_sort(begin, end, comp); + return; + } + + // Choose pivot as median of 3 or pseudomedian of 9. + diff_t s2 = size / 2; + if (size > ninther_threshold) { + sort3(begin, begin + s2, end - 1, comp); + sort3(begin + 1, begin + (s2 - 1), end - 2, comp); + sort3(begin + 2, begin + (s2 + 1), end - 3, comp); + sort3(begin + (s2 - 1), begin + s2, begin + (s2 + 1), comp); + std::iter_swap(begin, begin + s2); + } else + sort3(begin + s2, begin, end - 1, comp); + + // If *(begin - 1) is the end of the right partition of a previous partition + // operation there is no element in [begin, end) that is smaller than + // *(begin - 1). Then if our pivot compares equal to *(begin - 1) we change + // strategy, putting equal elements in the left partition, greater elements + // in the right partition. We do not have to recurse on the left partition, + // since it's sorted (all equal). + if (!leftmost && !comp(*(begin - 1), *begin)) { + begin = partition_left(begin, end, comp) + 1; + continue; + } + + // Partition and get results. + std::pair part_result = + Branchless ? partition_right_branchless(begin, end, comp) + : partition_right(begin, end, comp); + Iter pivot_pos = part_result.first; + bool already_partitioned = part_result.second; + + // Check for a highly unbalanced partition. + diff_t l_size = pivot_pos - begin; + diff_t r_size = end - (pivot_pos + 1); + bool highly_unbalanced = l_size < size / 8 || r_size < size / 8; + + // If we got a highly unbalanced partition we shuffle elements to break many + // patterns. + if (highly_unbalanced) { + // If we had too many bad partitions, switch to heapsort to guarantee O(n + // log n). + if (--bad_allowed == 0) { + std::nth_element(begin, mid, end, comp); + return; + } + + if (l_size >= insertion_sort_threshold) { + std::iter_swap(begin, begin + l_size / 4); + std::iter_swap(pivot_pos - 1, pivot_pos - l_size / 4); + + if (l_size > ninther_threshold) { + std::iter_swap(begin + 1, begin + (l_size / 4 + 1)); + std::iter_swap(begin + 2, begin + (l_size / 4 + 2)); + std::iter_swap(pivot_pos - 2, pivot_pos - (l_size / 4 + 1)); + std::iter_swap(pivot_pos - 3, pivot_pos - (l_size / 4 + 2)); + } + } + + if (r_size >= insertion_sort_threshold) { + std::iter_swap(pivot_pos + 1, pivot_pos + (1 + r_size / 4)); + std::iter_swap(end - 1, end - r_size / 4); + + if (r_size > ninther_threshold) { + std::iter_swap(pivot_pos + 2, pivot_pos + (2 + r_size / 4)); + std::iter_swap(pivot_pos + 3, pivot_pos + (3 + r_size / 4)); + std::iter_swap(end - 2, end - (1 + r_size / 4)); + std::iter_swap(end - 3, end - (2 + r_size / 4)); + } + } + } else { + // If we were decently balanced and we tried to sort an already + // partitioned sequence try to use insertion sort. + if (already_partitioned && + partial_insertion_sort(begin, pivot_pos, comp) && + partial_insertion_sort(pivot_pos + 1, end, comp)) + return; + } + // Sort the left partition first using recursion and do tail recursion + // elimination for the right-hand partition. + if (pivot_pos < mid) { + begin = pivot_pos + 1; + leftmost = false; + } else { + end = pivot_pos; + } + } + } + } // namespace pdqsort_detail + + template + inline void pdqsort(Iter begin, Iter end, Compare comp) { + if (begin == end) return; + +#if __cplusplus >= 201103L + pdqsort_detail::pdqsort_loop< + Iter, Compare, + pdqsort_detail::is_default_compare< + typename std::decay::type>::value && + std::is_arithmetic< + typename std::iterator_traits::value_type>::value>( + begin, end, comp, pdqsort_detail::log2(end - begin)); +#else + pdqsort_detail::pdqsort_loop( + begin, end, comp, pdqsort_detail::log2(end - begin)); +#endif + } + + template + inline void pdqsort(Iter begin, Iter end) { + typedef typename std::iterator_traits::value_type T; + pdqsort(begin, end, std::less()); + } + + template + inline void pdqsort_branchless(Iter begin, Iter end, Compare comp) { + if (begin == end) return; + pdqsort_detail::pdqsort_loop( + begin, end, comp, pdqsort_detail::log2(end - begin)); + } + + template + inline void pdqsort_branchless(Iter begin, Iter end) { + typedef typename std::iterator_traits::value_type T; + pdqsort_branchless(begin, end, std::less()); + } + + template + inline void pdqpartial_sort(Iter begin, Iter mid, Iter end, Compare comp) { + if (begin == end) return; + +#if __cplusplus >= 201103L + pdqsort_detail::pdqpartial_sort_loop< + Iter, Compare, + pdqsort_detail::is_default_compare< + typename std::decay::type>::value && + std::is_arithmetic< + typename std::iterator_traits::value_type>::value>( + begin, mid, end, comp, pdqsort_detail::log2(end - begin)); +#else + pdqsort_detail::pdqpartial_sort_loop( + begin, end, comp, pdqsort_detail::log2(end - begin)); +#endif + } + + template + inline void pdqpartial_sort(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + pdqpartial_sort(begin, mid, end, std::less()); + } + + template + inline void pdqpartial_sort_branchless(Iter begin, Iter mid, Iter end, + Compare comp) { + if (begin == end) return; + pdqsort_detail::pdqpartial_sort_loop( + begin, mid, end, comp, pdqsort_detail::log2(end - begin)); + } + + template + inline void pdqpartial_sort_branchless(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + pdqpartial_sort_branchless(begin, mid, end, std::less()); + } + + template + inline void pdqselect(Iter begin, Iter mid, Iter end, Compare comp) { + if (mid == end) return; + using CompType = typename median_common_detail::CompareRefType::type; + +#if __cplusplus >= 201103L + pdqsort_detail::pdqselect_loop< + Iter, CompType, + pdqsort_detail::is_default_compare< + typename std::decay::type>::value && + std::is_arithmetic< + typename std::iterator_traits::value_type>::value>( + begin, mid, end, comp, pdqsort_detail::log2(end - begin)); +#else + pdqsort_detail::pdqselect_loop( + begin, end, comp, pdqsort_detail::log2(end - begin)); +#endif + } + + template + inline void pdqselect(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + pdqselect(begin, mid, end, std::less()); + } + + template + inline void pdqselect_branchless(Iter begin, Iter mid, Iter end, Compare comp) { + if (mid == end) return; + using CompType = typename median_common_detail::CompareRefType::type; + pdqsort_detail::pdqselect_loop( + begin, mid, end, comp, pdqsort_detail::log2(end - begin)); + } + + template + inline void pdqselect_branchless(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + pdqselect_branchless(begin, mid, end, std::less()); + } + +#undef PDQSORT_PREFER_MOVE + +#endif + +} // namespace miniselect \ No newline at end of file diff --git a/src/cpp/include/sorting/readerwriterqueue.h b/src/cpp/include/sorting/readerwriterqueue.h new file mode 100644 index 00000000..098d47cf --- /dev/null +++ b/src/cpp/include/sorting/readerwriterqueue.h @@ -0,0 +1,979 @@ +// ©2013-2020 Cameron Desrochers. +// Distributed under the simplified BSD license (see the license file that +// should have come with this header). + +#pragma once + +#include "atomicops.h" +#include +#include +#include +#include +#include +#include +#include +#include // For malloc/free/abort & size_t +#include +#if __cplusplus > 199711L || _MSC_VER >= 1700 // C++11 or VS2012 +#include +#endif + + +// A lock-free queue for a single-consumer, single-producer architecture. +// The queue is also wait-free in the common path (except if more memory +// needs to be allocated, in which case malloc is called). +// Allocates memory sparingly, and only once if the original maximum size +// estimate is never exceeded. +// Tested on x86/x64 processors, but semantics should be correct for all +// architectures (given the right implementations in atomicops.h), provided +// that aligned integer and pointer accesses are naturally atomic. +// Note that there should only be one consumer thread and producer thread; +// Switching roles of the threads, or using multiple consecutive threads for +// one role, is not safe unless properly synchronized. +// Using the queue exclusively from one thread is fine, though a bit silly. + +#ifndef MOODYCAMEL_CACHE_LINE_SIZE +#define MOODYCAMEL_CACHE_LINE_SIZE 64 +#endif + +#ifndef MOODYCAMEL_EXCEPTIONS_ENABLED +#if (defined(_MSC_VER) && defined(_CPPUNWIND)) || (defined(__GNUC__) && defined(__EXCEPTIONS)) || (!defined(_MSC_VER) && !defined(__GNUC__)) +#define MOODYCAMEL_EXCEPTIONS_ENABLED +#endif +#endif + +#ifndef MOODYCAMEL_HAS_EMPLACE +#if !defined(_MSC_VER) || _MSC_VER >= 1800 // variadic templates: either a non-MS compiler or VS >= 2013 +#define MOODYCAMEL_HAS_EMPLACE 1 +#endif +#endif + +#ifndef MOODYCAMEL_MAYBE_ALIGN_TO_CACHELINE +#if defined (__APPLE__) && defined (__MACH__) && __cplusplus >= 201703L +// This is required to find out what deployment target we are using +#include +#if !defined(MAC_OS_X_VERSION_MIN_REQUIRED) || !defined(MAC_OS_X_VERSION_10_14) || MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_14 +// C++17 new(size_t, align_val_t) is not backwards-compatible with older versions of macOS, so we can't support over-alignment in this case +#define MOODYCAMEL_MAYBE_ALIGN_TO_CACHELINE +#endif +#endif +#endif + +#ifndef MOODYCAMEL_MAYBE_ALIGN_TO_CACHELINE +#define MOODYCAMEL_MAYBE_ALIGN_TO_CACHELINE AE_ALIGN(MOODYCAMEL_CACHE_LINE_SIZE) +#endif + +#ifdef AE_VCPP +#pragma warning(push) +#pragma warning(disable: 4324) // structure was padded due to __declspec(align()) +#pragma warning(disable: 4820) // padding was added +#pragma warning(disable: 4127) // conditional expression is constant +#endif + +namespace moodycamel { + + template + class MOODYCAMEL_MAYBE_ALIGN_TO_CACHELINE ReaderWriterQueue +{ + // Design: Based on a queue-of-queues. The low-level queues are just + // circular buffers with front and tail indices indicating where the + // next element to dequeue is and where the next element can be enqueued, + // respectively. Each low-level queue is called a "block". Each block + // wastes exactly one element's worth of space to keep the design simple + // (if front == tail then the queue is empty, and can't be full). + // The high-level queue is a circular linked list of blocks; again there + // is a front and tail, but this time they are pointers to the blocks. + // The front block is where the next element to be dequeued is, provided + // the block is not empty. The back block is where elements are to be + // enqueued, provided the block is not full. + // The producer thread owns all the tail indices/pointers. The consumer + // thread owns all the front indices/pointers. Both threads read each + // other's variables, but only the owning thread updates them. E.g. After + // the consumer reads the producer's tail, the tail may change before the + // consumer is done dequeuing an object, but the consumer knows the tail + // will never go backwards, only forwards. + // If there is no room to enqueue an object, an additional block (of + // equal size to the last block) is added. Blocks are never removed. + + public: + typedef T value_type; + + // Constructs a queue that can hold at least `size` elements without further + // allocations. If more than MAX_BLOCK_SIZE elements are requested, + // then several blocks of MAX_BLOCK_SIZE each are reserved (including + // at least one extra buffer block). + AE_NO_TSAN explicit ReaderWriterQueue(size_t size = 15) +#ifndef NDEBUG + : enqueuing(false) + ,dequeuing(false) +#endif +{ + assert(MAX_BLOCK_SIZE == ceilToPow2(MAX_BLOCK_SIZE) && "MAX_BLOCK_SIZE must be a power of 2"); + assert(MAX_BLOCK_SIZE >= 2 && "MAX_BLOCK_SIZE must be at least 2"); + + Block* firstBlock = nullptr; + + largestBlockSize = ceilToPow2(size + 1); // We need a spare slot to fit size elements in the block + if (largestBlockSize > MAX_BLOCK_SIZE * 2) { + // We need a spare block in case the producer is writing to a different block the consumer is reading from, and + // wants to enqueue the maximum number of elements. We also need a spare element in each block to avoid the ambiguity + // between front == tail meaning "empty" and "full". + // So the effective number of slots that are guaranteed to be usable at any time is the block size - 1 times the + // number of blocks - 1. Solving for size and applying a ceiling to the division gives us (after simplifying): + size_t initialBlockCount = (size + MAX_BLOCK_SIZE * 2 - 3) / (MAX_BLOCK_SIZE - 1); + largestBlockSize = MAX_BLOCK_SIZE; + Block* lastBlock = nullptr; + for (size_t i = 0; i != initialBlockCount; ++i) { + auto block = make_block(largestBlockSize); + if (block == nullptr) { +#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED + throw std::bad_alloc(); +#else + abort(); +#endif +} +if (firstBlock == nullptr) { +firstBlock = block; +} +else { +lastBlock->next = block; +} +lastBlock = block; +block->next = firstBlock; +} +} +else { +firstBlock = make_block(largestBlockSize); +if (firstBlock == nullptr) { +#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED +throw std::bad_alloc(); +#else +abort(); +#endif +} +firstBlock->next = firstBlock; +} +frontBlock = firstBlock; +tailBlock = firstBlock; + +// Make sure the reader/writer threads will have the initialized memory setup above: +fence(memory_order_sync); +} + +// Note: The queue should not be accessed concurrently while it's +// being moved. It's up to the user to synchronize this. +AE_NO_TSAN ReaderWriterQueue(ReaderWriterQueue&& other) +: frontBlock(other.frontBlock.load()), +tailBlock(other.tailBlock.load()), +largestBlockSize(other.largestBlockSize) +#ifndef NDEBUG +,enqueuing(false) +,dequeuing(false) +#endif +{ +other.largestBlockSize = 32; +Block* b = other.make_block(other.largestBlockSize); +if (b == nullptr) { +#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED +throw std::bad_alloc(); +#else +abort(); +#endif +} +b->next = b; +other.frontBlock = b; +other.tailBlock = b; +} + +// Note: The queue should not be accessed concurrently while it's +// being moved. It's up to the user to synchronize this. +ReaderWriterQueue& operator=(ReaderWriterQueue&& other) AE_NO_TSAN +{ +Block* b = frontBlock.load(); +frontBlock = other.frontBlock.load(); +other.frontBlock = b; +b = tailBlock.load(); +tailBlock = other.tailBlock.load(); +other.tailBlock = b; +std::swap(largestBlockSize, other.largestBlockSize); +return *this; +} + +// Note: The queue should not be accessed concurrently while it's +// being deleted. It's up to the user to synchronize this. +AE_NO_TSAN ~ReaderWriterQueue() +{ + // Make sure we get the latest version of all variables from other CPUs: + fence(memory_order_sync); + + // Destroy any remaining objects in queue and free memory + Block* frontBlock_ = frontBlock; + Block* block = frontBlock_; + do { + Block* nextBlock = block->next; + size_t blockFront = block->front; + size_t blockTail = block->tail; + + for (size_t i = blockFront; i != blockTail; i = (i + 1) & block->sizeMask) { + auto element = reinterpret_cast(block->data + i * sizeof(T)); + element->~T(); + (void)element; + } + + auto rawBlock = block->rawThis; + block->~Block(); + std::free(rawBlock); + block = nextBlock; + } while (block != frontBlock_); +} + + +// Enqueues a copy of element if there is room in the queue. +// Returns true if the element was enqueued, false otherwise. +// Does not allocate memory. +AE_FORCEINLINE bool try_enqueue(T const& element) AE_NO_TSAN +{ +return inner_enqueue(element); +} + +// Enqueues a moved copy of element if there is room in the queue. +// Returns true if the element was enqueued, false otherwise. +// Does not allocate memory. +AE_FORCEINLINE bool try_enqueue(T&& element) AE_NO_TSAN +{ +return inner_enqueue(std::forward(element)); +} + +#if MOODYCAMEL_HAS_EMPLACE +// Like try_enqueue() but with emplace semantics (i.e. construct-in-place). +template +AE_FORCEINLINE bool try_emplace(Args&&... args) AE_NO_TSAN +{ +return inner_enqueue(std::forward(args)...); +} +#endif + +// Enqueues a copy of element on the queue. +// Allocates an additional block of memory if needed. +// Only fails (returns false) if memory allocation fails. +AE_FORCEINLINE bool enqueue(T const& element) AE_NO_TSAN +{ +return inner_enqueue(element); +} + +// Enqueues a moved copy of element on the queue. +// Allocates an additional block of memory if needed. +// Only fails (returns false) if memory allocation fails. +AE_FORCEINLINE bool enqueue(T&& element) AE_NO_TSAN +{ +return inner_enqueue(std::forward(element)); +} + +#if MOODYCAMEL_HAS_EMPLACE +// Like enqueue() but with emplace semantics (i.e. construct-in-place). +template +AE_FORCEINLINE bool emplace(Args&&... args) AE_NO_TSAN +{ +return inner_enqueue(std::forward(args)...); +} +#endif + +// Attempts to dequeue an element; if the queue is empty, +// returns false instead. If the queue has at least one element, +// moves front to result using operator=, then returns true. +template +bool try_dequeue(U& result) AE_NO_TSAN +{ +#ifndef NDEBUG +ReentrantGuard guard(this->dequeuing); +#endif + +// High-level pseudocode: +// Remember where the tail block is +// If the front block has an element in it, dequeue it +// Else +// If front block was the tail block when we entered the function, return false +// Else advance to next block and dequeue the item there + +// Note that we have to use the value of the tail block from before we check if the front +// block is full or not, in case the front block is empty and then, before we check if the +// tail block is at the front block or not, the producer fills up the front block *and +// moves on*, which would make us skip a filled block. Seems unlikely, but was consistently +// reproducible in practice. +// In order to avoid overhead in the common case, though, we do a double-checked pattern +// where we have the fast path if the front block is not empty, then read the tail block, +// then re-read the front block and check if it's not empty again, then check if the tail +// block has advanced. + +Block* frontBlock_ = frontBlock.load(); +size_t blockTail = frontBlock_->localTail; +size_t blockFront = frontBlock_->front.load(); + +if (blockFront != blockTail || blockFront != (frontBlock_->localTail = frontBlock_->tail.load())) { +fence(memory_order_acquire); + +non_empty_front_block: +// Front block not empty, dequeue from here +auto element = reinterpret_cast(frontBlock_->data + blockFront * sizeof(T)); +result = std::move(*element); +element->~T(); + +blockFront = (blockFront + 1) & frontBlock_->sizeMask; + +fence(memory_order_release); +frontBlock_->front = blockFront; +} +else if (frontBlock_ != tailBlock.load()) { +fence(memory_order_acquire); + +frontBlock_ = frontBlock.load(); +blockTail = frontBlock_->localTail = frontBlock_->tail.load(); +blockFront = frontBlock_->front.load(); +fence(memory_order_acquire); + +if (blockFront != blockTail) { +// Oh look, the front block isn't empty after all +goto non_empty_front_block; +} + +// Front block is empty but there's another block ahead, advance to it +Block* nextBlock = frontBlock_->next; +// Don't need an acquire fence here since next can only ever be set on the tailBlock, +// and we're not the tailBlock, and we did an acquire earlier after reading tailBlock which +// ensures next is up-to-date on this CPU in case we recently were at tailBlock. + +size_t nextBlockFront = nextBlock->front.load(); +size_t nextBlockTail = nextBlock->localTail = nextBlock->tail.load(); +fence(memory_order_acquire); + +// Since the tailBlock is only ever advanced after being written to, +// we know there's for sure an element to dequeue on it +assert(nextBlockFront != nextBlockTail); +AE_UNUSED(nextBlockTail); + +// We're done with this block, let the producer use it if it needs +fence(memory_order_release); // Expose possibly pending changes to frontBlock->front from last dequeue +frontBlock = frontBlock_ = nextBlock; + +compiler_fence(memory_order_release); // Not strictly needed + +auto element = reinterpret_cast(frontBlock_->data + nextBlockFront * sizeof(T)); + +result = std::move(*element); +element->~T(); + +nextBlockFront = (nextBlockFront + 1) & frontBlock_->sizeMask; + +fence(memory_order_release); +frontBlock_->front = nextBlockFront; +} +else { +// No elements in current block and no other block to advance to +return false; +} + +return true; +} + + +// Returns a pointer to the front element in the queue (the one that +// would be removed next by a call to `try_dequeue` or `pop`). If the +// queue appears empty at the time the method is called, nullptr is +// returned instead. +// Must be called only from the consumer thread. +T* peek() const AE_NO_TSAN +{ +#ifndef NDEBUG +ReentrantGuard guard(this->dequeuing); +#endif +// See try_dequeue() for reasoning + +Block* frontBlock_ = frontBlock.load(); +size_t blockTail = frontBlock_->localTail; +size_t blockFront = frontBlock_->front.load(); + +if (blockFront != blockTail || blockFront != (frontBlock_->localTail = frontBlock_->tail.load())) { +fence(memory_order_acquire); +non_empty_front_block: +return reinterpret_cast(frontBlock_->data + blockFront * sizeof(T)); +} +else if (frontBlock_ != tailBlock.load()) { +fence(memory_order_acquire); +frontBlock_ = frontBlock.load(); +blockTail = frontBlock_->localTail = frontBlock_->tail.load(); +blockFront = frontBlock_->front.load(); +fence(memory_order_acquire); + +if (blockFront != blockTail) { +goto non_empty_front_block; +} + +Block* nextBlock = frontBlock_->next; + +size_t nextBlockFront = nextBlock->front.load(); +fence(memory_order_acquire); + +assert(nextBlockFront != nextBlock->tail.load()); +return reinterpret_cast(nextBlock->data + nextBlockFront * sizeof(T)); +} + +return nullptr; +} + +// Removes the front element from the queue, if any, without returning it. +// Returns true on success, or false if the queue appeared empty at the time +// `pop` was called. +bool pop() AE_NO_TSAN +{ +#ifndef NDEBUG +ReentrantGuard guard(this->dequeuing); +#endif +// See try_dequeue() for reasoning + +Block* frontBlock_ = frontBlock.load(); +size_t blockTail = frontBlock_->localTail; +size_t blockFront = frontBlock_->front.load(); + +if (blockFront != blockTail || blockFront != (frontBlock_->localTail = frontBlock_->tail.load())) { +fence(memory_order_acquire); + +non_empty_front_block: +auto element = reinterpret_cast(frontBlock_->data + blockFront * sizeof(T)); +element->~T(); + +blockFront = (blockFront + 1) & frontBlock_->sizeMask; + +fence(memory_order_release); +frontBlock_->front = blockFront; +} +else if (frontBlock_ != tailBlock.load()) { +fence(memory_order_acquire); +frontBlock_ = frontBlock.load(); +blockTail = frontBlock_->localTail = frontBlock_->tail.load(); +blockFront = frontBlock_->front.load(); +fence(memory_order_acquire); + +if (blockFront != blockTail) { +goto non_empty_front_block; +} + +// Front block is empty but there's another block ahead, advance to it +Block* nextBlock = frontBlock_->next; + +size_t nextBlockFront = nextBlock->front.load(); +size_t nextBlockTail = nextBlock->localTail = nextBlock->tail.load(); +fence(memory_order_acquire); + +assert(nextBlockFront != nextBlockTail); +AE_UNUSED(nextBlockTail); + +fence(memory_order_release); +frontBlock = frontBlock_ = nextBlock; + +compiler_fence(memory_order_release); + +auto element = reinterpret_cast(frontBlock_->data + nextBlockFront * sizeof(T)); +element->~T(); + +nextBlockFront = (nextBlockFront + 1) & frontBlock_->sizeMask; + +fence(memory_order_release); +frontBlock_->front = nextBlockFront; +} +else { +// No elements in current block and no other block to advance to +return false; +} + +return true; +} + +// Returns the approximate number of items currently in the queue. +// Safe to call from both the producer and consumer threads. +inline size_t size_approx() const AE_NO_TSAN +{ +size_t result = 0; +Block* frontBlock_ = frontBlock.load(); +Block* block = frontBlock_; +do { +fence(memory_order_acquire); +size_t blockFront = block->front.load(); +size_t blockTail = block->tail.load(); +result += (blockTail - blockFront) & block->sizeMask; +block = block->next.load(); +} while (block != frontBlock_); +return result; +} + +// Returns the total number of items that could be enqueued without incurring +// an allocation when this queue is empty. +// Safe to call from both the producer and consumer threads. +// +// NOTE: The actual capacity during usage may be different depending on the consumer. +// If the consumer is removing elements concurrently, the producer cannot add to +// the block the consumer is removing from until it's completely empty, except in +// the case where the producer was writing to the same block the consumer was +// reading from the whole time. +inline size_t max_capacity() const { + size_t result = 0; + Block* frontBlock_ = frontBlock.load(); + Block* block = frontBlock_; + do { + fence(memory_order_acquire); + result += block->sizeMask; + block = block->next.load(); + } while (block != frontBlock_); + return result; +} + + +private: +enum AllocationMode { CanAlloc, CannotAlloc }; + +#if MOODYCAMEL_HAS_EMPLACE +template +bool inner_enqueue(Args&&... args) AE_NO_TSAN +#else +template + bool inner_enqueue(U&& element) AE_NO_TSAN +#endif +{ +#ifndef NDEBUG +ReentrantGuard guard(this->enqueuing); +#endif + +// High-level pseudocode (assuming we're allowed to alloc a new block): +// If room in tail block, add to tail +// Else check next block +// If next block is not the head block, enqueue on next block +// Else create a new block and enqueue there +// Advance tail to the block we just enqueued to + +Block* tailBlock_ = tailBlock.load(); +size_t blockFront = tailBlock_->localFront; +size_t blockTail = tailBlock_->tail.load(); + +size_t nextBlockTail = (blockTail + 1) & tailBlock_->sizeMask; +if (nextBlockTail != blockFront || nextBlockTail != (tailBlock_->localFront = tailBlock_->front.load())) { +fence(memory_order_acquire); +// This block has room for at least one more element +char* location = tailBlock_->data + blockTail * sizeof(T); +#if MOODYCAMEL_HAS_EMPLACE +new (location) T(std::forward(args)...); +#else +new (location) T(std::forward(element)); +#endif + +fence(memory_order_release); +tailBlock_->tail = nextBlockTail; +} +else { +fence(memory_order_acquire); +if (tailBlock_->next.load() != frontBlock) { +// Note that the reason we can't advance to the frontBlock and start adding new entries there +// is because if we did, then dequeue would stay in that block, eventually reading the new values, +// instead of advancing to the next full block (whose values were enqueued first and so should be +// consumed first). + +fence(memory_order_acquire); // Ensure we get latest writes if we got the latest frontBlock + +// tailBlock is full, but there's a free block ahead, use it +Block* tailBlockNext = tailBlock_->next.load(); +size_t nextBlockFront = tailBlockNext->localFront = tailBlockNext->front.load(); +nextBlockTail = tailBlockNext->tail.load(); +fence(memory_order_acquire); + +// This block must be empty since it's not the head block and we +// go through the blocks in a circle +assert(nextBlockFront == nextBlockTail); +tailBlockNext->localFront = nextBlockFront; + +char* location = tailBlockNext->data + nextBlockTail * sizeof(T); +#if MOODYCAMEL_HAS_EMPLACE +new (location) T(std::forward(args)...); +#else +new (location) T(std::forward(element)); +#endif + +tailBlockNext->tail = (nextBlockTail + 1) & tailBlockNext->sizeMask; + +fence(memory_order_release); +tailBlock = tailBlockNext; +} +else if (canAlloc == CanAlloc) { +// tailBlock is full and there's no free block ahead; create a new block +auto newBlockSize = largestBlockSize >= MAX_BLOCK_SIZE ? largestBlockSize : largestBlockSize * 2; +auto newBlock = make_block(newBlockSize); +if (newBlock == nullptr) { +// Could not allocate a block! +return false; +} +largestBlockSize = newBlockSize; + +#if MOODYCAMEL_HAS_EMPLACE +new (newBlock->data) T(std::forward(args)...); +#else +new (newBlock->data) T(std::forward(element)); +#endif +assert(newBlock->front == 0); +newBlock->tail = newBlock->localTail = 1; + +newBlock->next = tailBlock_->next.load(); +tailBlock_->next = newBlock; + +// Might be possible for the dequeue thread to see the new tailBlock->next +// *without* seeing the new tailBlock value, but this is OK since it can't +// advance to the next block until tailBlock is set anyway (because the only +// case where it could try to read the next is if it's already at the tailBlock, +// and it won't advance past tailBlock in any circumstance). + +fence(memory_order_release); +tailBlock = newBlock; +} +else if (canAlloc == CannotAlloc) { +// Would have had to allocate a new block to enqueue, but not allowed +return false; +} +else { +assert(false && "Should be unreachable code"); +return false; +} +} + +return true; +} + + +// Disable copying +ReaderWriterQueue(ReaderWriterQueue const&) { } + +// Disable assignment +ReaderWriterQueue& operator=(ReaderWriterQueue const&) { } + + +AE_FORCEINLINE static size_t ceilToPow2(size_t x) +{ +// From http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 +--x; +x |= x >> 1; +x |= x >> 2; +x |= x >> 4; +for (size_t i = 1; i < sizeof(size_t); i <<= 1) { +x |= x >> (i << 3); +} +++x; +return x; +} + +template +static AE_FORCEINLINE char* align_for(char* ptr) AE_NO_TSAN +{ +const std::size_t alignment = std::alignment_of::value; +return ptr + (alignment - (reinterpret_cast(ptr) % alignment)) % alignment; +} +private: +#ifndef NDEBUG +struct ReentrantGuard +{ + AE_NO_TSAN ReentrantGuard(weak_atomic& _inSection) + : inSection(_inSection) + { + assert(!inSection && "Concurrent (or re-entrant) enqueue or dequeue operation detected (only one thread at a time may hold the producer or consumer role)"); + inSection = true; + } + + AE_NO_TSAN ~ReentrantGuard() { inSection = false; } + +private: + ReentrantGuard& operator=(ReentrantGuard const&); + +private: + weak_atomic& inSection; +}; +#endif + +struct Block +{ + // Avoid false-sharing by putting highly contended variables on their own cache lines + weak_atomic front; // (Atomic) Elements are read from here + size_t localTail; // An uncontended shadow copy of tail, owned by the consumer + + char cachelineFiller0[MOODYCAMEL_CACHE_LINE_SIZE - sizeof(weak_atomic) - sizeof(size_t)]; + weak_atomic tail; // (Atomic) Elements are enqueued here + size_t localFront; + + char cachelineFiller1[MOODYCAMEL_CACHE_LINE_SIZE - sizeof(weak_atomic) - sizeof(size_t)]; // next isn't very contended, but we don't want it on the same cache line as tail (which is) + weak_atomic next; // (Atomic) + + char* data; // Contents (on heap) are aligned to T's alignment + + const size_t sizeMask; + + + // size must be a power of two (and greater than 0) + AE_NO_TSAN Block(size_t const& _size, char* _rawThis, char* _data) + : front(0UL), localTail(0), tail(0UL), localFront(0), next(nullptr), data(_data), sizeMask(_size - 1), rawThis(_rawThis) + { + } + +private: + // C4512 - Assignment operator could not be generated + Block& operator=(Block const&); + +public: + char* rawThis; +}; + + +static Block* make_block(size_t capacity) AE_NO_TSAN +{ +// Allocate enough memory for the block itself, as well as all the elements it will contain +auto size = sizeof(Block) + std::alignment_of::value - 1; +size += sizeof(T) * capacity + std::alignment_of::value - 1; +auto newBlockRaw = static_cast(std::malloc(size)); +if (newBlockRaw == nullptr) { +return nullptr; +} + +auto newBlockAligned = align_for(newBlockRaw); +auto newBlockData = align_for(newBlockAligned + sizeof(Block)); +return new (newBlockAligned) Block(capacity, newBlockRaw, newBlockData); +} + +private: +weak_atomic frontBlock; // (Atomic) Elements are dequeued from this block + +char cachelineFiller[MOODYCAMEL_CACHE_LINE_SIZE - sizeof(weak_atomic)]; +weak_atomic tailBlock; // (Atomic) Elements are enqueued to this block + +size_t largestBlockSize; + +#ifndef NDEBUG +weak_atomic enqueuing; +mutable weak_atomic dequeuing; +#endif +}; + +// Like ReaderWriterQueue, but also providees blocking operations +template +class BlockingReaderWriterQueue +{ +private: + typedef ::moodycamel::ReaderWriterQueue ReaderWriterQueue; + +public: + explicit BlockingReaderWriterQueue(size_t size = 15) AE_NO_TSAN + : inner(size), sema(new spsc_sema::LightweightSemaphore()) + { } + + BlockingReaderWriterQueue(BlockingReaderWriterQueue&& other) AE_NO_TSAN + : inner(std::move(other.inner)), sema(std::move(other.sema)) + { } + + BlockingReaderWriterQueue& operator=(BlockingReaderWriterQueue&& other) AE_NO_TSAN + { + std::swap(sema, other.sema); + std::swap(inner, other.inner); + return *this; + } + + + // Enqueues a copy of element if there is room in the queue. + // Returns true if the element was enqueued, false otherwise. + // Does not allocate memory. + AE_FORCEINLINE bool try_enqueue(T const& element) AE_NO_TSAN + { + if (inner.try_enqueue(element)) { + sema->signal(); + return true; + } + return false; + } + + // Enqueues a moved copy of element if there is room in the queue. + // Returns true if the element was enqueued, false otherwise. + // Does not allocate memory. + AE_FORCEINLINE bool try_enqueue(T&& element) AE_NO_TSAN + { + if (inner.try_enqueue(std::forward(element))) { + sema->signal(); + return true; + } + return false; + } + +#if MOODYCAMEL_HAS_EMPLACE + // Like try_enqueue() but with emplace semantics (i.e. construct-in-place). + template + AE_FORCEINLINE bool try_emplace(Args&&... args) AE_NO_TSAN + { + if (inner.try_emplace(std::forward(args)...)) { + sema->signal(); + return true; + } + return false; + } +#endif + + + // Enqueues a copy of element on the queue. + // Allocates an additional block of memory if needed. + // Only fails (returns false) if memory allocation fails. + AE_FORCEINLINE bool enqueue(T const& element) AE_NO_TSAN + { + if (inner.enqueue(element)) { + sema->signal(); + return true; + } + return false; + } + + // Enqueues a moved copy of element on the queue. + // Allocates an additional block of memory if needed. + // Only fails (returns false) if memory allocation fails. + AE_FORCEINLINE bool enqueue(T&& element) AE_NO_TSAN + { + if (inner.enqueue(std::forward(element))) { + sema->signal(); + return true; + } + return false; + } + +#if MOODYCAMEL_HAS_EMPLACE + // Like enqueue() but with emplace semantics (i.e. construct-in-place). + template + AE_FORCEINLINE bool emplace(Args&&... args) AE_NO_TSAN + { + if (inner.emplace(std::forward(args)...)) { + sema->signal(); + return true; + } + return false; + } +#endif + + + // Attempts to dequeue an element; if the queue is empty, + // returns false instead. If the queue has at least one element, + // moves front to result using operator=, then returns true. + template + bool try_dequeue(U& result) AE_NO_TSAN + { + if (sema->tryWait()) { + bool success = inner.try_dequeue(result); + assert(success); + AE_UNUSED(success); + return true; + } + return false; + } + + + // Attempts to dequeue an element; if the queue is empty, + // waits until an element is available, then dequeues it. + template + void wait_dequeue(U& result) AE_NO_TSAN + { + while (!sema->wait()); + bool success = inner.try_dequeue(result); + AE_UNUSED(result); + assert(success); + AE_UNUSED(success); + } + + + // Attempts to dequeue an element; if the queue is empty, + // waits until an element is available up to the specified timeout, + // then dequeues it and returns true, or returns false if the timeout + // expires before an element can be dequeued. + // Using a negative timeout indicates an indefinite timeout, + // and is thus functionally equivalent to calling wait_dequeue. + template + bool wait_dequeue_timed(U& result, std::int64_t timeout_usecs) AE_NO_TSAN + { + if (!sema->wait(timeout_usecs)) { + return false; + } + bool success = inner.try_dequeue(result); + AE_UNUSED(result); + assert(success); + AE_UNUSED(success); + return true; + } + + +#if __cplusplus > 199711L || _MSC_VER >= 1700 + // Attempts to dequeue an element; if the queue is empty, + // waits until an element is available up to the specified timeout, + // then dequeues it and returns true, or returns false if the timeout + // expires before an element can be dequeued. + // Using a negative timeout indicates an indefinite timeout, + // and is thus functionally equivalent to calling wait_dequeue. + template + inline bool wait_dequeue_timed(U& result, std::chrono::duration const& timeout) AE_NO_TSAN + { + return wait_dequeue_timed(result, std::chrono::duration_cast(timeout).count()); + } +#endif + + + // Returns a pointer to the front element in the queue (the one that + // would be removed next by a call to `try_dequeue` or `pop`). If the + // queue appears empty at the time the method is called, nullptr is + // returned instead. + // Must be called only from the consumer thread. + AE_FORCEINLINE T* peek() const AE_NO_TSAN + { + return inner.peek(); + } + + // Removes the front element from the queue, if any, without returning it. + // Returns true on success, or false if the queue appeared empty at the time + // `pop` was called. + AE_FORCEINLINE bool pop() AE_NO_TSAN + { + if (sema->tryWait()) { + bool result = inner.pop(); + assert(result); + AE_UNUSED(result); + return true; + } + return false; + } + + // Returns the approximate number of items currently in the queue. + // Safe to call from both the producer and consumer threads. + AE_FORCEINLINE size_t size_approx() const AE_NO_TSAN + { + return sema->availableApprox(); + } + + // Returns the total number of items that could be enqueued without incurring + // an allocation when this queue is empty. + // Safe to call from both the producer and consumer threads. + // + // NOTE: The actual capacity during usage may be different depending on the consumer. + // If the consumer is removing elements concurrently, the producer cannot add to + // the block the consumer is removing from until it's completely empty, except in + // the case where the producer was writing to the same block the consumer was + // reading from the whole time. + AE_FORCEINLINE size_t max_capacity() const { + return inner.max_capacity(); + } + +private: + // Disable copying & assignment + BlockingReaderWriterQueue(BlockingReaderWriterQueue const&) { } + BlockingReaderWriterQueue& operator=(BlockingReaderWriterQueue const&) { } + +private: + ReaderWriterQueue inner; + std::unique_ptr sema; +}; + +} // end namespace moodycamel + +#ifdef AE_VCPP +#pragma warning(pop) +#endif \ No newline at end of file diff --git a/src/cpp/src/clustering.cpp b/src/cpp/src/clustering.cpp index ebbc88e0..4b947166 100644 --- a/src/cpp/src/clustering.cpp +++ b/src/cpp/src/clustering.cpp @@ -9,18 +9,146 @@ #include "faiss/Clustering.h" #include "index_partition.h" #include +#include +#include + +#ifdef QUAKE_ENABLE_GPU +#include +#include // RAFT resources (handle) +#include // RAFT device view (make_device_matrix_view, etc.) +#include // cuVS k-means API + +shared_ptr kmeans_cuvs_sample_and_predict( + Tensor vectors, Tensor ids, + shared_ptr build_params) +{ + /* ---------- unpack / sanity-check parameters ----------------------- */ + const int num_clusters = build_params->nlist; + const int niter = build_params->niter; + int gpu_batch_size = build_params->gpu_batch_size; + int gpu_sample_sz = build_params->gpu_sample_size; + const MetricType metric = str_to_metric_type(build_params->metric); + + TORCH_CHECK(vectors.dim() == 2, "vectors must be [N,D]"); + TORCH_CHECK(ids.dim() == 1, "ids must be [N]"); + + const int64_t N = vectors.size(0); + const int64_t D = vectors.size(1); + + gpu_sample_sz = std::min(gpu_sample_sz, (int)N); + gpu_batch_size = std::min(gpu_batch_size, (int)N); + TORCH_CHECK(gpu_sample_sz > 0 && gpu_sample_sz <= N, "invalid sample size"); + + /* ---------- copy to pinned host & (optionally) normalize ----------- */ + Tensor cpu_pts = vectors.contiguous().pin_memory(); + if (metric == faiss::METRIC_INNER_PRODUCT) { + // cuVS k-means is Euclidean; approximate cosine by L2 on the unit sphere + cpu_pts = cpu_pts.div(cpu_pts.norm(2, 1, /*keepdim=*/true)); + } -shared_ptr kmeans(Tensor vectors, + /* ---------- draw random sample for training ------------------------ */ + const Tensor samp_idx = torch::randperm(N, torch::kLong).slice(0, 0, gpu_sample_sz); + const Tensor samp_host = cpu_pts.index_select(0, samp_idx); + const Tensor samp_gpu = samp_host.to(torch::kCUDA, /*non_blocking=*/true).contiguous(); + + /* ---------- RAFT handle & cuVS parameters -------------------------- */ + raft::resources handle; + cudaStream_t stream = c10::cuda::getCurrentCUDAStream(); + raft::resource::set_cuda_stream(handle, stream); + + cuvs::cluster::kmeans::params params; + params.n_clusters = num_clusters; + params.init = cuvs::cluster::kmeans::params::InitMethod::Random; + params.max_iter = niter; + + /* ---------- centroids on device ------------------------------------ */ + Tensor cent_gpu = torch::empty({num_clusters, D}, + torch::dtype(torch::kFloat32).device(torch::kCUDA)) + .contiguous(); + + /* ---------- fit on the sample -------------------------------------- */ + { + float inertia = 0.0f; + int actual_iter = 0; + cuvs::cluster::kmeans::fit( + handle, params, + raft::make_device_matrix_view(samp_gpu.data_ptr(), + gpu_sample_sz, (int)D), + std::nullopt, + raft::make_device_matrix_view(cent_gpu.data_ptr(), + num_clusters, (int)D), + raft::make_host_scalar_view(&inertia), + raft::make_host_scalar_view(&actual_iter) + ); + } + + /* ---------- predict every point, in order, exactly once ------------ */ + Tensor all_labels = torch::empty({N}, torch::kLong); // on CPU + + auto run_predict = [&](Tensor batch_host, int64_t dst_off) + { + const int64_t bs = batch_host.size(0); + + Tensor batch_gpu = batch_host.to(torch::kCUDA, /*non_blocking=*/true) + .contiguous(); + Tensor lbl_gpu32 = torch::empty({bs}, + torch::dtype(torch::kInt32).device(torch::kCUDA)); + + float dummy_inertia = 0.0f; // storage required by the API + cuvs::cluster::kmeans::predict( + handle, params, + raft::make_device_matrix_view(batch_gpu.data_ptr(), + bs, (int)D), + std::nullopt, + raft::make_device_matrix_view(cent_gpu.data_ptr(), + num_clusters, (int)D), + raft::make_device_vector_view(lbl_gpu32.data_ptr(), bs), + /*verbose=*/false, + raft::make_host_scalar_view(&dummy_inertia) + ); + + all_labels.narrow(0, dst_off, bs) + .copy_(lbl_gpu32.to(torch::kLong).cpu(), /*non_blocking=*/false); + }; + + for (int64_t off = 0; off < N; off += gpu_batch_size) { + const int64_t bs = std::min(gpu_batch_size, N - off); + run_predict(cpu_pts.slice(0, off, off + bs), off); + } + + /* ---------- group vectors/ids by cluster on CPU -------------------- */ + Tensor lbl_sorted, idx_sorted; + std::tie(lbl_sorted, idx_sorted) = torch::sort(all_labels); + Tensor vecs_sorted = vectors.index_select(0, idx_sorted); + Tensor ids_sorted = ids.index_select(0, idx_sorted); + + Tensor counts = torch::bincount(lbl_sorted, /*weights=*/{}, num_clusters); + std::vector split_sz(counts.data_ptr(), + counts.data_ptr() + num_clusters); + + std::vector cluster_vecs = torch::split(vecs_sorted, split_sz, 0); + std::vector cluster_ids = torch::split(ids_sorted, split_sz, 0); + + /* ---------- package result ---------------------------------------- */ + auto out = std::make_shared(); + out->centroids = cent_gpu.cpu().contiguous(); + out->partition_ids = torch::arange(num_clusters, torch::kLong); + out->vectors = std::move(cluster_vecs); + out->vector_ids = std::move(cluster_ids); + return out; +} +#endif + +shared_ptr kmeans_cpu(Tensor vectors, Tensor ids, - int n_clusters, - MetricType metric_type, - int niter, - bool use_gpu /*=false*/, + shared_ptr build_params, Tensor /* initial_centroids */) { // Ensure enough vectors are available and sizes match. - assert(vectors.size(0) >= n_clusters * 2); + assert(vectors.size(0) >= build_params->nlist * 2); assert(vectors.size(0) == ids.size(0)); + MetricType metric_type = str_to_metric_type(build_params->metric); + // Normalize vectors for inner product if (metric_type == faiss::METRIC_INNER_PRODUCT) vectors = vectors / vectors.norm(2, 1).unsqueeze(1); @@ -29,33 +157,20 @@ shared_ptr kmeans(Tensor vectors, int d = vectors.size(1); faiss::Index* index_ptr = nullptr; - - if (use_gpu) { - // Check if GPU resources are available. - #ifdef FAISS_ENABLE_GPU - faiss::gpu::StandardGpuResources gpu_res; - if (metric_type == faiss::METRIC_INNER_PRODUCT) - index_ptr = new faiss::gpu::GpuIndexFlatIP(&gpu_res, d); - else - index_ptr = new faiss::gpu::GpuIndexFlatL2(&gpu_res, d); - #else - throw std::runtime_error("GPU resources are not available. Please compile with FAISS_ENABLE_GPU."); - #endif - } else { - if (metric_type == faiss::METRIC_INNER_PRODUCT) - index_ptr = new faiss::IndexFlatIP(d); - else - index_ptr = new faiss::IndexFlatL2(d); - } + if (metric_type == faiss::METRIC_INNER_PRODUCT) + index_ptr = new faiss::IndexFlatIP(d); + else + index_ptr = new faiss::IndexFlatL2(d); faiss::ClusteringParameters cp; - cp.niter = niter; + cp.niter = build_params->niter; + cp.spherical = (metric_type == faiss::METRIC_INNER_PRODUCT); - faiss::Clustering clus(d, n_clusters, cp); + faiss::Clustering clus(d, build_params->nlist, cp); clus.train(n, vectors.data_ptr(), *index_ptr); // Retrieve centroids as a torch Tensor. - Tensor centroids = torch::from_blob(clus.centroids.data(), {n_clusters, d}, torch::kFloat32).clone(); + Tensor centroids = torch::from_blob(clus.centroids.data(), {build_params->nlist, d}, torch::kFloat32).clone(); if (metric_type == faiss::METRIC_INNER_PRODUCT) centroids = centroids / centroids.norm(2, 1).unsqueeze(1); @@ -72,7 +187,7 @@ shared_ptr kmeans(Tensor vectors, Tensor sorted_ids = ids.index_select(0, sorted_indices); // Compute counts per cluster using bincount. - Tensor counts_tensor = torch::bincount(sorted_assignments, /*weights=*/{}, n_clusters); + Tensor counts_tensor = torch::bincount(sorted_assignments, /*weights=*/{}, build_params->nlist); // Ensure counts are on CPU to extract split sizes. counts_tensor = counts_tensor.to(torch::kCPU); // Convert counts tensor to std::vector @@ -83,7 +198,7 @@ shared_ptr kmeans(Tensor vectors, vector cluster_vectors = torch::split(sorted_vectors, counts_vector, 0); vector cluster_ids = torch::split(sorted_ids, counts_vector, 0); - Tensor partition_ids = torch::arange(n_clusters, torch::kInt64); + Tensor partition_ids = torch::arange(build_params->nlist, torch::kInt64); shared_ptr clustering = std::make_shared(); clustering->centroids = centroids; @@ -96,16 +211,45 @@ shared_ptr kmeans(Tensor vectors, return clustering; } +shared_ptr kmeans(Tensor vectors, + Tensor ids, + shared_ptr build_params, + Tensor /* initial_centroids */) { + if (build_params->use_gpu) { + #ifdef QUAKE_ENABLE_GPU + return kmeans_cuvs_sample_and_predict( + vectors, + ids, + build_params); + #else + throw std::runtime_error("GPU support is not enabled. Please compile with QUAKE_ENABLE_GPU."); + #endif + } else { + return kmeans_cpu(vectors, ids, build_params); + } +} + + tuple >> kmeans_refine_partitions( Tensor centroids, - vector> partitions, + vector> &partitions, MetricType metric, - int refinement_iterations) { + int refinement_iterations, + int num_threads) { + + size_t max_nq = 0; + for (auto &p : partitions) { + max_nq = std::max(max_nq, (size_t)p->num_vectors_); + } + max_nq = max_nq * 10; // Ensure we allocate enough capacity // Determine number of clusters and dimension. int n_clusters = centroids.size(0); int d = centroids.size(1); + vector > buffers = create_buffers(max_nq, 1, (metric == faiss::METRIC_INNER_PRODUCT), n_clusters); + + // Run for the desired number of iterations (if refinement_iterations==0, do one pass). int iterations = (refinement_iterations > 0) ? refinement_iterations : 1; @@ -113,14 +257,23 @@ tuple >> kmeans_refine_partitions( Tensor centroid_counts = torch::zeros({n_clusters}, torch::kInt64); auto centroid_sums_accessor = centroid_sums.accessor(); auto centroid_counts_accessor = centroid_counts.accessor(); + Tensor centroid_ids = torch::arange(n_clusters, torch::kInt64); + auto centroid_ids_ptr = centroid_ids.data_ptr(); vector> prev_partitions = partitions; vector> new_partitions; for (int iter = 0; iter < iterations; iter++) { - if (iter > 0) { centroids = centroid_sums / centroid_counts.unsqueeze(1).to(torch::kFloat32); + + // normalize centroids if using inner product metric + if (metric == faiss::METRIC_INNER_PRODUCT) { + centroids = centroids + / centroids.norm(2,1) + .unsqueeze(1) + .to(torch::kFloat32); + } } // Reset accumulators. @@ -133,11 +286,11 @@ tuple >> kmeans_refine_partitions( new_partitions[i] = make_shared(); new_partitions[i]->set_code_size(partitions[0]->code_size_); new_partitions[i]->resize(10); + new_partitions[i]->set_core_id(partitions[i]->core_id_); } float *centroids_ptr = centroids.data_ptr(); - // Process each existing partition. for (auto &part: partitions) { int64_t nvec = part->num_vectors_; if (nvec <= 0) continue; @@ -145,18 +298,21 @@ tuple >> kmeans_refine_partitions( float *part_vecs = (float *) part->codes_; int64_t *part_vec_ids = part->ids_; - // Create batched TopK buffers (k=1 for nearest centroid). - vector > buffers = create_buffers(nvec, 1, false); - // Use batched_scan_list to get nearest centroid for each vector. batched_scan_list(part_vecs, centroids_ptr, - nullptr, + centroid_ids_ptr, nvec, n_clusters, d, buffers, - metric); + metric, + nullptr, + nullptr, + nullptr, + 128, + BLAS_DB_BS, + {}); // For each vector in this partition, determine its assignment. for (int i = 0; i < nvec; i++) { @@ -173,10 +329,13 @@ tuple >> kmeans_refine_partitions( centroid_counts_accessor[assigned_cluster]++; new_partitions[assigned_cluster]->append(1, vec_id, (uint8_t *) vec_ptr); + + // reset the buffer for this slot + buffers[i]->reset(); } } // end for each partition + std::move(new_partitions.begin(), new_partitions.end(), partitions.begin()); } // end iterations - return std::make_tuple(centroids, partitions); } diff --git a/src/cpp/src/dynamic_inverted_list.cpp b/src/cpp/src/dynamic_inverted_list.cpp index 0ea36450..9101d557 100644 --- a/src/cpp/src/dynamic_inverted_list.cpp +++ b/src/cpp/src/dynamic_inverted_list.cpp @@ -40,7 +40,8 @@ namespace faiss { DynamicInvertedLists::DynamicInvertedLists(size_t nlist, size_t code_size) - : InvertedLists(nlist, code_size) { + : nlist(nlist), code_size(code_size) { + d_ = code_size / sizeof(float); code_size_ = code_size; // Initialize empty partitions @@ -48,6 +49,7 @@ namespace faiss { // IndexPartition ip; shared_ptr ip = std::make_shared(); ip->set_code_size(code_size); + ip->partition_id_ = static_cast(i); partitions_[i] = ip; } curr_list_id_ = nlist; @@ -68,7 +70,8 @@ namespace faiss { size_t DynamicInvertedLists::list_size(size_t list_no) const { auto it = partitions_.find(list_no); if (it == partitions_.end()) { - throw std::runtime_error("List does not exist in list_size"); + string err_message = "List " + std::to_string(list_no) + " does not exist in list_size"; + throw std::runtime_error(err_message); } return static_cast(it->second->num_vectors_); } @@ -76,7 +79,8 @@ namespace faiss { const uint8_t *DynamicInvertedLists::get_codes(size_t list_no) const { auto it = partitions_.find(list_no); if (it == partitions_.end()) { - throw std::runtime_error("List does not exist in get_codes"); + string err_message = "List " + std::to_string(list_no) + " does not exist in get_codes"; + throw std::runtime_error(err_message); } return it->second->codes_; } @@ -84,7 +88,8 @@ namespace faiss { const idx_t *DynamicInvertedLists::get_ids(size_t list_no) const { auto it = partitions_.find(list_no); if (it == partitions_.end()) { - throw std::runtime_error("List does not exist in get_ids"); + string err_message = "List " + std::to_string(list_no) + " does not exist in get_ids"; + throw std::runtime_error(err_message); } return it->second->ids_; } @@ -100,22 +105,30 @@ namespace faiss { void DynamicInvertedLists::remove_entry(size_t list_no, idx_t id) { auto it = partitions_.find(list_no); if (it == partitions_.end()) { - throw std::runtime_error("List does not exist in remove_entry"); + string err_message = "List " + std::to_string(list_no) + " does not exist in remove_entry"; + throw std::runtime_error(err_message); } - shared_ptr part = it->second; + auto& part = it->second; if (part->num_vectors_ == 0) return; - int64_t idx_to_remove = part->find_id(id); - if (idx_to_remove != -1) { - part->remove(idx_to_remove); + int64_t pos = part->find_id(id); + if (pos == -1) return; + + int64_t swapped = part->remove(pos); + map_erase(id); + + if (swapped != -1) { // someone moved into `pos` + idx_t moved_id = part->ids_[pos]; + map_swap(part.get(), pos, moved_id); } } void DynamicInvertedLists::remove_entries_from_partition(size_t list_no, vector vectors_to_remove) { auto it = partitions_.find(list_no); if (it == partitions_.end()) { - throw std::runtime_error("List does not exist in remove_entries_from_partition"); + string err_message = "List " + std::to_string(list_no) + " does not exist in remove_entries_from_partition"; + throw std::runtime_error(err_message); } shared_ptr part = it->second; @@ -125,50 +138,94 @@ namespace faiss { // We'll perform removals by scanning and removing matches. // Because remove() swaps last element in, we must be careful with iteration. for (int64_t i = 0; i < part->num_vectors_;) { - if (vectors_to_remove_set.find(part->ids_[i]) != vectors_to_remove_set.end()) { - part->remove(i); - // don't increment i, because we just swapped a new element into i + if (vectors_to_remove_set.count(part->ids_[i])) { + idx_t victim = part->ids_[i]; + int64_t swapped = part->remove(i); + map_erase(victim); + if (swapped != -1) + map_swap(part.get(), i, part->ids_[i]); } else { i++; } } } - void DynamicInvertedLists::remove_vectors(std::set vectors_to_remove) { - // Remove from all partitions - for (auto &kv: partitions_) { - shared_ptr part = kv.second; - for (int64_t i = 0; i < part->num_vectors_;) { - if (vectors_to_remove.find(part->ids_[i]) != vectors_to_remove.end()) { - part->remove(i); - } else { - i++; - } + void DynamicInvertedLists::remove_vectors(int64_t* vectors_to_remove, size_t num_vectors, bool update_delta) { + // Create a map of all the ids to remove for each partition + std::unordered_map> ids_to_delete; + + #pragma unroll + for(int i = 0; i < num_vectors; i++) { + idx_t vector_id = static_cast(vectors_to_remove[i]); + auto vector_details = id_to_location_[vector_id]; + assert(vector_details.second >= 0 && vector_details.second < vector_details.first->num_vectors_); + ids_to_delete[vector_details.first].push_back(vector_id); + } + + // Now perform the deletes for each partition + for(auto& curr_partition_deletes : ids_to_delete) { + IndexPartition* partition = curr_partition_deletes.first; + for(int64_t id_to_remove : curr_partition_deletes.second) { + // First lookup the offset of the id to remove + // The reason we access it here rather than storing it in the map because + // ids position previous deletes may have changed this + int64_t delete_idx = id_to_location_[id_to_remove].second; + + // Now actually delete the item at that offset + map_erase(id_to_remove); + int64_t swapped = partition->remove(delete_idx, update_delta); + + // Update the position of the vector we swapped into the deleted vectors position + if (swapped != -1) map_swap(partition, delete_idx, partition->ids_[delete_idx]); + } + + // Finally check if the index should be resized after these operations + partition->check_buffer_size(); + } + } + + void DynamicInvertedLists::build_map() { + id_to_location_.clear(); + for (auto& kv : partitions_) { + IndexPartition* part = kv.second.get(); + for (int64_t i = 0; i < part->num_vectors_; i++) { + id_to_location_[part->ids_[i]] = {part, i}; } } } + shared_ptr DynamicInvertedLists::get_partition(size_t list_no) { + if(partitions_.find(list_no) == partitions_.end()) { + string err_message = "List " + std::to_string(list_no) + " does not exist in add_entries"; + throw std::runtime_error(err_message); + } + return partitions_[list_no]; + } + size_t DynamicInvertedLists::add_entries( size_t list_no, size_t n_entry, const idx_t *ids, - const uint8_t *codes) { - if (n_entry == 0) { - return 0; - } + const uint8_t *codes, + bool update_delta) { + if (n_entry == 0) return 0; auto it = partitions_.find(list_no); if (it == partitions_.end()) { - throw std::runtime_error("List does not exist in add_entries"); + string err_message = "List " + std::to_string(list_no) + " does not exist in add_entries"; + throw std::runtime_error(err_message); } - shared_ptr part = it->second; - // Ensure code_size is set - if (part->code_size_ != static_cast(code_size)) { + auto& part = it->second; + if (part->code_size_ != static_cast(code_size)) part->set_code_size(static_cast(code_size)); - } - part->append((int64_t) n_entry, ids, codes); + const int64_t base = part->num_vectors_; // size *before* append + part->append(static_cast(n_entry), ids, codes, update_delta); + + for (size_t i = 0; i < n_entry; ++i) + map_add(part.get(), base + i, ids[i]); + return n_entry; } @@ -180,99 +237,102 @@ namespace faiss { const uint8_t *codes) { auto it = partitions_.find(list_no); if (it == partitions_.end()) { - throw std::runtime_error("List does not exist in update_entries"); + string err_message = "List " + std::to_string(list_no) + " does not exist in update_entries"; + throw std::runtime_error(err_message); } shared_ptr part = it->second; part->update((int64_t) offset, (int64_t) n_entry, ids, codes); } - void DynamicInvertedLists::batch_update_entries( - size_t old_vector_partition, - int64_t *new_vector_partitions, - uint8_t *new_vectors, - int64_t *new_vector_ids, - int num_vectors) { - // This logic will: - // 1. Remove all vectors from old_vector_partition that moved to a new partition - // 2. Append them to their new partitions - - // Identify which vectors belong to old_vector_partition and distribute them - // to new partitions. - std::unordered_map > vectors_for_new_partition; - - for (int i = 0; i < num_vectors; i++) { - size_t new_p = static_cast(new_vector_partitions[i]); - if (new_p != old_vector_partition) { - vectors_for_new_partition[new_p].push_back(i); +void DynamicInvertedLists::batch_update_entries( + size_t old_partition, + int64_t* new_partitions, + uint8_t* new_vectors, + int64_t* new_ids, + int num) +{ + /* 1. gather indices to move grouped by their *destination* */ + std::unordered_map> to_move; + for (int i = 0; i < num; ++i) { + size_t dst = static_cast(new_partitions[i]); + if (dst != old_partition) + to_move[dst].push_back(i); + } + + /* 2. FIRST remove them from the old partition + (updates map_erase + map_swap) */ + auto old_it = partitions_.find(old_partition); + if (old_it != partitions_.end()) { + auto& old_part = old_it->second; + for (auto& kv : to_move) + for (int idx : kv.second) { + idx_t victim = static_cast(new_ids[idx]); + int64_t pos = old_part->find_id(victim); + if (pos != -1) { + int64_t sw = old_part->remove(pos); + map_erase(victim); + if (sw != -1) + map_swap(old_part.get(), pos, old_part->ids_[pos]); + } } - } + } - // Append entries to new partitions - for (auto &kv: vectors_for_new_partition) { - size_t new_p = kv.first; - auto it = partitions_.find(new_p); - if (it == partitions_.end()) { - // Create a new partition if needed - add_list(new_p); - it = partitions_.find(new_p); - } - shared_ptr new_part = it->second; - if (new_part->code_size_ != static_cast(code_size)) { - new_part->set_code_size((int64_t) code_size); - } + /* 3. THEN append them to their new partitions + (updates map_add) */ + for (auto& kv : to_move) { + size_t dst = kv.first; + auto& idxs = kv.second; - // Gather all IDs and codes to append at once - std::vector tmp_ids; - tmp_ids.reserve(kv.second.size()); - std::vector tmp_codes; - tmp_codes.reserve(kv.second.size() * code_size); - - for (int idx: kv.second) { - tmp_ids.push_back((idx_t) new_vector_ids[idx]); - tmp_codes.insert(tmp_codes.end(), - new_vectors + idx * code_size, - new_vectors + (idx + 1) * code_size); - } + auto it = partitions_.find(dst); + if (it == partitions_.end()) { + add_list(dst); + it = partitions_.find(dst); + } + auto& new_part = it->second; + if (new_part->code_size_ != static_cast(code_size)) + new_part->set_code_size((int64_t)code_size); - new_part->append((int64_t) kv.second.size(), tmp_ids.data(), tmp_codes.data()); - } - - // If needed, remove them from old_vector_partition - auto old_it = partitions_.find(old_vector_partition); - if (old_it != partitions_.end()) { - shared_ptr old_part = old_it->second; - // remove vectors that moved - for (auto &kv: vectors_for_new_partition) { - for (int idx: kv.second) { - idx_t old_id = (idx_t) new_vector_ids[idx]; - int64_t pos = old_part->find_id(old_id); - if (pos != -1) { - old_part->remove(pos); - } - } - } + const int64_t base = new_part->num_vectors_; + std::vector ids_buf; ids_buf.reserve(idxs.size()); + std::vector codes_buf; codes_buf.reserve(idxs.size() * code_size); + + for (int j : idxs) { + ids_buf .push_back(static_cast(new_ids[j])); + codes_buf.insert(codes_buf.end(), + new_vectors + j * code_size, + new_vectors + (j + 1) * code_size); } + new_part->append((int64_t)idxs.size(), ids_buf.data(), codes_buf.data()); + + for (size_t k = 0; k < idxs.size(); ++k) + map_add(new_part.get(), base + k, ids_buf[k]); } +} void DynamicInvertedLists::remove_list(size_t list_no) { auto it = partitions_.find(list_no); if (it == partitions_.end()) { - // Already doesn't exist return; } + for (int64_t i = 0; i < it->second->num_vectors_; i++) { + map_erase(it->second->ids_[i]); + } partitions_.erase(it); nlist--; } void DynamicInvertedLists::add_list(size_t list_no) { if (partitions_.find(list_no) != partitions_.end()) { - throw std::runtime_error("List already exists in add_list"); + string err_message = "List " + std::to_string(list_no) + " already exists in add_list"; + throw std::runtime_error(err_message); } shared_ptr ip = std::make_shared(); ip->set_code_size((int64_t) code_size); + ip->partition_id_ = static_cast(list_no); partitions_[list_no] = ip; + ip->reset_delta(); nlist++; } @@ -285,47 +345,87 @@ namespace faiss { return part->find_id(id) != -1; } - bool DynamicInvertedLists::get_vector_for_id(idx_t id, float *vector_values) { - for (auto &kv: partitions_) { - shared_ptr part = kv.second; - int64_t pos = part->find_id(id); - if (pos != -1) { - // Found it, copy vector - // code_size_ is in bytes. Assuming float vectors of dimension (code_size_/sizeof(float)) - std::memcpy(vector_values, part->codes_ + pos * part->code_size_, part->code_size_); - return true; - } + bool DynamicInvertedLists::get_vector_for_id(idx_t id, float *out) { + if (id_to_location_.empty()) build_map(); + + if (id < 0) { + std::memset(out, 0, code_size_); + return true; } - return false; + + auto it = id_to_location_.find(id); + if (it == id_to_location_.end()) return false; + + IndexPartition* part = it->second.first; + int64_t pos = it->second.second; + std::memcpy(out, part->codes_ + pos * part->code_size_, part->code_size_); + return true; } - vector DynamicInvertedLists::get_vectors_by_id(vector ids) { + void DynamicInvertedLists::write_vector_by_id(idx_t id, float* vector_values) { + // Get the partition information + if (id_to_location_.empty()) build_map(); + auto it = id_to_location_.find(id); + if (it == id_to_location_.end()) { + string err_message = "Vector " + std::to_string(id) + " not in id map"; + throw std::runtime_error(err_message); + } + + // Copy the values from the input buffer to the index copy + IndexPartition* part = it->second.first; + int64_t pos = it->second.second; + std::memcpy(part->codes_ + pos * part->code_size_, reinterpret_cast(vector_values), part->code_size_); + } + + vector DynamicInvertedLists::get_vectors_by_id(vector ids) + { + if (id_to_location_.empty()) build_map(); + + vector ret; ret.reserve(ids.size()); - vector ret; for (int64_t id : ids) { - bool found = false; - for (auto &kv: partitions_) { - shared_ptr part = kv.second; - int64_t pos = part->find_id(id); - if (pos != -1) { - ret.push_back(reinterpret_cast(part->codes_ + pos * part->code_size_)); - found = true; - break; - } - } - if (!found) { - throw std::runtime_error("ID not found in any partition"); + if (id < 0) { + ret.push_back(nullptr); + continue; } + + auto it = id_to_location_.find(id); + if (it == id_to_location_.end()) + throw std::runtime_error("ID not found in any partition: " + std::to_string(id)); + + IndexPartition* part = it->second.first; + int64_t pos = it->second.second; + ret.push_back(reinterpret_cast(part->codes_ + pos * part->code_size_)); } return ret; } + // vector ret; + // for (int64_t id : ids) { + // bool found = false; + // for (auto &kv: partitions_) { + // shared_ptr part = kv.second; + // int64_t pos = part->find_id(id); + // if (pos != -1) { + // ret.push_back(reinterpret_cast(part->codes_ + pos * part->code_size_)); + // found = true; + // break; + // } + // } + // if (!found) { + // throw std::runtime_error("ID not found in any partition"); + // } + // } + // return ret; + // } + size_t DynamicInvertedLists::get_new_list_id() { return curr_list_id_++; } void DynamicInvertedLists::reset() { partitions_.clear(); + id_to_location_.clear(); nlist = 0; curr_list_id_ = 0; } @@ -517,6 +617,8 @@ namespace faiss { curr_list_id_ = max_list_id + 1; ifs.close(); + + build_map(); } Tensor DynamicInvertedLists::get_partition_ids() { diff --git a/src/cpp/src/index_partition.cpp b/src/cpp/src/index_partition.cpp index 8604a939..fd1358cf 100644 --- a/src/cpp/src/index_partition.cpp +++ b/src/cpp/src/index_partition.cpp @@ -6,6 +6,10 @@ #include +// Initialize the static defaults +float IndexPartition::delete_resize_threshold_ = 0.8; +float IndexPartition::capacity_resize_threshold_ = 1.1; + IndexPartition::IndexPartition(int64_t num_vectors, uint8_t* codes, idx_t* ids, @@ -17,6 +21,11 @@ IndexPartition::IndexPartition(int64_t num_vectors, ids_ = nullptr; numa_node_ = -1; core_id_ = -1; + last_snapshot_size_ = 0; + delta_count_ = 0; + + reset_delta(); + ensure_capacity(num_vectors); append(num_vectors, ids, codes); } @@ -39,6 +48,20 @@ IndexPartition::~IndexPartition() { clear(); } +void IndexPartition::allocate_delta_buffer() { + if(delta_vec_ == nullptr) { + delta_vec_ = allocate_memory(code_size_, numa_node_); + } +} + +void IndexPartition::reset_delta() { + allocate_delta_buffer(); + + last_snapshot_size_ = num_vectors_; + delta_count_ = 0; + std::memset(delta_vec_, 0, code_size_ * sizeof(uint8_t)); +} + void IndexPartition::set_code_size(int64_t code_size) { if (code_size <= 0) { throw std::runtime_error("Invalid code_size"); @@ -49,8 +72,27 @@ void IndexPartition::set_code_size(int64_t code_size) { code_size_ = code_size; } -void IndexPartition::append(int64_t n_entry, const idx_t* new_ids, const uint8_t* new_codes) { +void IndexPartition::append(int64_t n_entry, const idx_t* new_ids, const uint8_t* new_codes, bool update_delta) { if (n_entry <= 0) return; + + // Record the delta of the new vectors + if(update_delta) { + const int dimension = code_size_ / sizeof(float); + float* delta_values = reinterpret_cast(delta_vec_); + const float* new_values = reinterpret_cast(new_codes); + + for (int64_t i = 0; i < n_entry; i++) { + const float* inserted_vector = new_values + (i * dimension); + + #pragma unroll + for (int j = 0; j < dimension; j++) { + delta_values[j] += inserted_vector[j]; + } + } + delta_count_ += n_entry; + } + + // Now add them to the index ensure_capacity(num_vectors_ + n_entry); const size_t code_bytes = static_cast(code_size_); std::memcpy(codes_ + num_vectors_ * code_bytes, new_codes, n_entry * code_bytes); @@ -76,29 +118,37 @@ void IndexPartition::update(int64_t offset, int64_t n_entry, const idx_t* new_id std::memcpy(ids_ + offset, new_ids, n_entry * sizeof(idx_t)); } -void IndexPartition::remove(int64_t index) { - if (index < 0 || index >= num_vectors_) { +int64_t IndexPartition::remove(int64_t idx, bool update_delta) +{ + if (idx < 0 || idx >= num_vectors_) { throw std::runtime_error("Index out of range in remove"); } - if (index == num_vectors_ - 1) { - num_vectors_--; - return; - } - - int64_t last_idx = num_vectors_ - 1; - const size_t code_bytes = static_cast(code_size_); - // // Update id_to_index_ - // idx_t last_id = ids_[last_idx]; - // idx_t removed_id = ids_[index]; - // - // id_to_index_[last_id] = index; - // id_to_index_.erase(removed_id); + // Update the delta to not include the removed vector + if(update_delta) { + const int dimension = code_size_ / sizeof(float); + float* delta_values = reinterpret_cast(delta_vec_); + float* vector_to_delete = reinterpret_cast(codes_) + (idx * dimension); - std::memcpy(codes_ + index * code_bytes, codes_ + last_idx * code_bytes, code_bytes); - ids_[index] = ids_[last_idx]; + #pragma unroll + for (int j = 0; j < dimension; j++) { + delta_values[j] -= vector_to_delete[j]; + } + delta_count_ -= 1; + churn_count_ += 1; + } - num_vectors_--; + // Now remove this vector + const int64_t last = num_vectors_ - 1; + if (idx != last) { // swap last -> idx + std::memcpy(codes_ + idx * code_size_, + codes_ + last * code_size_, + code_size_); + ids_[idx] = ids_[last]; + } + --num_vectors_; + + return (idx == last) ? -1 : idx; // <‑‑ the new occupant of slot idx } void IndexPartition::resize(int64_t new_capacity) { @@ -126,6 +176,13 @@ void IndexPartition::clear() { ids_ = nullptr; } +void IndexPartition::check_buffer_size() { + float curr_occupancy = (1.0 * num_vectors_)/buffer_size_; + if(curr_occupancy <= IndexPartition::delete_resize_threshold_) { + resize(num_vectors_ * IndexPartition::capacity_resize_threshold_); + } +} + int64_t IndexPartition::find_id(idx_t id) const { // use map @@ -156,6 +213,7 @@ void IndexPartition::set_numa_node(int new_numa_node) { bool is_valid_numa = (new_numa_node == -1) || (numa_available() != -1 && new_numa_node <= numa_max_node()); + if (!is_valid_numa) { throw std::runtime_error("Invalid numa node specified"); } @@ -245,13 +303,9 @@ void IndexPartition::reallocate_memory(int64_t new_capacity) { } void IndexPartition::ensure_capacity(int64_t required) { - if (required > buffer_size_) { - int64_t new_capacity = std::max(1024, buffer_size_); - while (new_capacity < required) { - new_capacity *= 2; - } - reallocate_memory(new_capacity); - } + if(buffer_size_ <= required) { + reallocate_memory(required * IndexPartition::capacity_resize_threshold_); + } } template diff --git a/src/cpp/src/maintenance_cost_estimator.cpp b/src/cpp/src/maintenance_cost_estimator.cpp index 8fd2a9c1..ed4d2eb8 100644 --- a/src/cpp/src/maintenance_cost_estimator.cpp +++ b/src/cpp/src/maintenance_cost_estimator.cpp @@ -63,6 +63,8 @@ void ListScanLatencyEstimator::profile_scan_latency() { torch::Tensor ids = torch::randperm(max_n); torch::Tensor query = torch::rand({d_}); + + for (size_t i = 0; i < n_values_.size(); ++i) { for (size_t j = 0; j < k_values_.size(); ++j) { int n = n_values_[i]; @@ -70,7 +72,7 @@ void ListScanLatencyEstimator::profile_scan_latency() { torch::Tensor curr_vectors = vectors.narrow(0, 0, n); torch::Tensor curr_ids = ids.narrow(0, 0, n); - auto topk_buffer = make_shared(k, false); + auto topk_buffer = make_shared(k, false, k * 100, 0); const float *query_ptr = query.data_ptr(); const float *curr_vectors_ptr = curr_vectors.data_ptr(); @@ -80,7 +82,7 @@ void ListScanLatencyEstimator::profile_scan_latency() { for (int m = 0; m < n_trials_; ++m) { auto start = std::chrono::high_resolution_clock::now(); scan_list(query_ptr, curr_vectors_ptr, curr_ids_ptr, n, d_, - *topk_buffer); + *topk_buffer, faiss::METRIC_L2, 10000000.0); auto end = std::chrono::high_resolution_clock::now(); auto duration = diff --git a/src/cpp/src/maintenance_policies.cpp b/src/cpp/src/maintenance_policies.cpp index 21ced2bb..2bf15df7 100644 --- a/src/cpp/src/maintenance_policies.cpp +++ b/src/cpp/src/maintenance_policies.cpp @@ -30,107 +30,161 @@ MaintenancePolicy::MaintenancePolicy( params_->window_size, partition_manager_->ntotal()); } +constexpr bool LOG_WMA = true; shared_ptr MaintenancePolicy::perform_maintenance() { // only consider split/deletion once the window is full - int64_t num_queries = hit_count_tracker_->get_num_queries_recorded(); - if (hit_count_tracker_->get_num_queries_recorded() < params_->window_size) { - std::cout << "Window not full yet. " << num_queries << " queries recorded and " << params_->window_size - << " queries required." << std::endl; + auto start_total = steady_clock::now(); + if (partition_manager_->parent_ == nullptr) { return std::make_shared(); } - auto start_total = steady_clock::now(); - // STEP 1: Aggregate hit counts from the HitCountTracker. - vector > per_query_hits = hit_count_tracker_->get_per_query_hits(); - unordered_map aggregated_hits; - for (const auto &query_hits: per_query_hits) { - for (int64_t pid: query_hits) { - aggregated_hits[pid]++; - } - } + vector partitions_to_delete; + vector partitions_to_split; + vector partitions_to_recluster; Tensor all_partition_ids_tens = partition_manager_->get_partition_ids(); vector all_partition_ids = vector(all_partition_ids_tens.data_ptr(), all_partition_ids_tens.data_ptr() + all_partition_ids_tens.size(0)); - // STEP 2: Use cost estimation to decide which partitions to delete or split. - int total_partitions = partition_manager_->nlist(); - float current_scan_fraction = hit_count_tracker_->get_current_scan_fraction(); - vector partitions_to_delete; - vector partitions_to_split; + if (params_->max_partition_size != -1) { + if constexpr(debug_) std::cout << "Mainteance bounding partition sizes to [" << params_->min_partition_size << "," << params_->max_partition_size << "]" << std::endl; + for (const auto &partition_id: all_partition_ids) { + int partition_size = partition_manager_->get_partition_size(partition_id); - int avg_partition_size = partition_manager_->ntotal() / total_partitions; - for (const auto &partition_id: all_partition_ids) { - // Get hit count and hit rate for the partition. - int hit_count = aggregated_hits[partition_id]; - float hit_rate = static_cast(hit_count) / static_cast(params_->window_size); - int partition_size = partition_manager_->get_partition_size(partition_id); - - // Deletion decision. - float delete_delta = cost_estimator_->compute_delete_delta( - partition_size, hit_rate, total_partitions, current_scan_fraction, avg_partition_size); - - if (delete_delta < -params_->delete_threshold_ns) { - - if (params_->enable_delete_rejection && partition_size > params_->min_partition_size) { - // check the assignments of the partitions to be deleted. - auto search_params = make_shared(); - search_params->k = 2; // get the top 2 partitions, ignore the first one as it is the partition itself - search_params->batched_scan = true; - float *partition_vectors = (float *) partition_manager_->partition_store_->partitions_[partition_id]->codes_; - Tensor part_vecs = torch::from_blob(partition_vectors, {(int64_t) partition_manager_->partition_store_->list_size(partition_id), - partition_manager_->d()}, torch::kFloat32); - auto res = partition_manager_->parent_->search(part_vecs, search_params); - - Tensor reassign_ids = res->ids.flatten(); - - // remove the partition itself - reassign_ids = reassign_ids.masked_select(reassign_ids != partition_id); - - // Get A) the unique partitions, B) the number reassigned, C) the size of the partitions, D) hit rates of the partitions - Tensor uniques; - Tensor counts; - std::tie(uniques, std::ignore, counts) = torch::_unique2(reassign_ids, true, false, true); - Tensor part_sizes = partition_manager_->get_partition_sizes(uniques); - - // convert to vectors - vector reassign_id_vec = vector(uniques.data_ptr(), uniques.data_ptr() + uniques.size(0)); - - vector reassign_sizes = vector(part_sizes.data_ptr(), - part_sizes.data_ptr() + part_sizes.size(0)); - vector reassign_counts = vector(counts.data_ptr(), - counts.data_ptr() + counts.size(0)); - vector hit_rates; - for (int64_t reassign_id: reassign_id_vec) { - hit_rates.push_back(static_cast(aggregated_hits[reassign_id]) / static_cast(params_->window_size)); - } + if (partition_size > params_->max_partition_size) { + partitions_to_split.emplace_back(partition_id); + } else if (partition_size < params_->min_partition_size) { + partitions_to_delete.emplace_back(partition_id); + } + } + + } else { + if constexpr(debug_) std::cout << "Using the cost model to determine delete/split" << std::endl; + + int64_t num_queries = hit_count_tracker_->get_num_queries_recorded(); + if (hit_count_tracker_->get_num_queries_recorded() < params_->window_size) { + std::cout << "Window not full yet. " << num_queries << " queries recorded and " << params_->window_size + << " queries required." << std::endl; + return std::make_shared(); + } - float delta = cost_estimator_->compute_delete_delta_w_reassign(partition_manager_->get_partition_size(partition_id), - static_cast(aggregated_hits[partition_id]) / static_cast(params_->window_size), - total_partitions, - reassign_counts, - reassign_sizes, - hit_rates); + // STEP 1: Aggregate hit counts from the HitCountTracker. + vector > per_query_hits = hit_count_tracker_->get_per_query_hits(); + unordered_map aggregated_hits; + for (const auto &query_hits: per_query_hits) { + for (int64_t pid: query_hits) { + aggregated_hits[pid]++; + } + } - if (delta < -params_->delete_threshold_ns) { + // STEP 2: Use cost estimation to decide which partitions to delete or split. + int total_partitions = partition_manager_->nlist(); + float current_scan_fraction = hit_count_tracker_->get_current_scan_fraction(); + + float* new_centroids_buffer = reinterpret_cast(quake_alloc(partition_manager_->d() * sizeof(float), 0)); + int avg_partition_size = partition_manager_->ntotal() / total_partitions; + + for (const auto &partition_id: all_partition_ids) { + // Update the centroid for this vector if we have a delta + bool choose_partition = false; + float delete_factor = partition_manager_->get_delete_factor(partition_id); + partition_manager_->update_centroid(partition_id, new_centroids_buffer); + + // Get hit count and hit rate for the partition. + int hit_count = aggregated_hits[partition_id]; + float hit_rate = static_cast(hit_count) / static_cast(params_->window_size); + int partition_size = partition_manager_->get_partition_size(partition_id); + + // Deletion decision. + float delete_delta = cost_estimator_->compute_delete_delta( + partition_size, hit_rate, total_partitions, current_scan_fraction, avg_partition_size); + bool consider_partition_for_delete = delete_delta < -params_->delete_threshold_ns; + // if constexpr(debug_) std::cout << "For partition " << partition_id << " of size " << partition_size << " got delete delta " << delete_delta << " leading to delete decision of " << consider_partition_for_delete << std::endl; + + if (consider_partition_for_delete) { + + if (params_->enable_delete_rejection && partition_size > params_->min_partition_size) { + // check the assignments of the partitions to be deleted. + auto search_params = make_shared(); + search_params->k = 2; // get the top 2 partitions, ignore the first one as it is the partition itself + search_params->batched_scan = true; + search_params->track_hits = false; + float *partition_vectors = (float *) partition_manager_->partition_store_->partitions_[partition_id]->codes_; + Tensor part_vecs = torch::from_blob(partition_vectors, {(int64_t) partition_manager_->partition_store_->list_size(partition_id), + partition_manager_->d()}, torch::kFloat32); + auto res = partition_manager_->parent_->search(part_vecs, search_params); + + Tensor reassign_ids = res->ids.flatten(); + + // remove the partition itself + reassign_ids = reassign_ids.masked_select(reassign_ids != partition_id); + + // Get A) the unique partitions, B) the number reassigned, C) the size of the partitions, D) hit rates of the partitions + Tensor uniques; + Tensor counts; + std::tie(uniques, std::ignore, counts) = torch::_unique2(reassign_ids, true, false, true); + Tensor part_sizes = partition_manager_->get_partition_sizes(uniques); + + // convert to vectors + vector reassign_id_vec = vector(uniques.data_ptr(), uniques.data_ptr() + uniques.size(0)); + + vector reassign_sizes = vector(part_sizes.data_ptr(), + part_sizes.data_ptr() + part_sizes.size(0)); + vector reassign_counts = vector(counts.data_ptr(), + counts.data_ptr() + counts.size(0)); + vector hit_rates; + for (int64_t reassign_id: reassign_id_vec) { + hit_rates.push_back(static_cast(aggregated_hits[reassign_id]) / static_cast(params_->window_size)); + } + + float delta = cost_estimator_->compute_delete_delta_w_reassign(partition_manager_->get_partition_size(partition_id), + static_cast(aggregated_hits[partition_id]) / static_cast(params_->window_size), + total_partitions, + reassign_counts, + reassign_sizes, + hit_rates); + + if (delta < -params_->delete_threshold_ns) { + choose_partition = true; + partitions_to_delete.push_back(partition_id); + } + } else { partitions_to_delete.push_back(partition_id); + choose_partition = true; } } else { - partitions_to_delete.push_back(partition_id); - } - } else { - if (partition_size > params_->min_partition_size) { - float split_delta = cost_estimator_->compute_split_delta( - partition_size, hit_rate, total_partitions); - if (split_delta < -params_->split_threshold_ns) { - partitions_to_split.push_back(partition_id); + bool partition_large_enough = partition_size > params_->min_partition_size; + if (partition_size > params_->min_partition_size) { + float split_delta = cost_estimator_->compute_split_delta( + partition_size, hit_rate, total_partitions); + bool should_split = split_delta < -params_->split_threshold_ns; + if constexpr(debug_) std::cout << "For partition " << partition_id << " of size " << partition_size << " got split delta " << split_delta << " leading to split decision of " << should_split << std::endl; + if (should_split) { + partitions_to_split.push_back(partition_id); + choose_partition = true; + } } } - } + + // If it was not chosen then consider it for some tracking based optimizations + if(choose_partition) { + continue; + } + + // If a large chunk of the partition was deleted then mark it for deletion + bool perform_delete = delete_factor != -1.0 && delete_factor > params_->partition_reduction_threshold; + if(perform_delete) { + partitions_to_delete.push_back(partition_id); + continue; + } + } + + quake_free(new_centroids_buffer, partition_manager_->d() * sizeof(float)); } + // Convert partition ID vectors to Torch tensors. Tensor partitions_to_delete_tens = torch::from_blob( partitions_to_delete.data(), {static_cast(partitions_to_delete.size())}, @@ -142,37 +196,59 @@ shared_ptr MaintenancePolicy::perform_maintenance() { // STEP 3: Process deletions. auto start_delete = steady_clock::now(); if (partitions_to_delete_tens.numel() > 0) { - partition_manager_->delete_partitions(partitions_to_delete_tens); + partition_manager_->delete_partitions(partitions_to_delete_tens, true); } auto end_delete = steady_clock::now(); // STEP 4: Process splits. auto start_split = steady_clock::now(); shared_ptr split_partitions; + torch::Tensor refine_partitions; if (partitions_to_split_tens.numel() > 0) { - // split the partitions into two - split_partitions = partition_manager_->split_partitions(partitions_to_split_tens); + split_partitions = partition_manager_->split_partitions(partitions_to_split_tens, params_->split_knn_iterations); // remove old partitions partition_manager_->delete_partitions(partitions_to_split_tens, false); // add new partitions partition_manager_->add_partitions(split_partitions); + refine_partitions = split_partitions->partition_ids; + } else { + refine_partitions = torch::empty({0}, torch::kInt64); } auto end_split = steady_clock::now(); + // STEP 5: Perform local refinement on newly split partitions. - if (split_partitions && split_partitions->partition_ids.numel() > 0) { - local_refinement(split_partitions->partition_ids); + if (refine_partitions.numel() > 0) { + local_refinement(refine_partitions); } auto end_total = steady_clock::now(); + int64_t refinement_time_us = static_cast(duration_cast(end_total - end_split).count()); + + // Step 6: Recluster any partitions + + // STEP 7: Clean up any empty partitions + vector empty_ids = {}; + for (auto pair : partition_manager_->partition_store_->partitions_) { + if (pair.second->num_vectors_ <= 0) { + empty_ids.emplace_back(pair.first); + } + } + if (empty_ids.size() > 0) { + partition_manager_->delete_partitions(torch::from_blob(empty_ids.data(), {static_cast(empty_ids.size())}, torch::kInt64)); + } - // STEP 6: Fill in timing details. + // STEP 7: Fill in timing details. shared_ptr timing_info = std::make_shared(); timing_info->delete_time_us = duration_cast(end_delete - start_delete).count(); timing_info->split_time_us = duration_cast(end_split - start_split).count(); + timing_info->refinement_time_us = refinement_time_us; timing_info->total_time_us = duration_cast(end_total - start_total).count(); + timing_info->n_splits = static_cast(partitions_to_split.size()); + timing_info->n_deletes = static_cast(partitions_to_delete.size()); + return timing_info; } @@ -190,6 +266,8 @@ void MaintenancePolicy::local_refinement(const torch::Tensor &partition_ids) { auto search_params = std::make_shared(); search_params->nprobe = 1000; search_params->k = params_->refinement_radius; + search_params->batched_scan = true; + search_params->track_hits = false; if (params_->refinement_radius == 0) { return; diff --git a/src/cpp/src/partition_manager.cpp b/src/cpp/src/partition_manager.cpp index 3b738856..3ed6e72f 100644 --- a/src/cpp/src/partition_manager.cpp +++ b/src/cpp/src/partition_manager.cpp @@ -99,6 +99,8 @@ void PartitionManager::init_partitions( resident_ids_.insert(id_val); } } + + std::shared_ptr partition = partition_store_->get_partition(partition_ids_accessor[i]); partition_store_->add_entries( partition_ids_accessor[i], count, @@ -109,9 +111,15 @@ void PartitionManager::init_partitions( std::cout << "[PartitionManager] init_partitions: Added " << count << " entries to partition " << partition_ids_accessor[i] << std::endl; } + + // Also reset the index delta + partition->reset_delta(); + if (debug_) std::cout << "[PartitionManager] init_partitions: Reset delta finished" << std::endl; } } + partition_store_->build_map(); + if (debug_) { std::cout << "[PartitionManager] init_partitions: Created " << nlist << " partitions, dimension=" << dim << std::endl; @@ -124,7 +132,8 @@ shared_ptr PartitionManager::add( const Tensor &vectors, const Tensor &vector_ids, const Tensor &assignments, - bool check_uniques + bool check_uniques, + bool record_delta ) { auto timing_info = std::make_shared(); @@ -199,7 +208,12 @@ shared_ptr PartitionManager::add( // Determine partition assignments for each vector. vector partition_ids_for_each(n, -1); if (parent_ == nullptr) { - partition_ids_for_each.assign(n, 0); + // round robin assign + int64_t num_parts = nlist(); + for (int64_t i = 0; i < partition_ids_for_each.size(); i++) { + partition_ids_for_each[i] = i % num_parts; + } + if (debug_) { std::cout << "[PartitionManager] add: No parent index; assigning all vectors to partition 0." << std::endl; } @@ -218,10 +232,8 @@ shared_ptr PartitionManager::add( } auto search_params = make_shared(); search_params->k = 1; - search_params->nprobe = parent_->nlist(); - if (n > 10) { - search_params->batched_scan = true; - } + search_params->scan_all = true; + search_params->track_hits = false; auto parent_search_result = parent_->search(vectors, search_params); Tensor label_out = parent_search_result->ids; auto lbl_ptr = label_out.data_ptr(); @@ -244,6 +256,13 @@ shared_ptr PartitionManager::add( for (int64_t i = 0; i < n; i++) { int64_t pid = partition_ids_for_each[i]; + + if (pid < 0 || pid >= curr_partition_id_) { + std::string error_msg = "[PartitionManager] add: Invalid partition ID of " + std::to_string(pid) + "/" + std::to_string(curr_partition_id_); + error_msg = error_msg + " for vector " + std::to_string(i) + "/" + std::to_string(n); + throw runtime_error(error_msg); + } + if (debug_) { std::cout << "[PartitionManager] add: Inserting vector " << i << " with id " << id_accessor[i] << " into partition " << pid << std::endl; @@ -253,15 +272,17 @@ shared_ptr PartitionManager::add( pid, /*n_entry=*/1, id_ptr + i, - code_ptr + i * code_size_bytes + code_ptr + i * code_size_bytes, + record_delta ); } + auto e3 = std::chrono::high_resolution_clock::now(); timing_info->modify_time_us = std::chrono::duration_cast(e3 - s3).count(); return timing_info; } -shared_ptr PartitionManager::remove(const Tensor &ids) { +shared_ptr PartitionManager::remove(const Tensor &ids, bool record_delta) { shared_ptr timing_info = std::make_shared(); auto s1 = std::chrono::high_resolution_clock::now(); @@ -299,17 +320,8 @@ shared_ptr PartitionManager::remove(const Tensor &ids) { auto e1 = std::chrono::high_resolution_clock::now(); timing_info->input_validation_time_us = std::chrono::duration_cast(e1 - s1).count(); - auto s2 = std::chrono::high_resolution_clock::now(); - std::set to_remove; - auto ptr = ids.data_ptr(); - for (int64_t i = 0; i < ids.size(0); i++) { - to_remove.insert(static_cast(ptr[i])); - } - auto e2 = std::chrono::high_resolution_clock::now(); - timing_info->find_partition_time_us = std::chrono::duration_cast(e2 - s2).count(); - auto s3 = std::chrono::high_resolution_clock::now(); - partition_store_->remove_vectors(to_remove); + partition_store_->remove_vectors(ids.data_ptr(), ids.size(0), record_delta); if (debug_) { std::cout << "[PartitionManager] remove: Completed removal." << std::endl; } @@ -390,7 +402,7 @@ shared_ptr PartitionManager::select_partitions(const Tensor &select_ return clustering; } -shared_ptr PartitionManager::split_partitions(const Tensor &partition_ids) { +shared_ptr PartitionManager::split_partitions(const Tensor &partition_ids, int knn_iteration) { if (debug_) { std::cout << "[PartitionManager] split_partitions: Splitting " << partition_ids.size(0) << " partitions." << std::endl; @@ -407,7 +419,12 @@ shared_ptr PartitionManager::split_partitions(const Tensor &partitio split_vectors.reserve(total_new_partitions); split_ids.reserve(total_new_partitions); - shared_ptr clustering = select_partitions(partition_ids); + shared_ptr clustering = select_partitions(partition_ids, true); + + shared_ptr build_params = make_shared(); + build_params->niter = knn_iteration; + build_params->nlist = num_splits; + build_params->metric = metric_type_to_str(parent_->metric_); for (int64_t i = 0; i < partition_ids.size(0); ++i) { // Ensure enough vectors to split @@ -415,8 +432,7 @@ shared_ptr PartitionManager::split_partitions(const Tensor &partitio shared_ptr curr_split_clustering = kmeans( clustering->vectors[i], clustering->vector_ids[i], - num_splits, - parent_->metric_ + build_params ); for (size_t j = 0; j < curr_split_clustering->nlist(); ++j) { @@ -444,6 +460,72 @@ shared_ptr PartitionManager::split_partitions(const Tensor &partitio return split_clustering; } +float PartitionManager::get_churn_factor(int64_t partition_id) { + // Get the partition details + std::shared_ptr curr_partition = partition_store_->get_partition(partition_id); + int64_t num_changes = curr_partition->churn_count_; + int64_t previous_size = std::max(curr_partition->last_snapshot_size_, curr_partition->num_vectors_); + if(num_changes <= 0 || previous_size == 0) { // Deal with the case that there has been no deletes + return -1.0; + } + + return (1.0 * num_changes)/previous_size; +} + +float PartitionManager::get_delete_factor(int64_t partition_id) { + // Get the partition details + std::shared_ptr curr_partition = partition_store_->get_partition(partition_id); + int64_t num_deletes = -1 * curr_partition->delta_count_; + int64_t previous_size = curr_partition->last_snapshot_size_; + if(num_deletes <= 0 || previous_size == 0) { // Deal with the case that there has been no deletes + return -1.0; + } + + return (1.0 * num_deletes)/previous_size; +} + +int64_t PartitionManager::update_centroid(int64_t partition_id, float* centroid_buffer) { + // Load the specified partition + const int dimension = partition_store_->d_; + std::shared_ptr curr_partition = partition_store_->get_partition(partition_id); + int64_t delta_size = curr_partition->delta_count_; + if(delta_size == 0 || curr_partition->num_vectors_ == 0) { + curr_partition->reset_delta(); + return delta_size; + } + + // Get the centroid for this partition + std::shared_ptr centroid_store = parent_->partition_manager_->partition_store_; + bool found_centroid = centroid_store->get_vector_for_id(partition_id, centroid_buffer); + if(!found_centroid) { + std::string err_msg = std::string("Failed to load centroid for partition ") + std::to_string(partition_id); + throw std::runtime_error(err_msg); + } + + // Get the variables needed + int64_t old_size = curr_partition->last_snapshot_size_; + int64_t curr_size = curr_partition->num_vectors_; + if(old_size + delta_size != curr_size) { + std::string err_msg = std::string("Invalid sizes for partition ") + std::to_string(partition_id); + throw std::runtime_error(err_msg); + } + + // Update the centroid based on the delta + float* delta_vec = reinterpret_cast(curr_partition->delta_vec_); + #pragma unroll + for(int i = 0; i < dimension; i++) { + float old_val = centroid_buffer[i]; + centroid_buffer[i] = (old_size * centroid_buffer[i] + delta_vec[i])/curr_size; + } + + // Write back the new centroid + centroid_store->write_vector_by_id(partition_id, centroid_buffer); + + // Reset the delta as we have process this delta + curr_partition->reset_delta(); + return delta_size; +} + void PartitionManager::refine_partitions(Tensor partition_ids, int iterations) { if (debug_) { std::cout << "[PartitionManager] refine_partitions: Refining partitions with iterations = " @@ -479,9 +561,13 @@ void PartitionManager::refine_partitions(Tensor partition_ids, int iterations) { // replace partitions for (int i = 0; i < partition_ids.size(0); i++) { + index_partitions[i]->partition_id_ = pids[i]; partition_store_->partitions_[pids[i]] = index_partitions[i]; + index_partitions[i]->reset_delta(); } + partition_store_->build_map(); + if (debug_) { std::cout << "[PartitionManager] refine_partitions: Completed refinement." << std::endl; } @@ -503,12 +589,17 @@ void PartitionManager::add_partitions(shared_ptr partitions) { for (int64_t i = 0; i < nlist; i++) { int64_t list_no = p_ids_accessor[i]; partition_store_->add_list(list_no); + if (num_workers_ > 0) { + set_partition_core_id(list_no, list_no % num_workers_); + } + partition_store_->add_entries( list_no, partitions->vectors[i].size(0), partitions->vector_ids[i].data_ptr(), as_uint8_ptr(partitions->vectors[i]) ); + partition_store_->get_partition(list_no)->reset_delta(); if (debug_) { std::cout << "[PartitionManager] add_partitions: Added partition " << list_no << " with " << partitions->vectors[i].size(0) << " vectors." << std::endl; @@ -545,7 +636,7 @@ void PartitionManager::delete_partitions(const Tensor &partition_ids, bool reass if (vectors.size(0) == 0) { continue; } - add(vectors, ids, Tensor(), false); + add(vectors, ids, Tensor(), false, true); } } } else { @@ -554,13 +645,15 @@ void PartitionManager::delete_partitions(const Tensor &partition_ids, bool reass } -void PartitionManager::distribute_partitions(int num_workers) { +void PartitionManager::distribute_partitions(int num_workers, bool use_numa) { if (debug_) { std::cout << "[PartitionManager] distribute_partitions: Attempting to distribute partitions across " << num_workers << " workers." << std::endl; } - if (parent_ == nullptr) { + num_workers_ = num_workers; + + if (parent_ == nullptr && partition_store_->nlist == 1) { auto codes = (float *) partition_store_->get_codes(0); auto ids = (int64_t *) partition_store_->get_ids(0); int64_t ntotal = partition_store_->list_size(0); @@ -597,13 +690,21 @@ void PartitionManager::distribute_partitions(int num_workers) { } Tensor partition_ids = get_partition_ids(); - for (int i = 0; i < partition_store_->nlist; i++) { - set_partition_core_id(partition_ids[i].item(), i % num_workers); + auto partition_ids_accessor = partition_ids.accessor(); + for (int i = 0; i < partition_ids.size(0); i++) { + set_partition_core_id(partition_ids_accessor[i], partition_ids_accessor[i] % num_workers, use_numa); } } -void PartitionManager::set_partition_core_id(int64_t partition_id, int core_id) { - partition_store_->partitions_[partition_id]->core_id_ = core_id; +void PartitionManager::set_partition_core_id(int64_t partition_id, int core_id, bool use_numa) { + partition_store_->partitions_[partition_id]->set_core_id(core_id); + int node = cpu_numa_node(core_id); + + #ifdef QUAKE_USE_NUMA + if (use_numa) { + partition_store_->partitions_[partition_id]->set_numa_node(node); + } + #endif } int PartitionManager::get_partition_core_id(int64_t partition_id) { @@ -677,7 +778,12 @@ Tensor PartitionManager::get_partition_sizes(Tensor partition_ids) { auto partition_sizes_accessor = partition_sizes.accessor(); for (int i = 0; i < partition_ids.size(0); i++) { int64_t list_no = partition_ids_accessor[i]; - partition_sizes_accessor[i] = partition_store_->list_size(list_no); + if (list_no == -1) { + partition_sizes_accessor[i] = 0; + } else { + partition_sizes_accessor[i] = partition_store_->list_size(list_no); + } + if (debug_) { std::cout << "[PartitionManager] get_partition_sizes: Partition " << list_no << " size: " << partition_sizes_accessor[i] << std::endl; diff --git a/src/cpp/src/quake_index.cpp b/src/cpp/src/quake_index.cpp index 1f7cc853..cfdcd20d 100644 --- a/src/cpp/src/quake_index.cpp +++ b/src/cpp/src/quake_index.cpp @@ -39,15 +39,14 @@ shared_ptr QuakeIndex::build(Tensor x, Tensor ids, shared_ptrnum_workers << " workers." << std::endl; + query_coordinator_ = make_shared(parent_, partition_manager_, maintenance_policy_, metric_, current_level_, build_params_->num_workers, build_params_->use_numa, build_params_->num_merge_workers); auto end = std::chrono::high_resolution_clock::now(); timing_info->total_time_us = std::chrono::duration_cast(end - start).count(); return timing_info; } +void QuakeIndex::add_level(shared_ptr params) { + + if (!parent_) { + throw std::runtime_error("[QuakeIndex::add_level()] No parent index. Cannot add level."); + } + + if (parent_->parent_) { + parent_->add_level(params); + } else { + // create an index over the centroids + + // get all centroids and their ids + Tensor ids = parent_->partition_manager_->get_ids(); + Tensor centroids = parent_->partition_manager_->get(ids); + + parent_->build(centroids, ids, params); + } +} + shared_ptr QuakeIndex::search(Tensor x, shared_ptr search_params) { @@ -124,7 +150,7 @@ shared_ptr QuakeIndex::add(Tensor x, Tensor ids) { throw std::runtime_error("[QuakeIndex::add()] No partition manager. Build the index first."); } - auto modify_info = partition_manager_->add(x, ids); + auto modify_info = partition_manager_->add(x, ids, torch::Tensor(), false, true); modify_info->n_vectors = x.size(0); return modify_info; } @@ -134,7 +160,7 @@ shared_ptr QuakeIndex::remove(Tensor ids) { throw std::runtime_error("[QuakeIndex::remove()] No partition manager. Build the index first."); } - auto modify_info = partition_manager_->remove(ids); + auto modify_info = partition_manager_->remove(ids, true); modify_info->n_vectors = ids.size(0); return modify_info; } @@ -152,6 +178,10 @@ void QuakeIndex::initialize_maintenance_policy(shared_ptrmaintenance_policy_ = maintenance_policy_; } + + if (parent_ != nullptr) { + parent_->initialize_maintenance_policy(maintenance_policy_params_); + } } shared_ptr QuakeIndex::maintenance() { @@ -159,7 +189,13 @@ shared_ptr QuakeIndex::maintenance() { throw std::runtime_error("[QuakeIndex::maintenance()] No maintenance policy set."); } - return maintenance_policy_->perform_maintenance(); + auto maintenance_info = maintenance_policy_->perform_maintenance(); + + if (parent_ && parent_->maintenance_policy_) { + parent_->maintenance_policy_->perform_maintenance(); + } + + return maintenance_info; } bool QuakeIndex::validate() { @@ -205,7 +241,7 @@ void QuakeIndex::save(const std::string& dir_path) { std::cout << "[QuakeIndex::save] Index saved to directory: " << dir_path << "\n"; } -void QuakeIndex::load(const std::string& dir_path, int n_workers) { +void QuakeIndex::load(const std::string& dir_path, shared_ptr build_params) { namespace fs = std::filesystem; if (!fs::exists(dir_path) || !fs::is_directory(dir_path)) { @@ -250,7 +286,12 @@ void QuakeIndex::load(const std::string& dir_path, int n_workers) { std::string parent_dir = (fs::path(dir_path) / "parent").string(); if (fs::exists(parent_dir) && fs::is_directory(parent_dir)) { parent_ = std::make_shared(); - parent_->load(parent_dir, n_workers); + int n_parts = partition_manager_->nlist(); + auto parent_params = make_shared(); + if (build_params->parent_params != nullptr) { + parent_params = build_params->parent_params; + } + parent_->load(parent_dir, parent_params); partition_manager_->parent_ = parent_; } else { parent_ = nullptr; @@ -261,9 +302,16 @@ void QuakeIndex::load(const std::string& dir_path, int n_workers) { initialize_maintenance_policy(default_params); // 5. Create query coordinator - std::cout << "Loading coordinator with n_workers=" << n_workers << '\n'; - query_coordinator_ = std::make_shared(parent_, partition_manager_, maintenance_policy_, metric_, n_workers); - std::cout << "Loaded coordinator\n"; + std::cout << "Loading coordinator at level " << current_level_ << " with " + << build_params->num_workers << " workers, NUMA: " << (build_params->use_numa ? "enabled" : "disabled") << " and metric " << metric_ << std::endl; + query_coordinator_ = std::make_shared(parent_, + partition_manager_, + maintenance_policy_, + metric_, + current_level_, + build_params->num_workers, + build_params->use_numa, + build_params->num_merge_workers); } int64_t QuakeIndex::ntotal() { diff --git a/src/cpp/src/query_coordinator.cpp b/src/cpp/src/query_coordinator.cpp index 67112a1f..d81fbcd4 100644 --- a/src/cpp/src/query_coordinator.cpp +++ b/src/cpp/src/query_coordinator.cpp @@ -10,462 +10,1426 @@ #include #include #include +//#include "parallel_hashmap/btree.h" + +#include +#include +#include +#include + +int QueryCoordinator::batch_scan_partition_chunk_size_ = BLAS_DB_BS; +int QueryCoordinator::batch_scan_query_chunk_size_ = DEFAULT_BLAS_Q_BS; + +// Wrapper for the system call since glibc doesn't provide one +static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, + int cpu, int group_fd, unsigned long flags) { + return syscall(__NR_perf_event_open, hw_event, pid, cpu, group_fd, flags); +} + +class MetricTracker { +public: + // File descriptors + int fd_leader = -1; + int fd_cycles = -1; + int fd_cache_refs = -1; + int fd_cache_misses = -1; + + long long count_instr = 0; + long long count_cycles = 0; + long long cache_count_refs = 0; + long long cache_count_misses = 0; + + // Helper to configure the attributes + void configure_attr(struct perf_event_attr& pe, uint64_t config) { + memset(&pe, 0, sizeof(struct perf_event_attr)); + pe.type = PERF_TYPE_HARDWARE; + pe.size = sizeof(struct perf_event_attr); + pe.config = config; + pe.disabled = 1; // Start disabled + pe.exclude_kernel = 1; // Exclude kernel instructions + pe.exclude_hv = 1; // Exclude hypervisor + } + + MetricTracker() { + + } + + ~MetricTracker() { + if (fd_cache_misses != -1) close(fd_cache_misses); + if (fd_cache_refs != -1) close(fd_cache_refs); + if (fd_cycles != -1) close(fd_cycles); + if (fd_leader != -1) close(fd_leader); + } + + void perform_setup() { + struct perf_event_attr pe; + + // 1. Instructions (Leader) + configure_attr(pe, PERF_COUNT_HW_INSTRUCTIONS); + fd_leader = perf_event_open(&pe, 0, -1, -1, 0); + if (fd_leader == -1) { perror("Error opening leader"); exit(EXIT_FAILURE); } + + // 2. Cycles (Follower) + configure_attr(pe, PERF_COUNT_HW_CPU_CYCLES); + // We still group them via the 'group_fd' argument here to ensure they + // are scheduled on the CPU at the same time. + fd_cycles = perf_event_open(&pe, 0, -1, fd_leader, 0); + if (fd_cycles == -1) { perror("Error opening cycles"); exit(EXIT_FAILURE); } + + // 3. Cache References (Follower) + configure_attr(pe, PERF_COUNT_HW_CACHE_REFERENCES); + fd_cache_refs = perf_event_open(&pe, 0, -1, fd_leader, 0); + if (fd_cache_refs == -1) { perror("Error opening cache refs"); exit(EXIT_FAILURE); } + + // 4. Cache Misses (Follower) + configure_attr(pe, PERF_COUNT_HW_CACHE_MISSES); + fd_cache_misses = perf_event_open(&pe, 0, -1, fd_leader, 0); + if (fd_cache_misses == -1) { perror("Error opening cache misses"); exit(EXIT_FAILURE); } + } + + void start() { + // We reset and enable the WHOLE GROUP using the leader. + ioctl(fd_leader, PERF_EVENT_IOC_RESET, PERF_IOC_FLAG_GROUP); + ioctl(fd_leader, PERF_EVENT_IOC_ENABLE, PERF_IOC_FLAG_GROUP); + } + + void stop_and_record() { + // Disable the group atomically + ioctl(fd_leader, PERF_EVENT_IOC_DISABLE, PERF_IOC_FLAG_GROUP); + + // Read values individually + if (read(fd_leader, &count_instr, sizeof(long long)) == -1) { perror("Read instr"); exit(EXIT_FAILURE); } + if (read(fd_cycles, &count_cycles, sizeof(long long)) == -1) { perror("Read cycles"); exit(EXIT_FAILURE); } + if (read(fd_cache_refs, &cache_count_refs, sizeof(long long)) == -1) { perror("Read refs"); exit(EXIT_FAILURE); } + if (read(fd_cache_misses, &cache_count_misses, sizeof(long long)) == -1) { perror("Read misses"); exit(EXIT_FAILURE); } + } + + // Changed to double for better precision + double get_ipc() { + if(count_cycles == 0) return 0.0; + return (double)count_instr / (double)count_cycles; + } + + // Changed to double for better precision + double get_cache_miss_rate() { + if(cache_count_refs == 0) return 0.0; + return (100.0 * (double) cache_count_misses) / (double)cache_count_refs; + } +}; + +static void ensure_blas_buffers(QueryCoordinator::CoreResources& res, + size_t max_q, + size_t db_bs, + int node) +{ + const size_t ip_need = db_bs * max_q; + if (res.blas_ip_capacity < ip_need) { + if (res.blas_ip_block) quake_free(res.blas_ip_block, res.blas_ip_capacity * sizeof(float)); + res.blas_ip_block = static_cast(quake_alloc(ip_need * sizeof(float), node)); + res.blas_ip_capacity = ip_need; + } + if (res.blas_norms_x_cap < max_q) { + if (res.blas_norms_x) quake_free(res.blas_norms_x, res.blas_norms_x_cap * sizeof(float)); + res.blas_norms_x = static_cast(quake_alloc(max_q * sizeof(float), node)); + res.blas_norms_x_cap = max_q; + } + if (res.blas_norms_y_cap < db_bs) { + if (res.blas_norms_y) quake_free(res.blas_norms_y, res.blas_norms_y_cap * sizeof(float)); + res.blas_norms_y = static_cast(quake_alloc(db_bs * sizeof(float), node)); + res.blas_norms_y_cap = db_bs; + } +} // Constructor QueryCoordinator::QueryCoordinator(shared_ptr parent, shared_ptr partition_manager, shared_ptr maintenance_policy, MetricType metric, - int num_workers) + int current_level, + int num_workers, + bool use_numa, + int num_merge_workers) : parent_(parent), partition_manager_(partition_manager), maintenance_policy_(maintenance_policy), metric_(metric), + current_level_(current_level), num_workers_(num_workers), + num_merge_workers_(num_merge_workers), workers_initialized_(false) { + if (debug_) std::cout << "[QueryCoordinator::QueryCoordinator] Coordinator intiialized with " << num_workers_ << " workers" << std::endl; if (num_workers_ > 0) { - initialize_workers(num_workers_); + initialize_workers(num_workers_, num_merge_workers_, use_numa); } } // Destructor QueryCoordinator::~QueryCoordinator() { shutdown_workers(); + + // Free up numa node local buffers + for (int idx = 0; idx < numa_resources_.size(); idx++) { + auto &nr = numa_resources_[idx]; + if(nr.local_query_buffer != nullptr) quake_free(nr.local_query_buffer, nr.buffer_size); + } + + // Free up global merger buffers + if (global_heap_vals_buffer_ != nullptr) quake_free(global_heap_vals_buffer_, global_heap_buffer_capacity_ * sizeof(float)); + if (global_heap_ids_buffer_ != nullptr) quake_free(global_heap_ids_buffer_, global_heap_buffer_capacity_ * sizeof(int64_t)); } -void QueryCoordinator::allocate_core_resources(int core_idx, int num_queries, int k, int d) { - CoreResources &res = core_resources_[core_idx]; - res.core_id = core_idx; - res.local_query_buffer.resize(num_queries * d * sizeof(float)); - res.topk_buffer_pool.resize(num_queries); - for (int q = 0; q < num_queries; ++q) { - res.topk_buffer_pool[q] = make_shared(k, metric_ == faiss::METRIC_INNER_PRODUCT); - res.job_queue = moodycamel::BlockingConcurrentQueue(); +void QueryCoordinator::allocate_core_resources(int core_idx, + int num_queries, + int k, + int d) +{ + auto& CR = core_resources_[core_idx]; + CR.core_id = core_idx; + CR.topk_buffer_pool.clear(); + + // --- ZERO‐INITIALIZE our batched‐query buffers so we never free garbage pointers --- + CR.batch_queries = nullptr; + + CR.blas_ip_block = nullptr; + CR.blas_ip_capacity = 0; + CR.blas_norms_x = nullptr; + CR.blas_norms_x_cap = 0; + CR.blas_norms_y = nullptr; + CR.blas_norms_y_cap = 0; + + int numa_node = 0; +#ifdef QUAKE_USE_NUMA + numa_node = cpu_numa_node(core_idx); +#endif + + // job queue remains default‐constructed + numa_resources_.resize(get_num_numa_nodes()); + auto& numa_res = numa_resources_[numa_node]; + size_t bytes = size_t(num_queries) * d * sizeof(float); + if (numa_res.buffer_size != bytes) { + if(numa_res.local_query_buffer != nullptr) quake_free(numa_res.local_query_buffer, numa_res.buffer_size); + numa_res.local_query_buffer = static_cast(quake_alloc(bytes, numa_node)); + numa_res.buffer_size = bytes; } +} + + +// The heart of it: one function, two instantiations. +template +void QueryCoordinator::merge_worker_fn(int mid) { + auto& MR = merge_res_[mid]; + ResultJob rj; + + while (true) { + MR.queue.wait_dequeue(rj); + if (rj.query_id == -1) // poison pill + return; + + using Handler = typename faiss::HeapBlockResultHandler::SingleResultHandler; + auto h = std::static_pointer_cast(MR.handlers[rj.query_id]); + + if (!rj.distances.empty()) { + // feed all partial results + for (size_t i = 0; i < rj.distances.size(); ++i) { + h->add_result(rj.distances[i], rj.indices[i]); + } + // update pivot + query_dist_pivots_[rj.query_id].store(h->threshold, + std::memory_order_relaxed); + } + + // once all ranks for this query are in, finalize & sort + if (per_query_total_left_[rj.query_id].fetch_sub(1, std::memory_order_acq_rel) == 1) { + h->end(); + + // pack into pairs for sorting + int k = h->k; + std::vector> result; + result.reserve(k); + for (int i = 0; i < k; ++i) { + result.emplace_back(h->heap_dis[i], h->heap_ids[i]); + } + // for CMin (inner-product) we want descending distances + // for CMax (L2) we want ascending distances + auto cmp = [](auto& a, auto& b) { + return Compare::cmp(b.first, a.first); + }; + std::sort(result.begin(), result.end(), cmp); + + // write them back + for (int i = 0; i < k; ++i) { + h->heap_dis[i] = result[i].first; + h->heap_ids[i] = result[i].second; + } + } + --total_left_; + } } -// Initialize Worker Threads -void QueryCoordinator::initialize_workers(int num_cores) { - if (workers_initialized_) { - std::cerr << "[QueryCoordinator::initialize_workers] Workers already initialized." << std::endl; - return; +constexpr bool RUN_WITH_HARDWARE_COUNTERS = false; + +void QueryCoordinator::partition_scan_worker_fn(int core_index) { + CoreResources &res = core_resources_[core_index]; + int numa_node = 0; +#ifdef QUAKE_USE_NUMA + numa_node = cpu_numa_node(core_index); +#endif + NUMAResources &nr = numa_resources_[numa_node]; + + set_thread_affinity(core_index); + omp_set_num_threads(1); + + MetricTracker tracker; // Tracker used to track IPC for scan + nr.metric_tracker_ptr = (void*) &tracker; + if constexpr(RUN_WITH_HARDWARE_COUNTERS) { + tracker.perform_setup(); } - std::cout << "[QueryCoordinator::initialize_workers] Initializing " << num_cores << " worker threads." << - std::endl; + int i = 0; + res.wait_time_ns = 0; + res.process_time_ns = 0; + res.enqueue_time_ns = 0; + res.job_time_ns = 0; - partition_manager_->distribute_partitions(num_cores); + while (!stop_workers_) { + int64_t jid = 0; - core_resources_.resize(num_cores); - worker_threads_.resize(num_cores); - worker_job_counter_.reserve(num_cores); - for (int i = 0; i < num_cores; i++) { - if (!set_thread_affinity(i)) { - std::cout << "[QueryCoordinator::initialize_workers] Failed to set thread affinity on core " << i << std::endl; + auto start = std::chrono::high_resolution_clock::now(); + nr.job_queue.wait_dequeue(jid); + auto end = std::chrono::high_resolution_clock::now(); + + res.wait_time_ns += std::chrono::duration_cast(end - start).count(); + + if (jid == -1) { + break; } - allocate_core_resources(i, 1, 10, partition_manager_->d()); - worker_threads_[i] = std::thread(&QueryCoordinator::partition_scan_worker_fn, this, i); - worker_job_counter_[i] = 0; + + auto s2 = std::chrono::high_resolution_clock::now(); + process_scan_job(job_buffer_[jid], res); + i++; + end = std::chrono::high_resolution_clock::now(); + + res.process_time_ns += std::chrono::duration_cast(end - s2).count(); + res.job_time_ns += std::chrono::duration_cast(end - start).count(); + + // std::cout << "[partition_scan_worker_fn] Core: " << core_index + // << ", Job ID: " << jid + // << ", Processed: " << i + // << ", Wait time: " << res.wait_time_ns / 1e6 << " ms" + // << ", Process time: " << res.process_time_ns / 1e6 << " ms" + // << ", Enqueue time: " << res.enqueue_time_ns / 1e6 << " ms" + // << ", Job time: " << res.job_time_ns / 1e6 << " ms" << std::endl; } - workers_initialized_ = true; + + // Cleanup resources associated with worker + if (res.blas_ip_block) quake_free(res.blas_ip_block, res.blas_ip_capacity * sizeof(float)); + if (res.blas_norms_x) quake_free(res.blas_norms_x, res.blas_norms_x_cap * sizeof(float)); + if (res.blas_norms_y) quake_free(res.blas_norms_y, res.blas_norms_y_cap * sizeof(float)); + if (res.batch_queries) quake_free(res.batch_queries, res.batch_q_capacity * sizeof(float)); } -// Shutdown Worker Threads -void QueryCoordinator::shutdown_workers() { - if (!workers_initialized_) { +void QueryCoordinator::process_scan_job(ScanJob job, + CoreResources &res) { + + auto start = std::chrono::high_resolution_clock::now(); + int numa_node = 0; +#ifdef QUAKE_USE_NUMA + numa_node = cpu_numa_node(res.core_id); +#endif + NUMAResources &nr = numa_resources_[numa_node]; + + // Attempt to fetch partition data; if the list doesn't exist, catch and enqueue empty results. + const float *codes = nullptr; + const int64_t *ids = nullptr; + int64_t part_size = 0; + try { + codes = (float *)(partition_manager_->partition_store_->get_codes(job.partition_id)); + ids = (int64_t *) partition_manager_->partition_store_->get_ids(job.partition_id); + part_size = partition_manager_->partition_store_->list_size(job.partition_id); + } catch (const std::exception &e) { + std::cerr << "[process_scan_job] Partition " << job.partition_id + << " invalid: " << e.what() << ". Returning empty result(s).\n"; + if (job.is_batched) { + for (int64_t i = 0; i < job.num_queries; ++i) { + enqueue_result_job(ResultJob{(*job.query_ids)[i], (*job.ranks)[i], {}, {}}); + } + } else { + enqueue_result_job(ResultJob{job.query_id, job.rank, {}, {}}); + } return; } - stop_workers_.store(true); - // Enqueue a special shutdown job for each core. - for (auto &res : core_resources_) { - ScanJob termination_job; - termination_job.partition_id = -1; - res.job_queue.enqueue(termination_job); + if (part_size == 0) { + // empty => enqueue zero‐work per query + if (job.is_batched) { + for (int64_t i = 0; i < job.num_queries; ++i) { + enqueue_result_job(ResultJob{(*job.query_ids)[i], (*job.ranks)[i], {}, {}}); + } + } else { + enqueue_result_job(ResultJob{job.query_id, job.rank, {}, {}}); + } + return; } - // Join all worker threads. - for (auto &thr : worker_threads_) { - if (thr.joinable()) - thr.join(); + auto end = std::chrono::high_resolution_clock::now(); + res.process_preamble_time_ns += std::chrono::duration_cast(end - start).count(); + + if (!job.is_batched) { + handle_nonbatched_job(job, res, nr); + } else { + handle_batched_job(job, res, nr); } - worker_threads_.clear(); - workers_initialized_ = false; + res.queries_counter += job.num_queries; + res.job_counter++; } -// Worker Thread Function -void QueryCoordinator::partition_scan_worker_fn(int core_index) { +void QueryCoordinator::handle_nonbatched_job(const ScanJob &job, + CoreResources &res, + NUMAResources &nr) { - CoreResources &res = core_resources_[core_index]; + // check that the job has not been processed yet + if (query_done_flags_[job.query_id].load(std::memory_order_relaxed)) { + enqueue_result_job(ResultJob{job.query_id, job.rank, {}, {}}); + return; + } - if (!set_thread_affinity(core_index)) { - std::cout << "[QueryCoordinator::partition_scan_worker_fn] Failed to set thread affinity on core " << core_index << std::endl; + // check the job is not larger than the maximum rank + if (job.rank > max_rank_[job.query_id].load(std::memory_order_relaxed)) { + enqueue_result_job(ResultJob{job.query_id, job.rank, {}, {}}); + return; } + // ensure buffers + if (res.topk_buffer_pool.size() < 1) { + res.topk_buffer_pool.resize(1); + res.topk_buffer_pool[0] = std::make_shared( + job.k, + metric_ == faiss::METRIC_INNER_PRODUCT, + /*cap=*/std::min(100 * job.k, 10000), + /*node=*/cpu_numa_node(res.core_id) + ); + } else if (res.topk_buffer_pool[0]->k() != job.k) { + // check capacity + if (res.topk_buffer_pool[0]->capacity() < job.k) { + res.topk_buffer_pool[0] = std::make_shared( + job.k, + metric_ == faiss::METRIC_INNER_PRODUCT, + /*cap=*/std::min(100 * job.k, 10000), + /*node=*/cpu_numa_node(res.core_id) + ); + } + res.topk_buffer_pool[0]->set_k(job.k); + } + auto buf = res.topk_buffer_pool[0]; + res.topk_buffer_pool[0]->reset(); + + try { + const float* codes = (float*)partition_manager_->partition_store_->get_codes(job.partition_id); + const int64_t* ids = (int64_t*)partition_manager_->partition_store_->get_ids(job.partition_id); + int64_t part_size = partition_manager_->partition_store_->list_size(job.partition_id); + int D = partition_manager_->d(); + + // Defensive check for partition validity right before scan + if (!codes || !ids || part_size <= 0) { + std::cerr << "[QueryCoordinator::handle_nonbatched_job] Partition " << job.partition_id + << " invalid or empty before scan for query " << job.query_id + << ". Enqueuing empty result.\n"; + enqueue_result_job(ResultJob{job.query_id, job.rank, {}, {}}); + return; // Important to return after enqueueing the placeholder + } - while (true) { - ScanJob job; + job_flags_[job.query_id][job.rank].store(true, std::memory_order_relaxed); // Mark this job as processed + + auto start = std::chrono::high_resolution_clock::now(); + + scan_list(nr.local_query_buffer + (job.query_id * D), + codes, + ids, + part_size, + D, + *buf, + metric_, + query_dist_pivots_[job.query_id].load(std::memory_order_relaxed)); + + auto end = std::chrono::high_resolution_clock::now(); + + int64_t scan_time = std::chrono::duration_cast(end - start).count(); + res.scan_time_ns += scan_time; + + int64_t total_partition_bytes = part_size * D * sizeof(float) + part_size * sizeof(int64_t); + int64_t total_query_bytes = D * sizeof(float); + int64_t total_scan_bytes = total_partition_bytes + total_query_bytes; + res.bytes_scan_total += total_scan_bytes; + res.partition_size += part_size; + res.num_scan_jobs += 1; + res.per_job_scan_throughput += (1.0 * total_scan_bytes)/scan_time; + + start = std::chrono::high_resolution_clock::now(); + + // If scan_list completes, enqueue its results + auto tv = buf->get_topk(false); + auto ti = buf->get_topk_indices(false); + enqueue_result_job(ResultJob{job.query_id, job.rank, std::move(tv), std::move(ti)}); + + end = std::chrono::high_resolution_clock::now(); + res.enqueue_time_ns += std::chrono::duration_cast(end - start).count(); + + } catch (const std::exception& e) { + std::cerr << "[QueryCoordinator::handle_nonbatched_job] Exception during scan for partition " + << job.partition_id << ", query " << job.query_id << ": " << e.what() + << ". Enqueuing empty result.\n"; + enqueue_result_job(ResultJob{job.query_id, job.rank, {}, {}}); // Enqueue empty result on error + } catch (...) { + std::cerr << "[QueryCoordinator::handle_nonbatched_job] Unknown exception during scan for partition " + << job.partition_id << ", query " << job.query_id + << ". Enqueuing empty result.\n"; + enqueue_result_job(ResultJob{job.query_id, job.rank, {}, {}}); // Enqueue empty result on error + } +} - auto job_wait_start = std::chrono::high_resolution_clock::now(); - res.job_queue.wait_dequeue(job); - auto job_wait_end = std::chrono::high_resolution_clock::now(); +void QueryCoordinator::handle_batched_job(const ScanJob &job, + CoreResources &res, + NUMAResources &nr) { - job_pull_time_ns += std::chrono::duration_cast(job_wait_end - job_wait_start).count(); - auto job_process_start = std::chrono::high_resolution_clock::now(); - shared_ptr local_topk_buffer = res.topk_buffer_pool[0]; - // Shutdown signal: -1 indicates the worker should exit. - if (job.partition_id == -1) { - break; - } + auto start = std::chrono::high_resolution_clock::now(); + auto s1 = std::chrono::high_resolution_clock::now(); + // Total queries, Top-K, dimension, NUMA node + int64_t Q = job.num_queries; + int K = job.k; + int D = partition_manager_->d(); + int node = cpu_numa_node(res.core_id); + MetricTracker* metrics_tracker = (MetricTracker*) nr.metric_tracker_ptr; - // Ignore this job if the global buffer is not processing queries. - if (!global_topk_buffer_pool_[job.query_ids[0]]->currently_processing_query()) { - // decrement the job counter - global_topk_buffer_pool_[job.query_ids[0]]->record_empty_job(); - continue; + // Fetch partition data + const float *codes = (float *) partition_manager_->partition_store_->get_codes(job.partition_id); + const int64_t *ids = partition_manager_->partition_store_->get_ids(job.partition_id); + int64_t part_size = partition_manager_->partition_store_->list_size(job.partition_id); + if (!codes || !ids || part_size <= 0) { + for (int64_t i = 0; i < Q; ++i) { + enqueue_result_job(ResultJob{(*job.query_ids)[i], (*job.ranks)[i], {}, {}}); } + return; + } - worker_job_counter_[core_index]++; - - // Retrieve partition data. - const float *partition_codes = (float *) partition_manager_->partition_store_->get_codes(job.partition_id); - const int64_t *partition_ids = (int64_t *) partition_manager_->partition_store_->get_ids(job.partition_id); - int64_t partition_size = partition_manager_->partition_store_->list_size(job.partition_id); + // 1) Prepare per-thread buffers *once* + size_t cap = std::min(100 * K, 10000); + int64_t queries_req = Q; + + if (res.topk_buffer_pool.size() < (size_t)queries_req) { + res.topk_buffer_pool.resize(queries_req); + for (size_t i = 0; i < (size_t)queries_req; ++i) { + res.topk_buffer_pool[i] = + std::make_shared(K, + metric_ == faiss::METRIC_INNER_PRODUCT, + cap, + node); + } + } - // Branch for non-batched jobs. - if (!job.is_batched) { + ensure_blas_buffers(res, Q, BLAS_DB_BS, node); - // Allocate a thread-local query buffer if needed. - if (res.local_query_buffer.size() < partition_manager_->d() * sizeof(float)) { - res.local_query_buffer.resize(partition_manager_->d() * sizeof(float)); - } + size_t max_q = size_t(queries_req) * D; + if (res.batch_q_capacity < max_q) { + quake_free(res.batch_queries, res.batch_q_capacity * sizeof(float)); + res.batch_queries = static_cast(quake_alloc(max_q * sizeof(float), node)); + res.batch_q_capacity = max_q; + } - // Copy the contents of the query vector to the local buffer using memcpy. - if (memcpy(res.local_query_buffer.data(), job.query_vector, partition_manager_->d() * sizeof(float)) == nullptr) { - throw std::runtime_error("[partition_scan_worker_fn] memcpy failed."); - } + // reset only the first 'chunk' TopK buffers + for (int64_t i = 0; i < Q; ++i) { + auto &buf = res.topk_buffer_pool[i]; + buf->set_k(K); + buf->reset(); + } - if (local_topk_buffer == nullptr) { - throw std::runtime_error("[partition_scan_worker_fn] local_topk_buffer is null."); - } else { - local_topk_buffer->set_k(job.k); - local_topk_buffer->reset(); - } - // Perform the scan on the partition. - scan_list((float *) res.local_query_buffer.data(), - partition_codes, - partition_ids, - partition_size, - partition_manager_->d(), - *local_topk_buffer, - metric_); - - vector topk = local_topk_buffer->get_topk(); - vector topk_indices = local_topk_buffer->get_topk_indices(); - int64_t n_results = topk_indices.size(); - - // Merge local results into the global query buffer. - global_topk_buffer_pool_[job.query_ids[0]]->batch_add(topk.data(), topk_indices.data(), n_results); - job_flags_[job.query_ids[0]][job.rank] = true; - } - // Batched job branch. - else { - if (!job.query_vector || job.num_queries == 0) { - throw std::runtime_error("[partition_scan_worker_fn] Invalid batched job."); - } + auto s2 = std::chrono::high_resolution_clock::now(); + + // // init only the first chunk*K slots in scratch + // float init_val = (metric_ == faiss::METRIC_INNER_PRODUCT) + // ? -std::numeric_limits::infinity() + // : std::numeric_limits::infinity(); + // std::fill_n(res.batch_distances, Q * K, init_val); + // std::fill_n(res.batch_ids, Q * K, -1LL); + + // auto + + // gather queries + float *qptr = nullptr; + float *dst = res.batch_queries; + vector query_ids; + vector ranks; + query_ids.reserve(Q); + ranks.reserve(Q); + int64_t offset = 0; + for (int64_t i = 0; i < Q; ++i) { + + int qid = (*job.query_ids)[i]; + int qrank = (*job.ranks)[i]; + + // check that the job has not been processed yet + if (job_flags_[qid][qrank]) { + // already processed, skip + enqueue_result_job(ResultJob{qid, qrank, {}, {}});; + } else { + // copy query vector to the local buffer + const float *src = nr.local_query_buffer + size_t(qid) * D; + std::memcpy(dst + offset, src, D * sizeof(float)); + query_ids.push_back(qid); + ranks.push_back(qrank); + offset += D; + } + } + qptr = dst; - // Allocate a thread-local query buffer if needed. - if (res.local_query_buffer.size() < partition_manager_->d() * sizeof(float) * job.num_queries) { - res.local_query_buffer.resize(partition_manager_->d() * sizeof(float) * job.num_queries); - } + vector *> pivots; + pivots.resize(query_ids.size()); + for (int64_t i = 0; i < query_ids.size(); ++i) { + int qid = query_ids[i]; + pivots[i] = &query_dist_pivots_[qid]; + } - int64_t d = partition_manager_->d(); - std::vector query_subset(job.num_queries * d); - for (int i = 0; i < job.num_queries; i++) { - int64_t global_q = job.query_ids[i]; - memcpy(&query_subset[i * d], - job.query_vector + global_q * d, - d * sizeof(float)); - } - // Then copy query_subset into your local buffer if needed: - if(memcpy(res.local_query_buffer.data(), - query_subset.data(), - query_subset.size() * sizeof(float)) == nullptr) { - throw std::runtime_error("[partition_scan_worker_fn] memcpy failed."); - } + auto s3 = std::chrono::high_resolution_clock::now(); + + // check that things are on the proper NUMA node + // bool ok = true; + // ok = ok && verify_numa_locality(qptr, "qptr"); + // ok = ok && verify_numa_locality(codes, "codes"); + // ok = ok && verify_numa_locality(ids, "ids"); + // ok = ok && verify_numa_locality(res.batch_queries, "batch_queries"); + // ok = ok && verify_numa_locality(res.blas_ip_block, "blas_ip_block"); + // ok = ok && verify_numa_locality(res.blas_norms_x, "blas_norms_x"); + // ok = ok && verify_numa_locality(res.blas_norms_y, "blas_norms_y"); + // for (int64_t i = 0; i < Q; ++i) { + // ok = ok && verify_numa_locality(res.topk_buffer_pool[i]->ord_, "ord_"); + // ok = ok && verify_numa_locality(res.topk_buffer_pool[i]->vals_, "vals_"); + // ok = ok && verify_numa_locality(res.topk_buffer_pool[i]->ids_, "ids_"); + // } + // if (!ok) { + // std::cerr << "[QueryCoordinator::handle_batched_job] NUMA locality check failed.\n"; + // // throw std::runtime_error("NUMA locality check failed"); + // } + + // run the scan on this chunk + if constexpr(RUN_WITH_HARDWARE_COUNTERS) { + metrics_tracker->start(); + } - // Use a thread_local vector to hold one buffer per query. - if (res.topk_buffer_pool.size() < static_cast(job.num_queries)) { - res.topk_buffer_pool.resize(job.num_queries); - for (int64_t q = 0; q < job.num_queries; ++q) { - res.topk_buffer_pool[q] = std::make_shared(job.k, metric_ == faiss::METRIC_INNER_PRODUCT); - } - } else { - for (int64_t q = 0; q < job.num_queries; ++q) { - res.topk_buffer_pool[q]->set_k(job.k); - res.topk_buffer_pool[q]->reset(); - } - } + batched_scan_list( + qptr, + codes, ids, + query_ids.size(), part_size, D, + res.topk_buffer_pool, + metric_, + res.blas_ip_block, + res.blas_norms_x, + res.blas_norms_y, + QueryCoordinator::batch_scan_partition_chunk_size_, + Q, + pivots); + + if constexpr(RUN_WITH_HARDWARE_COUNTERS) { + metrics_tracker->stop_and_record(); + } - // Process the batched job. - batched_scan_list((float *) res.local_query_buffer.data(), - partition_codes, - partition_ids, - job.num_queries, - partition_size, - partition_manager_->d(), - res.topk_buffer_pool, - metric_); - - vector> topk_list(job.num_queries); - vector> topk_indices_list(job.num_queries); - for (int64_t q = 0; q < job.num_queries; q++) { - topk_list[q] = res.topk_buffer_pool[q]->get_topk(); - topk_indices_list[q] = res.topk_buffer_pool[q]->get_topk_indices(); - } + auto s4 = std::chrono::high_resolution_clock::now(); + auto end = std::chrono::high_resolution_clock::now(); + + int64_t scan_time = std::chrono::duration_cast(end - start).count(); + res.scan_time_ns += scan_time; + + // Record batch scan metadata + int64_t partition_bytes = part_size * D * sizeof(float) + part_size * sizeof(int64_t); + int64_t query_bytes = query_ids.size() * D * sizeof(float); + int64_t total_bytes = partition_bytes + query_bytes; + res.bytes_scan_total += total_bytes; + res.partition_size += part_size; + res.num_scan_jobs += 1; + res.per_job_scan_throughput += (1.0 * total_bytes)/scan_time; + + /* + res.faiss_norms_x_time_ns += batch_scan_info->faiss_norms_x_time_ns; + res.faiss_norms_y_time_ns += batch_scan_info->faiss_norms_y_time_ns; + res.sgemm_time_ns += batch_scan_info->sgemm_time_ns; + res.ip_to_l2_time_ns += batch_scan_info->ip_to_l2_time_ns; + res.top_k_buffer_add_ns += batch_scan_info->top_k_buffer_add_ns; + */ + + if constexpr(RUN_WITH_HARDWARE_COUNTERS) { + float measured_ipc = metrics_tracker->get_ipc(); + if(measured_ipc != -1.0) { + res.per_job_ipc += measured_ipc; + res.measured_ipc_count += 1; + } - for (int64_t q = 0; q < job.num_queries; q++) { - int64_t global_q = job.query_ids[q]; - int n_results = topk_indices_list[q].size(); - global_topk_buffer_pool_[global_q]->batch_add(topk_list[q].data(), topk_indices_list[q].data(), n_results); - } + float measured_cache_miss_rate = metrics_tracker->get_cache_miss_rate(); + if(measured_cache_miss_rate != -1.0) { + res.per_job_cache_miss_rate += measured_cache_miss_rate; + res.measured_cache_count += 1; } - auto job_process_end = std::chrono::high_resolution_clock::now(); - job_process_time_ns += std::chrono::duration_cast(job_process_end - job_process_start).count(); } -} -// Worker-Based Scan Implementation -shared_ptr QueryCoordinator::worker_scan( - Tensor x, - Tensor partition_ids, - shared_ptr search_params) { - if (!partition_manager_) { - throw std::runtime_error("[QueryCoordinator::worker_scan] partition_manager_ is null."); + start = std::chrono::high_resolution_clock::now(); + // collect results for this chunk + std::vector results_batch; + results_batch.reserve(query_ids.size()); + for (int64_t i = 0; i < query_ids.size(); ++i) { + int global_q = query_ids[i]; + int rank_q = ranks[i]; + job_flags_[global_q][rank_q].store(true, std::memory_order_relaxed); // Mark that the job has been processed + + auto tv = res.topk_buffer_pool[i]->get_topk(false); + auto ti = res.topk_buffer_pool[i]->get_topk_indices(false); + enqueue_result_job(ResultJob{global_q, rank_q, std::move(tv), std::move(ti)}); } - if (!x.defined() || x.size(0) == 0) { - auto empty_result = std::make_shared(); - empty_result->ids = torch::empty({0}, torch::kInt64); - empty_result->distances = torch::empty({0}, torch::kFloat32); - empty_result->timing_info = std::make_shared(); - return empty_result; + end = std::chrono::high_resolution_clock::now(); + auto s5 = std::chrono::high_resolution_clock::now(); + res.enqueue_time_ns += std::chrono::duration_cast(end - start).count(); + + // // print out debug timing info s1, ... s5 + // std::cout << "QueryCoordinator::handle_batched_job: " + // << "job_id: " << job.job_id + // << ", core_id: " << res.core_id + // << ", num_queries: " << job.num_queries + // << ", partition_id: " << job.partition_id + // << ", k: " << job.k + // << ", rank: " << job.rank + // << ", numa_node: " << node + // << ", preamble: " << std::chrono::duration_cast(s2 - s1).count() + // << ", query copy: " << std::chrono::duration_cast(s3 - s2).count() + // << ", scan: " << std::chrono::duration_cast(s4 - s3).count() + // << ", enqueue: " << std::chrono::duration_cast(end - s4).count() + // << std::endl; +} + +// --- replace old enqueue ------------------------------------------------- +inline void QueryCoordinator::enqueue_result_job(ResultJob job) +{ + if (job.query_id < 0) { + // Poison pill to stop the merge worker + for (auto& mr : merge_res_) { + mr.queue.enqueue(ResultJob{-1, 0, {}, {}}); + } + return; } + const size_t mid = static_cast(job.query_id) % num_merge_workers_; + merge_res_[mid].queue.enqueue(std::move(job)); +} - int64_t num_queries = x.size(0); - int64_t dimension = x.size(1); - int k = search_params->k; - int64_t nlist = partition_manager_->nlist(); - bool use_aps = (search_params->recall_target > 0.0 && !search_params->batched_scan); - auto timing_info = make_shared(); - timing_info->n_queries = num_queries; - timing_info->n_clusters = nlist; - timing_info->search_params = search_params; - float *x_ptr = x.data_ptr(); +void QueryCoordinator::init_global_buffers(int64_t nQ, + int K, + Tensor &partition_ids, + shared_ptr params) { + std::lock_guard lg(global_mutex_); - auto start_time = high_resolution_clock::now(); + // resize or reset + size_t query_capacity = nQ * K; + if(global_heap_vals_buffer_ == nullptr || global_heap_ids_buffer_ == nullptr || global_heap_buffer_capacity_ < query_capacity) { + // Free any existing buffers + if (global_heap_vals_buffer_ != nullptr) quake_free(global_heap_vals_buffer_, global_heap_buffer_capacity_ * sizeof(float)); + if (global_heap_ids_buffer_ != nullptr) quake_free(global_heap_ids_buffer_, global_heap_buffer_capacity_ * sizeof(int64_t)); - if (partition_ids.dim() == 1) { - partition_ids = partition_ids.unsqueeze(0).expand({num_queries, partition_ids.size(0)}); + // Allocate the new buffers + global_heap_vals_buffer_ = (float *) quake_alloc(query_capacity * sizeof(float), 0); + global_heap_ids_buffer_ = (int64_t *) quake_alloc(query_capacity * sizeof(int64_t), 0); + global_heap_buffer_capacity_ = query_capacity; } - job_flags_.clear(); - job_flags_.resize(num_queries); - for (int64_t q = 0; q < num_queries; q++) { - job_flags_[q] = vector>(partition_ids.size(1)); - for (int64_t p = 0; p < partition_ids.size(1); p++) { - job_flags_[q][p] = false; - } + std::fill_n(global_heap_ids_buffer_, query_capacity, -1); + + float max_val = (metric_ == faiss::METRIC_INNER_PRODUCT) + ? -std::numeric_limits::infinity() + : std::numeric_limits::infinity(); + + std::fill_n(global_heap_vals_buffer_, query_capacity, max_val); + + if (metric_ == faiss::METRIC_INNER_PRODUCT) { + global_max_heaps_ = std::make_shared< + faiss::HeapBlockResultHandler< + faiss::CMin>>(nQ, global_heap_vals_buffer_, global_heap_ids_buffer_, K); + } else { + global_min_heaps_ = std::make_shared< + faiss::HeapBlockResultHandler< + faiss::CMax>>(nQ, global_heap_vals_buffer_, global_heap_ids_buffer_, K); } - job_pull_time_ns = 0; - job_process_time_ns = 0; + for (auto& mr : merge_res_) { + mr.handlers.clear(); + mr.handlers.resize(nQ, nullptr); - { - std::lock_guard lock(global_mutex_); - if (global_topk_buffer_pool_.size() < static_cast(num_queries)) { - std::cout << "Resizing query_topk_buffers_ from " << global_topk_buffer_pool_.size() - << " to " << num_queries << std::endl; - int old_size = global_topk_buffer_pool_.size(); - global_topk_buffer_pool_.resize(num_queries); - for (int64_t q = old_size; q < num_queries; q++) { - global_topk_buffer_pool_[q] = std::make_shared(k, metric_ == faiss::METRIC_INNER_PRODUCT); + /* allocate handler objects once per query --------------------------- */ + if (metric_ == faiss::METRIC_INNER_PRODUCT) { + using H = faiss::HeapBlockResultHandler< + faiss::CMin>::SingleResultHandler; + for (int64_t q = 0; q < nQ; ++q) { + mr.handlers[q] = std::make_shared(*global_max_heaps_); + std::shared_ptr ip_ptr = std::static_pointer_cast(mr.handlers[q]); + ip_ptr->begin(q); } } else { - for (int64_t q = 0; q < num_queries; q++) { - global_topk_buffer_pool_[q]->set_k(k); - global_topk_buffer_pool_[q]->reset(); - global_topk_buffer_pool_[q]->set_processing_query(true); + using H = faiss::HeapBlockResultHandler< + faiss::CMax>::SingleResultHandler; + for (int64_t q = 0; q < nQ; ++q) { + mr.handlers[q] = std::make_shared(*global_min_heaps_); + std::shared_ptr lp_ptr = std::static_pointer_cast(mr.handlers[q]); + lp_ptr->begin(q); } } - for (int64_t q = 0; q < num_queries; q++) { - global_topk_buffer_pool_[q]->set_jobs_left(partition_ids.size(1)); + } + + query_dist_pivots_ = vector>(nQ); + query_done_flags_ = vector>(nQ); + for (int64_t q = 0; q < nQ; ++q) { + query_dist_pivots_[q].store(max_val, std::memory_order_relaxed); + query_done_flags_[q].store(false, std::memory_order_relaxed); + } +} + +void QueryCoordinator::copy_query_to_numa(const float *xptr, int64_t nQ, int64_t D) { + + for (int node = 0; node < get_num_numa_nodes(); ++node) { + auto &nr = numa_resources_[node]; + if (nr.buffer_size < size_t(nQ) * size_t(D) * sizeof(float)) { + quake_free(nr.local_query_buffer, nr.buffer_size); + nr.local_query_buffer = static_cast(quake_alloc( + size_t(nQ) * size_t(D) * sizeof(float), + node)); + nr.buffer_size = size_t(nQ) * size_t(D) * sizeof(float); } + std::memcpy(nr.local_query_buffer, + xptr, + size_t(nQ) * size_t(D) * sizeof(float)); } - auto end_time = high_resolution_clock::now(); - timing_info->buffer_init_time_ns = - duration_cast(end_time - start_time).count(); +} - start_time = high_resolution_clock::now(); - if (search_params->batched_scan) { - auto partition_ids_accessor = partition_ids.accessor(); +void QueryCoordinator::enqueue_scan_jobs(Tensor x, + Tensor partition_ids, + shared_ptr params) +{ + int64_t nQ = x.size(0), D = x.size(1); + float *xptr = x.data_ptr(); - std::unordered_map> per_partition_query_ids; - for (int64_t q = 0; q < num_queries; q++) { - for (int64_t p = 0; p < partition_ids.size(1); p++) { - int64_t pid = partition_ids_accessor[q][p]; - if (pid < 0) continue; - per_partition_query_ids[pid].push_back(q); + auto partition_ids_acc = partition_ids.accessor(); + + vector core_to_numa(num_workers_); + for (int i = 0; i < num_workers_; ++i) { + core_to_numa[i] = cpu_numa_node(i); + } + + // std::cout << "[enqueue_scan_jobs] Enqueuing jobs for " << nQ + // << " queries, " << partition_ids.size(1) + // << " partitions, k = " << params->k + // << ", batched scan: " << (params->batched_scan ? "yes" : "no") << std::endl; + + // Reset job state + next_job_id_ = 0; + total_left_.store(0, std::memory_order_relaxed); + job_flags_.clear(); + job_flags_.resize(nQ); + per_query_total_left_ = vector>(nQ); + max_rank_ = vector>(nQ); + for (int64_t q = 0; q < nQ; ++q) { + job_flags_[q] = vector>(partition_ids.size(1)); + + int valid_count = 0; + for (int p = 0; p < partition_ids.size(1); ++p) { + job_flags_[q][p].store(false); + if (partition_ids_acc[q][p] < 0) { + job_flags_[q][p] = true; + } else { + valid_count++; } } - for (auto &kv : per_partition_query_ids) { - ScanJob job; - job.is_batched = true; - job.partition_id = kv.first; - job.k = k; - job.query_vector = x.data_ptr(); - job.num_queries = kv.second.size(); - job.query_ids = kv.second; - int core_id = partition_manager_->get_partition_core_id(kv.first); - core_resources_[core_id].job_queue.enqueue(job); + per_query_total_left_[q].store(valid_count, std::memory_order_relaxed); + max_rank_[q].store(partition_ids.size(1) - 1, std::memory_order_relaxed); + } + job_buffer_.clear(); + job_buffer_.reserve(nQ * partition_ids.size(1)); + + auto pid_acc = partition_ids.accessor(); + if (!params->batched_scan) { + // one job per (q,p) + + // // order by query id then partition rank + // for (int64_t q = 0; q < nQ; ++q) { + // const float* qptr = xptr + q*D; + // for (int p = 0; p < partition_ids.size(1); ++p) { + // int64_t pid = pid_acc[q][p]; + // if (pid < 0) continue; + // ScanJob job; + // job.job_id = next_job_id_; + // job.is_batched = false; + // job.query_id = (int)q; + // job.partition_id = pid; + // job.k = params->k; + // job.rank = p; + // job_buffer_.push_back(job); + // numa_resources_[core_to_numa[pid % num_workers_]].job_queue.enqueue(next_job_id_); + // next_job_id_++; + // total_left_.fetch_add(1, std::memory_order_relaxed); + // } + // } + + // order by partition rank then query id + for (int p = 0; p < partition_ids.size(1); ++p) { + for (int64_t q = 0; q < nQ; ++q) { + int64_t pid = pid_acc[q][p]; + if (pid < 0) continue; + ScanJob job; + job.job_id = next_job_id_; + job.is_batched = false; + job.query_id = (int)q; + job.partition_id = pid; + job.k = params->k; + job.rank = p; + job_buffer_.push_back(job); + numa_resources_[core_to_numa[pid % num_workers_]].job_queue.enqueue(next_job_id_); + next_job_id_++; + total_left_.fetch_add(1, std::memory_order_relaxed); + } } } else { - auto partition_ids_accessor = partition_ids.accessor(); + /* Build per-partition query lists -------------------------------------- */ + std::unordered_map>> qlist; + for (int64_t q = 0; q < nQ; ++q) { + for (int p = 0; p < partition_ids.size(1); ++p) { + int64_t pid = pid_acc[q][p]; + if (pid >= 0) qlist[pid].emplace_back(q, p); + } + } - int64_t start = 0; - int64_t end = num_queries; - parallel_for(start, end, [&](int64_t q) { - for (int64_t p = 0; p < partition_ids.size(1); p++) { - int64_t pid = partition_ids_accessor[q][p]; - if (pid == -1) continue; + int nlist = partition_manager_->nlist(); + bool scan_all = false; + if (nlist == 1) { + scan_all = true; + } - ScanJob job; - job.is_batched = false; - job.query_ids = {q}; - job.partition_id = pid; - job.k = k; - job.query_vector = x_ptr + q * dimension; - job.num_queries = 1; - job.rank = p; - - int core_id = partition_manager_->get_partition_core_id(pid); - core_resources_[core_id].job_queue.enqueue(job); + /* Emit ScanJobs, already split into ≤ params->batch_size chunks -------------- */ + for (int64_t pid = 0; pid < (int64_t)qlist.size(); ++pid) { + auto &pairs = qlist[pid]; + if (pairs.empty()) continue; + + for (size_t off = 0; off < pairs.size(); off += params->batch_size) { + size_t chunk = std::min(params->batch_size, pairs.size() - off); + + // Split query / rank vectors for this chunk. + auto qids = std::make_shared>(); + auto ranks = std::make_shared>(); + qids ->reserve(chunk); + ranks->reserve(chunk); + for (size_t i = 0; i < chunk; ++i) { + qids ->push_back(pairs[off + i].first); + ranks->push_back(pairs[off + i].second); + } + + if (chunk < MIN_BATCH_SCAN_SIZE) { + for (size_t i = 0; i < chunk; ++i) { + int qid = qids->at(i); + int rank = ranks->at(i); + ScanJob job; + job.is_batched = false; + job.job_id = next_job_id_; + job.partition_id = pid; + job.k = params->k; + job.rank = rank; + job.query_id = qid; + job_buffer_.push_back(job); + numa_resources_[core_to_numa[pid % num_workers_]].job_queue.enqueue(next_job_id_); + next_job_id_++; + total_left_.fetch_add(1, std::memory_order_relaxed); + } + } else { + ScanJob job; + job.is_batched = true; + job.job_id = next_job_id_; + job.partition_id = pid; + job.k = params->k; + job.num_queries = static_cast(chunk); + job.query_ids = qids; + job.ranks = ranks; + job.scan_all = scan_all; + + job_buffer_.push_back(job); + + // Choose NUMA queue by partition-to-core mapping. + int core = pid % num_workers_; + int node = core_to_numa[core]; + numa_resources_[node].job_queue.enqueue(next_job_id_); + next_job_id_++; + total_left_.fetch_add(chunk, std::memory_order_relaxed); + } } - }, search_params->num_threads); + } } - end_time = high_resolution_clock::now(); - timing_info->job_enqueue_time_ns = duration_cast(end_time - start_time).count(); +} - auto last_flush_time = high_resolution_clock::now(); - vector> boundary_distances(num_queries); + +void QueryCoordinator::drain_and_apply_aps(Tensor queries, + Tensor partition_ids, + shared_ptr search_params, + std::shared_ptr timing) +{ + + int64_t nQ = queries.size(0), D = queries.size(1); + + auto start = std::chrono::high_resolution_clock::now(); + // compute boundary distances + bool use_aps = (search_params->recall_target > 0 && parent_); + vector> boundary_distances; + vector curr_radii; + vector> recall_profiles; + vector recall_profile_set; if (use_aps) { - for (int64_t q = 0; q < num_queries; q++) { - vector partition_ids_to_scan_vec = vector(partition_ids[q].data_ptr(), - partition_ids[q].data_ptr() + partition_ids[q].size(0)); - vector cluster_centroids = parent_->partition_manager_->get_vectors(partition_ids_to_scan_vec); - boundary_distances[q] = compute_boundary_distances(x[q], - cluster_centroids, - metric_ == faiss::METRIC_L2); + boundary_distances.resize(nQ); + recall_profiles.resize(nQ); + if (metric_ == faiss::METRIC_INNER_PRODUCT) { + curr_radii.resize(nQ, -std::numeric_limits::infinity()); + } else { + curr_radii.resize(nQ, std::numeric_limits::infinity()); + } + + recall_profile_set.resize(nQ, false); + for (int64_t q = 0; q < nQ; ++q) { + vector curr_pids_vec(partition_ids[q].data_ptr(), + partition_ids[q].data_ptr() + partition_ids[q].size(0)); + vector curr_centroids_vec = parent_->partition_manager_->get_vectors(curr_pids_vec); + boundary_distances[q] = compute_boundary_distances(queries[q], + curr_centroids_vec, + metric_ == faiss::METRIC_L2); } } + auto end = std::chrono::high_resolution_clock::now(); + timing->boundary_distance_time_ns = + std::chrono::duration_cast(end - start).count(); + + timing->aps_time_ns = 0; + while (total_left_.load(std::memory_order_relaxed) > 0) { + std::this_thread::sleep_for(std::chrono::microseconds(search_params->aps_flush_period_us)); - start_time = high_resolution_clock::now(); - last_flush_time = high_resolution_clock::now(); + if (use_aps) { + // compute recall profile and mark jobs complete + start = std::chrono::high_resolution_clock::now(); + for (int64_t q = 0; q < nQ; ++q) { + int n_left = per_query_total_left_[q].load(std::memory_order_relaxed); + bool query_done = query_done_flags_[q].load(std::memory_order_relaxed); + if (!query_done && n_left < partition_ids.size(1)) { - vector query_radius(num_queries, 0.0f); - vector> probs(num_queries); + float query_radius = query_dist_pivots_[q].load(std::memory_order_relaxed); - while (true) { - // check if all jobs have been processed - bool all_done = true; - for (int64_t q = 0; q < num_queries; q++) { - all_done = all_done && (global_topk_buffer_pool_[q]->jobs_left_ == 0); - } + // check if the query radius has changed + float rel_difference = std::abs((curr_radii[q] - query_radius) / query_radius); - if (all_done) { - break; - } + if (rel_difference > search_params->recompute_threshold) { + // compute the recall profile for queries that are in progress + vector curr_pids_vec(partition_ids[q].data_ptr(), + partition_ids[q].data_ptr() + partition_ids[q].size(0)); + + recall_profiles[q] = compute_recall_profile(boundary_distances[q], + query_radius, + D, + {}, + search_params->use_precomputed, + metric_ == faiss::METRIC_L2); + + curr_radii[q] = query_radius; + recall_profile_set[q] = true; - // check if recall target has been reached - if (use_aps && duration_cast(high_resolution_clock::now() - last_flush_time).count() - > search_params->aps_flush_period_us) { - for (int64_t q = 0; q < num_queries; q++) { - auto curr_buffer = global_topk_buffer_pool_[q]; - int scanned = curr_buffer->get_num_partitions_scanned(); - if (curr_buffer->currently_processing_query() && - scanned > 0 && scanned < (int) boundary_distances[q].size()) { - float radius = curr_buffer->get_kth_distance(); - - if (query_radius[q] != radius) { - query_radius[q] = radius; - - // recompute recall profile if the radius has changed - probs[q] = compute_recall_profile(boundary_distances[q], - radius, - dimension, - {}, - search_params->use_precomputed, - metric_ == faiss::METRIC_L2); } - float cum = 0.0f; - for (int i = 0; i < partition_ids.size(1); i++) { - if (job_flags_[q][i]) { - cum += probs[q][i]; + if (recall_profile_set[q]) { + float recall_estimate = 0.0f; + float sum = 0.0f; + int max_rank = 0; + + for (int p = 0; p < partition_ids.size(1); ++p) { + sum += recall_profiles[q][p]; + if (sum >= search_params->recall_target) { + max_rank = p; + break; + } + } + max_rank_[q].store(max_rank, std::memory_order_relaxed); + + int n_scanned = 0; + float partition_recall_in_flight = 0.0f; + int count_in_flight = 0; + for (int p = 0; p < partition_ids.size(1); ++p) { + if (job_flags_[q][p].load(std::memory_order_relaxed)) { + n_scanned++; + recall_estimate += recall_profiles[q][p]; + } else { + if (count_in_flight < num_workers_) { + count_in_flight++; + partition_recall_in_flight += recall_profiles[q][p]; + } + + } + } + + // add the in-flight recall to the estimate + recall_estimate += partition_recall_in_flight; + + if (recall_estimate >= search_params->recall_target) { + query_done_flags_[q].store(true, std::memory_order_relaxed); } - } - if (cum > search_params->recall_target) { - curr_buffer->set_processing_query(false); } } } - last_flush_time = high_resolution_clock::now(); + end = std::chrono::high_resolution_clock::now(); + timing->aps_time_ns += + std::chrono::duration_cast(end - start).count(); } - std::this_thread::sleep_for(microseconds(1)); + } - end_time = high_resolution_clock::now(); - timing_info->job_wait_time_ns = - duration_cast(end_time - start_time).count(); - // Aggregate results. - start_time = high_resolution_clock::now(); - auto topk_ids = torch::full({num_queries, k}, -1, torch::kInt64); - auto topk_dists = torch::full({num_queries, k}, - std::numeric_limits::infinity(), torch::kFloat32); - auto ids_accessor = topk_ids.accessor(); - auto dists_accessor = topk_dists.accessor(); - { - std::lock_guard lock(global_mutex_); - for (int64_t q = 0; q < num_queries; q++) { - auto topk = global_topk_buffer_pool_[q]->get_topk(); - auto ids = global_topk_buffer_pool_[q]->get_topk_indices(); - for (int i = 0; i < k; i++) { - if (i < (int) ids.size()) { - ids_accessor[q][i] = ids[i]; - dists_accessor[q][i] = topk[i]; - } else { - ids_accessor[q][i] = -1; - dists_accessor[q][i] = (metric_ == faiss::METRIC_INNER_PRODUCT) - ? -std::numeric_limits::infinity() - : std::numeric_limits::infinity(); - } + if (search_params->track_hits && maintenance_policy_) { + size_t partitions_per_query = partition_ids.size(1); + for (int64_t q = 0; q < nQ; ++q) { + std::vector scanned_ids; + scanned_ids.reserve(partitions_per_query); + for (int p = 0; p < partitions_per_query; ++p) { + if (job_flags_[q][p].load(std::memory_order_relaxed)) { + int64_t pid = partition_ids[q][p].item(); + if (pid < 0) continue; + scanned_ids.emplace_back(pid); + } } - timing_info->partitions_scanned = global_topk_buffer_pool_[q]->get_num_partitions_scanned(); + timing->partitions_scanned += scanned_ids.size(); + maintenance_policy_->record_query_hits(scanned_ids); } } - end_time = high_resolution_clock::now(); - timing_info->result_aggregate_time_ns = duration_cast(end_time - start_time).count(); - auto search_result = std::make_shared(); - search_result->ids = topk_ids; - search_result->distances = topk_dists; - search_result->timing_info = timing_info; +} - return search_result; +std::shared_ptr +QueryCoordinator::aggregate_scan_results(int64_t nQ, + int K, + shared_ptr timing, + Tensor out_ids, + Tensor out_dists) { + auto id_acc = out_ids.accessor(); + auto d_acc = out_dists.accessor(); + + + + + // copy results from the global heaps to the output tensors + if (metric_ == faiss::METRIC_INNER_PRODUCT) { + for (int64_t q = 0; q < nQ; ++q) { + + for (int64_t i = 0; i < K; ++i) { + id_acc[q][i] = global_max_heaps_->heap_ids_tab[q * K + i]; + d_acc [q][i] = global_max_heaps_->heap_dis_tab[q * K + i]; + } + } + } else { + for (int64_t q = 0; q < nQ; ++q) { + for (int64_t i = 0; i < K; ++i) { + id_acc[q][i] = global_min_heaps_->heap_ids_tab[q * K + i]; + d_acc [q][i] = global_min_heaps_->heap_dis_tab[q * K + i]; + } + } + } + + auto res = std::make_shared(); + res->ids = out_ids; + res->distances = out_dists; + res->timing_info = timing; + return res; +} + +std::shared_ptr QueryCoordinator::worker_scan( + Tensor x, + Tensor partition_ids, + std::shared_ptr params) +{ + int64_t nQ = x.size(0), D = x.size(1); + int K = params->k; + bool use_aps = (params->recall_target>0 && parent_); + + int64_t nJobsExpected = 0; + + auto timing = std::make_shared(); + timing->n_queries = nQ; + timing->n_clusters = partition_manager_->nlist(); + timing->search_params = params; + + // get initial values of the per-core resource timers; + vector core_wait_time_ns(num_workers_, 0); + vector core_process_time_ns(num_workers_, 0); + vector core_process_preamble_time_ns(num_workers_, 0); + vector core_enqueue_time_ns(num_workers_, 0); + vector core_job_time_ns(num_workers_, 0); + vector core_scan_time_ns(num_workers_, 0); + + for (int i = 0; i < num_workers_; ++i) { + core_wait_time_ns[i] = core_resources_[i].wait_time_ns; + core_process_time_ns[i] = core_resources_[i].process_time_ns; + core_process_preamble_time_ns[i] = core_resources_[i].process_preamble_time_ns; + core_enqueue_time_ns[i] = core_resources_[i].enqueue_time_ns; + core_job_time_ns[i] = core_resources_[i].job_time_ns; + core_scan_time_ns[i] = core_resources_[i].scan_time_ns; + + core_resources_[i].bytes_scan_total = 0; + core_resources_[i].partition_size = 0; + core_resources_[i].num_scan_jobs = 0; + core_resources_[i].per_job_scan_throughput = 0; + + core_resources_[i].per_job_ipc = 0; + core_resources_[i].measured_ipc_count = 0; + + core_resources_[i].per_job_cache_miss_rate = 0; + core_resources_[i].measured_cache_count = 0; + + core_resources_[i].faiss_norms_x_time_ns = 0; + core_resources_[i].faiss_norms_y_time_ns = 0; + core_resources_[i].sgemm_time_ns = 0; + core_resources_[i].ip_to_l2_time_ns = 0; + core_resources_[i].top_k_buffer_add_ns = 0; + } + + auto s1 = high_resolution_clock::now(); + + // 1) init global buffers & jobs_left + init_global_buffers(nQ, K, partition_ids, params); + + auto s2 = high_resolution_clock::now(); + + // 2) copy query vec to NUMA buffers + copy_query_to_numa(x.data_ptr(), nQ, D); + + auto s3 = high_resolution_clock::now(); + + // 3) enqueue jobs + enqueue_scan_jobs(x, partition_ids, params); + + auto s4 = high_resolution_clock::now(); + + Tensor out_ids = torch::empty({nQ, K}, torch::kLong); + Tensor out_dists = torch::empty({nQ, K}, torch::kFloat); + + // 4) drain results + APS + drain_and_apply_aps(x, partition_ids, params, timing); + + auto s5 = high_resolution_clock::now(); + + auto res = aggregate_scan_results(nQ, K, timing, out_ids, out_dists); + + auto s6 = high_resolution_clock::now(); + + res->timing_info->buffer_init_time_ns = + duration_cast(s2 - s1).count(); + res->timing_info->copy_query_time_ns = + duration_cast(s3 - s2).count(); + res->timing_info->job_enqueue_time_ns = + duration_cast(s4 - s3).count(); + res->timing_info->job_wait_time_ns = + duration_cast(s5 - s4).count(); + res->timing_info->result_aggregate_time_ns = + duration_cast(s6 - s5).count(); + // + // // retrieve the final values of the per-core resource timers; + for (int i = 0; i < num_workers_; ++i) { + core_wait_time_ns[i] = core_resources_[i].wait_time_ns - core_wait_time_ns[i]; + core_process_time_ns[i] = core_resources_[i].process_time_ns - core_process_time_ns[i]; + core_process_preamble_time_ns[i] = core_resources_[i].process_preamble_time_ns - core_process_preamble_time_ns[i]; + core_enqueue_time_ns[i] = core_resources_[i].enqueue_time_ns - core_enqueue_time_ns[i]; + core_job_time_ns[i] = core_resources_[i].job_time_ns - core_job_time_ns[i]; + core_scan_time_ns[i] = core_resources_[i].scan_time_ns - core_scan_time_ns[i]; + } + + // add timers to timing info. average across all workers + int64_t num_workers = num_workers_; + double wait_time_ns = 0; + double process_time_ns = 0; + double process_preamble_time_ns = 0; + double enqueue_time_ns = 0; + double job_time_ns = 0; + double scan_time_ns = 0; + double scan_bytes = 0; + double scan_throughput = 0; + double partition_size = 0; + double worker_scan_time = 0; + double local_throughput = 0; + + double local_ipc = 0; int local_ipc_count = 0; + double local_cache_miss_rate = 0; double local_cache_count = 0; + int num_scan_jobs = 0; + + double faiss_norms_x_total = 0; double faiss_norms_y_total = 0; + double sgemm_time_total = 0; double ip_to_l2_time_total = 0; + double top_k_add_time_total = 0; + + for (int i = 0; i < num_workers; ++i) { + wait_time_ns += core_wait_time_ns[i]; + process_time_ns += core_process_time_ns[i]; + process_preamble_time_ns += core_process_preamble_time_ns[i]; + enqueue_time_ns += core_enqueue_time_ns[i]; + job_time_ns += core_job_time_ns[i]; + scan_time_ns += core_scan_time_ns[i]; + + partition_size += core_resources_[i].partition_size; + + if(core_resources_[i].bytes_scan_total > 0.0) { + scan_bytes += core_resources_[i].bytes_scan_total; + num_scan_jobs += core_resources_[i].num_scan_jobs; + worker_scan_time += core_scan_time_ns[i]; + local_throughput += core_resources_[i].per_job_scan_throughput; + + local_ipc += core_resources_[i].per_job_ipc; + local_ipc_count += core_resources_[i].measured_ipc_count; + + local_cache_miss_rate += core_resources_[i].per_job_cache_miss_rate; + local_cache_count += core_resources_[i].measured_cache_count; + + faiss_norms_x_total += core_resources_[i].faiss_norms_x_time_ns; + faiss_norms_y_total += core_resources_[i].faiss_norms_y_time_ns; + sgemm_time_total += core_resources_[i].sgemm_time_ns; + ip_to_l2_time_total += core_resources_[i].ip_to_l2_time_ns; + top_k_add_time_total += core_resources_[i].top_k_buffer_add_ns; + } + } + + res->timing_info->worker_wait_time_ns = wait_time_ns / num_workers; + res->timing_info->worker_process_time_ns = process_time_ns / num_workers; + res->timing_info->worker_process_preamble_time_ns = process_preamble_time_ns / num_workers; + res->timing_info->worker_enqueue_time_ns = enqueue_time_ns / num_workers; + res->timing_info->worker_job_time_ns = job_time_ns / num_workers; + res->timing_info->worker_scan_time_ns = scan_time_ns / num_workers; + res->timing_info->worker_partition_size_bytes = scan_bytes / num_scan_jobs; + res->timing_info->worker_partition_size = partition_size / num_scan_jobs; + res->timing_info->total_worker_jobs = num_scan_jobs; + res->timing_info->local_scan_throughput = local_throughput / num_scan_jobs; + res->timing_info->worker_batch_scan_ipc = local_ipc / local_ipc_count; + res->timing_info->single_scan_job_time_ns = scan_time_ns / num_scan_jobs; + + res->timing_info->worker_batch_scan_miss_rate = local_cache_miss_rate / local_cache_count; + + if(res->timing_info->worker_scan_time_ns != 0) { + res->timing_info->worker_scan_throughput = scan_bytes / worker_scan_time; + } else { + res->timing_info->worker_scan_throughput = 0; + } + + res->timing_info->faiss_norms_x_time_ns = faiss_norms_x_total / num_scan_jobs; + res->timing_info->faiss_norms_y_time_ns = faiss_norms_y_total / num_scan_jobs; + res->timing_info->sgemm_time_ns = sgemm_time_total / num_scan_jobs; + res->timing_info->ip_to_l2_time_ns = ip_to_l2_time_total / num_scan_jobs; + res->timing_info->top_k_buffer_add_ns = top_k_add_time_total / num_scan_jobs; + + // // print out the per-core resource timers; + // for (int i = 0; i < num_workers_; ++i) { + // std::cout << "[QueryCoordinator::worker_scan] Core " << i << ": " + // << "job_counter=" << core_resources_[i].job_counter << " " + // << "queries_counter=" << core_resources_[i].queries_counter << " " + // << "wait_time_ms=" << (float) core_wait_time_ns[i] / 1e6 << " " + // << "scan_setup_time_ms=" << (float) core_scan_setup_time_ns[i] / 1e6 << " " + // << "scan_time_ms=" << (float) core_scan_time_ns[i] / 1e6 << " " + // << "scan_push_time_ms=" << (float) core_scan_push_time_ns[i] / 1e6 << " " + // << "process_time_ms=" << (float) core_process_time_ns[i] / 1e6 << " " + // << "process_preamble_time_ms=" << (float) core_process_preamble_time_ns[i] / 1e6 << " " + // << "enqueue_time_ms=" << (float) core_enqueue_time_ns[i] / 1e6 << " " + // << "job_time_ms=" << (float) core_job_time_ns[i] / 1e6 << std::endl; + // } + // + // // print out the main thread timers; + // std::cout << "[QueryCoordinator::worker_scan] Main thread: " + // << "buffer_init_time_ms=" << (float) res->timing_info->buffer_init_time_ns / 1e6 << " " + // << "copy_query_time_ms=" << (float) res->timing_info->copy_query_time_ns / 1e6 << " " + // << "job_enqueue_time_ms=" << (float) res->timing_info->job_enqueue_time_ns / 1e6 << " " + // << "job_wait_time_ms=" << (float) res->timing_info->job_wait_time_ns / 1e6 << " " + // << "result_aggregate_time_ms=" << (float) res->timing_info->result_aggregate_time_ns / 1e6 + // << std::endl; + + return res; +} + +// Initialize Worker Threads +void QueryCoordinator::initialize_workers(int num_workers, int num_merge_workers, bool use_numa) { + if (workers_initialized_) { + std::cerr << "[QueryCoordinator::initialize_workers] Workers already initialized." << std::endl; + return; + } + + int num_cores = std::thread::hardware_concurrency(); + std::cout << "[QueryCoordinator::initialize_workers] Initializing " << num_workers << " worker threads with use_numa=" << use_numa + << " with " << num_cores << " cores" << std::endl; + + partition_manager_->distribute_partitions(num_workers, use_numa); + core_resources_.resize(num_workers); + worker_threads_.resize(num_workers); + stop_workers_.store(false); + for (int i = 0; i < num_workers; i++) { + if (!set_thread_affinity(i % num_cores)) { + std::cout << "[QueryCoordinator::initialize_workers] Failed to set thread affinity on core " << i << std::endl; + } + allocate_core_resources(i, 1, 10, partition_manager_->d()); + worker_threads_[i] = std::thread(&QueryCoordinator::partition_scan_worker_fn, this, i); + } + + merge_threads_.resize(num_merge_workers_); + merge_res_.resize(num_merge_workers_); + if (metric_ == faiss::METRIC_INNER_PRODUCT) { + // CMin: we want largest-inner-product first + for (int i = 0; i < num_merge_workers_; ++i) { + merge_threads_[i] = std::thread( + &QueryCoordinator::merge_worker_fn>, + this, i); + } + } else { + // CMax: we want smallest-L2 first + for (int i = 0; i < num_merge_workers_; ++i) { + merge_threads_[i] = std::thread( + &QueryCoordinator::merge_worker_fn>, + this, i); + } + } + + workers_initialized_ = true; + + // set main thread on separate thread from workers + int num_cores_on_machine = std::thread::hardware_concurrency(); + set_thread_affinity(num_workers + num_merge_workers); + // set_thread_affinity(0); +} + +// Shutdown Worker Threads +void QueryCoordinator::shutdown_workers() { + if (!workers_initialized_) { + return; + } + + stop_workers_.store(true); + // Enqueue poison pills to all worker threads. + for (auto &res : numa_resources_) { + for (int i = 0; i < num_workers_; ++i) + res.job_queue.enqueue(-1); + } + + // Enqueue poison pills to all merge workers. + for (int m = 0; m < num_merge_workers_; ++m) { + merge_res_[m].queue.enqueue(ResultJob{-1, 0, {}, {}}); + } + + // Join all worker threads. + for (auto &thr : worker_threads_) { + if (thr.joinable()) + thr.join(); + } + + for (auto &t : merge_threads_) { + if (t.joinable()) + t.join(); + } + + + merge_threads_.clear(); + worker_threads_.clear(); + workers_initialized_ = false; } shared_ptr QueryCoordinator::serial_scan(Tensor x, Tensor partition_ids, @@ -514,7 +1478,12 @@ shared_ptr QueryCoordinator::serial_scan(Tensor x, Tensor partitio // Use our custom parallel_for to process queries in parallel. parallel_for(0, num_queries, [&](int64_t q) { // Create a local TopK buffer for query q. - auto topk_buf = std::make_shared(k, is_descending); + + auto t1 = high_resolution_clock::now(); + + auto topk_buf = std::make_shared(k, is_descending, + /*cap=*/10 * k, + /*node=*/0); const float* query_vec = x_ptr + q * dimension; int num_parts = partition_ids.size(1); @@ -525,16 +1494,38 @@ shared_ptr QueryCoordinator::serial_scan(Tensor x, Tensor partitio query_radius = -1000000.0; } + auto t2 = high_resolution_clock::now(); + Tensor partition_sizes = partition_manager_->get_partition_sizes(partition_ids[q]); + vector partition_sizes_vec = vector(partition_sizes.data_ptr(), + partition_sizes.data_ptr() + partition_sizes.size(0)); + auto t3 = high_resolution_clock::now(); if (use_aps) { vector partition_ids_to_scan_vec = std::vector(partition_ids[q].data_ptr(), partition_ids[q].data_ptr() + partition_ids[q].size(0)); + vector cluster_centroids = parent_->partition_manager_->get_vectors(partition_ids_to_scan_vec); + t3 = high_resolution_clock::now(); + + // trim nullptrs + cluster_centroids.erase(std::remove(cluster_centroids.begin(), cluster_centroids.end(), nullptr), + cluster_centroids.end()); + + boundary_distances = compute_boundary_distances(x[q], cluster_centroids, metric_ == faiss::METRIC_L2); } + auto t4 = high_resolution_clock::now(); + + int64_t scan_time = 0; + int64_t aps_time = 0; + + vector scanned_ids; + for (int p = 0; p < num_parts; p++) { + + auto curr_time = high_resolution_clock::now(); int64_t pi = partition_ids_accessor[q][p]; if (pi == -1) { @@ -552,37 +1543,70 @@ shared_ptr QueryCoordinator::serial_scan(Tensor x, Tensor partitio partition_manager_->partition_store_->list_size(pi), dimension, *topk_buf, - metric_); + metric_, + NULL); + scanned_ids.push_back(pi); float curr_radius = topk_buf->get_kth_distance(); float percent_change = abs(curr_radius - query_radius) / curr_radius; + auto end_time = high_resolution_clock::now(); + + scan_time += duration_cast(end_time - start_time).count(); + start_time = high_resolution_clock::now(); - if (use_aps) { - if (percent_change > search_params->recompute_threshold) { + bool first_list = (p == 0); + if (use_aps && curr_radius != 0) { + if (first_list || percent_change > search_params->recompute_threshold) { query_radius = curr_radius; - partition_probs = compute_recall_profile(boundary_distances, - query_radius, - dimension, - {}, - search_params->use_precomputed, - metric_ == faiss::METRIC_L2); + if (search_params->use_auncel) { + partition_probs = compute_recall_profile_auncel(boundary_distances, + query_radius, + search_params->k, + search_params->auncel_a, + search_params->auncel_b); + } else { + partition_probs = compute_recall_profile(boundary_distances, + query_radius, + dimension, + partition_sizes_vec, + search_params->use_precomputed, + metric_ == faiss::METRIC_L2); + } } float recall_estimate = 0.0; - for (int i = 0; i < p; i++) { + for (int i = 0; i < p + 1; i++) { recall_estimate += partition_probs[i]; } + end_time = high_resolution_clock::now(); + aps_time += duration_cast(end_time - start_time).count(); if (recall_estimate >= search_params->recall_target) { break; } } } + + timing_info->partitions_scanned = scanned_ids.size(); + + if (search_params->track_hits && maintenance_policy_) { + if (debug_) std::cout << "[QueryCoordinator::serial_scan] record_query_hits being called with " << scanned_ids.size() << " ids" << std::endl; + maintenance_policy_->record_query_hits(std::vector(scanned_ids.begin(), scanned_ids.end())); + } + // Retrieve the top-k results for query q. all_topk_dists[q] = topk_buf->get_topk(); all_topk_ids[q] = topk_buf->get_topk_indices(); + auto t5 = high_resolution_clock::now(); + + // std::cout << "Query " << q << " times: " << duration_cast(t2 - t1).count() << " " + // << duration_cast(t3 - t2).count() << " " + // << duration_cast(t4 - t3).count() << " " + // << duration_cast(t5 - t4).count() << std::endl; + // std::cout << "Scan time: " << scan_time / 1000.0 << " APS time: " << aps_time / 1000.0 << std::endl; }, search_params->num_threads); + // Aggregate per-query results into output tensors. auto ret_ids_accessor = ret_ids.accessor(); auto ret_dists_accessor = ret_dists.accessor(); @@ -616,24 +1640,38 @@ shared_ptr QueryCoordinator::search(Tensor x, shared_ptrk <= 0) { + throw std::runtime_error("[QueryCoordinator::search] k must be greater than 0."); + } + auto parent_timing_info = std::make_shared(); auto start = high_resolution_clock::now(); // if there is no parent, then the coordinator is operating on a flat index and we need to scan all partitions Tensor partition_ids_to_scan; - if (parent_ == nullptr) { + Tensor partition_distances; + if (parent_ == nullptr || search_params->scan_all) { // scan all partitions for each query - partition_ids_to_scan = torch::arange(partition_manager_->nlist(), torch::kInt64); + partition_ids_to_scan = partition_manager_->get_partition_ids(); + search_params->batched_scan = true; } else { auto parent_search_params = make_shared(); - - parent_search_params->recall_target = search_params->recall_target; - parent_search_params->use_precomputed = search_params->use_precomputed; - parent_search_params->recompute_threshold = search_params->recompute_threshold; - parent_search_params->batched_scan = true; + if (search_params->parent_params == nullptr) { +// parent_search_params->recall_target = .99; + parent_search_params->use_precomputed = search_params->use_precomputed; + parent_search_params->recompute_threshold = search_params->recompute_threshold; +// parent_search_params->initial_search_fraction = .5; + parent_search_params->batched_scan = false; + if (x.size(0) > 10) { + parent_search_params->batched_scan = true; + } + } else { + parent_search_params = search_params->parent_params; + } // if recall_target is set, we need an initial set of partitions to consider - if (parent_search_params->recall_target > 0.0 && !search_params->batched_scan) { + if (search_params->recall_target > 0.0) { int initial_num_partitions_to_search = std::max( (int) (partition_manager_->nlist() * search_params->initial_search_fraction), 1); parent_search_params->k = initial_num_partitions_to_search; @@ -643,9 +1681,20 @@ shared_ptr QueryCoordinator::search(Tensor x, shared_ptrsearch(x, parent_search_params); partition_ids_to_scan = parent_search_result->ids; + partition_distances = parent_search_result->distances; parent_timing_info = parent_search_result->timing_info; } + if (search_params->use_spann && partition_distances.defined()) { + // prune partitions based on relative distance compared to nearest centroid + partition_distances = partition_distances / partition_distances.select(1, 0).unsqueeze(0); + + Tensor mask = partition_distances.ge(search_params->spann_eps); + + // set mask partition ids to -1 + partition_ids_to_scan.masked_fill_(mask, -1); + } + auto search_result = scan_partitions(x, partition_ids_to_scan, search_params); search_result->timing_info->parent_info = parent_timing_info; @@ -653,11 +1702,27 @@ shared_ptr QueryCoordinator::search(Tensor x, shared_ptrtiming_info->total_time_ns = duration_cast(end - start). count(); + // print out the results and report curren_level_ + // std::cout << "[QueryCoordinator::search] " + // << "current_level_=" << current_level_ << " " + // << "n_queries=" << x.size(0) << " " + // << "k=" << search_params->k << " " + // << "batched_scan=" << search_params->batched_scan << " " + // << "total_time_ms=" << (float) search_result->timing_info->total_time_ns / 1e6 << " " + // << "parent_total_time_ms=" << (float) parent_timing_info->total_time_ns / 1e6 << " " << std::endl; + return search_result; } shared_ptr QueryCoordinator::scan_partitions(Tensor x, Tensor partition_ids, shared_ptr search_params) { + + if (partition_ids.dim() == 0) { + throw std::runtime_error("[QueryCoordinator::scan_partitions] partition_ids is empty."); + } + if (partition_ids.dim() == 1) { + partition_ids = partition_ids.unsqueeze(0).expand({x.size(0), partition_ids.size(0)}); + } if (workers_initialized_) { if (debug_) std::cout << "[QueryCoordinator::scan_partitions] Using worker-based scan." << std::endl; return worker_scan(x, partition_ids, search_params); @@ -691,109 +1756,120 @@ shared_ptr QueryCoordinator::batched_serial_scan( auto timing_info = std::make_shared(); auto start = high_resolution_clock::now(); - int64_t num_queries = x.size(0); + auto s1 = high_resolution_clock::now(); + x = x.contiguous(); + int num_queries = x.size(0); int k = (search_params && search_params->k > 0) ? search_params->k : 1; + int64_t d = partition_manager_->d(); + float* all_queries_ptr = x.data_ptr(); + + // Get the partition information + int64_t pid = partition_ids[0].item(); + const float *list_codes = (float *) partition_manager_->partition_store_->get_codes(pid); + const int64_t *list_ids = partition_manager_->partition_store_->get_ids(pid); + int64_t list_size = partition_manager_->partition_store_->list_size(pid); + + // Buffers for each query batch + int query_batch_size = search_params->batch_size; + int num_loop_iterations = (num_queries + query_batch_size - 1)/query_batch_size; + vector> global_buffers = create_buffers(query_batch_size, k, (metric_ == faiss::METRIC_INNER_PRODUCT), + /*cap=*/10 * k); + + // Initialize the result buffer + auto topk_ids = torch::full({num_queries, k}, -1, torch::kInt64); + auto topk_dists = torch::full({num_queries, k}, + (metric_ == faiss::METRIC_INNER_PRODUCT ? + -std::numeric_limits::infinity() : + std::numeric_limits::infinity()), torch::kFloat32); + auto topk_ids_accessor = topk_ids.accessor(); + auto topk_dists_accessor = topk_dists.accessor(); - // Global Top-K buffers: one for each query. - vector> global_buffers = create_buffers(num_queries, k, (metric_ == faiss::METRIC_INNER_PRODUCT)); - - // Ensure partition_ids is 2D. If it’s 1D, assume every query scans the same set. - if (partition_ids.dim() == 1) { - partition_ids = partition_ids.unsqueeze(0).expand({num_queries, partition_ids.size(0)}); - } - auto part_ids_accessor = partition_ids.accessor(); - int num_parts = partition_ids.size(1); - - // Group queries by partition ID. - std::unordered_map> queries_by_partition; - for (int64_t q = 0; q < num_queries; q++) { - for (int p = 0; p < num_parts; p++) { - int64_t pid = part_ids_accessor[q][p]; - if (pid < 0) continue; - queries_by_partition[pid].push_back(q); - } - } - - std::vector>> queries_vec; - queries_vec.reserve(queries_by_partition.size()); - for (const auto &entry : queries_by_partition) { - queries_vec.push_back(entry); - } - - parallel_for((int64_t) 0, (int64_t) queries_by_partition.size(), [&](int64_t i) { - int64_t pid = queries_vec[i].first; - auto query_indices = queries_vec[i].second; + auto e1 = high_resolution_clock::now(); + timing_info->buffer_init_time_ns = duration_cast(e1 - s1).count(); - // Create a tensor for the indices and then a subset of the queries. - Tensor indices_tensor = torch::tensor(query_indices, torch::kInt64); - Tensor x_subset = x.index_select(0, indices_tensor); - int64_t batch_size = x_subset.size(0); + int64_t job_setup_time = 0; + int64_t job_scan_time = 0; + int64_t job_enque_time = 0; + int64_t job_scan_bytes = 0; - // Get the partition’s data. - const float *list_codes = (float *) partition_manager_->partition_store_->get_codes(pid); - const int64_t *list_ids = partition_manager_->partition_store_->get_ids(pid); - int64_t list_size = partition_manager_->partition_store_->list_size(pid); - int64_t d = partition_manager_->d(); + for (int i = 0; i < num_loop_iterations; i++) { + auto s2 = high_resolution_clock::now(); + // Determine the queries this batch is processing + int batch_start_offset = i * query_batch_size; + int num_batch_queries = std::min(query_batch_size, num_queries - batch_start_offset); + float* batch_queries = all_queries_ptr + batch_start_offset * d; - // Create temporary Top-K buffers for this sub-batch. - vector> local_buffers = create_buffers(batch_size, k, (metric_ == faiss::METRIC_INNER_PRODUCT)); + // Reset the buffer where we will write the results + for(int j = 0; j < num_batch_queries; j++) { + global_buffers[j]->reset(); + } + auto e2 = high_resolution_clock::now(); + job_setup_time += duration_cast(e2 - s2).count(); // Perform a single batched scan on the partition. - batched_scan_list(x_subset.data_ptr(), + auto s3 = high_resolution_clock::now(); + batched_scan_list(batch_queries, list_codes, list_ids, - batch_size, + num_batch_queries, list_size, d, - local_buffers, - metric_); + global_buffers, + metric_, + nullptr, + nullptr, + nullptr, + BLAS_DB_BS, + DEFAULT_BLAS_Q_BS, + {}); + + auto e3 = high_resolution_clock::now(); + job_scan_bytes += (num_batch_queries + list_size) * d * sizeof(float) + list_size * sizeof(idx_t); + job_scan_time += duration_cast(e3 - s3).count(); // Merge the local results into the corresponding global buffers. - for (int i = 0; i < batch_size; i++) { - int global_q = query_indices[i]; - vector local_dists = local_buffers[i]->get_topk(); - vector local_ids = local_buffers[i]->get_topk_indices(); - // Merge: global buffer adds the new candidate distances/ids. - global_buffers[global_q]->batch_add(local_dists.data(), local_ids.data(), local_ids.size()); - } - - - }, search_params->num_threads); - - // Aggregate the final results into output tensors. - auto topk_ids = torch::full({num_queries, k}, -1, torch::kInt64); - auto topk_dists = torch::full({num_queries, k}, - (metric_ == faiss::METRIC_INNER_PRODUCT ? - -std::numeric_limits::infinity() : - std::numeric_limits::infinity()), torch::kFloat32); - auto topk_ids_accessor = topk_ids.accessor(); - auto topk_dists_accessor = topk_dists.accessor(); + auto s4 = high_resolution_clock::now(); + for(int j = 0; j < num_batch_queries; j++) { + int query_offset = batch_start_offset + j; + vector best_dists = global_buffers[j]->get_topk(); + vector best_ids = global_buffers[j]->get_topk_indices(); + const int n_results = std::min((int) best_dists.size(), k); + + #pragma unroll + for (int result_idx = 0; result_idx < n_results; result_idx++) { + topk_ids_accessor[query_offset][result_idx] = best_ids[result_idx]; + topk_dists_accessor[query_offset][result_idx] = best_dists[result_idx]; + } - for (int64_t q = 0; q < num_queries; q++) { - vector best_dists = global_buffers[q]->get_topk(); - vector best_ids = global_buffers[q]->get_topk_indices(); - int n_results = std::min((int) best_dists.size(), k); - for (int i = 0; i < n_results; i++) { - topk_ids_accessor[q][i] = best_ids[i]; - topk_dists_accessor[q][i] = best_dists[i]; - } - // Fill in remaining slots with defaults. - for (int i = n_results; i < k; i++) { - topk_ids_accessor[q][i] = -1; - topk_dists_accessor[q][i] = (metric_ == faiss::METRIC_INNER_PRODUCT) ? - -std::numeric_limits::infinity() : - std::numeric_limits::infinity(); + // Fill in remaining slots with defaults. + #pragma unroll + for (int result_idx = n_results; result_idx < k; result_idx++) { + topk_ids_accessor[query_offset][result_idx] = -1; + topk_dists_accessor[query_offset][result_idx] = (metric_ == faiss::METRIC_INNER_PRODUCT) ? + -std::numeric_limits::infinity() : + std::numeric_limits::infinity(); + } } - // Optionally record per-query partition scan counts here. + auto e4 = high_resolution_clock::now(); + job_enque_time += duration_cast(e4 - s4).count(); } + // Aggregate the final results into output tensors. auto end = high_resolution_clock::now(); timing_info->total_time_ns = duration_cast(end - start).count(); + timing_info->worker_process_preamble_time_ns = job_setup_time; + timing_info->worker_scan_time_ns = job_scan_time; + timing_info->worker_enqueue_time_ns = job_enque_time; + timing_info->worker_partition_size_bytes = job_scan_bytes; + timing_info->worker_scan_throughput = (1.0 * job_scan_bytes)/job_scan_time; + timing_info->worker_partition_size = list_size; + timing_info->total_worker_jobs = 1; + // Prepare and return the final search result. auto search_result = std::make_shared(); search_result->ids = topk_ids; search_result->distances = topk_dists; search_result->timing_info = timing_info; return search_result; -} +} \ No newline at end of file diff --git a/src/cpp/third_party/cmake/fetch_rapids.cmake b/src/cpp/third_party/cmake/fetch_rapids.cmake new file mode 100644 index 00000000..0ec5fa04 --- /dev/null +++ b/src/cpp/third_party/cmake/fetch_rapids.cmake @@ -0,0 +1,22 @@ +# ============================================================================= +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. + +# Use this variable to update RAPIDS and cuVS versions +set(RAPIDS_VERSION "25.06") +set(rapids-cmake-version ${RAPIDS_VERSION}) + +if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/CUVS_RAPIDS.cmake) + file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION}/RAPIDS.cmake + ${CMAKE_CURRENT_BINARY_DIR}/CUVS_RAPIDS.cmake) +endif() +include(${CMAKE_CURRENT_BINARY_DIR}/CUVS_RAPIDS.cmake) diff --git a/src/cpp/third_party/cmake/get_cuvs.cmake b/src/cpp/third_party/cmake/get_cuvs.cmake new file mode 100644 index 00000000..a2a4e884 --- /dev/null +++ b/src/cpp/third_party/cmake/get_cuvs.cmake @@ -0,0 +1,56 @@ +# ============================================================================= +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. + +# Use RAPIDS_VERSION from cmake/thirdparty/fetch_rapids.cmake +set(CUVS_VERSION "${RAPIDS_VERSION}") +set(CUVS_FORK "rapidsai") +set(CUVS_PINNED_TAG "branch-${RAPIDS_VERSION}") + +function(find_and_configure_cuvs) + set(oneValueArgs VERSION FORK PINNED_TAG ENABLE_NVTX BUILD_CUVS_C_LIBRARY) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN} ) + + + set(CUVS_COMPONENTS "") + if(PKG_BUILD_CUVS_C_LIBRARY) + string(APPEND CUVS_COMPONENTS " c_api") + endif() + #----------------------------------------------------- + # Invoke CPM find_package() + #----------------------------------------------------- + rapids_cpm_find(cuvs ${PKG_VERSION} + GLOBAL_TARGETS cuvs::cuvs + BUILD_EXPORT_SET cuvs-examples-exports + INSTALL_EXPORT_SET cuvs-examples-exports + COMPONENTS ${CUVS_COMPONENTS} + CPM_ARGS + GIT_REPOSITORY https://github.com/${PKG_FORK}/cuvs.git + GIT_TAG ${PKG_PINNED_TAG} + SOURCE_SUBDIR cpp + OPTIONS + "BUILD_C_LIBRARY ${PKG_BUILD_CUVS_C_LIBRARY}" + "BUILD_TESTS OFF" + "CUVS_NVTX ${PKG_ENABLE_NVTX}" + ) +endfunction() + +# Change pinned tag here to test a commit in CI +# To use a different CUVS locally, set the CMake variable +# CPM_cuvs_SOURCE=/path/to/local/cuvs +find_and_configure_cuvs(VERSION ${CUVS_VERSION}.00 + FORK ${CUVS_FORK} + PINNED_TAG ${CUVS_PINNED_TAG} + ENABLE_NVTX OFF + BUILD_CUVS_C_LIBRARY ${BUILD_CUVS_C_LIBRARY} +) diff --git a/src/python/datasets/ann_datasets.py b/src/python/datasets/ann_datasets.py index 89297392..44e7bdc9 100644 --- a/src/python/datasets/ann_datasets.py +++ b/src/python/datasets/ann_datasets.py @@ -5,8 +5,9 @@ import numpy as np import torch +import os -from quake.utils import download_url, extract_file, fvecs_to_tensor, ivecs_to_tensor, to_path +from quake.utils import download_url, extract_file, fvecs_to_tensor, ivecs_to_tensor, to_path, knn DEFAULT_DOWNLOAD_DIR = Path("data/") @@ -71,14 +72,301 @@ def load_queries(self) -> Union[np.ndarray, torch.Tensor]: def load_ground_truth(self) -> Union[np.ndarray, torch.Tensor]: return ivecs_to_tensor(self.data_dir / "sift_groundtruth.ivecs") +class UniformDataset(Dataset): + def __init__(self, num_vectors: int, dim: int, download_dir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR): + self.num_vectors = num_vectors + self.dim = dim + self.download_dir = to_path(download_dir) + + def is_downloaded(self) -> bool: + # check if files exist + vecs = self.download_dir / "vectors.pt" + queries = self.download_dir / "queries.pt" + gt = self.download_dir / "ground_truth.pt" + + return vecs.exists() and queries.exists() and gt.exists() + + def download(self, overwrite: bool = False): + + vecs = np.random.uniform(0, 1, (self.num_vectors, self.dim)).astype(np.float32) + queries = np.random.uniform(0, 1, (1000, self.dim)).astype(np.float32) + gt = knn(queries, vecs, k=100)[0] + + # vecs = torch.randn(self.num_vectors, self.dim) + # queries = torch.randn(1000, self.dim) + # gt = knn(queries, vecs, k=100)[0] + + + np.save(self.download_dir / "vectors.npy", vecs) + np.save(self.download_dir / "queries.npy", queries) + np.save(self.download_dir / "ground_truth.npy", gt) + + def load_vectors(self) -> Union[np.ndarray, torch.Tensor]: + # return torch.randn(self.num_vectors, self.dim) + # Load the vectors from disk + return torch.from_numpy(np.load(self.download_dir / "vectors.npy")) + + def load_queries(self) -> Union[np.ndarray, torch.Tensor]: + # return torch.randn(self.num_vectors, self.dim) + # Load the queries from disk + return torch.from_numpy(np.load(self.download_dir / "queries.npy")) + + def load_ground_truth(self) -> Union[np.ndarray, torch.Tensor]: + # return torch.randint(0, self.num_vectors, (self.num_vectors, 100)) + # Load the ground truth from disk + return torch.from_numpy(np.load(self.download_dir / "ground_truth.npy")) + + +class MSTuring10m(Dataset): + def __init__(self, download_dir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR): + self.download_dir = to_path(download_dir) + self.data_dir = self.download_dir / "MSTuringANNS10M" + self.downloaded = False + self.metric = "l2" + + def is_downloaded(self) -> bool: + base = self.data_dir / "base1b.fbin.crop_nb_10000000" + query = self.data_dir / "query100K.fbin" + gt = self.data_dir / "msturing-gt-10M" + downloaded = base.exists() and query.exists() and gt.exists() + + if not downloaded: + raise RuntimeError("MSTuring dataset is not available. Please download it manually using big-ann-benchmarks.") + + return downloaded + + def download(self, overwrite: bool = False): + pass + + def load_vectors(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "base1b.fbin.crop_nb_10000000" + n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) + # return to_torch(np.fromfile(fname, dtype=np.float32, offset=8).reshape((n, d))) + return torch.from_numpy(np.fromfile(fname, dtype=np.float32, offset=8).reshape((n, d))) + + def load_queries(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "query100K.fbin" + n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) + # return to_torch(np.fromfile(fname, dtype=np.float32, offset=8).reshape((n, d))) + return torch.from_numpy(np.fromfile(fname, dtype=np.float32, offset=8).reshape((n, d))) + + def load_ground_truth(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "msturing-gt-10M" + n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) + assert os.stat(fname).st_size == 8 + n * d * (4 + 4) + f = open(fname, "rb") + f.seek(4 + 4) + ids = np.fromfile(f, dtype="int32", count=n * d).reshape(n, d) + # D = np.fromfile(f, dtype="float32", count=n * d).reshape(n, d) + + return torch.from_numpy(ids).long() + +class MSTuring100m(Dataset): + def __init__(self, download_dir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR): + self.download_dir = to_path(download_dir) + self.data_dir = self.download_dir / "MSTuringANNS100M" + self.downloaded = False + self.metric = "l2" + + def is_downloaded(self) -> bool: + base = self.data_dir / "base1b.fbin.crop_nb_100000000" + query = self.data_dir / "query100K.fbin" + gt = self.data_dir / "msturing-gt-100M" + downloaded = base.exists() and query.exists() and gt.exists() + + if not downloaded: + raise RuntimeError("MSTuring dataset is not available. Please download it manually using big-ann-benchmarks.") + + return downloaded + + def download(self, overwrite: bool = False): + pass + + def load_vectors(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "base1b.fbin.crop_nb_100000000" + n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) + # return to_torch(np.fromfile(fname, dtype=np.float32, offset=8).reshape((n, d))) + return torch.from_numpy(np.fromfile(fname, dtype=np.float32, offset=8).reshape((n, d))) + + def load_queries(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "query100K.fbin" + n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) + # return to_torch(np.fromfile(fname, dtype=np.float32, offset=8).reshape((n, d))) + return torch.from_numpy(np.fromfile(fname, dtype=np.float32, offset=8).reshape((n, d))) + + def load_ground_truth(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "msturing-gt-100M" + n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) + assert os.stat(fname).st_size == 8 + n * d * (4 + 4) + f = open(fname, "rb") + f.seek(4 + 4) + ids = np.fromfile(f, dtype="int32", count=n * d).reshape(n, d) + # D = np.fromfile(f, dtype="float32", count=n * d).reshape(n, d) + + return torch.from_numpy(ids).long() + +class Sift10m(Dataset): + def __init__(self, download_dir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR): + self.download_dir = to_path(download_dir) + self.data_dir = self.download_dir / "sift10m" + self.downloaded = False + self.metric = "l2" + + def is_downloaded(self) -> bool: + base = self.data_dir / "base.1B.u8bin.crop_nb_10000000" + query = self.data_dir / "query.public.10K.u8bin" + gt = self.data_dir / "bigann-10M" + return base.exists() and query.exists() and gt.exists() + + def download(self, overwrite: bool = False): + pass + + def load_vectors(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "base.1B.u8bin.crop_nb_10000000" + n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) + # return to_torch(np.fromfile(fname, dtype=np.uint8, offset=8).reshape((n, d))) + return torch.from_numpy(np.fromfile(fname, dtype=np.uint8, offset=8).reshape((n, d))).to(torch.float32) + + def load_queries(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "query.public.10K.u8bin" + n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) + # return to_torch(np.fromfile(fname, dtype=np.uint8, offset=8).reshape((n, d))) + return torch.from_numpy(np.fromfile(fname, dtype=np.uint8, offset=8).reshape((n, d))).to(torch.float32) + + def load_ground_truth(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "bigann-10M" + n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) + assert os.stat(fname).st_size == 8 + n * d * (4 + 4) + f = open(fname, "rb") + f.seek(4 + 4) + ids = np.fromfile(f, dtype="int32", count=n * d).reshape(n, d) + # D = np.fromfile(f, dtype="float32", count=n * d).reshape(n, d) + + return torch.from_numpy(ids).long() + +class Wiki13M(Dataset): + def __init__(self, download_dir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR): + self.download_dir = to_path(download_dir) + self.data_dir = self.download_dir / "wiki13m" + self.downloaded = False + self.metric = "ip" + + def is_downloaded(self) -> bool: + base = self.data_dir / "wiki.npy" + query = self.data_dir / "wiki_queries.npy" + gt = self.data_dir / "wiki_gt.npy" + return base.exists() and query.exists() and gt.exists() + + def download(self, overwrite: bool = False): + pass + + def load_vectors(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "wiki.npy" + return torch.from_numpy(np.load(fname)).to(torch.float32) + + def load_queries(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "wiki_queries.npy" + return torch.from_numpy(np.load(fname)).to(torch.float32) + + def load_ground_truth(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "wiki_gt.npy" + return torch.from_numpy(np.load(fname)).long() + + +class WikiCohere1M(Dataset): + def __init__(self, download_dir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR): + self.download_dir = to_path(download_dir) + self.data_dir = self.download_dir / "wikipedia_cohere" + self.downloaded = False + self.metric = "ip" + + def is_downloaded(self) -> bool: + base = self.data_dir / "wikipedia_base.bin.crop_nb_1000000" + query = self.data_dir / "wikipedia_query.bin" + gt = self.data_dir / " wikipedia-1M" + return base.exists() and query.exists() and gt.exists() + + def download(self, overwrite: bool = False): + pass + + def load_vectors(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "wikipedia_base.bin.crop_nb_1000000" + n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) + return torch.from_numpy(np.fromfile(fname, dtype=np.float32, offset=8).reshape((n, d))) + + def load_queries(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "wikipedia_query.bin" + n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) + return torch.from_numpy(np.fromfile(fname, dtype=np.float32, offset=8).reshape((n, d))) + + def load_ground_truth(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "wikipedia-1M" + n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) + assert os.stat(fname).st_size == 8 + n * d * (4 + 4) + f = open(fname, "rb") + f.seek(4 + 4) + ids = np.fromfile(f, dtype="int32", count=n * d).reshape(n, d) + return torch.from_numpy(ids).long() + +class T2I1M(Dataset): + def __init__(self, download_dir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR): + self.download_dir = to_path(download_dir) + self.data_dir = self.download_dir / "text2image1B" + self.downloaded = False + self.metric = "ip" + + def is_downloaded(self) -> bool: + base = self.data_dir / "base.1B.fbin.crop_nb_1000000" + query = self.data_dir / "query.heldout.30K.fbin" + gt = self.data_dir / "gt100-heldout.30K.fbin" + return base.exists() and query.exists() and gt.exists() + + def download(self, overwrite: bool = False): + pass + + def load_vectors(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "base.1B.fbin.crop_nb_1000000" + n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) + return torch.from_numpy(np.fromfile(fname, dtype=np.float32, offset=8).reshape((n, d))) + + def load_queries(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "query.heldout.30K.fbin" + n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) + return torch.from_numpy(np.fromfile(fname, dtype=np.float32, offset=8).reshape((n, d))) + + def load_ground_truth(self) -> Union[np.ndarray, torch.Tensor]: + fname = self.data_dir / "gt100-heldout.30K.fbin" + n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) + assert os.stat(fname).st_size == 8 + n * d * (4 + 4) + f = open(fname, "rb") + f.seek(4 + 4) + ids = np.fromfile(f, dtype="int32", count=n * d).reshape(n, d) + return torch.from_numpy(ids).long() + + + def load_dataset( name: str, download_dir: str = DEFAULT_DOWNLOAD_DIR, overwrite_download: bool = False ) -> List[Union[np.ndarray, torch.Tensor]]: if name.lower() == "sift1m": dataset = Sift1m(download_dir=download_dir) + elif name.lower() == "uniform": + dataset = UniformDataset(num_vectors=100000, dim=8, download_dir=download_dir) + elif name.lower() == "msturing10m": + dataset = MSTuring10m(download_dir=download_dir) + elif name.lower() == "msturing100m": + dataset = MSTuring100m(download_dir=download_dir) + elif name.lower() == "sift10m": + dataset = Sift10m(download_dir=download_dir) + elif name.lower() == "wiki13m": + dataset = Wiki13M(download_dir=download_dir) + elif name.lower() == "t2i1m": + dataset = T2I1M(download_dir=download_dir) + elif name.lower() == "wikicohere1m": + dataset = WikiCohere1M(download_dir=download_dir) else: - raise RuntimeError("Unimplemented dataset") + raise RuntimeError("Unimplemented dataset + " + name) dataset.download(overwrite=overwrite_download) return dataset.load() diff --git a/src/python/index_wrappers/diskann.py b/src/python/index_wrappers/diskann.py index 9b0d834f..340e6e86 100644 --- a/src/python/index_wrappers/diskann.py +++ b/src/python/index_wrappers/diskann.py @@ -5,8 +5,9 @@ import diskannpy as dap import numpy as np import torch +from pathlib import Path -from quake import SearchTimingInfo +from quake import SearchTimingInfo, SearchResult from quake.index_wrappers.wrapper import IndexWrapper from quake.utils import to_numpy, to_torch @@ -101,16 +102,28 @@ def search( timing_info = SearchTimingInfo() start = time.time() - indices, distances = self.index.batch_search( - query, k_neighbors=k, complexity=complexity, num_threads=num_threads - ) + if query.shape[0] == 1: + indices, distances = self.index.search( + query.flatten(), k_neighbors=k, complexity=complexity) + indices = indices.reshape(1, -1) + distances = distances.reshape(1, -1) + else: + # batch search + indices, distances = self.index.batch_search( + query, k_neighbors=k, complexity=complexity, num_threads=num_threads + ) end = time.time() - timing_info.total_time_us = int((end - start) * 1e6) + timing_info.total_time_ns = int((end - start) * 1e9) indices = to_torch(indices.astype(np.int64)) indices = indices - 1 # ids are 1-indexed distances = to_torch(distances) - return indices, distances, timing_info + search_result = SearchResult() + search_result.ids = to_torch(indices) + search_result.distances = to_torch(distances) + search_result.timing_info = timing_info + + return search_result def save(self, path: str): """ @@ -119,13 +132,10 @@ def save(self, path: str): :param path: The path to save the index to. :type path: str """ - # assert self.index is not None - # if not Path(path).exists(): - # Path(path).mkdir(exist_ok=True) - # self.index.save(path) - - # TODO: save/load index - raise RuntimeError("DiskANNDynamic.save() not implemented") + assert self.index is not None + if not Path(path).exists(): + Path(path).mkdir(exist_ok=True) + self.index.save(path) def load( self, @@ -141,16 +151,16 @@ def load( :param path: The path to load the index from. :type path: str """ - # self.index = dap.DynamicMemoryIndex.from_file( - # index_directory=path, - # max_vectors=max_vectors, - # complexity=complexity, - # graph_degree=graph_degree, - # num_threads=num_threads, - # ) + self.index = dap.DynamicMemoryIndex.from_file( + index_directory=path, + max_vectors=max_vectors, + complexity=complexity, + graph_degree=graph_degree, + num_threads=num_threads, + ) - # TODO: save/load index - raise RuntimeError("DiskANNDynamic.save() not implemented") + # # TODO: save/load index + # raise RuntimeError("DiskANNDynamic.save() not implemented") def add(self, vectors: torch.Tensor, ids: torch.Tensor, num_threads: int = 0): """ @@ -190,3 +200,9 @@ def remove(self, ids: torch.Tensor, lazy: bool = True): def centroids(self) -> torch.Tensor | None: return super().centroids() + + def index_state(self) -> str: + return "" + + def maintenance(self): + pass \ No newline at end of file diff --git a/src/python/index_wrappers/faiss_hnsw.py b/src/python/index_wrappers/faiss_hnsw.py index 9fc7dcc2..eed4a1ea 100644 --- a/src/python/index_wrappers/faiss_hnsw.py +++ b/src/python/index_wrappers/faiss_hnsw.py @@ -4,7 +4,7 @@ import faiss import torch -from quake import SearchTimingInfo +from quake import SearchTimingInfo, SearchResult from quake.index_wrappers.faiss_ivf import metric_str_to_faiss from quake.index_wrappers.wrapper import IndexWrapper from quake.utils import to_numpy, to_torch @@ -85,11 +85,15 @@ def search(self, query: torch.Tensor, k: int, ef_search: int = 16) -> Tuple[torc distances, indices = self.index.search(query, k) end = time.time() - timing_info.total_time_us = int((end - start) * 1e6) - distances = to_torch(distances) - indices = to_torch(indices) + timing_info.total_time_ns = int((end - start) * 1e9) + + search_result = SearchResult() + search_result.ids = to_torch(indices) + search_result.distances = to_torch(distances) + search_result.timing_info = timing_info + + return search_result - return indices, distances, timing_info def save(self, filename: str): """ @@ -130,3 +134,9 @@ def remove(self, ids: torch.Tensor): """ # throw a runtime error raise RuntimeError("Faiss HNSW does not support removal of vectors.") + + def index_state(self) -> str: + return "" + + def maintenance(self): + pass \ No newline at end of file diff --git a/src/python/index_wrappers/faiss_ivf.py b/src/python/index_wrappers/faiss_ivf.py index 4326cfac..225a3ea3 100644 --- a/src/python/index_wrappers/faiss_ivf.py +++ b/src/python/index_wrappers/faiss_ivf.py @@ -233,6 +233,7 @@ def search( search_result.ids = to_torch(indices) search_result.distances = to_torch(distances) search_result.timing_info = timing_info + search_result.timing_info.parent_info = SearchTimingInfo() return search_result @@ -259,7 +260,7 @@ def save(self, filename: str): """ faiss.write_index(self.index, str(filename)) - def load(self, filename: str): + def load(self, filename: str, **kwargs): """ Load the index from a file. diff --git a/src/python/index_wrappers/quake.py b/src/python/index_wrappers/quake.py index 6c1fffb8..d6558d59 100644 --- a/src/python/index_wrappers/quake.py +++ b/src/python/index_wrappers/quake.py @@ -42,10 +42,21 @@ def index_state(self) -> dict: :return: The state of the index as a dictionary. """ - return { + state = { "n_list": self.index.nlist(), "n_total": self.index.ntotal(), } + # # if has parent get the parent nlist + # if self.index.parent.parent: + # state["n_list1"] = self.index.parent.parent.nlist() + + # get all the parents + curr = self.index.parent + while curr.parent: + state["n_list1"] = curr.nlist() + curr = curr.parent + + return state def build( self, @@ -54,8 +65,14 @@ def build( metric: str = "l2", ids: Optional[torch.Tensor] = None, num_workers: int = 0, + num_merge_workers: int = 1, m: int = -1, code_size: int = 8, + parent=None, + use_gpu=False, + use_numa=False, + gpu_batch_size=100000, + gpu_sample_size=1000000 ): """ Build the index with the given vectors and arguments. @@ -78,6 +95,19 @@ def build( build_params.metric = metric build_params.nlist = nc build_params.num_workers = num_workers + build_params.num_merge_workers = num_merge_workers + build_params.use_numa = use_numa + + if parent is not None: + build_params.parent_params = quake.IndexBuildParams() + build_params.parent_params.nlist = parent.get("nc", 1) + build_params.parent_params.num_workers = parent.get("num_workers", 0) + build_params.parent_params.num_merge_workers = parent.get("num_merge_workers", 1) + + + build_params.use_gpu = use_gpu + build_params.gpu_batch_size = gpu_batch_size + build_params.gpu_sample_size = gpu_sample_size self.index = QuakeIndex() @@ -125,6 +155,10 @@ def search( recompute_threshold=0.1, aps_flush_period_us=50, n_threads=1, + parent=None, + sample_prefix=0, + sample_stride=5, + batch_size=128, ) -> Tuple[torch.Tensor, torch.Tensor]: """ Find the k-nearest neighbors of the query vectors. @@ -145,6 +179,17 @@ def search( search_params.aps_flush_period_us = aps_flush_period_us search_params.k = k search_params.num_threads = n_threads + search_params.sample_prefix = sample_prefix + search_params.sample_stride = sample_stride + search_params.batch_size = batch_size + + if parent is not None: + search_params.parent_params = quake.SearchParams() + search_params.parent_params.nprobe = parent.get("nprobe", 1) + search_params.parent_params.recall_target = parent.get("recall_target", -1) + search_params.parent_params.initial_search_fraction = parent.get("initial_search_fraction", 0.05) + search_params.parent_params.batch_size = parent.get("batch_size", 128) + search_params.parent_params.batched_scan = parent.get("batched_scan", search_params.batched_scan) return self.index.search(query, search_params) @@ -166,13 +211,11 @@ def save(self, filename: str): def load( self, filename: str, - n_workers: int = 0, + num_workers: int = 0, + num_merge_workers: int = 1, use_numa: bool = False, + parent: dict = None, verbose: bool = False, - verify_numa: bool = False, - same_core: bool = True, - use_centroid_workers: bool = False, - use_adaptive_n_probe: bool = False, ): """ Load the index from a file. @@ -180,11 +223,18 @@ def load( :param filename: The name of the file to load the index from. """ print( - f"Loading index from {filename}, with {n_workers} workers, use_numa={use_numa}, verbose={verbose}, " - f"verify_numa={verify_numa}, same_core={same_core}, use_centroid_workers={use_centroid_workers}" + f"Loading index from {filename}, with {num_workers} workers, use_numa={use_numa}, parent={parent}" ) self.index = QuakeIndex() - self.index.load(str(filename), n_workers) + build_params = quake.IndexBuildParams() + build_params.num_workers = num_workers + build_params.use_numa = use_numa + build_params.parent_params = quake.IndexBuildParams() + if parent is not None: + build_params.parent_params.num_workers = parent.get("num_workers", 0) + build_params.parent_params.num_merge_workers = parent.get("num_merge_workers", 1) + build_params.parent_params.use_numa = parent.get("use_numa", build_params.use_numa) + self.index.load(str(filename), build_params) def centroids(self) -> torch.Tensor: """ diff --git a/src/python/index_wrappers/scann.py b/src/python/index_wrappers/scann.py index 2b224026..1c437d5f 100644 --- a/src/python/index_wrappers/scann.py +++ b/src/python/index_wrappers/scann.py @@ -7,8 +7,9 @@ import torch from quake.index_wrappers.wrapper import IndexWrapper +from quake import SearchTimingInfo, SearchResult from quake.utils import to_numpy, to_torch - +import time class Scann(IndexWrapper): index: scann.scann_ops_pybind.ScannSearcher @@ -129,6 +130,8 @@ def search( if not n_threads == 0: self.index.set_num_threads(n_threads) + start_time = time.time() + query = to_numpy(query) indices, distances = self.index.search_batched( query, @@ -137,8 +140,16 @@ def search( pre_reorder_num_neighbors=pre_reorder_num_neighbors, ) indices = np.array(indices) + end_time = time.time() + timing_info = SearchTimingInfo() + timing_info.total_time_ns = int((end_time - start_time) * 1e9) + + search_result = SearchResult() + search_result.ids = to_torch(indices) + search_result.distances = to_torch(distances) + search_result.timing_info = timing_info - return to_torch(indices), to_torch(distances) + return search_result def add(self, vectors: torch.Tensor, ids: Optional[torch.Tensor] = None): """ @@ -187,5 +198,11 @@ def load(self, filename: str): """ self.index = scann.scann_ops_pybind.load_searcher(str(filename)) + def index_state(self) -> str: + return self.index.config() + + def maintenance(self): + pass + def centroids(self) -> torch.Tensor | None: return super().centroids() diff --git a/src/python/index_wrappers/vamana.py b/src/python/index_wrappers/vamana.py index fde8877e..0d880379 100644 --- a/src/python/index_wrappers/vamana.py +++ b/src/python/index_wrappers/vamana.py @@ -6,6 +6,8 @@ from quake.index_wrappers.wrapper import IndexWrapper from quake.utils import to_numpy, to_path, to_torch +from quake import SearchTimingInfo, SearchResult +import time class Vamana(IndexWrapper): @@ -53,11 +55,24 @@ def build( self.index = svs.DynamicVamana.build( parameters=parameters, data=vectors, ids=ids, distance_type=distance, num_threads=num_threads ) - self.index.search_window_size = 128 - def search(self, queries: torch.Tensor, k: int) -> Tuple[np.ndarray, np.ndarray]: + def search(self, queries: torch.Tensor, k: int, search_window_size: int) -> Tuple[np.ndarray, np.ndarray]: + + start_time = time.time() + self.index.search_window_size = search_window_size indices, distances = self.index.search(queries=to_numpy(queries).astype(np.float32), n_neighbors=k) - return to_torch(indices.astype(np.int64)), to_torch(distances) + + end_time = time.time() + timing_info = SearchTimingInfo() + timing_info.total_time_ns = int((end_time - start_time) * 1e9) + + search_result = SearchResult() + search_result.ids = to_torch(indices) + search_result.distances = to_torch(distances) + search_result.timing_info = timing_info + + return search_result + def add(self, vectors: torch.Tensor, ids: Optional[torch.Tensor] = None): if ids is None: @@ -99,3 +114,9 @@ def load(self, directory: str, metrics: str = "ip"): def centroids(self) -> torch.Tensor | None: return super().centroids() + + def index_state(self) -> str: + return "" + + def maintenance(self): + pass \ No newline at end of file diff --git a/src/python/workload_generator.py b/src/python/workload_generator.py index 3e90c814..2b127b19 100644 --- a/src/python/workload_generator.py +++ b/src/python/workload_generator.py @@ -2,10 +2,12 @@ import time from abc import ABC, abstractmethod from pathlib import Path -from typing import Optional, Union +from typing import Optional, Union, Dict, List import matplotlib.pyplot as plt import numpy as np +import pandas as pd +import quake import torch from quake import SearchParams @@ -212,13 +214,13 @@ def initialize_clustered_index(self): index_dir = self.workload_dir / "clustered_index.bin" if index_dir.exists(): index = QuakeWrapper() - index.load(index_dir) + index.load(index_dir, num_workers=1, parent={"num_workers": 1}) n_clusters = index.index.nlist() else: n_clusters = self.base_vectors.shape[0] // self.cluster_size index = QuakeWrapper() index.build( - self.base_vectors, nc=n_clusters, metric=self.metric, ids=torch.arange(self.base_vectors.shape[0]) + self.base_vectors, nc=n_clusters, metric=self.metric, ids=torch.arange(self.base_vectors.shape[0]), num_workers=1, parent={"num_workers": 1} ) index.save(str(self.workload_dir / "clustered_index.bin")) @@ -386,221 +388,259 @@ def generate_workload(self): class WorkloadEvaluator: - """ - Evaluates a generated workload on a given index and produces summary statistics and plots. - """ - - def __init__( - self, - workload_dir: Union[str, Path], - output_dir: Union[str, Path], - base_vectors_path: Optional[Union[str, Path]] = None, - ): - self.workload_dir = to_path(workload_dir) - self.output_dir = to_path(output_dir) - self.runbook_path = self.workload_dir / "runbook.json" - self.operations_dir = self.workload_dir / "operations" - self.initial_indices_path = self.workload_dir / "initial_indices.pt" - self.base_vectors_path = ( - to_path(base_vectors_path) if base_vectors_path else self.workload_dir / "base_vectors.pt" - ) - self.runbook = None - - def initialize_index(self, name, index, build_params, m_params): - index_dir = self.workload_dir / "init_indexes" - index_dir.mkdir(parents=True, exist_ok=True) - index_path = index_dir / f"{name}.index" - vectors = torch.load(self.base_vectors_path, weights_only=True).to(torch.float32) - initial_indices = torch.load(self.initial_indices_path, weights_only=True).to(torch.int64) - vectors = vectors[initial_indices] - if not index_path.exists(): - index.build(vectors, ids=initial_indices, **build_params) - index.save(index_path) - print(f"Index {name} built and saved to {index_path}") + # --------------------------------------------------------------------- + def __init__(self, + workload_dir: Union[str, Path], + output_dir: Union[str, Path], + base_vectors_path: Optional[Union[str, Path]] = None): + + self.workload_dir = to_path(workload_dir) + self.output_dir = to_path(output_dir) + self.runbook_path = self.workload_dir / "runbook.json" + self.ops_dir = self.workload_dir / "operations" + self.init_ids_path = self.workload_dir / "initial_indices.pt" + self.base_vec_path = (to_path(base_vectors_path) + if base_vectors_path + else self.workload_dir / "base_vectors.pt") + + # --------------------------------------------------------------------- + def _init_index(self, name: str, wrapper, build_params: Dict, + m_params: Optional[Dict]): + + idx_dir = self.workload_dir / "init_indexes" + idx_dir.mkdir(parents=True, exist_ok=True) + idx_file = idx_dir / f"{name}.index" + + vecs = torch.load(self.base_vec_path, weights_only=True).float() + init_ids = torch.load(self.init_ids_path, weights_only=True).long() + vecs_init = vecs[init_ids] + + if not idx_file.exists(): + print(f"[{name}] building base index …") + wrapper.build(vecs_init, ids=init_ids, **build_params) + wrapper.save(idx_file) + print(f"[{name}] stored → {idx_file}") else: - index.load(index_path, n_workers=build_params.get("num_workers", 0)) - print(f"Index {name} loaded from {index_path}") + num_workers = build_params.get("num_workers", 0) + use_numa = build_params.get("use_numa", False) + parent_params = build_params.get("parent", {}) + wrapper.load(idx_file, num_workers=num_workers, + use_numa=use_numa, parent=parent_params) + print(f"[{name}] loaded ← {idx_file}") + + if isinstance(wrapper, QuakeWrapper) and m_params: + mp = quake.MaintenancePolicyParams() + for k, v in m_params.items(): + setattr(mp, k, v) + wrapper.index.initialize_maintenance_policy(mp) + print(f"[{name}] maintenance policy: {m_params}") + + return wrapper + + # --------------------------------------------------------------------- + def evaluate_workload(self, *, + name: str, + index, + build_params: Dict, + search_params: Dict, + do_maintenance: bool = False, + m_params: Optional[Dict] = None, + batch: bool = False) -> List[Dict]: - if isinstance(index, QuakeWrapper) and m_params is not None: - index.index.initialize_maintenance_policy(m_params) - print(f"Maintenance policy initialized: {m_params}") + self.output_dir.mkdir(parents=True, exist_ok=True) + index = self._init_index(name, index, build_params, m_params) - return index + runbook = json.load(open(self.runbook_path)) + base_vecs = torch.load(self.base_vec_path, weights_only=True).float() + queries = (base_vecs if runbook["parameters"]["sample_queries"] + else torch.load(self.workload_dir / "query_vectors.pt", + weights_only=True).float()) - def evaluate_workload( - self, name, index, build_params, search_params, do_maintenance=False, m_params=None, batch=False - ): - """ - Evaluate the workload on the index. At the end a summary is printed and a multi-panel plot is saved. - """ - # validate search_params - assert "k" in search_params, "search_params must contain 'k' for number of neighbors" - - # --- Load Workload and Index --- - base_vectors = torch.load(self.base_vectors_path, weights_only=True).to(torch.float32) - initial_indices = torch.load(self.initial_indices_path, weights_only=True).to(torch.int64) - - index = self.initialize_index(name, index, build_params, m_params) - - self.runbook = json.load(open(self.runbook_path, "r")) - query_vectors = ( - base_vectors - if self.runbook["parameters"]["sample_queries"] - else torch.load(self.workload_dir / "query_vectors.pt", weights_only=True) - ) - query_vectors = query_vectors.to(torch.float32) - start_time = time.time() - init_time = time.time() - start_time - self.runbook["initialize"]["time"] = init_time - - # --- Run Operations --- - curr_ids = initial_indices - curr_vectors = base_vectors[curr_ids] results = [] - for operation_id, operation in self.runbook["operations"].items(): - print(f"Operation {operation_id}/{len(self.runbook['operations'])}...") - operation_id_int = int(operation_id) - operation_type = operation["type"] - operation_ids = torch.load(self.operations_dir / f"{operation_id}.pt", weights_only=True) - if operation_type == "insert": - curr_ids = torch.cat([curr_ids, operation_ids]) - curr_vectors = torch.cat([curr_vectors, base_vectors[operation_ids]]) - start_time = time.time() - index.add(base_vectors[operation_ids], ids=operation_ids, num_threads=16) - op_time = time.time() - start_time - mean_recall = None - elif operation_type == "delete": - start_time = time.time() - index.remove(operation_ids) - op_time = time.time() - start_time - mean_recall = None - elif operation_type == "query": - gt_ids = torch.load(self.operations_dir / f"{operation_id}_gt_ids.pt", weights_only=True) - # gt_dist = torch.load(self.operations_dir / f"{operation_id}_gt_dists.pt", weights_only=True) - queries = query_vectors[operation_ids] - - start_time = time.time() + totals = dict(query=0., insert=0., delete=0., maintain=0.) + + print(f"─ Evaluating workload on {name} ({len(runbook['operations'])} ops) ─") + for op_id, op in runbook["operations"].items(): + op_no = int(op_id) + op_type = op["type"] + ids = torch.load(self.ops_dir / f"{op_id}.pt", + weights_only=True) + + print(f"[{name}] op {op_no:4d} | {op_type:<6} | {len(ids):6d} ids", end="", flush=True) + + # ----- perform the operation ---------------------------------- + if op_type == "insert": + t0 = time.perf_counter() + index.add(base_vecs[ids], ids=ids) + latency_ms = (time.perf_counter() - t0) * 1e3 + recall = None + elif op_type == "delete": + t0 = time.perf_counter() + index.remove(ids) + latency_ms = (time.perf_counter() - t0) * 1e3 + recall = None + elif op_type == "query": + qs = queries[ids] + t0 = time.perf_counter() + t_infos = [] if batch: - search_result = index.search(queries, **search_params) - pred_ids = search_result.ids - timing_infos = [search_result.timing_info] + sr = index.search(qs, **search_params) + pred_ids = sr.ids else: - Is, Ds, timing_infos = [], [], [] - - for query in queries: - query = query.unsqueeze(0) - search_result = index.search(query, **search_params) - Is.append(search_result.ids) - Ds.append(search_result.distances) - timing_infos.append(search_result.timing_info) - pred_ids = torch.cat(Is) - op_time = time.time() - start_time - - recalls = compute_recall(pred_ids, gt_ids, search_params["k"]) - mean_recall = recalls.mean().item() - self.runbook["operations"][operation_id]["recall"] = mean_recall - total_time = sum([ti.total_time_ns for ti in timing_infos]) - mean_time = total_time / len(timing_infos) - print(f"Query Time: {mean_time:.2f} ns, Recall: {mean_recall:.2f}") + p = [] + for q in qs: + sr = index.search(q.unsqueeze(0), **search_params) + p.append(sr.ids) + t_infos.append(sr.timing_info) + pred_ids = torch.cat(p) + # pred_ids = torch.cat([ + # index.search(q.unsqueeze(0), **search_params).ids + # for q in qs + # ]) + latency_ms = (time.perf_counter() - t0) * 1e3 + gt_ids = torch.load(self.ops_dir / f"{op_id}_gt_ids.pt", + weights_only=True) + + total_parent_time = 0 + total_time = 0 + total_boundary_time = 0 + total_aps_time = 0 + total_scan_time = 0 + + for t_info in t_infos: + total_parent_time += t_info.parent_info.total_time_ns / 1e6 + total_time += t_info.total_time_ns / 1e6 + total_boundary_time += t_info.boundary_distance_time_ns / 1e6 + total_aps_time += t_info.aps_time_ns / 1e6 + total_scan_time += t_info.scan_time_ns / 1e6 + + print(f" | parent {total_parent_time:.2f} ms" + f" | total {total_time:.2f} ms" + f" | boundary {total_boundary_time:.2f} ms" + f" | aps {total_aps_time:.2f} ms" + f" | scan {total_scan_time:.2f} ms") + recall = compute_recall(pred_ids, gt_ids, + search_params["k"]).mean().item() + op["recall"] = recall + else: + raise ValueError(op_type) + + totals[op_type] += latency_ms + # ----- maintenance ------------------------------------------- + maint_ms = nsplits = ndeletes = 0 if do_maintenance: - print("Running maintenance...") - index.maintenance() - - n_resident = operation.get("n_resident", None) - index_state = index.index_state() - result = { - "operation_number": operation_id_int, - "operation_type": operation_type, - "latency_ms": op_time * 1000, - "recall": mean_recall, - "n_resident": n_resident, + mi = index.maintenance() + maint_ms = mi.total_time_us / 1e3 + nsplits = mi.n_splits + ndeletes = mi.n_deletes + totals["maintain"] += maint_ms + + print(f" | lat {latency_ms:8.2f} ms" + f" | maint {maint_ms:7.2f} ms" + f" | splits {nsplits:4d}" + f" | dels {ndeletes:4d}" + f"{' | rec {:.3f}'.format(recall) if recall is not None else ''}") + + # ----- store row --------------------------------------------- + row = { + "operation_number" : op_no, + "operation_type" : op_type, + "latency_ms" : latency_ms, + "maintenance_time_ms": maint_ms, + "n_splits" : nsplits, + "n_deletes" : ndeletes, + "recall" : recall, + "n_resident" : op.get("n_resident"), } - result.update(index_state) - result.update(search_params) - results.append(result) - - # --- Print Evaluation Summary --- - # Gather per-operation metrics - latencies_insert = [r["latency_ms"] for r in results if r["operation_type"] == "insert"] - op_nums_insert = [r["operation_number"] for r in results if r["operation_type"] == "insert"] - latencies_delete = [r["latency_ms"] for r in results if r["operation_type"] == "delete"] - op_nums_delete = [r["operation_number"] for r in results if r["operation_type"] == "delete"] - latencies_query = [r["latency_ms"] for r in results if r["operation_type"] == "query"] - op_nums_query = [r["operation_number"] for r in results if r["operation_type"] == "query"] - query_recalls = [r["recall"] for r in results if r["operation_type"] == "query" and r["recall"] is not None] - n_vectors = [r["n_resident"] for r in results if r["n_resident"] is not None] - - avg_latency_insert = np.mean(latencies_insert) if latencies_insert else None - avg_latency_delete = np.mean(latencies_delete) if latencies_delete else None - avg_latency_query = np.mean(latencies_query) if latencies_query else None - avg_query_recall = np.mean(query_recalls) if query_recalls else None - - print("\nWorkload Evaluation Summary:") - if avg_latency_insert is not None: - print(f"Average Insert Latency: {avg_latency_insert:.2f} ms") - if avg_latency_delete is not None: - print(f"Average Delete Latency: {avg_latency_delete:.2f} ms") - if avg_latency_query is not None: - print(f"Average Query Latency: {avg_latency_query:.2f} ms") - if avg_query_recall is not None: - print(f"Average Query Recall: {avg_query_recall:.2f}") - - # --- Generate Multi-Panel Plot --- + row.update(index.index_state()) # keeps existing behaviour + row.update(search_params) # 〃 + results.append(row) + + # ---- original four-panel figure (unchanged) --------------------- + df = pd.DataFrame(results) + self._four_panel_plot(df) + + # ---- cumulative-time bar chart --------------------------------- + self._time_breakdown_plot(totals, name) + + # ---- CSV -------------------------------------------------------- + df.to_csv(self.output_dir / "results.csv", index=False) + print(f"Results → {self.output_dir / 'results.csv'}") + return results + + # --------------------------------------------------------------------- + def _four_panel_plot(self, df: pd.DataFrame): + """ + Re-implementation of the original four-panel plot – byte-for-byte + identical file name (*evaluation_plots.png*). + """ + lat_insert = df[df.operation_type == "insert"] + lat_delete = df[df.operation_type == "delete"] + lat_query = df[df.operation_type == "query" ] + fig, axs = plt.subplots(2, 2, figsize=(12, 10)) - # Plot A: Latency per operation type + + # A: latency ax = axs[0, 0] - if op_nums_insert: - ax.plot(op_nums_insert, latencies_insert, label="Insert", marker="o") - if op_nums_delete: - ax.plot(op_nums_delete, latencies_delete, label="Delete", marker="s") - if op_nums_query: - ax.plot(op_nums_query, latencies_query, label="Query", marker="^") - ax.set_xlabel("Operation Number") - ax.set_ylabel("Latency (ms)") - ax.set_title("Operation Latency") - ax.legend() - # Plot B: Number of partitions per operation (if available) + if not lat_insert.empty: + ax.plot(lat_insert.operation_number, lat_insert.latency_ms, + label="Insert", marker="o") + if not lat_delete.empty: + ax.plot(lat_delete.operation_number, lat_delete.latency_ms, + label="Delete", marker="s") + if not lat_query.empty: + ax.plot(lat_query.operation_number, lat_query.latency_ms, + label="Query", marker="^") + ax.set(xlabel="Operation Number", ylabel="Latency (ms)", + title="Operation Latency"); ax.legend() + + # B: partitions + part = df[df.n_list.notna()] ax = axs[0, 1] - partitions = [r["n_list"] for r in results if r["n_list"] is not None] - op_nums_part = [r["operation_number"] for r in results if r["n_list"] is not None] - if partitions: - ax.plot(op_nums_part, partitions, marker="o") - ax.set_xlabel("Operation Number") - ax.set_ylabel("Number of Partitions") - ax.set_title("Partitions per Operation") + if not part.empty: + ax.plot(part.operation_number, part.n_list, marker="o") + ax.set(xlabel="Operation Number", ylabel="Number of Partitions", + title="Partitions per Operation") else: - ax.text(0.5, 0.5, "No partition info", ha="center", va="center") - ax.axis("off") - # Plot C: Resident set size per operation + ax.text(0.5, 0.5, "No partition info", + ha="center", va="center"); ax.axis("off") + + # C: resident set size + res = df[df.n_resident.notna()] ax = axs[1, 0] - if n_vectors: - op_nums_vect = [r["operation_number"] for r in results if r["n_resident"] is not None] - ax.plot(op_nums_vect, n_vectors, marker="o") - ax.set_xlabel("Operation Number") - ax.set_ylabel("Resident Vectors") - ax.set_title("Resident Set Size") + if not res.empty: + ax.plot(res.operation_number, res.n_resident, marker="o") + ax.set(xlabel="Operation Number", ylabel="Resident Vectors", + title="Resident Set Size") else: - ax.text(0.5, 0.5, "No resident set info", ha="center", va="center") - ax.axis("off") - # Plot D: Query recall per query operation + ax.text(0.5, 0.5, "No resident set info", + ha="center", va="center"); ax.axis("off") + + # D: recall + rec = df[(df.operation_type == "query") & df.recall.notna()] ax = axs[1, 1] - if op_nums_query and query_recalls: - ax.plot(op_nums_query, query_recalls, marker="o") - ax.set_xlabel("Operation Number") - ax.set_ylabel("Query Recall") - ax.set_title("Query Recall") + if not rec.empty: + ax.plot(rec.operation_number, rec.recall, marker="o") + ax.set(xlabel="Operation Number", ylabel="Query Recall", + title="Query Recall") else: - ax.text(0.5, 0.5, "No query recall info", ha="center", va="center") - ax.axis("off") - plt.tight_layout() + ax.text(0.5, 0.5, "No query recall info", + ha="center", va="center"); ax.axis("off") - # create output directory if it doesn't exist - self.output_dir.mkdir(parents=True, exist_ok=True) - - plot_path = self.output_dir / "evaluation_plots.png" - plt.savefig(plot_path) - print(f"Saved evaluation plots to {plot_path}") + plt.tight_layout() + plt.savefig(self.output_dir / "evaluation_plots.png") plt.close() - return results + # --------------------------------------------------------------------- + def _time_breakdown_plot(self, totals: Dict[str, float], title: str): + plt.figure(figsize=(6, 4)) + bars = [totals["query"], totals["insert"], + totals["delete"], totals["maintain"], + sum(totals.values())] + plt.bar(["Query", "Insert", "Delete", "Maintain", "Total"], bars) + plt.ylabel("Cumulative time (ms)") + plt.title(f"Time budget – {title}") + plt.tight_layout() + plt.savefig(self.output_dir / "time_breakdown.png", dpi=150) + plt.close() \ No newline at end of file diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/cpp/benchmark.cpp b/test/cpp/benchmark.cpp index 71426869..ed3497cb 100644 --- a/test/cpp/benchmark.cpp +++ b/test/cpp/benchmark.cpp @@ -28,11 +28,12 @@ using torch::Tensor; // Global benchmark parameters static const int64_t DIM = 128; static const int64_t NUM_VECTORS = 100000; // number of database vectors -static const int64_t N_LIST = 100; // number of clusters for IVF -static const int64_t NUM_QUERIES = 10; // number of queries for search benchmark -static const int64_t K = 10; // top-K neighbors -static const int64_t N_PROBE = 32; // number of probes for IVF +static const int64_t N_LIST = 1000; // number of clusters for IVF +static const int64_t NUM_QUERIES = 1000; // number of queries for search benchmark +static const int64_t K = 1; // top-K neighbors +static const int64_t N_PROBE = 20; // number of probes for IVF static const int64_t N_WORKERS = 12; // number of workers for parallel query coordinator +static const int64_t N_PARENT_WORKERS = 12; // Helper functions to generate random data and sequential IDs static Tensor generate_data(int64_t num, int64_t dim) { @@ -117,6 +118,8 @@ class QuakeWorkerIVFBenchmark : public ::testing::Test { build_params->metric = "l2"; build_params->niter = 3; build_params->num_workers = N_WORKERS; + build_params->parent_params = std::make_shared(); + build_params->parent_params->num_workers = N_PARENT_WORKERS; index_->build(data_, ids_, build_params); } }; @@ -305,6 +308,8 @@ TEST_F(QuakeWorkerIVFBenchmark, SearchBatch) { search_params->k = K; search_params->nprobe = N_PROBE; search_params->batched_scan = true; + search_params->parent_params = std::make_shared(); + search_params->parent_params->batched_scan = true; index_->search(queries, search_params); @@ -314,6 +319,14 @@ TEST_F(QuakeWorkerIVFBenchmark, SearchBatch) { auto elapsed = duration_cast(end - start).count(); std::cout << "[Quake IVF Worker] Batched search time: " << elapsed << " ms" << std::endl; + + // print out timing info + auto timing_info = result->timing_info; + std::cout << "Total time: " << timing_info->total_time_ns / 1e6 << " ms" << std::endl; + std::cout << "Parent search time: " << timing_info->parent_info->total_time_ns / 1e6 << " ms" << std::endl; + std::cout << "Job enqueue time: " << timing_info->job_enqueue_time_ns / 1e6 << " ms" << std::endl; + std::cout << "Result aggregate time: " << timing_info->result_aggregate_time_ns / 1e6 << " ms" << std::endl; + std::cout << "Job Wait time: " << timing_info->job_wait_time_ns / 1e6 << " ms" << std::endl; ASSERT_GT(elapsed, 0); } @@ -487,4 +500,197 @@ TEST_F(FaissIVFBenchmark, Remove) { auto elapsed = duration_cast(end - start).count(); std::cout << "[Faiss IVF] Remove time: " << elapsed << " ms" << std::endl; SUCCEED(); -} \ No newline at end of file +} + +// ------------------------------------------------------------------------- +// SEARCH, ADD, REMOVE, AND MAINTENANCE TEST +// ------------------------------------------------------------------------- +TEST(QuakeIndexStressTest, SearchAddRemoveMaintenanceTest) { + // Repeatedly search, add, remove, and perform maintenance to see if the index remains consistent. + + int64_t dimension = 128; + int64_t num_vectors = 10000; + int64_t num_queries = 100; + int64_t batch_size = 10000; + int n_ops = 100; + + QuakeIndex index; + auto build_params = std::make_shared(); + build_params->nlist = 100; + build_params->metric = "ip"; + build_params->niter = 5; + build_params->num_workers = 1; + + auto maintenance_params = std::make_shared(); + maintenance_params->refinement_radius = 50; + maintenance_params->refinement_iterations = 3; + maintenance_params->split_threshold_ns = 1; + maintenance_params->delete_threshold_ns = 1; + maintenance_params->enable_delete_rejection = false; + maintenance_params->enable_split_rejection = false; + maintenance_params->window_size = 1000; + + Tensor data_vectors = torch::randn({num_vectors, dimension}, torch::kFloat32); + Tensor data_ids = torch::arange(num_vectors, torch::kInt64); + + + // add level + auto parent_index_build_params = std::make_shared(); + parent_index_build_params->nlist = 100; + parent_index_build_params->metric = "ip"; + parent_index_build_params->niter = 5; + parent_index_build_params->num_workers = 1; + // build_params->parent_params = parent_index_build_params; + + // auto grandparent_index_build_params = std::make_shared(); + // grandparent_index_build_params->nlist = 1; + // grandparent_index_build_params->metric = "l2"; + // grandparent_index_build_params->num_workers = 0; + // build_params->parent_params->parent_params = grandparent_index_build_params; + + index.build(data_vectors, data_ids, build_params); + // + index.initialize_maintenance_policy(maintenance_params); + + // timers + int64_t search_time = 0; + int64_t add_time = 0; + int64_t remove_time = 0; + int64_t maintenance_time = 0; + + auto start = std::chrono::high_resolution_clock::now(); + auto end = std::chrono::high_resolution_clock::now(); + + int64_t num_deleted = 0; + + Tensor origin = torch::zeros(dimension); + Tensor drift = .1 * torch::ones(dimension); + + for (int i = 0; i < n_ops; i++) { + std::cout << "[SearchAddRemoveMaintenanceTest] Iteration " << i << "\n"; + // Search + auto query_vectors = torch::randn({num_queries, dimension}, torch::kFloat32) + origin; + auto search_params = std::make_shared(); + search_params->nprobe = 12; + search_params->k = 10; + search_params->batched_scan = true; + search_params->num_threads = 1; + + auto parent_search_params = std::make_shared(); + parent_search_params->nprobe = 50; + parent_search_params->batched_scan = true; + search_params->parent_params = parent_search_params; + + start = std::chrono::high_resolution_clock::now(); + auto search_result = index.search(query_vectors, search_params); + end = std::chrono::high_resolution_clock::now(); + search_time += std::chrono::duration_cast(end - start).count(); + ASSERT_EQ(search_result->ids.size(0), query_vectors.size(0)); + ASSERT_EQ(search_result->ids.size(1), search_params->k); + + // Add + auto add_vectors = torch::randn({batch_size, dimension}, torch::kFloat32) + origin; + origin += drift; + auto add_ids = torch::arange(batch_size, torch::kInt64) + num_vectors; + start = std::chrono::high_resolution_clock::now(); + auto add_info = index.add(add_vectors, add_ids); + end = std::chrono::high_resolution_clock::now(); + add_time += std::chrono::duration_cast(end - start).count(); + num_vectors += batch_size; + ASSERT_EQ(add_info->n_vectors, batch_size); + + // Remove + auto remove_ids = torch::arange(batch_size) + num_deleted; + num_deleted += batch_size; + start = std::chrono::high_resolution_clock::now(); + auto remove_info = index.remove(remove_ids); + end = std::chrono::high_resolution_clock::now(); + remove_time += std::chrono::duration_cast(end - start).count(); + ASSERT_EQ(remove_info->n_vectors, batch_size); + + start = std::chrono::high_resolution_clock::now(); + auto timing_info = index.maintenance(); + end = std::chrono::high_resolution_clock::now(); + maintenance_time += std::chrono::duration_cast(end - start).count(); + + std::cout << "n_splits=" << timing_info->n_splits << ", n_deletes=" << timing_info->n_deletes + << ", delete_time=" << timing_info->delete_time_us << " μs" + << ", split_time=" << timing_info->split_time_us << " μs" + << ", total_time=" << timing_info->total_time_us << " μs\n"; + } + + // print out mean times per operation + float mean_search_time = static_cast(search_time) / n_ops; + float mean_add_time = static_cast(add_time) / n_ops; + float mean_remove_time = static_cast(remove_time) / n_ops; + float mean_maintenance_time = static_cast(maintenance_time) / n_ops; + + std::cout << "[SearchAddRemoveMaintenanceTest] Mean search time: " << mean_search_time << " μs\n"; + std::cout << "[SearchAddRemoveMaintenanceTest] Mean add time: " << mean_add_time << " μs\n"; + std::cout << "[SearchAddRemoveMaintenanceTest] Mean remove time: " << mean_remove_time << " μs\n"; + std::cout << "[SearchAddRemoveMaintenanceTest] Mean maintenance time: " << mean_maintenance_time << " μs\n"; + + SUCCEED(); +} + +// TEST(FaissIndexStressTest, SearchAddRemoveMaintenanceTest) { +// // Repeatedly search, add, remove, and perform maintenance to see if the index remains consistent. +// +// int64_t dimension = 128; +// int64_t num_vectors = 100000; +// int64_t num_queries = 1; +// int64_t batch_size = 10000; +// int n_ops = 100; +// +// Tensor data_vectors = torch::randn({num_vectors, dimension}, torch::kFloat32); +// Tensor data_ids = torch::arange(num_vectors, torch::kInt64); +// +// auto quantizer = new faiss::IndexFlatL2(dimension); +// auto index = new faiss::IndexIVFFlat(quantizer, dimension, 1000, faiss::METRIC_L2); +// index->train(num_vectors, data_vectors.data_ptr()); +// index->add(num_vectors, data_vectors.data_ptr()); +// +// // timers +// int64_t search_time = 0; +// int64_t add_time = 0; +// int64_t remove_time = 0; +// int64_t maintenance_time = 0; +// +// auto start = std::chrono::high_resolution_clock::now(); +// auto end = std::chrono::high_resolution_clock::now(); +// +// for (int i = 0; i < n_ops; i++) { +// // Search +// auto query_vectors = generate_data(num_queries, dimension); +// std::vector distances(num_queries * K); +// std::vector labels(num_queries * K); +// +// start = std::chrono::high_resolution_clock::now(); +// index->nprobe = 12; +// index->search(num_queries, query_vectors.data_ptr(), 12, distances.data(), labels.data()); +// end = std::chrono::high_resolution_clock::now(); +// search_time += std::chrono::duration_cast(end - start).count(); +// +// // Add +// auto add_vectors = generate_data(batch_size, dimension); +// auto add_ids = generate_ids(batch_size, num_vectors); +// start = std::chrono::high_resolution_clock::now(); +// index->add(batch_size, add_vectors.data_ptr()); +// end = std::chrono::high_resolution_clock::now(); +// add_time += std::chrono::duration_cast(end - start).count(); +// num_vectors += batch_size; +// } +// // print out mean times per operation +// float mean_search_time = static_cast(search_time) / n_ops; +// float mean_add_time = static_cast(add_time) / n_ops; +// float mean_remove_time = static_cast(remove_time) / n_ops; +// float mean_maintenance_time = static_cast(maintenance_time) / n_ops; +// +// std::cout << "[SearchAddRemoveMaintenanceTest] Mean search time: " << mean_search_time << " μs\n"; +// std::cout << "[SearchAddRemoveMaintenanceTest] Mean add time: " << mean_add_time << " μs\n"; +// std::cout << "[SearchAddRemoveMaintenanceTest] Mean remove time: " << mean_remove_time << " μs\n"; +// std::cout << "[SearchAddRemoveMaintenanceTest] Mean maintenance time: " << mean_maintenance_time << " μs\n"; +// +// SUCCEED(); +// } + diff --git a/test/cpp/clustering.cpp b/test/cpp/clustering.cpp new file mode 100644 index 00000000..bd373ba0 --- /dev/null +++ b/test/cpp/clustering.cpp @@ -0,0 +1,141 @@ +#include +#include +#include "clustering.h" + +// Helpers to generate random data and sequential ids +static torch::Tensor generate_random_data(int64_t N, int64_t D) { + return torch::randn({N, D}, torch::kFloat32).contiguous(); +} +static torch::Tensor generate_sequential_ids(int64_t N, int64_t start = 0) { + return torch::arange(start, start + N, torch::kInt64).contiguous(); +} + +// Compute mean squared error for clustering (for CPU sanity) +static double compute_mse(const torch::Tensor& centroids, + const std::vector& clusters) { + double total_err = 0.0; + int64_t count = 0; + auto C = centroids.to(torch::kCPU); + for (size_t i = 0; i < clusters.size(); ++i) { + auto cl = clusters[i].to(torch::kCPU); + if (cl.size(0) == 0) continue; + auto diff = cl - C[i].unsqueeze(0); + total_err += diff.pow(2).sum().item(); + count += cl.size(0); + } + return count>0 ? total_err / count : 0.0; +} + +// Fixture +class ClusteringTest : public ::testing::Test { + protected: + const int64_t num_vectors = 5000; + const int64_t dim = 64; + const int num_clusters= 20; + + torch::Tensor vectors_cpu, ids_cpu; +#ifdef QUAKE_ENABLE_GPU + torch::Tensor vectors_cuda, ids_cuda; +#endif + + void SetUp() override { + vectors_cpu = generate_random_data(num_vectors, dim); + ids_cpu = generate_sequential_ids(num_vectors); + +#ifdef QUAKE_ENABLE_GPU + if (!torch::cuda::is_available()) { + GTEST_SKIP() << "CUDA not available"; + } + vectors_cuda = vectors_cpu.to(torch::kCUDA).contiguous(); + ids_cuda = ids_cpu.to(torch::kCUDA).contiguous(); +#endif + } +}; + +// Test existing CPU kmeans +TEST_F(ClusteringTest, KMeansCPU_L2) { + shared_ptr build_params = std::make_shared(); + build_params->nlist = num_clusters; + build_params->metric = "l2"; + build_params->niter = 10; + auto cl = kmeans_cpu(vectors_cpu, ids_cpu, build_params, torch::Tensor()); + ASSERT_EQ(cl->centroids.sizes(), (std::vector{num_clusters, dim})); + int64_t tot=0; + for (int i=0;ivectors[i].size(0), cl->vector_ids[i].size(0)); + tot += cl->vectors[i].size(0); + } + ASSERT_EQ(tot, num_vectors); +} + +// Compare CPU vs CPU wrapper +TEST_F(ClusteringTest, KMeansWrapper_CPU) { + shared_ptr build_params = std::make_shared(); + build_params->nlist = num_clusters; + build_params->metric = "l2"; + build_params->niter = 10; + build_params->use_gpu = false; + auto cl = kmeans(vectors_cpu, ids_cpu, build_params, torch::Tensor()); + ASSERT_EQ(cl->centroids.sizes(), (std::vector{num_clusters, dim})); + int64_t tot=0; + for (int i=0;ivectors[i].size(0); + } + ASSERT_EQ(tot, num_vectors); +} + +#ifdef QUAKE_ENABLE_GPU +TEST_F(ClusteringTest, SampleAndPredict_GPU_L2) { + shared_ptr build_params = std::make_shared(); + build_params->nlist = num_clusters; + build_params->metric = "l2"; + build_params->niter = 10; + build_params->use_gpu = true; + build_params->gpu_sample_size = 2000; + build_params->gpu_batch_size = 100; + + auto cl = kmeans_cuvs_sample_and_predict( + vectors_cpu, ids_cpu, build_params); + + // centroids must live on CPU and have correct shape + ASSERT_EQ(cl->centroids.device().type(), torch::kCPU); + ASSERT_EQ(cl->centroids.sizes(), (std::vector{num_clusters, dim})); + + // all vectors accounted for + int64_t tot=0; + for (int i=0;ivectors[i]; + ASSERT_EQ(part.device().type(), torch::kCPU); + ASSERT_EQ(part.size(0), cl->vector_ids[i].size(0)); + tot += part.size(0); + } + ASSERT_EQ(tot, num_vectors); + + build_params->use_gpu = false; + + // Optional quality check: rough MSE vs CPU run + auto cl_cpu = kmeans_cpu(vectors_cpu, ids_cpu, build_params, torch::Tensor()); + double mse_cpu = compute_mse(cl_cpu->centroids, cl_cpu->vectors); + double mse_gpu = compute_mse(cl->centroids, cl->vectors); + ASSERT_NEAR(mse_cpu, mse_gpu, mse_cpu * 0.30); +} + +// Full wrapper test for GPU +TEST_F(ClusteringTest, KMeansWrapper_GPU) { + shared_ptr build_params = std::make_shared(); + build_params->nlist = num_clusters; + build_params->metric = "l2"; + build_params->niter = 10; + build_params->use_gpu = true; + build_params->gpu_sample_size = 2000; + build_params->gpu_batch_size = 100; + + auto cl = kmeans(vectors_cpu, ids_cpu, build_params, + torch::Tensor()); + ASSERT_EQ(cl->centroids.device().type(), torch::kCPU); + ASSERT_EQ(cl->vectors.size(), size_t(num_clusters)); + int64_t tot=0; + for (auto &p : cl->vectors) tot += p.size(0); + ASSERT_EQ(tot, num_vectors); +} +#endif // QUAKE_ENABLE_GPU diff --git a/test/cpp/dynamic_inverted_list.cpp b/test/cpp/dynamic_inverted_list.cpp index f7325e49..1473d1ec 100644 --- a/test/cpp/dynamic_inverted_list.cpp +++ b/test/cpp/dynamic_inverted_list.cpp @@ -556,6 +556,136 @@ TEST_F(DynamicInvertedListTest, SerializationTest) { remove(filename.c_str()); } +// --------------------------------------------------------------------- +// Map‑consistency tests +// --------------------------------------------------------------------- + +// After removing a middle element the map must point the *swapped* id +TEST_F(DynamicInvertedListTest, MapConsistencyAfterRemoveSwap) +{ + const size_t list_no = 0; + const size_t n_entries = 3; + + std::vector codes; + std::vector ids; + generate_random_codes(n_entries, codes); + generate_sequential_ids(n_entries, ids, 10); // ids = 10,11,12 + + invlists->add_entries(list_no, n_entries, ids.data(), codes.data()); + + // Remove id 11 (middle) – id 12 will be swapped into slot 1 + invlists->remove_entry(list_no, 11); + + // --- removed id must be gone + EXPECT_THROW( { invlists->get_vectors_by_id({11}); }, std::runtime_error ); + + // --- swapped id 12 must still be retrievable and vector intact + auto vec_ptrs = invlists->get_vectors_by_id({12}); + ASSERT_EQ(vec_ptrs.size(), 1u); + EXPECT_EQ( + std::memcmp(vec_ptrs[0], + codes.data() + 2 * code_size, // original bytes of id 12 + code_size), + 0); +} + + +// Removing many IDs in one pass must keep the map sound +TEST_F(DynamicInvertedListTest, MapConsistencyBulkRemove) +{ + const size_t list_no = 1; + const size_t n_entries = 10; + + std::vector codes; + std::vector ids; + generate_random_codes(n_entries, codes); + generate_sequential_ids(n_entries, ids, 100); // ids = 100..109 + + invlists->add_entries(list_no, n_entries, ids.data(), codes.data()); + + // Remove every even id (5 victims) + std::vector victims; + for (idx_t id : ids) + if (id % 2 == 0) victims.push_back(id); + + invlists->remove_entries_from_partition(list_no, victims); + + // All survivors must be accessible and the bytes match + std::vector survivors; + for (idx_t id : ids) + if (id % 2) survivors.push_back(id); + + auto vecs = invlists->get_vectors_by_id(survivors); + ASSERT_EQ(vecs.size(), survivors.size()); + for (size_t k = 0; k < survivors.size(); ++k) { + size_t orig_idx = survivors[k] - 100; // 0‑based + EXPECT_EQ( + std::memcmp(vecs[k], + codes.data() + orig_idx * code_size, + code_size), + 0); + } +} + + +// batch_update_entries must leave every moved id valid in the map +TEST_F(DynamicInvertedListTest, MapConsistencyBatchUpdate) +{ + const size_t old_p = 2; + const size_t new_p = 3; + const size_t n = 6; // vectors + + std::vector codes; + std::vector ids; + generate_random_codes(n, codes); + generate_sequential_ids(n, ids, 500); // 500..505 + invlists->add_entries(old_p, n, ids.data(), codes.data()); + + // mark last 3 vectors to move + std::vector part_of(n, (int64_t)old_p); + for (size_t i = 3; i < n; ++i) part_of[i] = (int64_t)new_p; + + std::vector new_codes = codes; // unchanged payload + std::vector ids_i64(ids.begin(), ids.end()); + + invlists->batch_update_entries(old_p, + part_of.data(), + new_codes.data(), + ids_i64.data(), + (int)n); + + // all ids must still retrieve the identical vector bytes + auto vecs = invlists->get_vectors_by_id({ids.begin(), ids.end()}); + ASSERT_EQ(vecs.size(), n); + + for (size_t k = 0; k < n; ++k) + EXPECT_EQ( + std::memcmp(vecs[k], + codes.data() + k * code_size, + code_size), + 0); +} + +TEST_F(DynamicInvertedListTest, LazyBuildMapGuard) +{ + const size_t list_no = 4; + std::vector codes; + std::vector ids; + generate_random_codes(1, codes); + generate_sequential_ids(1, ids, 900); + + invlists->add_entries(list_no, 1, ids.data(), codes.data()); + + // Brutally wipe the internal map (undefined behaviour normally, but + // we do it via the public API by resetting and rebuilding). + invlists->build_map(); // ensure filled first + invlists->id_to_location_.clear(); // <-- direct access because we are a friend in unit test + + // the read accessor should transparently rebuild + float tmp[4]; + EXPECT_TRUE(invlists->get_vector_for_id(ids[0], tmp)); +} + // NUMA related tests (only if QUAKE_USE_NUMA is defined) #ifdef QUAKE_USE_NUMA TEST_F(DynamicInvertedListTest, NumaTests) { diff --git a/test/cpp/latency_estimator.cpp b/test/cpp/latency_estimator.cpp index 80ed570a..5cb95220 100644 --- a/test/cpp/latency_estimator.cpp +++ b/test/cpp/latency_estimator.cpp @@ -19,7 +19,7 @@ static float measure_actual_latency(const ListScanLatencyEstimator& estimator, torch::Tensor ids = torch::randperm(n); torch::Tensor query = torch::rand({estimator.d_}); - auto topk_buffer = make_shared(k, false); + auto topk_buffer = make_shared(k, false, k * 100, 0); const float* query_ptr = query.data_ptr(); const float* vectors_ptr = vectors.data_ptr(); @@ -28,7 +28,8 @@ static float measure_actual_latency(const ListScanLatencyEstimator& estimator, uint64_t total_latency_ns = 0; for (int m = 0; m < estimator.n_trials_; ++m) { auto start = std::chrono::high_resolution_clock::now(); - scan_list(query_ptr, vectors_ptr, ids_ptr, n, estimator.d_, *topk_buffer); + scan_list(query_ptr, vectors_ptr, ids_ptr, n, estimator.d_, *topk_buffer, + faiss::METRIC_L2, 10000000.0); auto end = std::chrono::high_resolution_clock::now(); auto duration = std::chrono::duration_cast(end - start); diff --git a/test/cpp/list_scanning.cpp b/test/cpp/list_scanning.cpp index 5c5e82d8..c90442ef 100644 --- a/test/cpp/list_scanning.cpp +++ b/test/cpp/list_scanning.cpp @@ -1,562 +1,562 @@ -// list_scanning_test.cpp - -#include -#include -#include -#include -#include -#include - -#include "list_scanning.h" // Ensure this path is correct - -// Test fixture for list scanning functions -class ListScanningTest : public ::testing::Test { -protected: - // Helper function to create a TopkBuffer - std::shared_ptr create_buffer(int k, bool is_descending) { - return std::make_shared>(k, is_descending); - } -}; - -// Test scan_list with single query and L2 metric -TEST_F(ListScanningTest, ScanList_SingleQuery_L2) { - int k = 2; - bool is_descending = false; // L2 - auto buffer = create_buffer(k, is_descending); - - // Define a query vector and a list of vectors - float query_vec[3] = {1.0f, 0.0f, 0.0f}; - float list_vecs[4 * 3] = { - 1.0f, 0.0f, 0.0f, // Distance: 0.0 - 0.0f, 1.0f, 0.0f, // Distance: 2.0 - 1.0f, 1.0f, 0.0f, // Distance: 1.0 - 2.0f, 0.0f, 0.0f // Distance: 1.0 - }; - int64_t list_ids[4] = {10, 20, 30, 40}; - - // Expected top-k: 0.0 (id=10), 1.0 (id=30 or 40) - scan_list(query_vec, list_vecs, list_ids, 4, 3, *buffer, faiss::METRIC_L2); - - std::vector expected_dists = {0.0f, 1.0f}; - std::vector expected_ids = {10, 30}; // Could also be {10, 40} based on order - - auto topk_dists = buffer->get_topk(); - auto topk_ids = buffer->get_topk_indices(); - - ASSERT_EQ(topk_dists.size(), expected_dists.size()); - ASSERT_EQ(topk_ids.size(), expected_ids.size()); - - EXPECT_FLOAT_EQ(topk_dists[0], expected_dists[0]); - EXPECT_EQ(topk_ids[0], expected_ids[0]); - - // Second element could be id=30 or id=40 - EXPECT_FLOAT_EQ(topk_dists[1], expected_dists[1]); - EXPECT_TRUE(topk_ids[1] == 30 || topk_ids[1] == 40); -} - -// Test scan_list with single query and Inner Product metric -TEST_F(ListScanningTest, ScanList_SingleQuery_InnerProduct) { - int k = 2; - bool is_descending = true; // Inner Product - auto buffer = create_buffer(k, is_descending); - - // Define a query vector and a list of vectors - float query_vec[3] = {1.0f, 0.0f, 0.0f}; - float list_vecs[4 * 3] = { - 1.0f, 0.0f, 0.0f, // Inner product: 1.0 - 0.0f, 1.0f, 0.0f, // Inner product: 0.0 - 1.0f, 1.0f, 0.0f, // Inner product: 1.0 - 2.0f, 0.0f, 0.0f // Inner product: 2.0 - }; - int64_t list_ids[4] = {10, 20, 30, 40}; - - // Expected top-k: 2.0 (id=40), 1.0 (id=10 or 30) - scan_list(query_vec, list_vecs, list_ids, 4, 3, *buffer, faiss::METRIC_INNER_PRODUCT); - - std::vector expected_dists = {2.0f, 1.0f}; - std::vector expected_ids = {40, 10}; // Could also be {40, 30} - - auto topk_dists = buffer->get_topk(); - auto topk_ids = buffer->get_topk_indices(); - - ASSERT_EQ(topk_dists.size(), expected_dists.size()); - ASSERT_EQ(topk_ids.size(), expected_ids.size()); - - EXPECT_FLOAT_EQ(topk_dists[0], expected_dists[0]); - EXPECT_EQ(topk_ids[0], expected_ids[0]); - - // Second element could be id=10 or id=30 - EXPECT_FLOAT_EQ(topk_dists[1], expected_dists[1]); - EXPECT_TRUE(topk_ids[1] == 10 || topk_ids[1] == 30); -} - -// Test batched_scan_list with multiple queries and L2 metric -TEST_F(ListScanningTest, BatchedScanList_MultipleQueries_L2) { - int k = 2; - MetricType metric = faiss::METRIC_L2; - - // Define two query vectors (dim=3) - float query_vecs[2 * 3] = { - 1.0f, 0.0f, 0.0f, // Query 0 - 0.0f, 1.0f, 0.0f // Query 1 - }; - - // Define a list of vectors (dim=3) - float list_vecs[4 * 3] = { - 1.0f, 0.0f, 0.0f, // Q0: 0.0, Q1: 2.0 - 0.0f, 1.0f, 0.0f, // Q0: 2.0, Q1: 0.0 - 1.0f, 1.0f, 0.0f, // Q0: 1.0, Q1: 1.0 - 2.0f, 0.0f, 0.0f // Q0: 4.0, Q1: 2.0 - }; - int64_t list_ids[4] = {10, 20, 30, 40}; - - // Create TopkBuffers for each query - std::vector> topk_buffers = create_buffers(2, k, false); // L2 metric (is_descending=false) - - // Perform batched_scan_list - batched_scan_list( - query_vecs, - list_vecs, - list_ids, - 2, // num_queries - 4, // list_size - 3, // dim - topk_buffers, - metric - ); - - // Expected results: - // Query 0: top-k distances: 0.0 (id=10), 1.0 (id=30) - // Query 1: top-k distances: 0.0 (id=20), 1.0 (id=30) - - // Verify Query 0 - auto topk_q0 = topk_buffers[0]->get_topk(); - auto topk_q0_ids = topk_buffers[0]->get_topk_indices(); - ASSERT_EQ(topk_q0.size(), 2); - EXPECT_FLOAT_EQ(topk_q0[0], 0.0f); - EXPECT_EQ(topk_q0_ids[0], 10); - EXPECT_FLOAT_EQ(topk_q0[1], 1.0f); - EXPECT_EQ(topk_q0_ids[1], 30); - - // Verify Query 1 - auto topk_q1 = topk_buffers[1]->get_topk(); - auto topk_q1_ids = topk_buffers[1]->get_topk_indices(); - ASSERT_EQ(topk_q1.size(), 2); - EXPECT_FLOAT_EQ(topk_q1[0], 0.0f); - EXPECT_EQ(topk_q1_ids[0], 20); - EXPECT_FLOAT_EQ(topk_q1[1], 1.0f); - EXPECT_EQ(topk_q1_ids[1], 30); -} - -// Test batched_scan_list with multiple queries and Inner Product metric -TEST_F(ListScanningTest, BatchedScanList_MultipleQueries_InnerProduct) { - int k = 2; - MetricType metric = faiss::METRIC_INNER_PRODUCT; - - // Define two query vectors (dim=3) - float query_vecs[2 * 3] = { - 1.0f, 0.0f, 0.0f, // Query 0 - 0.0f, 1.0f, 0.0f // Query 1 - }; - - // Define a list of vectors (dim=3) - float list_vecs[4 * 3] = { - 1.0f, 0.0f, 0.0f, // Inner products [1.0, 0.0] (id=10) - 0.0f, 1.0f, 0.0f, // Inner products [0.0, 1.0] (id=20) - .5f, 1.5f, 0.0f, // Inner products [.5, 1.5] (id=30) - 2.0f, 0.0f, 0.0f // Inner products [2.0, 0.0] (id=40) - }; - int64_t list_ids[4] = {10, 20, 30, 40}; - - // Create TopkBuffers for each query - std::vector> topk_buffers = create_buffers(2, k, true); // Inner Product metric (is_descending=true) - - // Perform batched_scan_list - batched_scan_list( - query_vecs, - list_vecs, - list_ids, - 2, // num_queries - 4, // list_size - 3, // dim - topk_buffers, - metric - ); - - // Expected results: - // Query 0: top-k scores: 2.0 (id=40), 1.0 (id=10) - // Query 1: top-k scores: 1.5 (id=30), 1.0 (id=20) - - // Verify Query 0 - auto topk_q0 = topk_buffers[0]->get_topk(); - auto topk_q0_ids = topk_buffers[0]->get_topk_indices(); - - std::cout << "topk_q0: " << topk_q0[0] << " " << topk_q0[1] << std::endl; - std::cout << "topk_q0_ids: " << topk_q0_ids[0] << " " << topk_q0_ids[1] << std::endl; - - ASSERT_EQ(topk_q0.size(), 2); - EXPECT_FLOAT_EQ(topk_q0[0], 2.0f); - EXPECT_TRUE(topk_q0_ids[0] == 40); - EXPECT_FLOAT_EQ(topk_q0[1], 1.0f); - EXPECT_TRUE(topk_q0_ids[1] == 10 || topk_q0_ids[1] == 20); - - // Verify Query 1 - auto topk_q1 = topk_buffers[1]->get_topk(); - auto topk_q1_ids = topk_buffers[1]->get_topk_indices(); - ASSERT_EQ(topk_q1.size(), 2); - EXPECT_FLOAT_EQ(topk_q1[0], 1.5f); - EXPECT_TRUE(topk_q1_ids[0] == 30); - EXPECT_FLOAT_EQ(topk_q1[1], 1.0f); - EXPECT_TRUE(topk_q1_ids[1] == 20); -} - -// Test batched_scan_list without list_ids (list_ids == nullptr) -TEST_F(ListScanningTest, BatchedScanList_NoListIds) { - int k = 2; - MetricType metric = faiss::METRIC_L2; - - // Define two query vectors - float query_vecs[2 * 2] = { - 1.0f, 0.0f, // Query 0 - 0.0f, 1.0f // Query 1 - }; - - // Define a list of vectors - float list_vecs[3 * 2] = { - 1.0f, 0.0f, // Distance for Q0: 0.0, Q1: 2.0 - 0.0f, 1.0f, // Distance for Q0: 2.0, Q1: 0.0 - 1.0f, 1.0f // Distance for Q0: 1.0, Q1: 1.0 - }; - // list_ids == nullptr - - // Create TopkBuffers for each query - std::vector> topk_buffers = create_buffers(2, k, false); // Inner Product metric (is_descending=true) - - - // Perform batched_scan_list - batched_scan_list( - query_vecs, - list_vecs, - nullptr, // list_ids - 2, // num_queries - 3, // list_size - 2, // dim - topk_buffers, - metric - ); - - // Expected results: - // Query 0: top-k distances: 0.0 (id=0), 1.0 (id=2) - // Query 1: top-k distances: 0.0 (id=1), 1.0 (id=2) - - std::vector expected_dists_q0 = {0.0f, 1.0f}; - std::vector expected_ids_q0 = {0, 2}; - std::vector expected_dists_q1 = {0.0f, 1.0f}; - std::vector expected_ids_q1 = {1, 2}; - - // Verify Query 0 - auto topk_q0 = topk_buffers[0]->get_topk(); - auto topk_q0_ids = topk_buffers[0]->get_topk_indices(); - - std::cout << "topk_q0: " << topk_q0[0] << " " << topk_q0[1] << std::endl; - std::cout << "topk_q0_ids: " << topk_q0_ids[0] << " " << topk_q0_ids[1] << std::endl; - - ASSERT_EQ(topk_q0.size(), 2); - EXPECT_FLOAT_EQ(topk_q0[0], 0.0f); - EXPECT_EQ(topk_q0_ids[0], 0); - EXPECT_FLOAT_EQ(topk_q0[1], 1.0f); - EXPECT_EQ(topk_q0_ids[1], 2); - - // Verify Query 1 - auto topk_q1 = topk_buffers[1]->get_topk(); - auto topk_q1_ids = topk_buffers[1]->get_topk_indices(); - ASSERT_EQ(topk_q1.size(), 2); - EXPECT_FLOAT_EQ(topk_q1[0], 0.0f); - EXPECT_EQ(topk_q1_ids[0], 1); - EXPECT_FLOAT_EQ(topk_q1[1], 1.0f); - EXPECT_EQ(topk_q1_ids[1], 2); -} - -// Test batched_scan_list with batch_size parameter -TEST_F(ListScanningTest, BatchedScanList_WithBatchSize) { - int k = 1; - MetricType metric = faiss::METRIC_L2; - - // Define three query vectors (dim=2) - float query_vecs[3 * 2] = { - 1.0f, 0.0f, // Query 0 - 0.0f, 1.0f, // Query 1 - 1.0f, 1.0f // Query 2 - }; - - // Define four list vectors (dim=2) - float list_vecs[4 * 2] = { - 1.0f, 0.0f, // Distance for Q0: 0.0, Q1: 2.0 - 0.0f, 1.0f, // Distance for Q0: 2.0, Q1: 0.0 - 1.0f, 1.0f, // Distance for Q0: 1.0, Q1: 1.0 - 2.0f, 2.0f // Distance for Q0: 4.0, Q1: 2.0 - }; - int64_t list_ids[4] = {100, 200, 300, 400}; - - // Create TopkBuffers for each query - std::vector> topk_buffers = create_buffers(3, k, false); // L2 metric (is_descending=false) - - // Perform batched_scan_list with batch_size=2 - batched_scan_list( - query_vecs, - list_vecs, - list_ids, - 3, // num_queries - 4, // list_size - 2, // dim - topk_buffers, - metric - ); - - // Expected top-k per query: - // Query 0: min distance: 0.0 (id=100) - // Query 1: min distance: 0.0 (id=200) - // Query 2: min distance: 1.0 (id=300) - - // Verify Query 0 - auto topk_q0 = topk_buffers[0]->get_topk(); - auto topk_q0_ids = topk_buffers[0]->get_topk_indices(); - ASSERT_EQ(topk_q0.size(), 1); - EXPECT_FLOAT_EQ(topk_q0[0], 0.0f); - EXPECT_EQ(topk_q0_ids[0], 100); - - // Verify Query 1 - auto topk_q1 = topk_buffers[1]->get_topk(); - auto topk_q1_ids = topk_buffers[1]->get_topk_indices(); - ASSERT_EQ(topk_q1.size(), 1); - EXPECT_FLOAT_EQ(topk_q1[0], 0.0f); - EXPECT_EQ(topk_q1_ids[0], 200); - - // Verify Query 2 - auto topk_q2 = topk_buffers[2]->get_topk(); - auto topk_q2_ids = topk_buffers[2]->get_topk_indices(); - ASSERT_EQ(topk_q2.size(), 1); - EXPECT_FLOAT_EQ(topk_q2[0], 0.0f); - EXPECT_EQ(topk_q2_ids[0], 300); -} - -// Test batched_scan_list with empty list -TEST_F(ListScanningTest, BatchedScanList_EmptyList) { - int k = 2; - MetricType metric = faiss::METRIC_L2; - - // Define one query vector (dim=2) - float query_vecs[1 * 2] = { - 1.0f, 1.0f // Query 0 - }; - int num_queries = 1; - - // Empty list - float list_vecs[0 * 2] = {}; // No list vectors - int64_t list_ids[0] = {}; - - // Create TopkBuffers for the single query - std::vector> topk_buffers = create_buffers(num_queries, k, false); // L2 metric - - // Perform batched_scan_list - batched_scan_list( - query_vecs, - list_vecs, - list_ids, - num_queries, - 0, // list_size - 2, // dim - topk_buffers, - metric - ); - - // Expected top-k: all distances remain +infinity, ids remain -1 - auto topk_q0 = topk_buffers[0]->get_topk(); - auto topk_q0_ids = topk_buffers[0]->get_topk_indices(); - ASSERT_EQ(topk_q0.size(), 0); // No valid top-k elements - - // Since there are no list vectors, all top-k entries should be default values - // However, based on TopkBuffer implementation, if no elements were inserted, get_topk() returns an empty vector - // To verify default values, you might need to adjust TopkBuffer's get_topk method to include default entries - // Alternatively, ensure that the test logic handles empty top-k appropriately -} - -// Test batched_scan_list with less than k elements -// Test batched_scan_list with less than k elements -TEST_F(ListScanningTest, BatchedScanList_LessThanKElements) { - int k = 5; - MetricType metric = faiss::METRIC_L2; - - // Define one query vector (dim=3) - float query_vecs[1 * 3] = {1.0f, 1.0f, 1.0f}; // Query 0 - int num_queries = 1; - - // Define a list with only 2 vectors (dim=3) - float list_vecs[2 * 3] = { - 1.0f, 1.0f, 1.0f, // Distance: 0.0 - 2.0f, 2.0f, 2.0f // Distance: sqrt(3) - }; - int64_t list_ids[2] = {100, 200}; - - // Create TopkBuffers for the single query - std::vector> topk_buffers = create_buffers(num_queries, k, false); // L2 metric - - // Perform batched_scan_list - batched_scan_list( - query_vecs, - list_vecs, - list_ids, - num_queries, - 2, // list_size=2 - 3, // dim=3 - topk_buffers, - metric - ); - - // Expected top-k: 0.0 (id=100), sqrt(3) (~1.73205) (id=200), inf, inf, inf - // Since TopkBuffer maintains only inserted elements, we need to handle default values separately - // Modify TopkBuffer's get_topk and get_topk_indices to include default entries if necessary - - // Retrieve top-k elements - auto topk_q0 = topk_buffers[0]->get_topk(); - auto topk_q0_ids = topk_buffers[0]->get_topk_indices(); - - // Verify inserted elements - ASSERT_EQ(topk_q0.size(), 2); // Only 2 elements were inserted - EXPECT_FLOAT_EQ(topk_q0[0], 0.0f); - EXPECT_EQ(topk_q0_ids[0], 100); - EXPECT_FLOAT_EQ(topk_q0[1], sqrt(3.0)); - EXPECT_EQ(topk_q0_ids[1], 200); -} - -TEST_F(ListScanningTest, LargeListCorrectnessInnerProduct) { - int num_queries = 100; - int64_t list_size = 10000; - int d = 128; - int k = 10; - Tensor query_vectors = torch::randn({num_queries, d}, torch::kFloat32); - Tensor list_vectors = torch::randn({list_size, d}, torch::kFloat32); - Tensor list_ids = torch::arange(0, list_size, torch::kInt64); - - // compute ground truth using pytorch matmul + topk - Tensor distances = torch::matmul(query_vectors, list_vectors.t()); - auto topk = torch::topk(distances, k, 1, true); - auto gt_ids = std::get<1>(topk); - auto gt_dists = std::get<0>(topk); - - auto gt_ids_accessor = gt_ids.accessor(); - auto gt_dists_accessor = gt_dists.accessor(); - - // perform single query scan first - auto buffer = make_shared(k, true); - for (int i = 0; i < num_queries; i++) { - scan_list( - query_vectors[i].data_ptr(), - list_vectors.data_ptr(), - list_ids.data_ptr(), - list_size, - d, - *buffer, - faiss::METRIC_INNER_PRODUCT - ); - - // check result is correct - auto topk_dist = buffer->get_topk(); - auto topk_ids = buffer->get_topk_indices(); - - for (int j = 0; j < k; j++) { - EXPECT_NEAR(topk_dist[j], gt_dists_accessor[i][j], .01); - EXPECT_EQ(topk_ids[j], gt_ids_accessor[i][j]); - } - buffer->reset(); - } - - // now perform batched scan - auto buffers = create_buffers(num_queries, k, true); - batched_scan_list( - query_vectors.data_ptr(), - list_vectors.data_ptr(), - list_ids.data_ptr(), - num_queries, - list_size, - d, - buffers, - faiss::METRIC_INNER_PRODUCT - ); - - for (int i = 0; i < num_queries; i++) { - auto topk_dist = buffers[i]->get_topk(); - auto topk_ids = buffers[i]->get_topk_indices(); - - for (int j = 0; j < k; j++) { - EXPECT_NEAR(topk_dist[j], gt_dists_accessor[i][j], .01); - EXPECT_EQ(topk_ids[j], gt_ids_accessor[i][j]); - } - } -} - -TEST_F(ListScanningTest, LargeListCorrectnessL2) { - int num_queries = 100; - int64_t list_size = 10000; - int d = 128; - int k = 10; - Tensor query_vectors = torch::randn({num_queries, d}, torch::kFloat32); - Tensor list_vectors = torch::randn({list_size, d}, torch::kFloat32); - Tensor list_ids = torch::arange(0, list_size, torch::kInt64); - - // compute ground truth using pytorch matmul + topk - Tensor distances = torch::cdist(query_vectors, list_vectors); - auto topk = torch::topk(distances, k, 1, false); - auto gt_ids = std::get<1>(topk); - auto gt_dists = std::get<0>(topk); - - auto gt_ids_accessor = gt_ids.accessor(); - auto gt_dists_accessor = gt_dists.accessor(); - - // perform single query scan first - auto buffer = make_shared(k, false); - for (int i = 0; i < num_queries; i++) { - scan_list( - query_vectors[i].data_ptr(), - list_vectors.data_ptr(), - list_ids.data_ptr(), - list_size, - d, - *buffer, - faiss::METRIC_L2 - ); - - // check result is correct - auto topk_dist = buffer->get_topk(); - auto topk_ids = buffer->get_topk_indices(); - - for (int j = 0; j < k; j++) { - EXPECT_NEAR(topk_dist[j], gt_dists_accessor[i][j], .01); - EXPECT_EQ(topk_ids[j], gt_ids_accessor[i][j]); - } - buffer->reset(); - } - - // now perform batched scan - auto buffers = create_buffers(num_queries, k, false); - batched_scan_list( - query_vectors.data_ptr(), - list_vectors.data_ptr(), - list_ids.data_ptr(), - num_queries, - list_size, - d, - buffers, - faiss::METRIC_L2 - ); - - for (int i = 0; i < num_queries; i++) { - auto topk_dist = buffers[i]->get_topk(); - auto topk_ids = buffers[i]->get_topk_indices(); - - for (int j = 0; j < k; j++) { - EXPECT_NEAR(topk_dist[j], gt_dists_accessor[i][j], .01); - EXPECT_EQ(topk_ids[j], gt_ids_accessor[i][j]); - } - } -} \ No newline at end of file +// // list_scanning_test.cpp +// +// #include +// #include +// #include +// #include +// #include +// #include +// +// #include "list_scanning.h" // Ensure this path is correct +// +// // Test fixture for list scanning functions +// class ListScanningTest : public ::testing::Test { +// protected: +// // Helper function to create a TopkBuffer +// std::shared_ptr create_buffer(int k, bool is_descending) { +// return std::make_shared>(k, is_descending, k * 100, 0); +// } +// }; +// +// // Test scan_list with single query and L2 metric +// TEST_F(ListScanningTest, ScanList_SingleQuery_L2) { +// int k = 2; +// bool is_descending = false; // L2 +// auto buffer = create_buffer(k, is_descending); +// +// // Define a query vector and a list of vectors +// float query_vec[3] = {1.0f, 0.0f, 0.0f}; +// float list_vecs[4 * 3] = { +// 1.0f, 0.0f, 0.0f, // Distance: 0.0 +// 0.0f, 1.0f, 0.0f, // Distance: 2.0 +// 1.0f, 1.0f, 0.0f, // Distance: 1.0 +// 2.0f, 0.0f, 0.0f // Distance: 1.0 +// }; +// int64_t list_ids[4] = {10, 20, 30, 40}; +// +// // Expected top-k: 0.0 (id=10), 1.0 (id=30 or 40) +// scan_list(query_vec, list_vecs, list_ids, 4, 3, *buffer, faiss::METRIC_L2, 10000000.0f); +// +// std::vector expected_dists = {0.0f, 1.0f}; +// std::vector expected_ids = {10, 30}; // Could also be {10, 40} based on order +// +// auto topk_dists = buffer->get_topk(); +// auto topk_ids = buffer->get_topk_indices(); +// +// ASSERT_EQ(topk_dists.size(), expected_dists.size()); +// ASSERT_EQ(topk_ids.size(), expected_ids.size()); +// +// EXPECT_FLOAT_EQ(topk_dists[0], expected_dists[0]); +// EXPECT_EQ(topk_ids[0], expected_ids[0]); +// +// // Second element could be id=30 or id=40 +// EXPECT_FLOAT_EQ(topk_dists[1], expected_dists[1]); +// EXPECT_TRUE(topk_ids[1] == 30 || topk_ids[1] == 40); +// } +// +// // Test scan_list with single query and Inner Product metric +// TEST_F(ListScanningTest, ScanList_SingleQuery_InnerProduct) { +// int k = 2; +// bool is_descending = true; // Inner Product +// auto buffer = create_buffer(k, is_descending); +// +// // Define a query vector and a list of vectors +// float query_vec[3] = {1.0f, 0.0f, 0.0f}; +// float list_vecs[4 * 3] = { +// 1.0f, 0.0f, 0.0f, // Inner product: 1.0 +// 0.0f, 1.0f, 0.0f, // Inner product: 0.0 +// 1.0f, 1.0f, 0.0f, // Inner product: 1.0 +// 2.0f, 0.0f, 0.0f // Inner product: 2.0 +// }; +// int64_t list_ids[4] = {10, 20, 30, 40}; +// +// // Expected top-k: 2.0 (id=40), 1.0 (id=10 or 30) +// scan_list(query_vec, list_vecs, list_ids, 4, 3, *buffer, faiss::METRIC_INNER_PRODUCT); +// +// std::vector expected_dists = {2.0f, 1.0f}; +// std::vector expected_ids = {40, 10}; // Could also be {40, 30} +// +// auto topk_dists = buffer->get_topk(); +// auto topk_ids = buffer->get_topk_indices(); +// +// ASSERT_EQ(topk_dists.size(), expected_dists.size()); +// ASSERT_EQ(topk_ids.size(), expected_ids.size()); +// +// EXPECT_FLOAT_EQ(topk_dists[0], expected_dists[0]); +// EXPECT_EQ(topk_ids[0], expected_ids[0]); +// +// // Second element could be id=10 or id=30 +// EXPECT_FLOAT_EQ(topk_dists[1], expected_dists[1]); +// EXPECT_TRUE(topk_ids[1] == 10 || topk_ids[1] == 30); +// } +// +// // Test batched_scan_list with multiple queries and L2 metric +// TEST_F(ListScanningTest, BatchedScanList_MultipleQueries_L2) { +// int k = 2; +// MetricType metric = faiss::METRIC_L2; +// +// // Define two query vectors (dim=3) +// float query_vecs[2 * 3] = { +// 1.0f, 0.0f, 0.0f, // Query 0 +// 0.0f, 1.0f, 0.0f // Query 1 +// }; +// +// // Define a list of vectors (dim=3) +// float list_vecs[4 * 3] = { +// 1.0f, 0.0f, 0.0f, // Q0: 0.0, Q1: 2.0 +// 0.0f, 1.0f, 0.0f, // Q0: 2.0, Q1: 0.0 +// 1.0f, 1.0f, 0.0f, // Q0: 1.0, Q1: 1.0 +// 2.0f, 0.0f, 0.0f // Q0: 4.0, Q1: 2.0 +// }; +// int64_t list_ids[4] = {10, 20, 30, 40}; +// +// // Create TopkBuffers for each query +// std::vector> topk_buffers = create_buffers(2, k, false); // L2 metric (is_descending=false) +// +// // Perform batched_scan_list +// batched_scan_list( +// query_vecs, +// list_vecs, +// list_ids, +// 2, // num_queries +// 4, // list_size +// 3, // dim +// topk_buffers, +// metric +// ); +// +// // Expected results: +// // Query 0: top-k distances: 0.0 (id=10), 1.0 (id=30) +// // Query 1: top-k distances: 0.0 (id=20), 1.0 (id=30) +// +// // Verify Query 0 +// auto topk_q0 = topk_buffers[0]->get_topk(); +// auto topk_q0_ids = topk_buffers[0]->get_topk_indices(); +// ASSERT_EQ(topk_q0.size(), 2); +// EXPECT_FLOAT_EQ(topk_q0[0], 0.0f); +// EXPECT_EQ(topk_q0_ids[0], 10); +// EXPECT_FLOAT_EQ(topk_q0[1], 1.0f); +// EXPECT_EQ(topk_q0_ids[1], 30); +// +// // Verify Query 1 +// auto topk_q1 = topk_buffers[1]->get_topk(); +// auto topk_q1_ids = topk_buffers[1]->get_topk_indices(); +// ASSERT_EQ(topk_q1.size(), 2); +// EXPECT_FLOAT_EQ(topk_q1[0], 0.0f); +// EXPECT_EQ(topk_q1_ids[0], 20); +// EXPECT_FLOAT_EQ(topk_q1[1], 1.0f); +// EXPECT_EQ(topk_q1_ids[1], 30); +// } +// +// // Test batched_scan_list with multiple queries and Inner Product metric +// TEST_F(ListScanningTest, BatchedScanList_MultipleQueries_InnerProduct) { +// int k = 2; +// MetricType metric = faiss::METRIC_INNER_PRODUCT; +// +// // Define two query vectors (dim=3) +// float query_vecs[2 * 3] = { +// 1.0f, 0.0f, 0.0f, // Query 0 +// 0.0f, 1.0f, 0.0f // Query 1 +// }; +// +// // Define a list of vectors (dim=3) +// float list_vecs[4 * 3] = { +// 1.0f, 0.0f, 0.0f, // Inner products [1.0, 0.0] (id=10) +// 0.0f, 1.0f, 0.0f, // Inner products [0.0, 1.0] (id=20) +// .5f, 1.5f, 0.0f, // Inner products [.5, 1.5] (id=30) +// 2.0f, 0.0f, 0.0f // Inner products [2.0, 0.0] (id=40) +// }; +// int64_t list_ids[4] = {10, 20, 30, 40}; +// +// // Create TopkBuffers for each query +// std::vector> topk_buffers = create_buffers(2, k, true); // Inner Product metric (is_descending=true) +// +// // Perform batched_scan_list +// batched_scan_list( +// query_vecs, +// list_vecs, +// list_ids, +// 2, // num_queries +// 4, // list_size +// 3, // dim +// topk_buffers, +// metric +// ); +// +// // Expected results: +// // Query 0: top-k scores: 2.0 (id=40), 1.0 (id=10) +// // Query 1: top-k scores: 1.5 (id=30), 1.0 (id=20) +// +// // Verify Query 0 +// auto topk_q0 = topk_buffers[0]->get_topk(); +// auto topk_q0_ids = topk_buffers[0]->get_topk_indices(); +// +// std::cout << "topk_q0: " << topk_q0[0] << " " << topk_q0[1] << std::endl; +// std::cout << "topk_q0_ids: " << topk_q0_ids[0] << " " << topk_q0_ids[1] << std::endl; +// +// ASSERT_EQ(topk_q0.size(), 2); +// EXPECT_FLOAT_EQ(topk_q0[0], 2.0f); +// EXPECT_TRUE(topk_q0_ids[0] == 40); +// EXPECT_FLOAT_EQ(topk_q0[1], 1.0f); +// EXPECT_TRUE(topk_q0_ids[1] == 10 || topk_q0_ids[1] == 20); +// +// // Verify Query 1 +// auto topk_q1 = topk_buffers[1]->get_topk(); +// auto topk_q1_ids = topk_buffers[1]->get_topk_indices(); +// ASSERT_EQ(topk_q1.size(), 2); +// EXPECT_FLOAT_EQ(topk_q1[0], 1.5f); +// EXPECT_TRUE(topk_q1_ids[0] == 30); +// EXPECT_FLOAT_EQ(topk_q1[1], 1.0f); +// EXPECT_TRUE(topk_q1_ids[1] == 20); +// } +// +// // Test batched_scan_list without list_ids (list_ids == nullptr) +// TEST_F(ListScanningTest, BatchedScanList_NoListIds) { +// int k = 2; +// MetricType metric = faiss::METRIC_L2; +// +// // Define two query vectors +// float query_vecs[2 * 2] = { +// 1.0f, 0.0f, // Query 0 +// 0.0f, 1.0f // Query 1 +// }; +// +// // Define a list of vectors +// float list_vecs[3 * 2] = { +// 1.0f, 0.0f, // Distance for Q0: 0.0, Q1: 2.0 +// 0.0f, 1.0f, // Distance for Q0: 2.0, Q1: 0.0 +// 1.0f, 1.0f // Distance for Q0: 1.0, Q1: 1.0 +// }; +// // list_ids == nullptr +// +// // Create TopkBuffers for each query +// std::vector> topk_buffers = create_buffers(2, k, false); // Inner Product metric (is_descending=true) +// +// +// // Perform batched_scan_list +// batched_scan_list( +// query_vecs, +// list_vecs, +// nullptr, // list_ids +// 2, // num_queries +// 3, // list_size +// 2, // dim +// topk_buffers, +// metric +// ); +// +// // Expected results: +// // Query 0: top-k distances: 0.0 (id=0), 1.0 (id=2) +// // Query 1: top-k distances: 0.0 (id=1), 1.0 (id=2) +// +// std::vector expected_dists_q0 = {0.0f, 1.0f}; +// std::vector expected_ids_q0 = {0, 2}; +// std::vector expected_dists_q1 = {0.0f, 1.0f}; +// std::vector expected_ids_q1 = {1, 2}; +// +// // Verify Query 0 +// auto topk_q0 = topk_buffers[0]->get_topk(); +// auto topk_q0_ids = topk_buffers[0]->get_topk_indices(); +// +// std::cout << "topk_q0: " << topk_q0[0] << " " << topk_q0[1] << std::endl; +// std::cout << "topk_q0_ids: " << topk_q0_ids[0] << " " << topk_q0_ids[1] << std::endl; +// +// ASSERT_EQ(topk_q0.size(), 2); +// EXPECT_FLOAT_EQ(topk_q0[0], 0.0f); +// EXPECT_EQ(topk_q0_ids[0], 0); +// EXPECT_FLOAT_EQ(topk_q0[1], 1.0f); +// EXPECT_EQ(topk_q0_ids[1], 2); +// +// // Verify Query 1 +// auto topk_q1 = topk_buffers[1]->get_topk(); +// auto topk_q1_ids = topk_buffers[1]->get_topk_indices(); +// ASSERT_EQ(topk_q1.size(), 2); +// EXPECT_FLOAT_EQ(topk_q1[0], 0.0f); +// EXPECT_EQ(topk_q1_ids[0], 1); +// EXPECT_FLOAT_EQ(topk_q1[1], 1.0f); +// EXPECT_EQ(topk_q1_ids[1], 2); +// } +// +// // Test batched_scan_list with batch_size parameter +// TEST_F(ListScanningTest, BatchedScanList_WithBatchSize) { +// int k = 1; +// MetricType metric = faiss::METRIC_L2; +// +// // Define three query vectors (dim=2) +// float query_vecs[3 * 2] = { +// 1.0f, 0.0f, // Query 0 +// 0.0f, 1.0f, // Query 1 +// 1.0f, 1.0f // Query 2 +// }; +// +// // Define four list vectors (dim=2) +// float list_vecs[4 * 2] = { +// 1.0f, 0.0f, // Distance for Q0: 0.0, Q1: 2.0 +// 0.0f, 1.0f, // Distance for Q0: 2.0, Q1: 0.0 +// 1.0f, 1.0f, // Distance for Q0: 1.0, Q1: 1.0 +// 2.0f, 2.0f // Distance for Q0: 4.0, Q1: 2.0 +// }; +// int64_t list_ids[4] = {100, 200, 300, 400}; +// +// // Create TopkBuffers for each query +// std::vector> topk_buffers = create_buffers(3, k, false); // L2 metric (is_descending=false) +// +// // Perform batched_scan_list with batch_size=2 +// batched_scan_list( +// query_vecs, +// list_vecs, +// list_ids, +// 3, // num_queries +// 4, // list_size +// 2, // dim +// topk_buffers, +// metric +// ); +// +// // Expected top-k per query: +// // Query 0: min distance: 0.0 (id=100) +// // Query 1: min distance: 0.0 (id=200) +// // Query 2: min distance: 1.0 (id=300) +// +// // Verify Query 0 +// auto topk_q0 = topk_buffers[0]->get_topk(); +// auto topk_q0_ids = topk_buffers[0]->get_topk_indices(); +// ASSERT_EQ(topk_q0.size(), 1); +// EXPECT_FLOAT_EQ(topk_q0[0], 0.0f); +// EXPECT_EQ(topk_q0_ids[0], 100); +// +// // Verify Query 1 +// auto topk_q1 = topk_buffers[1]->get_topk(); +// auto topk_q1_ids = topk_buffers[1]->get_topk_indices(); +// ASSERT_EQ(topk_q1.size(), 1); +// EXPECT_FLOAT_EQ(topk_q1[0], 0.0f); +// EXPECT_EQ(topk_q1_ids[0], 200); +// +// // Verify Query 2 +// auto topk_q2 = topk_buffers[2]->get_topk(); +// auto topk_q2_ids = topk_buffers[2]->get_topk_indices(); +// ASSERT_EQ(topk_q2.size(), 1); +// EXPECT_FLOAT_EQ(topk_q2[0], 0.0f); +// EXPECT_EQ(topk_q2_ids[0], 300); +// } +// +// // Test batched_scan_list with empty list +// TEST_F(ListScanningTest, BatchedScanList_EmptyList) { +// int k = 2; +// MetricType metric = faiss::METRIC_L2; +// +// // Define one query vector (dim=2) +// float query_vecs[1 * 2] = { +// 1.0f, 1.0f // Query 0 +// }; +// int num_queries = 1; +// +// // Empty list +// float list_vecs[0 * 2] = {}; // No list vectors +// int64_t list_ids[0] = {}; +// +// // Create TopkBuffers for the single query +// std::vector> topk_buffers = create_buffers(num_queries, k, false); // L2 metric +// +// // Perform batched_scan_list +// batched_scan_list( +// query_vecs, +// list_vecs, +// list_ids, +// num_queries, +// 0, // list_size +// 2, // dim +// topk_buffers, +// metric +// ); +// +// // Expected top-k: all distances remain +infinity, ids remain -1 +// auto topk_q0 = topk_buffers[0]->get_topk(); +// auto topk_q0_ids = topk_buffers[0]->get_topk_indices(); +// ASSERT_EQ(topk_q0.size(), 0); // No valid top-k elements +// +// // Since there are no list vectors, all top-k entries should be default values +// // However, based on TopkBuffer implementation, if no elements were inserted, get_topk() returns an empty vector +// // To verify default values, you might need to adjust TopkBuffer's get_topk method to include default entries +// // Alternatively, ensure that the test logic handles empty top-k appropriately +// } +// +// // Test batched_scan_list with less than k elements +// // Test batched_scan_list with less than k elements +// TEST_F(ListScanningTest, BatchedScanList_LessThanKElements) { +// int k = 5; +// MetricType metric = faiss::METRIC_L2; +// +// // Define one query vector (dim=3) +// float query_vecs[1 * 3] = {1.0f, 1.0f, 1.0f}; // Query 0 +// int num_queries = 1; +// +// // Define a list with only 2 vectors (dim=3) +// float list_vecs[2 * 3] = { +// 1.0f, 1.0f, 1.0f, // Distance: 0.0 +// 2.0f, 2.0f, 2.0f // Distance: sqrt(3) +// }; +// int64_t list_ids[2] = {100, 200}; +// +// // Create TopkBuffers for the single query +// std::vector> topk_buffers = create_buffers(num_queries, k, false); // L2 metric +// +// // Perform batched_scan_list +// batched_scan_list( +// query_vecs, +// list_vecs, +// list_ids, +// num_queries, +// 2, // list_size=2 +// 3, // dim=3 +// topk_buffers, +// metric +// ); +// +// // Expected top-k: 0.0 (id=100), sqrt(3) (~1.73205) (id=200), inf, inf, inf +// // Since TopkBuffer maintains only inserted elements, we need to handle default values separately +// // Modify TopkBuffer's get_topk and get_topk_indices to include default entries if necessary +// +// // Retrieve top-k elements +// auto topk_q0 = topk_buffers[0]->get_topk(); +// auto topk_q0_ids = topk_buffers[0]->get_topk_indices(); +// +// // Verify inserted elements +// ASSERT_EQ(topk_q0.size(), 2); // Only 2 elements were inserted +// EXPECT_FLOAT_EQ(topk_q0[0], 0.0f); +// EXPECT_EQ(topk_q0_ids[0], 100); +// EXPECT_FLOAT_EQ(topk_q0[1], sqrt(3.0)); +// EXPECT_EQ(topk_q0_ids[1], 200); +// } +// +// TEST_F(ListScanningTest, LargeListCorrectnessInnerProduct) { +// int num_queries = 100; +// int64_t list_size = 10000; +// int d = 128; +// int k = 10; +// Tensor query_vectors = torch::randn({num_queries, d}, torch::kFloat32); +// Tensor list_vectors = torch::randn({list_size, d}, torch::kFloat32); +// Tensor list_ids = torch::arange(0, list_size, torch::kInt64); +// +// // compute ground truth using pytorch matmul + topk +// Tensor distances = torch::matmul(query_vectors, list_vectors.t()); +// auto topk = torch::topk(distances, k, 1, true); +// auto gt_ids = std::get<1>(topk); +// auto gt_dists = std::get<0>(topk); +// +// auto gt_ids_accessor = gt_ids.accessor(); +// auto gt_dists_accessor = gt_dists.accessor(); +// +// // perform single query scan first +// auto buffer = make_shared(k, true, k * 100, 0); +// for (int i = 0; i < num_queries; i++) { +// scan_list( +// query_vectors[i].data_ptr(), +// list_vectors.data_ptr(), +// list_ids.data_ptr(), +// list_size, +// d, +// *buffer, +// faiss::METRIC_INNER_PRODUCT +// ); +// +// // check result is correct +// auto topk_dist = buffer->get_topk(); +// auto topk_ids = buffer->get_topk_indices(); +// +// for (int j = 0; j < k; j++) { +// EXPECT_NEAR(topk_dist[j], gt_dists_accessor[i][j], .01); +// EXPECT_EQ(topk_ids[j], gt_ids_accessor[i][j]); +// } +// buffer->reset(); +// } +// +// // now perform batched scan +// auto buffers = create_buffers(num_queries, k, true); +// batched_scan_list( +// query_vectors.data_ptr(), +// list_vectors.data_ptr(), +// list_ids.data_ptr(), +// num_queries, +// list_size, +// d, +// buffers, +// faiss::METRIC_INNER_PRODUCT +// ); +// +// for (int i = 0; i < num_queries; i++) { +// auto topk_dist = buffers[i]->get_topk(); +// auto topk_ids = buffers[i]->get_topk_indices(); +// +// for (int j = 0; j < k; j++) { +// EXPECT_NEAR(topk_dist[j], gt_dists_accessor[i][j], .01); +// EXPECT_EQ(topk_ids[j], gt_ids_accessor[i][j]); +// } +// } +// } +// +// TEST_F(ListScanningTest, LargeListCorrectnessL2) { +// int num_queries = 100; +// int64_t list_size = 10000; +// int d = 128; +// int k = 10; +// Tensor query_vectors = torch::randn({num_queries, d}, torch::kFloat32); +// Tensor list_vectors = torch::randn({list_size, d}, torch::kFloat32); +// Tensor list_ids = torch::arange(0, list_size, torch::kInt64); +// +// // compute ground truth using pytorch matmul + topk +// Tensor distances = torch::cdist(query_vectors, list_vectors); +// auto topk = torch::topk(distances, k, 1, false); +// auto gt_ids = std::get<1>(topk); +// auto gt_dists = std::get<0>(topk); +// +// auto gt_ids_accessor = gt_ids.accessor(); +// auto gt_dists_accessor = gt_dists.accessor(); +// +// // perform single query scan first +// auto buffer = make_shared(k, false, k * 100, 0); +// for (int i = 0; i < num_queries; i++) { +// scan_list( +// query_vectors[i].data_ptr(), +// list_vectors.data_ptr(), +// list_ids.data_ptr(), +// list_size, +// d, +// *buffer, +// faiss::METRIC_L2 +// ); +// +// // check result is correct +// auto topk_dist = buffer->get_topk(); +// auto topk_ids = buffer->get_topk_indices(); +// +// for (int j = 0; j < k; j++) { +// EXPECT_NEAR(topk_dist[j], gt_dists_accessor[i][j], .01); +// EXPECT_EQ(topk_ids[j], gt_ids_accessor[i][j]); +// } +// buffer->reset(); +// } +// +// // now perform batched scan +// auto buffers = create_buffers(num_queries, k, false); +// batched_scan_list( +// query_vectors.data_ptr(), +// list_vectors.data_ptr(), +// list_ids.data_ptr(), +// num_queries, +// list_size, +// d, +// buffers, +// faiss::METRIC_L2 +// ); +// +// for (int i = 0; i < num_queries; i++) { +// auto topk_dist = buffers[i]->get_topk(); +// auto topk_ids = buffers[i]->get_topk_indices(); +// +// for (int j = 0; j < k; j++) { +// EXPECT_NEAR(topk_dist[j], gt_dists_accessor[i][j], .01); +// EXPECT_EQ(topk_ids[j], gt_ids_accessor[i][j]); +// } +// } +// } \ No newline at end of file diff --git a/test/cpp/quake_index.cpp b/test/cpp/quake_index.cpp index 0521ea02..d7e6c057 100644 --- a/test/cpp/quake_index.cpp +++ b/test/cpp/quake_index.cpp @@ -7,6 +7,9 @@ #include #include "quake_index.h" #include +#include +#include +#include // Helper functions for random data static torch::Tensor generate_random_data(int64_t num_vectors, int64_t dim) { @@ -22,7 +25,7 @@ class QuakeIndexTest : public ::testing::Test { // Example parameters int64_t dimension_ = 16; int64_t nlist_ = 8; - int64_t num_vectors_ = 100; + int64_t num_vectors_ = 1000; int64_t num_queries_ = 5; // Data & IDs @@ -243,7 +246,8 @@ TEST_F(QuakeIndexTest, SaveLoadTest) { // Load into a new index QuakeIndex loaded_index; - loaded_index.load(path); + shared_ptr default_params = std::make_shared(); + loaded_index.load(path, default_params); // minimal checks EXPECT_EQ(loaded_index.ntotal(), index.ntotal()); @@ -283,17 +287,19 @@ TEST(QuakeIndexStressTest, LargeBuildTest) { << " vectors took " << build_duration_ms << " ms.\n"; } -#ifdef FAISS_ENABLE_GPU +#ifdef QUAKE_ENABLE_GPU TEST(QuakeIndexStressTestGPU, LargeBuildTest) { // Attempt to build an index with a large number of vectors. // Adjust these numbers based on your available memory/compute. int64_t dimension = 128; // Medium-high dimension int64_t num_vectors = 1e6; // 1 million vectors - auto data_vectors = generate_random_data(num_vectors, dimension); - auto data_ids = generate_sequential_ids(num_vectors, 0); + auto data_vectors = generate_random_data(num_vectors, dimension).contiguous(); + auto data_ids = generate_sequential_ids(num_vectors, 0).contiguous(); QuakeIndex index; + std::cout << "generated\n"; + auto build_params = std::make_shared(); build_params->nlist = 512; build_params->metric = "l2"; @@ -476,63 +482,44 @@ TEST(QuakeIndexStressTest, HighDimensionTest) { ASSERT_EQ(result->ids.size(1), search_params->k); } -// ------------------------------------------------------------------------- -// SEARCH, ADD, REMOVE, AND MAINTENANCE TEST -// ------------------------------------------------------------------------- -TEST(QuakeIndexStressTest, SearchAddRemoveMaintenanceTest) { - // Repeatedly search, add, remove, and perform maintenance to see if the index remains consistent. - - int64_t dimension = 16; - int64_t num_vectors = 100000; - int64_t num_queries = 100; - int64_t batch_size = 10; - +TEST_F(QuakeIndexTest, AddLevelTest) +{ QuakeIndex index; - auto build_params = std::make_shared(); - build_params->nlist = 100; - build_params->metric = "l2"; - build_params->niter = 3; - Tensor data_vectors = generate_random_data(num_vectors, dimension); - Tensor data_ids = generate_sequential_ids(num_vectors, 0); - - index.build(data_vectors, data_ids, build_params); - - for (int i = 0; i < 100; i++) { - // Search - std::cout << "Iteration " << i << std::endl; - auto query_vectors = generate_random_data(num_queries, dimension) * .1; - auto search_params = std::make_shared(); - search_params->nprobe = 1; - search_params->k = 5; - auto search_result = index.search(query_vectors, search_params); - ASSERT_EQ(search_result->ids.size(0), query_vectors.size(0)); - ASSERT_EQ(search_result->ids.size(1), search_params->k); - - // Add - auto add_vectors = generate_random_data(batch_size, dimension); - auto add_ids = generate_sequential_ids(batch_size, (i * batch_size) + num_vectors); - auto add_info = index.add(add_vectors, add_ids); - ASSERT_EQ(add_info->n_vectors, batch_size); - - // Remove - auto remove_ids = add_ids.slice(0, 0, batch_size / 2); - auto remove_info = index.remove(remove_ids); - ASSERT_EQ(remove_info->n_vectors, batch_size / 2); - - index.maintenance(); - } - - SUCCEED(); + // 1. Build a 2‑level index (leaf + one parent) + auto bp = std::make_shared(); + bp->nlist = 100; + bp->metric = "l2"; + index.build(data_vectors_, data_ids_, bp); + + const int64_t leaf_partitions = index.nlist(); // = nlist_ + + // 2. Add a *new* level on top + auto top_bp = std::make_shared(); + top_bp->nlist = 10; // flat over centroids + top_bp->metric = "l2"; + index.add_level(top_bp); + + // root now has a parent + ASSERT_NE(index.parent_, nullptr); + // that parent should have exactly `leaf_partitions` vectors (one per centroid) + EXPECT_EQ(index.parent_->ntotal(), leaf_partitions); + + // 3. Ensure we can still search and retrieve sane results + auto sp = std::make_shared(); + sp->k = 3; + auto res = index.search(query_vectors_, sp); + ASSERT_EQ(res->ids.size(0), query_vectors_.size(0)); + ASSERT_EQ(res->ids.size(1), sp->k); } // Define the GPU related test only if FAISS GPU support is enabled -#ifdef FAISS_ENABLE_GPU +#ifdef QUAKE_ENABLE_GPU // Test build with GPU enabled TEST(QuakeIndexGPUTest, BuildWithGPUTest) { int64_t dimension = 32; - int64_t num_vectors = 200; - int64_t nlist = 5; + int64_t num_vectors = 10000; + int64_t nlist = 10; torch::Tensor data_vectors = generate_random_data(num_vectors, dimension); torch::Tensor data_ids = generate_sequential_ids(num_vectors, 0); @@ -554,4 +541,4 @@ TEST(QuakeIndexGPUTest, BuildWithGPUTest) { EXPECT_EQ(timing_info->n_vectors, data_vectors.size(0)); EXPECT_EQ(timing_info->d, data_vectors.size(1)); } -#endif \ No newline at end of file +#endif diff --git a/test/cpp/quake_multi_level.cpp b/test/cpp/quake_multi_level.cpp new file mode 100644 index 00000000..9b5c787a --- /dev/null +++ b/test/cpp/quake_multi_level.cpp @@ -0,0 +1,103 @@ + +#include +#include "quake_index.h" + +using torch::Tensor; + +//--------------------------------------------------------------- +// Helpers +//--------------------------------------------------------------- +static Tensor rand_vectors(int64_t n, int64_t d, uint64_t seed) +{ + torch::manual_seed(seed); + return torch::randn({n, d}, torch::kFloat32); +} +static Tensor seq_ids(int64_t n, int64_t start = 0) +{ + return torch::arange(start, start + n, torch::kInt64); +} + +//--------------------------------------------------------------- +// Constants for all tests +//--------------------------------------------------------------- +constexpr int64_t DIM = 32; +constexpr int64_t NVEC = 20'000; +constexpr int64_t LEAF = 64; +constexpr int64_t MID = 16; +constexpr int64_t ROOT = 4; + +//--------------------------------------------------------------- +// 1. Build a 3‑level index, search, maintain, search again. +//--------------------------------------------------------------- +TEST(QuakeMultiLevel, EndToEndMaintenance) +{ + Tensor data = rand_vectors(NVEC, DIM, 0); + Tensor ids = seq_ids(NVEC); + + QuakeIndex ix; + auto build = std::make_shared(); + build->nlist = LEAF; + build->metric = "l2"; + ix.build(data, ids, build); + + // add 2nd level + auto mid = std::make_shared(); + mid->nlist = MID; + mid->metric = "l2"; + ix.add_level(mid); + + // add 3rd (root) level + auto root = std::make_shared(); + root->nlist = ROOT; + root->metric = "l2"; + ix.add_level(root); + + // quick sanity on hierarchy depth + ASSERT_NE(ix.parent_, nullptr); + ASSERT_NE(ix.parent_->parent_, nullptr); + + auto sp = std::make_shared(); + sp->k = 10; + + // baseline search + auto baseline = ix.search(rand_vectors(50, DIM, 1), sp); + ASSERT_EQ(baseline->ids.size(1), sp->k); + + // hammer with 2000 queries to populate hit tracker then run maintenance + ix.search(rand_vectors(2000, DIM, 2), sp); + auto mi = ix.maintenance(); + EXPECT_GE(mi->total_time_us, 0); + EXPECT_EQ(ix.ntotal(), NVEC); + + // final search should still work + auto after = ix.search(rand_vectors(50, DIM, 3), sp); + EXPECT_EQ(after->ids.size(1), sp->k); +} + +//--------------------------------------------------------------- +// 2. Calling add_level() on a 1‑level index must throw. +//--------------------------------------------------------------- +TEST(QuakeMultiLevel, AddLevelWithoutParentThrows) +{ + Tensor data = rand_vectors(1000, DIM, 4); + Tensor ids = seq_ids(1000); + + QuakeIndex flat; + flat.build(data, ids, std::make_shared()); // nlist=0 ⇒ flat + + EXPECT_THROW(flat.add_level(std::make_shared()), std::runtime_error); +} + +//--------------------------------------------------------------- +// 3. Root maintenance should be a safe no‑op (flat index). +//--------------------------------------------------------------- +TEST(QuakeMultiLevel, RootMaintenanceSafe) +{ + Tensor data = rand_vectors(800, DIM, 5); + Tensor ids = seq_ids(800); + + QuakeIndex flat; + flat.build(data, ids, std::make_shared()); // nlist=0 + + EXPECT_NO_THROW(flat.maintenance()); +} \ No newline at end of file diff --git a/test/cpp/query_coordinator.cpp b/test/cpp/query_coordinator.cpp index d272f2d0..0ed914b2 100644 --- a/test/cpp/query_coordinator.cpp +++ b/test/cpp/query_coordinator.cpp @@ -169,34 +169,6 @@ TEST_F(QueryCoordinatorTest, WorkerInitializationTest) { ASSERT_TRUE(coordinator->workers_initialized_); } -TEST_F(QueryCoordinatorTest, FlatWorkerScan) { - int num_workers = 4; - - // create flat index - auto flat_index = std::make_shared(); - auto build_params = std::make_shared(); - build_params->nlist = 1; - build_params->metric = "l2"; - flat_index->build(torch::randn({20, dimension_}), torch::arange(20), build_params); - - // create coordinator with workers - auto coordinator = std::make_shared( - flat_index->parent_, - flat_index->partition_manager_, - nullptr, - faiss::METRIC_L2, - num_workers - ); - - auto search_params = std::make_shared(); - search_params->k = 2; - - auto result_worker = coordinator->search(queries_, search_params); - ASSERT_TRUE(result_worker != nullptr); - ASSERT_EQ(result_worker->ids.sizes(), (std::vector{queries_.size(0), search_params->k})); - ASSERT_EQ(result_worker->distances.sizes(), (std::vector{queries_.size(0), search_params->k})); -} - // Test that worker-based scan produces the same results as serial scan TEST_F(QueryCoordinatorTest, WorkerScanCorrectnessTest) { // Initialize QueryCoordinator with workers diff --git a/test/cpp/scan_benchmark.cpp b/test/cpp/scan_benchmark.cpp new file mode 100644 index 00000000..df2b467c --- /dev/null +++ b/test/cpp/scan_benchmark.cpp @@ -0,0 +1,264 @@ +#include +#include +#include +#include +#include +#include // For std::iota +#include // For std::fixed, std::setprecision +#include // For std::thread +#include // For std::atomic + +#include // Assuming torch is available +#include "concurrentqueue.h" // Assuming moodycamel's queue header is findable + +using namespace std::chrono; +using torch::Tensor; + +// Namespace for this specific microbenchmark's helpers +namespace JobEnqueueMicrobenchmarkWithContention { + +// --- Struct Definitions (ScanJob, CoreResources, MockPartitionManager - same as before) --- + struct ScanJob { + int64_t partition_id; + int k_val; + const float* query_vector; + int query_id; + bool is_batched = false; + int64_t num_queries_in_job = 0; + int rank = 0; + + ScanJob() = default; + ScanJob(ScanJob&& other) noexcept = default; + ScanJob& operator=(ScanJob&& other) noexcept = default; + ScanJob(const ScanJob&) = delete; + ScanJob& operator=(const ScanJob&) = delete; + }; + + struct CoreResources { + int core_id_val; + moodycamel::ConcurrentQueue job_queue; + // For worker thread to count its dequeued items: + std::atomic dequeued_items_by_worker{0}; + + + // Explicitly define constructors for clarity if needed, + // or rely on aggregate initialization / default members. + CoreResources() : core_id_val(-1), job_queue(), dequeued_items_by_worker(0) {} // Default + CoreResources(int id) : core_id_val(id), job_queue(), dequeued_items_by_worker(0) {} // With ID + + // Since moodycamel::ConcurrentQueue is not copyable, + // CoreResources becomes non-copyable. We need move semantics. + CoreResources(CoreResources&& other) noexcept + : core_id_val(other.core_id_val), + job_queue(std::move(other.job_queue)), // Moodycamel queue is movable + dequeued_items_by_worker(other.dequeued_items_by_worker.load()) + {} + + CoreResources& operator=(CoreResources&& other) noexcept { + if (this != &other) { + core_id_val = other.core_id_val; + job_queue = std::move(other.job_queue); + dequeued_items_by_worker.store(other.dequeued_items_by_worker.load()); + } + return *this; + } + // Delete copy operations + CoreResources(const CoreResources&) = delete; + CoreResources& operator=(const CoreResources&) = delete; + }; + + + class MockPartitionManager { + public: + int num_cores_manager_ = 1; + MockPartitionManager(int cores) : num_cores_manager_(cores) { + if (cores <= 0) num_cores_manager_ = 1; + } + int get_partition_core_id(int64_t pid) const { + return static_cast(pid % num_cores_manager_); + } + }; + +// --- Parameters --- + static const int64_t BENCH_DIMENSION = 128; + static const int BENCH_K = 10; + static const int BENCH_NUM_CORES = 4; // Number of worker threads and queues + +// --- Worker Thread Function --- + void dequeuing_worker_function(CoreResources& core_res, std::atomic& stop_signal) { + long items_this_worker = 0; + int64_t job_id_val; // Renamed to avoid conflict + + while (!stop_signal.load(std::memory_order_acquire)) { + if (core_res.job_queue.try_dequeue(job_id_val)) { + items_this_worker++; + // Simulate minimal work with the job_id_val if needed + volatile int64_t sink = job_id_val; // "Use" job_id_val + (void)sink; + } else { + // Yield if the queue is often empty to reduce busy-waiting, + // though try_dequeue is quite efficient. + std::this_thread::yield(); + } + } + // After stop_signal is true, drain any remaining items from the queue + while (core_res.job_queue.try_dequeue(job_id_val)) { + items_this_worker++; + volatile int64_t sink = job_id_val; + (void)sink; + } + core_res.dequeued_items_by_worker.fetch_add(items_this_worker, std::memory_order_relaxed); + } + + +// --- Function with the snippet to benchmark (Updated jid logic) --- + int run_job_creation_and_enqueue_snippet( + std::vector& job_buffer, + std::vector& core_resources_list, + const MockPartitionManager& partition_manager, + const Tensor& partition_ids_tensor, + const float* x_ptr, + long num_queries, + long num_partitions_per_query, + int k_param, + int dimension_param) + { + auto partition_ids_accessor = partition_ids_tensor.accessor(); + job_buffer.resize(num_queries * num_partitions_per_query); // Max possible + int jid = 0; // Current index for job_buffer, also basis for enqueued ID + + for (long q = 0; q < num_queries; q++) { + for (long p = 0; p < num_partitions_per_query; p++) { + int64_t pid_val = partition_ids_accessor[q][p]; + if (pid_val == -1) continue; + + ScanJob job_item; + job_item.is_batched = false; + job_item.query_id = static_cast(q); + job_item.partition_id = pid_val; + job_item.k_val = k_param; + job_item.query_vector = x_ptr + q * dimension_param; + job_item.num_queries_in_job = 1; + job_item.rank = static_cast(p); + + int core_id = partition_manager.get_partition_core_id(pid_val); + + // Original snippet logic: place job, THEN increment jid, THEN enqueue the NEW jid. + job_buffer[jid] = std::move(job_item); + jid++; // jid is now the 1-based count of jobs created, or index for the *next* job. + core_resources_list[core_id].job_queue.enqueue(jid); // Enqueue this 1-based count. + } + } + return jid; // Returns the total count of jobs created (which is the last enqueued jid value) + } + + +// --- GTest TEST Case with Contention --- + TEST(JobEnqueueMicrobenchmark, SnippetPerformanceWithContention) { + long num_queries_test = 100; // Increase for more work: e.g., 1000 + long num_partitions_test = 1000; // Increase for more work: e.g., 1000 + int test_iterations = 5; + int warmup_iterations = 1; + + std::cout << std::endl << "[JobEnqueueMicrobenchmark::SnippetPerformanceWithContention]" << std::endl; + std::cout << " Config: Queries=" << num_queries_test + << ", Partitions/Query=" << num_partitions_test + << ", Iterations=" << test_iterations + << ", Worker Threads=" << BENCH_NUM_CORES << std::endl; + + // 1. Setup data structures + std::vector local_job_buffer; + std::vector local_core_resources; + local_core_resources.reserve(BENCH_NUM_CORES); + for (int i = 0; i < BENCH_NUM_CORES; ++i) { + local_core_resources.emplace_back(i); // Use constructor CoreResources(int id) + } + JobEnqueueMicrobenchmarkWithContention::MockPartitionManager local_partition_manager(BENCH_NUM_CORES); + + std::vector local_queries_data_storage(num_queries_test * BENCH_DIMENSION); + float* local_x_ptr = local_queries_data_storage.data(); + Tensor local_partition_ids_tensor = torch::empty({num_queries_test, num_partitions_test}, torch::TensorOptions().dtype(torch::kInt64)); + auto accessor = local_partition_ids_tensor.accessor(); + for (long i = 0; i < num_queries_test; ++i) { + for (long j = 0; j < num_partitions_test; ++j) { + accessor[i][j] = i * num_partitions_test + j; + } + } + + std::vector durations_ms; + durations_ms.reserve(test_iterations); + long total_jobs_actually_created = 0; + + + // 2. Run benchmark loop + for (int iter = 0; iter < test_iterations + warmup_iterations; ++iter) { + // Reset/drain queues and worker counts for each iteration + for (auto& cr : local_core_resources) { + int64_t dummy_val; + while (cr.job_queue.try_dequeue(dummy_val)); + cr.dequeued_items_by_worker.store(0, std::memory_order_relaxed); + } + local_job_buffer.clear(); + + std::atomic stop_worker_threads_signal{false}; + std::vector worker_threads; + worker_threads.reserve(BENCH_NUM_CORES); + + // Launch worker threads + for (int i = 0; i < BENCH_NUM_CORES; ++i) { + worker_threads.emplace_back(dequeuing_worker_function, std::ref(local_core_resources[i]), std::ref(stop_worker_threads_signal)); + } + + auto iter_start_time = high_resolution_clock::now(); + + // This is the timed operation (producer) + int jobs_created_this_iteration = run_job_creation_and_enqueue_snippet( + local_job_buffer, local_core_resources, local_partition_manager, + local_partition_ids_tensor, local_x_ptr, + num_queries_test, num_partitions_test, + BENCH_K, BENCH_DIMENSION); + + auto iter_end_time = high_resolution_clock::now(); + + // Signal workers to stop and wait for them to finish + stop_worker_threads_signal.store(true, std::memory_order_release); + for (auto& t : worker_threads) { + if (t.joinable()) { + t.join(); + } + } + + if (iter >= warmup_iterations) { + durations_ms.push_back(duration_cast(iter_end_time - iter_start_time).count() / 1000.0); + total_jobs_actually_created += jobs_created_this_iteration; + } + + // Sanity check: total dequeued items should match jobs created + long total_dequeued_this_iteration = 0; + for (const auto& cr : local_core_resources) { + total_dequeued_this_iteration += cr.dequeued_items_by_worker.load(); + } + ASSERT_EQ(total_dequeued_this_iteration, jobs_created_this_iteration) + << "Mismatch in created vs dequeued jobs for iteration " << iter; + } + + // 3. Report results + double total_duration_ms = 0; + for (double d : durations_ms) total_duration_ms += d; + double avg_duration_ms = durations_ms.empty() ? 0 : total_duration_ms / durations_ms.size(); + + std::cout << " Average time for snippet iteration (with contention): " + << std::fixed << std::setprecision(3) << avg_duration_ms << " ms" << std::endl; + + if (test_iterations > 0 && total_jobs_actually_created > 0) { + long avg_jobs_per_iteration = total_jobs_actually_created / test_iterations; + if (avg_jobs_per_iteration > 0 && avg_duration_ms > 0) { + std::cout << " Average jobs created per iteration: " << avg_jobs_per_iteration << std::endl; + std::cout << " Average time per job (snippet part, with contention): " + << (avg_duration_ms * 1000.0 / avg_jobs_per_iteration) << " us" << std::endl; + } + } + ASSERT_GT(avg_duration_ms, -0.00001); + } + +} // namespace JobEnqueueMicrobenchmarkWithContention \ No newline at end of file diff --git a/test/cpp/topk_buffer.cpp b/test/cpp/topk_buffer.cpp index c2afaaec..88225a65 100644 --- a/test/cpp/topk_buffer.cpp +++ b/test/cpp/topk_buffer.cpp @@ -13,7 +13,7 @@ class TypedTopKBufferTest : public ::testing::Test { protected: // Helper function to create a TopkBuffer std::shared_ptr create_buffer(int k, bool is_descending) { - return std::make_shared>(k, is_descending); + return std::make_shared>(k, is_descending, k* 100, 0); } }; diff --git a/test/experiments/__init__.py b/test/experiments/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/experiments/adaptive_partition_scanning/aps.py b/test/experiments/adaptive_partition_scanning/aps.py deleted file mode 100644 index 78ee9e7f..00000000 --- a/test/experiments/adaptive_partition_scanning/aps.py +++ /dev/null @@ -1,698 +0,0 @@ -import logging -from pathlib import Path - -import hydra -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -import seaborn as sns -import torch -from hydra.utils import get_original_cwd -from omegaconf import DictConfig - -from quake import IndexBuildParams, QuakeIndex, SearchParams -from quake.datasets.ann_datasets import load_dataset -from quake.utils import compute_recall, to_path - -# Set up logging -log = logging.getLogger(__name__) - -# Constants -MIN_FANOUT = 8 - - -# Dataset Management -def get_dataset(cfg): - dataset_path = get_original_cwd() / to_path(cfg.dataset.path) - vectors, queries, gt = load_dataset(cfg.dataset.name, dataset_path) - return vectors, queries[: cfg.experiment.nq], gt[: cfg.experiment.nq] - - -# Index Management -def build_or_load_index(cfg, num_workers): - index_dir = get_original_cwd() / to_path(cfg.paths.index_dir) - index_path = index_dir / f"{cfg.dataset.name}_dynamic_ivf{cfg.index.nc}.index" - if not index_path.exists() or cfg.overwrite.index: - # Build and save index - vectors, _, _ = get_dataset(cfg) - build_index = QuakeIndex() - build_params = IndexBuildParams() - build_params.nlist = cfg.index.nc - build_params.metric = cfg.index.metric - build_index.build(vectors, torch.arange(vectors.size(0)), build_params) - build_index.save(str(index_path.absolute())) - log.info(f"Index built and saved to {index_path}") - - # Load existing index - index = QuakeIndex() - index.load(str(index_path.absolute()), num_workers) - - log.info(f"Index loaded from {index_path} with {num_workers} workers") - - return index - - -# Experiment Execution -def run_single_experiment(args): - method, recall_target, recompute_ratio, use_precompute, cfg, action, n_workers = args - index = build_or_load_index(cfg, n_workers) - _, queries, gt = get_dataset(cfg) - k = cfg.experiment.k - - result_dir = get_original_cwd() / cfg.paths.results_dir / method - result_dir.mkdir(parents=True, exist_ok=True) - - print(f"Running experiment for {method} at recall {recall_target} with action {action}") - - if method == "Oracle": - result_path = result_dir / f"recall_{recall_target:.2f}.csv" - - data_df = run_experiment_for_configuration( - index=index, - queries=queries, - gt=gt, - k=k, - recall_target=recall_target, - oracle=True, - ) - # Save per-query data - data_df.to_csv(result_path, index=False) - log.info(f"Results saved to {result_path}") - elif method == "FixedNProbe": - result_path = result_dir / f"recall_{recall_target:.2f}.csv" - if result_path.exists() and not cfg.overwrite.results: - log.info(f"Results for {method} at recall {recall_target} already exist. Skipping.") - return - - data_df = run_experiment_for_configuration( - index=index, - queries=queries, - gt=gt, - k=k, - recall_target=recall_target, - fixed_nprobe=True, - ) - # Save per-query data - data_df.to_csv(result_path, index=False) - log.info(f"Results saved to {result_path}") - elif method.startswith("APS"): - result_path = result_dir / f"recall_{recall_target:.2f}.csv" - if result_path.exists() and not cfg.overwrite.results: - log.info(f"Results for {method} at recall {recall_target} already exist. Skipping.") - return - - data_df = run_experiment_for_configuration( - index=index, - queries=queries, - gt=gt, - k=k, - recall_target=recall_target, - recompute_ratio=recompute_ratio, - use_precompute=use_precompute, - ) - # Save per-query data - data_df.to_csv(result_path, index=False) - log.info(f"Results saved to {result_path}") - else: - raise ValueError(f"Unknown method: {method}") - - -# Results Management and Plotting -def collect_and_plot_results(cfg): - methods = cfg.methods - recall_targets = cfg.experiment.recall_targets - all_data = [] - - for method in methods: - for recall_target in recall_targets: - result_dir = cfg.paths.results_dir / method - result_path = result_dir / f"recall_{recall_target:.2f}.csv" - if not result_path.exists(): - log.warning(f"Result file {result_path} does not exist. Skipping.") - continue - data_df = pd.read_csv(result_path) - data_df["Recall Target"] = recall_target - - complete_name = method - data_df["Method"] = complete_name - all_data.append(data_df) - - if not all_data: - log.error("No data available for plotting.") - return - - df_plot = pd.concat(all_data, ignore_index=True) - - # Clean data - df_plot = df_plot.dropna(subset=["total_time_ms", "nprobe", "recall"]) - df_plot = df_plot[(df_plot["total_time_ms"] >= 0) & (df_plot["nprobe"] >= 0) & (df_plot["recall"] >= 0)] - - df_plot["Query Time (ms)"] = df_plot["total_time_ms"] - df_plot["Recall"] = df_plot["recall"] - - # Compute stats - grouped = df_plot.groupby(["Recall Target", "Method"]) - stats = grouped.agg( - { - "Query Time (ms)": ["min", "mean", "max"], - "nprobe": ["min", "mean", "max"], - "Recall": ["min", "mean", "max"], - "buffer_init_time_ms": ["min", "mean", "max"], - "job_enqueue_time_ms": ["min", "mean", "max"], - "boundary_distance_time_ms": ["min", "mean", "max"], - "job_wait_time_ms": ["min", "mean", "max"], - "result_aggregate_time_ms": ["min", "mean", "max"], - } - ).reset_index() - stats.columns = [" ".join(col).strip() for col in stats.columns.values] - - # Compute 'other_time_ms' and include in stats - df_plot["other_time_ms"] = df_plot["total_time_ms"] - ( - df_plot["buffer_init_time_ms"] - + df_plot["job_enqueue_time_ms"] - + df_plot["boundary_distance_time_ms"] - + df_plot["job_wait_time_ms"] - + df_plot["result_aggregate_time_ms"] - ) - df_plot["other_time_ms"] = df_plot["other_time_ms"].apply(lambda x: x if x >= 0 else 0) - other_time_stats = grouped["other_time_ms"].agg(["min", "mean", "max"]).reset_index() - other_time_stats.columns = [ - "Recall Target", - "Method", - "other_time_ms min", - "other_time_ms mean", - "other_time_ms max", - ] - stats = pd.merge(stats, other_time_stats, on=["Recall Target", "Method"]) - - # Compute p99 for 'Query Time (ms)' and 'Recall' - p95_latency = grouped["Query Time (ms)"].quantile(0.95).reset_index() - p95_recall = grouped["Recall"].quantile(0.05).reset_index() - p95_latency.rename(columns={"Query Time (ms)": "Query Time (ms) p95"}, inplace=True) - p95_recall.rename(columns={"Recall": "Recall p95"}, inplace=True) - stats = stats.merge(p95_latency, on=["Recall Target", "Method"]) - stats = stats.merge(p95_recall, on=["Recall Target", "Method"]) - - # Save this as intermediate results - stats_save_path = cfg.paths.plot_dir / "all_intermediate_stats.csv" - stats.to_csv(stats_save_path, index=False) - - # Plotting - plot_recall_only(df_plot, stats, cfg.paths.plot_dir) - plot_mean_line_plots(df_plot, stats, cfg.paths.plot_dir) - plot_query_overheads(stats, cfg.paths.plot_dir) - - -palette = {"Oracle": "C0", "APS": "C1"} - - -def plot_recall_only(df_plot, stats, plot_dir): - sns.set_style("whitegrid") - sns.set_context("talk", font_scale=0.8) - plt.rcParams["font.weight"] = "bold" - plt.rcParams["axes.labelweight"] = "bold" - - fig, ax = plt.subplots(figsize=(6, 4)) - stats["Method"] = stats["Method"].replace("Adaptive nprobe", "APS") - - # Shared x-axis range - x_min = stats["Recall Target"].min() - x_max = stats["Recall Target"].max() - - # Generate detailed x-axis ticks - x_ticks = [0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1.0] - - ax.set_title("Sift1M: Measured Recall vs. Recall Target", fontsize=14, fontweight="bold") - - for method in stats["Method"].unique(): - method_stats = stats[stats["Method"] == method].sort_values("Recall Target") - ax.plot( - method_stats["Recall Target"], - method_stats["Recall mean"], - label=method, - color=palette[method], - linewidth=4, - marker="o", - markersize=10, - ) - - # plot p95 recall - # p95_recall = method_stats['Recall p95'] - # ax.plot(method_stats['Recall Target'], p95_recall, color=palette[method], linestyle='--', linewidth=8) - - x_vals = np.linspace(x_min, x_max, 100) - ax.plot(x_vals, x_vals, color="black", linestyle="--", linewidth=4, label="Recall Target") - - ax.set_ylabel("Recall", fontsize=16, fontweight="bold") - ax.tick_params(axis="both", which="major", labelsize=16) - ax.set_yticks([0.7, 0.8, 0.9, 1.0]) # Custom y-axis ticks - ax.set_xticks(x_ticks) - ax.legend(fontsize=8, prop={"weight": "bold"}, loc="lower right") # Legend moved to top-left - ax.grid(True) - - plt.tight_layout() - plot_path = plot_dir / "recall_plot.pdf" - plt.savefig(plot_path, bbox_inches="tight") - log.info(f"Plot saved to {plot_path}") - plt.show() - - -# Plotting Functions -def plot_mean_line_plots(df_plot, stats, plot_dir): - sns.set_style("whitegrid") - sns.set_context("talk", font_scale=0.8) - plt.rcParams["font.weight"] = "bold" - plt.rcParams["axes.labelweight"] = "bold" - - # Create a vertically stacked layout with shared x-axis - fig, axes = plt.subplots(3, 1, figsize=(8, 6), sharex=True) # Stacked plots with shared x-axis - - stats["Method"] = stats["Method"].replace("Adaptive nprobe", "APS") - - # Shared x-axis range - x_min = stats["Recall Target"].min() - x_max = stats["Recall Target"].max() - - # Generate detailed x-axis ticks - x_ticks = [0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1.0] - - axes[0].set_title("Sift1M: Recall, Query Latency, and Nprobe vs. Recall Target", fontsize=20, fontweight="bold") - - # Plot 1: Recall - for method in stats["Method"].unique(): - method_stats = stats[stats["Method"] == method].sort_values("Recall Target") - axes[0].plot( - method_stats["Recall Target"], - method_stats["Recall mean"], - label=method, - color=palette[method], - linewidth=8, - marker="o", - markersize=10, - ) - # plot p95 recall - # p95_recall = method_stats['Recall p95'] - # axes[0].plot(method_stats['Recall Target'], p95_recall, color=palette[method], linestyle='--', linewidth=8) - - # Diagonal reference line - x_vals = np.linspace(x_min, x_max, 100) - axes[0].plot(x_vals, x_vals, color="black", linestyle="--", linewidth=8, label="Recall Target") - - axes[0].set_ylabel("Recall", fontsize=16, fontweight="bold") - axes[0].tick_params(axis="both", which="major", labelsize=16) - axes[0].set_yticks([0.7, 0.8, 0.9, 1.0]) # Custom y-axis ticks - axes[0].set_xticks(x_ticks) - axes[0].legend(fontsize=8, prop={"weight": "bold"}, loc="lower right") # Legend moved to top-left - axes[0].grid(True) - - # Plot 2: Query Time - for method in stats["Method"].unique(): - method_stats = stats[stats["Method"] == method].sort_values("Recall Target") - axes[1].plot( - method_stats["Recall Target"], - method_stats["Query Time (ms) mean"], - label=method, - color=palette[method], - linewidth=8, - marker="o", - markersize=10, - ) - - # plot p95 query time - p95_query_time = method_stats["Query Time (ms) p95"] - axes[1].plot(method_stats["Recall Target"], p95_query_time, color=palette[method], linestyle="--", linewidth=8) - - # axes[1].set_ylabel('Query Time (ms)', fontsize=20, fontweight='bold') - axes[1].set_yscale("log") - axes[1].tick_params(axis="both", which="major", labelsize=16) - axes[1].set_yticks([0.2, 0.4, 0.8, 1.6, 3.2]) # Custom y-axis ticks - axes[1].set_yticklabels(["0.2ms", "0.4ms", "0.8ms", "1.6ms", "3.2ms"]) # Add units to tick labels - axes[1].set_xticks(x_ticks) - # axes[1].legend(fontsize=16, prop={'weight': 'bold'}, loc='upper left') # Legend moved to top-left - axes[1].grid(True) - # - # # Plot 3: nprobe - for method in stats["Method"].unique(): - method_stats = stats[stats["Method"] == method].sort_values("Recall Target") - axes[2].plot( - method_stats["Recall Target"], - method_stats["nprobe mean"], - label=method, - color=palette[method], - linewidth=8, - marker="o", - markersize=10, - ) - - axes[2].set_xlabel("Recall Target", fontsize=16, fontweight="bold") # Shared x-axis - axes[2].set_ylabel("nprobe", fontsize=16, fontweight="bold") - axes[2].set_yscale("log") - axes[2].tick_params(axis="both", which="major", labelsize=16) - axes[2].set_yticks([4, 8, 16, 32, 64]) # Custom y-axis ticks - axes[2].set_yticklabels(["4", "8", "16", "32", "64"]) # Add units to tick labels - axes[2].set_xticks(x_ticks) - # axes[2].legend(fontsize=16, prop={'weight': 'bold'}, loc='upper left') # Legend moved to top-left - axes[2].grid(True) - - plt.tight_layout() - plot_path = plot_dir / "mean_line_plots_stacked.pdf" - plt.savefig(plot_path, bbox_inches="tight") - log.info(f"Plot saved to {plot_path}") - plt.show() - - -def plot_query_overheads(stats, plot_dir): - """ - Plot stacked bar charts showing the mean time taken by each component for each method and recall target, - including an 'Other' category representing the remaining time. - """ - import matplotlib.pyplot as plt - import numpy as np - from matplotlib.patches import Patch - - # Define the components and their labels - components = [ - "buffer_init_time_ms", - "job_enqueue_time_ms", - "boundary_distance_time_ms", - "job_wait_time_ms", - "result_aggregate_time_ms", - "other_time_ms", # Added 'Other' category - ] - component_labels = [ - "Buffer Init", - "Job Enqueue", - "Boundary Distance", - "Job Wait", - "Result Aggregate", - "Other", # Label for 'Other' - ] - # Blue, Orange, Green, Red, Purple, Gray] - component_colors = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#8c564b", "#7f7f7f"] - - # remap the method names: Adaptive nprobe -> Adaptive Partition Scanning - # stats[methods == 'Adaptive nprobe'] = 'Adaptive Partition Scanning' - stats["Method"] = stats["Method"].replace("Adaptive nprobe", "APS") - - # Define methods and their hatching patterns - methods = stats["Method"].unique() - - method_hatches = {method: "" if method == "Oracle" else "//" for method in methods} - - # Prepare data for plotting - recall_targets = sorted(stats["Recall Target"].unique()) - - # Initialize plotting - fig, ax = plt.subplots(figsize=(14, 8)) - - bar_width = 0.1 - opacity = 0.8 - - n_groups = len(recall_targets) - index = np.arange(n_groups) - - # Set positions for each method's bars - # method_offsets = np.linspace(-bar_width/2, bar_width/2, len(methods)) - positions = {} - positions["Oracle"] = index - 3 * bar_width / 2 - positions["APS"] = index - bar_width / 2 - positions["APS-R"] = index + bar_width / 2 - positions["APS-RP"] = index + 3 * bar_width / 2 - positions["FixedNProbe"] = index + 5 * bar_width / 2 - - # Initialize cumulative bottoms for stacking - cumulative_bottoms = {method: np.zeros(n_groups) for method in methods} - - # Plot each component - for comp, comp_label, comp_color in zip(components, component_labels, component_colors): - for method in methods: - print(f"Plotting {comp} for {method}") - - comp_means = [] - for rt in recall_targets: - row = stats[(stats["Recall Target"] == rt) & (stats["Method"] == method)] - if not row.empty: - comp_time = row[f"{comp} mean"].values[0] - else: - comp_time = 0 - comp_means.append(comp_time) - print(f"Component means: {comp_means}") - - method_lookup = method.split("+")[0].strip() - pos = positions[method_lookup] - ax.bar( - pos, - comp_means, - bar_width, - bottom=cumulative_bottoms[method], - color=comp_color, - alpha=opacity, - hatch=method_hatches[method], - edgecolor="black", - label=comp_label if (method == methods[0]) else "", - ) - cumulative_bottoms[method] += comp_means - - # Set x-axis labels and ticks - ax.set_xlabel("Recall Target", fontsize=14, weight="bold") - ax.set_ylabel("Time (ms)", fontsize=14, weight="bold") - ax.set_title("Query Overheads by Method and Recall Target", fontsize=16, weight="bold") - ax.set_xticks(index) - ax.set_xticklabels([f"{rt:.2f}" for rt in recall_targets], fontsize=12) - ax.tick_params(axis="y", labelsize=12) - - # Create custom legend for components - component_handles = [Patch(facecolor=color, edgecolor="black") for color in component_colors] - component_legend = ax.legend( - component_handles, component_labels, title="Components", title_fontsize=14, fontsize=12, loc="upper left" - ) - - # Create custom legend for methods - method_handles = [ - Patch(facecolor="white", edgecolor="black", hatch=method_hatches[method], label=method) for method in methods - ] - ax.legend(method_handles, methods, title="Method", title_fontsize=14, fontsize=12, loc="upper right") - - # Add the component legend back to the plot - ax.add_artist(component_legend) - - # Add grid for better readability - ax.yaxis.grid(True, linestyle="--", which="major", color="grey", alpha=0.5) - - # set ylimit - ax.set_ylim(0, 2.5) - - plt.tight_layout() - plot_path = plot_dir / "query_overheads.png" - plt.savefig(plot_path) - log.info(f"Overhead plot saved to {plot_path}") - plt.show() - - -# Helper Functions -def run_experiment_for_configuration( - index, queries, gt, k, recall_target, oracle=False, recompute_ratio=0.05, use_precompute=True, fixed_nprobe=False -): - - timing_infos = [] - - if oracle: - nprobes = [] - for i, query in enumerate(queries): - max_nprobe = index.nlist() - min_nprobe = 1 - best_nprobe = max_nprobe - - # find minimum nprobe that achieves recall_target - while min_nprobe <= max_nprobe: - curr_nprobe = (min_nprobe + max_nprobe) // 2 - - search_params = SearchParams() - search_params.nprobe = curr_nprobe - search_params.k = k - search_params.recall_target = -1 - - search_result = index.search(query.unsqueeze(0), search_params) - ids, dist, timing_info = search_result.ids, search_result.distances, search_result.timing_info - recall = compute_recall(ids, gt[i].unsqueeze(0), k) - - if float(recall) >= recall_target: - max_nprobe = curr_nprobe - 1 - best_nprobe = curr_nprobe - else: - min_nprobe = curr_nprobe + 1 - - nprobes.append(best_nprobe) - - # Second run to execute queries - all_ids = [] - all_dists = [] - - for query, nprobe in zip(queries, nprobes): - - search_params = SearchParams() - search_params.nprobe = nprobe - search_params.k = k - search_params.recall_target = -1 - - search_result = index.search(query.unsqueeze(0), search_params) - - ids, dist, timing_info = search_result.ids, search_result.distances, search_result.timing_info - all_ids.append(ids) - all_dists.append(dist) - timing_infos.append(timing_info) - recalls = compute_recall(torch.cat(all_ids, dim=0), gt, k) - elif fixed_nprobe: - print("FIXED NPROBE") - # compute nprobe for recall target for all queries, rather than per-query - - # first use binary search to find the nprobe that achieves the recall target - max_nprobe = index.nlist() - min_nprobe = 1 - best_nprobe = max_nprobe - - while min_nprobe <= max_nprobe: - curr_nprobe = (min_nprobe + max_nprobe) // 2 - all_ids = [] - all_dists = [] - - search_params = SearchParams() - search_params.nprobe = curr_nprobe - search_params.k = k - search_params.recall_target = -1 - - for i, query in enumerate(queries): - search_result = index.search(query.unsqueeze(0), search_params) - ids, dist, timing_info = search_result.ids, search_result.distances, search_result.timing_info - all_ids.append(ids) - all_dists.append(dist) - recalls = compute_recall(torch.cat(all_ids, dim=0), gt, k) - avg_recall = recalls.mean().item() - if avg_recall >= recall_target: - max_nprobe = curr_nprobe - 1 - best_nprobe = curr_nprobe - else: - min_nprobe = curr_nprobe + 1 - - # Second run to execute queries - all_ids = [] - all_dists = [] - - search_params = SearchParams() - search_params.nprobe = best_nprobe - search_params.k = k - search_params.recall_target = -1 - - for query in queries: - search_result = index.search(query.unsqueeze(0), search_params) - ids, dist, timing_info = search_result.ids, search_result.distances, search_result.timing_info - all_ids.append(ids) - all_dists.append(dist) - timing_infos.append(timing_info) - recalls = compute_recall(torch.cat(all_ids, dim=0), gt, k) - else: - all_ids = [] - all_dists = [] - - search_params = SearchParams() - search_params.nprobe = -1 - search_params.k = k - search_params.initial_search_fraction = 0.1 - search_params.recall_target = recall_target - search_params.recompute_threshold = 0.01 - search_params.use_precomputed = use_precompute - search_params.num_threads = 1 - - # debug print search params - print( - f"Search Params: {search_params.nprobe}, {search_params.k}, {search_params.recall_target}, " - f"{search_params.recompute_threshold}, {search_params.use_precomputed}" - ) - - for query in queries: - search_result = index.search(query.unsqueeze(0), search_params) - ids, dist, timing_info = search_result.ids, search_result.distances, search_result.timing_info - all_ids.append(ids) - all_dists.append(dist) - timing_infos.append(timing_info) - - ids = torch.cat(all_ids, dim=0) - # dists = torch.cat(all_dists, dim=0) - recalls = compute_recall(ids, gt, k) - - per_query_data = [] - for i, timing_info in enumerate(timing_infos): - per_query_nprobe = timing_info.partitions_scanned - buffer_init_time_ms = timing_info.buffer_init_time_ns / 1e6 - job_enqueue_time_ms = timing_info.job_enqueue_time_ns / 1e6 - boundary_distance_time_ms = timing_info.boundary_distance_time_ns / 1e6 - job_wait_time_ms = timing_info.job_wait_time_ns / 1e6 - result_aggregate_time_ms = timing_info.result_aggregate_time_ns / 1e6 - total_time_ms = timing_info.total_time_ns / 1e6 - - per_query_recall = float(recalls[i]) - per_query_data.append( - { - "nprobe": per_query_nprobe, - "recall": per_query_recall, - "buffer_init_time_ms": buffer_init_time_ms, - "job_enqueue_time_ms": job_enqueue_time_ms, - "boundary_distance_time_ms": boundary_distance_time_ms, - "job_wait_time_ms": job_wait_time_ms, - "result_aggregate_time_ms": result_aggregate_time_ms, - "total_time_ms": total_time_ms, - } - ) - - per_query_data_df = pd.DataFrame(per_query_data) - # per_query_data_df = per_query_data_df[1:] # Skip the first query - return per_query_data_df - - -def get_nprobe_for_recall_target(recall_target, nlist): - nprobe = int(nlist * recall_target / 2) - nprobe = max(1, min(nprobe, nlist)) - return nprobe - - -@hydra.main(config_path="configs", config_name="sift1m") -def main(cfg: DictConfig): - # Set up directories - base_dir = get_original_cwd() / Path(cfg.base_dir) - - cfg.paths.index_dir = Path(cfg.paths.index_dir) - cfg.paths.index_dir.mkdir(parents=True, exist_ok=True) - cfg.paths.results_dir = base_dir / cfg.paths.results_dir - cfg.paths.results_dir.mkdir(parents=True, exist_ok=True) - cfg.paths.plot_dir = base_dir / cfg.paths.plot_dir - cfg.paths.plot_dir.mkdir(parents=True, exist_ok=True) - - if cfg.mode == "run": - # Prepare experiment parameters - methods = cfg.methods - recall_targets = cfg.experiment.recall_targets - n_workers = cfg.experiment.n_workers - experiment_args = [] - for method in methods: - for recall_target in recall_targets: - if method == "Oracle": - experiment_args.append((method, recall_target, 0.05, True, cfg, "execute_queries", n_workers)) - elif method == "APS": - experiment_args.append((method, recall_target, 0.0, True, cfg, "execute_queries", n_workers)) - elif method == "APS-R": - experiment_args.append((method, recall_target, -1, True, cfg, "execute_queries", n_workers)) - elif method == "APS-RP": - experiment_args.append((method, recall_target, -1, False, cfg, "execute_queries", n_workers)) - elif method == "FixedNProbe": - experiment_args.append((method, recall_target, -1, True, cfg, "execute_queries", n_workers)) - - # Run experiments - for args in experiment_args: - run_single_experiment(args) - - elif cfg.mode == "plot": - collect_and_plot_results(cfg) - - -if __name__ == "__main__": - main() diff --git a/test/experiments/adaptive_partition_scanning/configs/sift1m.yaml b/test/experiments/adaptive_partition_scanning/configs/sift1m.yaml deleted file mode 100644 index 51e8f982..00000000 --- a/test/experiments/adaptive_partition_scanning/configs/sift1m.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# configs/experiment.yaml -mode: run # options: build, run, plot -base_dir: data/aps -dataset: - name: sift1m - path: data -experiment: - nq: 100 - k: 100 - n_workers: 0 - recall_targets: [0.7, 0.75, 0.8, 0.85, 0.9, 0.95, .99] -index: - metric: l2 - nc: 1000 -overwrite: - index: false - results: true -methods: ['Oracle', 'APS'] -#methods: ['APS'] -#methods: ['Oracle'] - -paths: - index_dir: data/sift/indexes - results_dir: data/sift/results - plot_dir: data/sift/plots \ No newline at end of file diff --git a/test/experiments/osdi2025/README.md b/test/experiments/osdi2025/README.md new file mode 100644 index 00000000..a3759705 --- /dev/null +++ b/test/experiments/osdi2025/README.md @@ -0,0 +1,181 @@ + +# Quake OSDI 2025 Artifact Evaluation + +This artifact provides the experimental setup for the Quake vector search system, as described in the OSDI 2025 submission. + +## Structure + +The experiments are organized as follows: + + + +```plaintext +quake/test/experiments/osdi2025/ +├── experiment_runner.py \# Main script to run any experiment +├── experiment_utils.py \# Shared functionality across experiments +├── README.md \# This file +├── paper/ \# PDF of the paper +│ ├── Quake.pdf \# Original Submission +│ ├── Quake-Revision.pdf \# Revised Version +├── / \# Directory for each specific experiment +│ ├── configs/ \# YAML configuration files for the experiment +│ │ └── .yaml +│ └── run.py \# Python script to execute the experiment logic +└── ... \# Other experiment directories +``` + +Each experiment directory (e.g., `aps_recall_targets`, `early_termination`) contains its specific logic in `run.py` and configurations in its `configs/` subdirectory. The `experiment_utils.py` module provides shared functionality for loading data, configurations, preparing indexes, and plotting, aiming to simplify individual experiment scripts. + +## Installation + +**Requirements:** +- Python 3.9 or later +- Linux (Tested on Ubuntu 22.04) +- Conda package manager + +### Install from Scratch + +This will install Quake, the baselines and all necessary dependencies. + +Run `sudo install.sh` + +This will: +- Install required system packages (GCC/G++ 11, CMake, Miniconda, Intel oneAPI, etc.) +- Set up a Conda environment (quake-env) with all Python dependencies +- Build and install the Quake library and baselines (Faiss, SCANN, DiskANN, SVS) +- Automatically clone, build, and install all required repositories + +### Quake Installation Only (No Baselines) + +To just evaluate Quake use this. + +1. **Clone the Repository:** + ```bash + git clone https://github.com/marius-team/quake.git + cd quake + git checkout osdi2025 + git submodule update --init --recursive + + ``` + +2. **Set Up the Conda Environment:** + Create and activate the environment using the provided YAML file: + ```bash + conda env create -f environments/ubuntu-cpu/conda.yaml + conda activate quake-env + ``` + +3. **Install Quake:** + Ensure you are in the `quake` repository root. + ```bash + pip install --no-use-pep517 . + ``` + +## Running Experiments + +All experiments are launched using the central `experiment_runner.py` script located in the `test/experiments/osdi2025/` directory. + +**General Command Structure:** +Make sure to run all commands from the **repository root directory (`quake/`)**. + +```bash +OMP_NUM_THREADS=1 python3 -m test.experiments.osdi2025.experiment_runner \ + --experiment \ + --config \ + --output-dir # Optional +```` + +**Explanation:** + +* `OMP_NUM_THREADS=1`: Recommended for consistent performance measurement as OMP interferes with our custom parallelism. +* `--experiment `: Specifies which experiment to run (e.g., `kick_the_tires`, `aps_recall_targets`). +* `--config `: Specifies the configuration file from `test/experiments/osdi2025//configs/`. For `sift1m.yaml`, use `sift1m`. +* `--output-dir ` (Optional): If provided, results (CSVs, plots) will be saved here. Default is `test/experiments/osdi2025//results//`. + +### Kick the Tires (Quick Verification) + +1. **Run the Quickstart Example:** + Tests Quake's core operations. + + ```bash + python3 examples/quickstart.py + ``` + +2. **Run the `kick_the_tires` Benchmark:** + Tests basic system functionality and performance with a dynamic workload. + + ```bash + OMP_NUM_THREADS=1 python3 -m test.experiments.osdi2025.experiment_runner \ + --experiment kick_the_tires \ + --config sift1m + ``` + Outputs will be in `test/experiments/osdi2025/kick_the_tires/results/sift1m/`. + +### Reproduction of Main Quake Components + +These experiments should take less than hour (mostly in maintenance_ablation) to run in total if the datasets/indexes have been prepared ahead of time. Otherwise it will take 2 hours on the first run. + +1. Reproduction of APS (Figure 6 in `paper/Quake.pdf`) + + This compares an Oracle against APS at various recall targets. + + ```bash + OMP_NUM_THREADS=1 python3 -m test.experiments.osdi2025.experiment_runner \ + --experiment aps_recall_targets \ + --config sift1m + ``` + +2. Reproduction of NUMA-aware searching (Figure 5 in `paper/Quake.pdf`) + + This measures query latency and varies the number of workers used in searching and whether partitions are pinned to NUMA regions. + + ```bash + OMP_NUM_THREADS=1 python3 -m test.experiments.osdi2025.experiment_runner \ + --experiment numa_single_query \ + --config msturing10m + ``` + +3. Maintenance Ablation Study (Table 6 in `paper/Quake-Revision.pdf`) + + This removes critical pieces of the Quake maintenance policy and measure its effect on workload performance. + + ```bash + OMP_NUM_THREADS=1 python3 -m test.experiments.osdi2025.experiment_runner \ + --experiment maintenance_ablation \ + --config sift1m + ``` + + +### Full Reproduction (In progress) + +## Experiment Summaries and Outputs + +The following table summarizes each experiment and their figure/table in paper. Output files are generated within the experiment's specific output directory (refer to the `--output-dir` option or its default value). + +Some experiments are added as revisions for the camera ready. See `paper/Quake.pdf` for the initial submission and `paper/Quake-Revision.pdf` for the revised version + +| Experiment Name | Description | Reproduces in Paper | +|------------------------|----------------------------------------------------------------------------------------------------|---------------------------| +| `aps_recall_targets` | Evaluates Quake's Adaptive Partition Scanning (APS) against baselines for various recall targets. | Figure 6 in Quake | +| `early_termination` | Compares APS with early termination strategies like LAET and Auncel. | Table 5 in Quake-Revision | +| `kick_the_tires` | Basic dynamic workload stress test focusing on latency, recall, and system state over operations. | | +| `maintenance_ablation` | Evaluates the impact of Quake's maintenance operations under a dynamic workload. | Table 6 in Quake-Revision | +| `numa_multi_query` | Tests Quake's multi-query batch performance under different NUMA and worker configurations. | Revision Experiment | +| `numa_single_query` | Tests Quake's single-query performance under different NUMA and worker configurations. | Figure 5 in Quake | + +**To run any of these, replace `` and `` in the general command structure.** For example, to run the `aps_recall_targets` experiment with its `sift1m` configuration: + +```bash +OMP_NUM_THREADS=1 python3 -m test.experiments.osdi2025.experiment_runner \ + --experiment aps_recall_targets \ + --config sift1m +``` + +Results, including CSV files and plots as detailed above, will be saved to the specified (or default) output directory. Check the terminal output for the exact location. + +## Notes + +* **Dataset Downloads:** The `quake.datasets.ann_datasets.load_dataset` utility will attempt to download datasets if they are not found locally (typically in a `data/` directory in the repository root). An internet connection is required for initial downloads. +* **Execution Time:** Full experimental runs can be time-consuming, ranging from minutes for small configurations to many hours for large-scale evaluations. The `kick_the_tires` experiment with `sift1m` is designed for relatively quick verification. + + diff --git a/test/experiments/osdi2025/__init__.py b/test/experiments/osdi2025/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/experiments/osdi2025/aps_recall_targets/configs/sift1m.yaml b/test/experiments/osdi2025/aps_recall_targets/configs/sift1m.yaml new file mode 100644 index 00000000..6672d1af --- /dev/null +++ b/test/experiments/osdi2025/aps_recall_targets/configs/sift1m.yaml @@ -0,0 +1,29 @@ +# configs/experiment.yaml +mode: run # options: build, run, plot +base_dir: data/aps +dataset: + name: sift1m + path: data +experiment: + nq: 100 + k: 100 + n_workers: 1 + recall_targets: [.2, .3, .4, .5, .6, .7, .8, .9, .95, .99, .999] + recompute_ratio: 0.00001 + use_precompute: true + initial_search_fraction: .05 + aps_flush_period_us: 10 +index: + metric: l2 + nlist: 1000 +overwrite: + index: false + results: true +#methods: ['Oracle', 'APS'] +methods: ['APS'] +#methods: ['Oracle'] + +paths: + index_dir: data/sift1m/indexes + results_dir: data/sift1m/results + plot_dir: data/sift1m/plots \ No newline at end of file diff --git a/test/experiments/osdi2025/aps_recall_targets/run.py b/test/experiments/osdi2025/aps_recall_targets/run.py new file mode 100644 index 00000000..8ee5d7be --- /dev/null +++ b/test/experiments/osdi2025/aps_recall_targets/run.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +""" +Adaptive‐Partition‐Scanning Recall‐Target Runner (Simplified) +Supports: Oracle, FixedNProbe, APS +""" +import logging +from pathlib import Path +import numpy as np +import torch +import pandas as pd + +from test.experiments.osdi2025 import experiment_utils as common_utils +from quake import SearchParams # Keep direct import for clarity if used extensively + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") + +def run_experiment(cfg_path: str, output_dir_str: str): + cfg = common_utils.load_config(cfg_path) + out_dir = Path(output_dir_str) + + dataset_cfg = cfg["dataset"] + exp_cfg = cfg["experiment"] + idx_build_cfg = cfg["index"] + paths_cfg = cfg["paths"] + + vecs, queries, gt = common_utils.load_data( + dataset_cfg["name"], dataset_cfg.get("path", ""), exp_cfg["nq"] + ) + + idx_path = Path(paths_cfg["index_dir"]) / f"{dataset_cfg['name']}_ivf{idx_build_cfg['nlist']}.index" + quake_idx = common_utils.prepare_quake_index( + vecs, + {"nlist": idx_build_cfg["nlist"], "metric": idx_build_cfg["metric"], "num_workers": exp_cfg["n_workers"]}, + idx_path, + force_rebuild=cfg["overwrite"].get("index", False) + ) + logger.info(f"Index ready (workers={exp_cfg['n_workers']})") + + records = [] + k = exp_cfg["k"] + max_nlist_val = quake_idx.nlist() + + for method in cfg["methods"]: + for rt in exp_cfg["recall_targets"]: + per_query_stats = [] # (nprobe, recall, time_ms) + + if method == "Oracle": + for i, q_vec in enumerate(queries): + lo, hi, best_np = 1, max_nlist_val, max_nlist_val + while lo <= hi: + mid = (lo + hi) // 2 + sp = common_utils.create_search_params(nprobe=mid, k=k, recall_target=-1) + _, rec, _ = common_utils.run_search_trial(quake_idx, q_vec, gt[i], k, sp) + if rec >= rt: + best_np, hi = mid, mid - 1 + else: + lo = mid + 1 + sp_final = common_utils.create_search_params(nprobe=best_np, k=k, recall_target=-1) + _, final_rec, final_time_ms = common_utils.run_search_trial(quake_idx, q_vec, gt[i], k, sp_final) + per_query_stats.append((best_np, final_rec, final_time_ms)) + + elif method == "FixedNProbe": + lo, hi, best_global_np = 1, max_nlist_val, max_nlist_val + while lo <= hi: + mid = (lo + hi) // 2 + sp = common_utils.create_search_params(nprobe=mid, k=k, recall_target=-1) + current_ids = [] + for q_vec_fixed in queries: # Use all queries for tuning this + res_ids = quake_idx.search(q_vec_fixed.unsqueeze(0), sp).ids + current_ids.append(res_ids) + avg_rec = common_utils.quake_compute_recall(torch.cat(current_ids,0), gt, k).mean().item() + if avg_rec >= rt: + best_global_np, hi = mid, mid - 1 + else: + lo = mid + 1 + + sp_final = common_utils.create_search_params(nprobe=best_global_np, k=k, recall_target=-1) + for i, q_vec in enumerate(queries): + _, final_rec, final_time_ms = common_utils.run_search_trial(quake_idx, q_vec, gt[i], k, sp_final) + per_query_stats.append((best_global_np, final_rec, final_time_ms)) + + elif method == "APS": + sp = common_utils.create_search_params( + nprobe=-1, k=k, recall_target=rt, + recompute_threshold=exp_cfg["recompute_ratio"], + use_precomputed=exp_cfg["use_precompute"], + initial_search_fraction=exp_cfg["initial_search_fraction"], + aps_flush_period_us=exp_cfg["aps_flush_period_us"] + ) + # for i, q_vec in enumerate(queries): + # res = quake_idx.search(q_vec.unsqueeze(0), sp) + # nprobe_scan = res.timing_info.partitions_scanned + # final_rec = common_utils.quake_compute_recall(res.ids, gt[i].unsqueeze(0), k).item() + # final_time_ms = res.timing_info.total_time_ns / 1e6 + # boundary_time_ms = res.timing_info.boundary_time_ns / 1e6 if hasattr(res.timing_info, 'boundary_time_ns') else 0.0 + # aps_time_ms = res.timing_info.aps_time_ns / 1e6 if hasattr(res.timing_info, 'aps_time_ns') else 0.0 + # + # logger.info(f"APS recall: {final_rec:.4f} (target={rt}), nprobe_scanned={nprobe_scan}, boundary_time={boundary_time_ms:.2f}ms, aps_time={aps_time_ms:.2f}ms, total_time={final_time_ms:.2f}ms") + # per_query_stats.append((nprobe_scan, final_rec, final_time_ms)) + + res = quake_idx.search(queries, sp) + nprobe_scan = res.timing_info.partitions_scanned + final_recall = common_utils.quake_compute_recall(res.ids, gt, k) + final_time_ms = res.timing_info.total_time_ns / 1e6 + boundary_time_ms = res.timing_info.boundary_distance_time_ns / 1e6 + aps_time_ms = res.timing_info.aps_time_ns / 1e6 + + logger.info(f"APS recall: {final_recall.mean():.4f} (target={rt}), nprobe_scanned={nprobe_scan}, " + f"boundary_time={boundary_time_ms:.2f}ms, aps_time={aps_time_ms:.2f}ms, total_time={final_time_ms:.2f}ms") + per_query_stats.append((nprobe_scan, final_recall.mean().item(), final_time_ms)) + + else: + raise ValueError(f"Unknown method: {method}") + + + arr = np.array(per_query_stats, dtype=float) + mean_nprobe = arr[:,0].mean() + std_nprobe = arr[:,0].std() + mean_recall = arr[:,1].mean() + std_recall = arr[:,1].std() + mean_time_ms = arr[:,2].mean() + std_time_ms = arr[:,2].std() + + logger.info(f"Method={method} RecallTarget={rt}, Recall={mean_recall:.4f} (std={std_recall:.4f}), " + f"Time={mean_time_ms:.2f}ms (std={std_time_ms:.2f}ms), NProbe={mean_nprobe:.1f} (std={std_nprobe:.1f})") + + records.append({ + "Method": method, "RecallTarget": rt, + "mean_nprobe": mean_nprobe, "std_nprobe": std_nprobe, + "mean_recall": mean_recall, "std_recall": std_recall, + "mean_time_ms": mean_time_ms, "std_time_ms": std_time_ms, + }) + + results_df = pd.DataFrame(records) + common_utils.save_results_csv(results_df, out_dir / "aps_recall_targets_summary.csv") + + plot_path = out_dir / "aps_recall_vs_metrics.png" + common_utils.plot_recall_performance( + results_df, + x_metric="RecallTarget", + y_metrics=["mean_time_ms", "mean_recall", "mean_nprobe"], + y_labels=["Mean Query Time (ms)", "Mean Recall Achieved", "Mean NProbe"], + y_scales=["linear", "linear", "log"], + plot_title_prefix=f"{dataset_cfg['name']} APS Experiment", + output_path=plot_path, + recall_targets_for_line=exp_cfg["recall_targets"] + ) \ No newline at end of file diff --git a/test/experiments/osdi2025/early_termination/configs/sift1m.yaml b/test/experiments/osdi2025/early_termination/configs/sift1m.yaml new file mode 100644 index 00000000..df1c96a1 --- /dev/null +++ b/test/experiments/osdi2025/early_termination/configs/sift1m.yaml @@ -0,0 +1,38 @@ +# File: configs/experiment.yaml +mode: run +base_dir: data/aps +dataset: + name: sift1m + path: data +experiment: + nq: 10000 + k: 100 + n_workers: 0 + recall_targets: [0.8, 0.9, 0.99] + recompute_ratio: 0.0001 + use_precompute: true + initial_search_fraction: .1 +index: + metric: l2 + nlist: 1000 +overwrite: + index: false + results: true +methods: ['LAET', 'Auncel', 'Oracle', 'FixedNProbe', 'APS', 'SPANN'] +#methods: ['Auncel'] + +paths: + index_dir: data/sift1m/indexes + results_dir: data/sift1m/results + plot_dir: data/sift1m/plots + laet_artifacts_base_dir: data/laet_files + +laet_config: + laet_index_key_full: "IVF1000" + pred_thresh_for_model_features: 1 + feature_idx_for_model: 0 + base_vectors_path_for_laet_train: data/sift/sift_base.fvecs # Base vectors for LAET training + train_query_vectors_path_for_laet_train: data/sift/sift_query.fvecs # Training query vectors for LAET training + gt_train_path_for_laet_train: data/sift/sift_groundtruth.ivecs # Ground truth for LAET training + gt_format_for_laet_train: "ivecs" # Format for ground truth + faiss_training_vectors_path_for_laet_train: data/sift/sift_learn.fvecs # FAISS training vectors for LAET training \ No newline at end of file diff --git a/test/experiments/osdi2025/early_termination/laet.py b/test/experiments/osdi2025/early_termination/laet.py new file mode 100644 index 00000000..2484033d --- /dev/null +++ b/test/experiments/osdi2025/early_termination/laet.py @@ -0,0 +1,315 @@ +# File: laet.py +import faiss +import lightgbm as lgb +import numpy as np +import pandas as pd +import os +import time +import csv +import sys +import math +import re +from pathlib import Path +from sklearn.model_selection import train_test_split +import torch +from sklearn.metrics import mean_squared_error, mean_absolute_error + +def _mmap_fvecs(fname): + x = np.memmap(fname, dtype='int32', mode='r'); d = x[0]; return x.view('float32').reshape(-1, d + 1)[:, 1:] +def _mmap_bvecs(fname): + x = np.memmap(fname, dtype='uint8', mode='r'); d = x[:4].view('int32')[0]; return x.reshape(-1, d + 4)[:, 4:] +def _read_gt_file(fname, format_type, k_load=1): + gt_list = [] + if format_type == "ivecs": + if not os.path.exists(fname) or os.path.getsize(fname) == 0: + return [] + a = np.fromfile(fname, dtype='int32') + if a.size == 0: return [] + d_gt = a[0] + if a.size % (d_gt + 1) != 0: + return [] + data = a.reshape(-1, d_gt + 1) + for i in range(data.shape[0]): + count = data[i,0]; num_to_take = min(count, k_load if k_load is not None else count) + gt_list.append(list(data[i, 1:1 + num_to_take])) + elif format_type == "tsv": + if not os.path.exists(fname): + return [] + with open(fname, 'r', newline='') as f: + for row in csv.reader(f, delimiter='\t'): + try: + ids = [int(x) for x in row if x.strip()] + gt_list.append(ids[:k_load] if k_load is not None else ids) + except ValueError: + pass + return gt_list +def _write_tsv(data, filename): + if os.path.exists(filename): os.remove(filename) + with open(filename, 'w', newline='') as f: csv.writer(f, delimiter='\t').writerows(data) +# --- End I/O --- + +class LAETPipeline: + def __init__(self, vector_dim, laet_artifacts_config): + self.vector_dim = vector_dim + self.config = laet_artifacts_config + self.gbdt_model = None + self.log_target_prediction = False + + base_dir = Path(self.config["base_dir"]) + self.training_data_output_dir = base_dir / self.config.get("training_data_subdir", "laet_training_data") + self.model_output_dir = base_dir / self.config.get("model_subdir", "laet_models") + + self.training_data_output_dir.mkdir(parents=True, exist_ok=True) + self.model_output_dir.mkdir(parents=True, exist_ok=True) + + self.model_name_prefix = f"{self.config['dataset_name_for_files']}_{self.config['index_key_for_files'].replace(',', '_')}" + _log_suffix_for_naming = "_Log" if self.config.get("model_was_trained_on_log_target", False) else "" + self.gbdt_model_filename = f"{self.model_name_prefix}_gbdt_model{_log_suffix_for_naming}.txt" + self.full_gbdt_model_path = self.model_output_dir / self.gbdt_model_filename + + def _find_min_param_faiss(self, faiss_idx, q_vec, gt_set, k_chk, is_hnsw, max_p, init_p=1, step=1): + if not gt_set: return max_p + curr_p = init_p + while curr_p <= max_p: + if is_hnsw: faiss_idx.hnsw.efSearch = curr_p + else: faiss_idx.nprobe = curr_p + try: + _, I = faiss_idx.search(q_vec, k_chk) + if I.size > 0 and any(idx in gt_set for idx in I[0] if idx >= 0): return curr_p + except Exception: pass + curr_p += step + if curr_p > max_p and curr_p - step <= max_p : + if curr_p - step == max_p: continue + break + return max_p + + def _generate_training_data_tsv_internal(self, tsv_path, data_paths_cfg, gen_params_cfg, faiss_params_cfg): + print(f"LAET: Generating training data to {tsv_path}...") + xb = _mmap_fvecs(data_paths_cfg["base_vectors"]) if data_paths_cfg["base_vectors"].endswith('.fvecs') else _mmap_bvecs(data_paths_cfg["base_vectors"]) + xq_all = _mmap_fvecs(data_paths_cfg["train_query_vectors"]) if data_paths_cfg["train_query_vectors"].endswith('.fvecs') else _mmap_bvecs(data_paths_cfg["train_query_vectors"]) + + num_q = gen_params_cfg.get("num_train_queries_to_process", -1) + xq = xq_all[:num_q] if num_q > 0 else xq_all + xq_f32 = xq.astype(np.float32) + + gt_all = _read_gt_file(data_paths_cfg["gt_train"], data_paths_cfg["gt_format"], gen_params_cfg.get("k_gt_load_for_target", 1)) + gt = gt_all[:num_q] if num_q > 0 else gt_all + if len(xq) != len(gt): raise ValueError(f"Query ({len(xq)})/GT ({len(gt)}) length mismatch in training data gen.") + + faiss_idx_key = faiss_params_cfg["faiss_index_key_for_gen"] + is_hnsw = "hnsw" in faiss_idx_key.lower() + faiss_idx = faiss.index_factory(self.vector_dim, faiss_idx_key) + if not faiss_idx.is_trained: + xt_idx_path = data_paths_cfg.get("training_vectors_for_faiss_index") + if not xt_idx_path: raise ValueError("FAISS index needs training, but no training vectors path provided.") + xt_idx = _mmap_fvecs(xt_idx_path) if xt_idx_path.endswith('.fvecs') else _mmap_bvecs(xt_idx_path) + faiss_idx.train(xt_idx.astype(np.float32)) + faiss_idx.add(xb.astype(np.float32)) + + max_p = gen_params_cfg.get("max_efsearch_scan", 512) if is_hnsw else gen_params_cfg.get("max_nprobe_scan", 256) + if not is_hnsw and hasattr(faiss_idx, 'nlist'): max_p = min(max_p, faiss_idx.nlist) + + k_search_target = gen_params_cfg.get("k_search_for_target", 10) + param_scan_step = gen_params_cfg.get("param_scan_step", 1) + out_data = [] + start_gen_time = time.time() + print(f"LAET Training Data Gen: Max param to scan for optimal_p: {max_p}") + for i in range(len(xq)): + opt_p = self._find_min_param_faiss(faiss_idx, xq_f32[i:i+1], set(gt[i]), k_search_target, is_hnsw, max_p, step=param_scan_step) + out_data.append([opt_p, i] + xq_f32[i].tolist()) + if (i + 1) % 500 == 0 or (i+1) == len(xq): + elapsed_gen = time.time() - start_gen_time + _write_tsv(out_data, str(tsv_path)) + # Log distribution of optimal_p found + if out_data: + optimal_ps_found = np.array([row[0] for row in out_data]) + print(f"LAET Training Data Gen: Distribution of optimal_p found (target for GBDT): " + f"Min={np.min(optimal_ps_found)}, Mean={np.mean(optimal_ps_found):.2f}, Median={np.median(optimal_ps_found)}, Max={np.max(optimal_ps_found)}, " + f"StdDev={np.std(optimal_ps_found):.2f}") + + print(f"LAET: Training data saved to {tsv_path}") + + + def _train_gbdt_model_internal(self, tsv_path, model_save_path, train_params_cfg): + print(f"LAET: Training GBDT model from {tsv_path}...") + data_df = pd.read_csv(tsv_path, sep='\t', header=None) + target = data_df.iloc[:, 0].values.astype(np.float32) + features = data_df.iloc[:, 2:].values.astype(np.float32) + + log_target_training = train_params_cfg.get("log_target_training", False) + if log_target_training: target = np.log2(np.maximum(1, target)) + self.log_target_prediction = log_target_training + + X_tr, X_te, y_tr, y_te = train_test_split(features, target, test_size=train_params_cfg.get("test_size", 0.1), random_state=42) + + lgb_tr_data = lgb.Dataset(X_tr, y_tr) + lgb_te_data = lgb.Dataset(X_te, y_te, reference=lgb_tr_data) + + lgbm_params = { + 'objective': 'regression_l1', 'metric': 'mae', + 'num_leaves': train_params_cfg.get("num_leaves", 31), + 'learning_rate': train_params_cfg.get("learning_rate", 0.05), + 'feature_fraction': train_params_cfg.get("feature_fraction", 0.9), + 'bagging_fraction': train_params_cfg.get("bagging_fraction", 0.8), + 'bagging_freq': train_params_cfg.get("bagging_freq", 5), + 'verbose': -1, 'n_jobs': -1, 'seed': 42, + 'n_estimators': train_params_cfg.get("n_estimators", 100) + } + num_boost_round_val = lgbm_params['n_estimators'] + + gbm = lgb.train(lgbm_params, lgb_tr_data, num_boost_round=num_boost_round_val, + valid_sets=[lgb_tr_data, lgb_te_data], + callbacks=[lgb.early_stopping(15, verbose=False), lgb.log_evaluation(period=0)]) + + self.gbdt_model = gbm + current_log_suffix = "_Log" if self.log_target_prediction else "" + effective_model_filename = f"{self.model_name_prefix}_gbdt_model{current_log_suffix}.txt" + effective_model_save_path = self.model_output_dir / effective_model_filename + + gbm.save_model(str(effective_model_save_path)) + self.full_gbdt_model_path = effective_model_save_path + print(f"LAET: GBDT model saved to {self.full_gbdt_model_path}") + + def get_or_train_model(self, overwrite_model=False, + data_paths_config=None, + generation_params_config=None, + faiss_params_config=None, + training_params_config=None + ): + potential_log_target = training_params_config.get("log_target_training", False) if training_params_config else self.config.get("model_was_trained_on_log_target", False) + _log_suffix_check = "_Log" if potential_log_target else "" + expected_model_filename_if_trained_now = f"{self.model_name_prefix}_gbdt_model{_log_suffix_check}.txt" + path_to_check_or_create = self.model_output_dir / expected_model_filename_if_trained_now + + if not overwrite_model and path_to_check_or_create.exists(): + print(f"LAET: Loading existing GBDT model: {path_to_check_or_create}") + self.gbdt_model = lgb.Booster(model_file=str(path_to_check_or_create)) + self.log_target_prediction = "_Log" in path_to_check_or_create.name + self.full_gbdt_model_path = path_to_check_or_create + print(f"LAET: Model loaded. Actual log_target_prediction: {self.log_target_prediction}") + else: + print(f"LAET: Training GBDT model (overwrite={overwrite_model}, exists={path_to_check_or_create.exists()})...") + if not all([data_paths_config, generation_params_config, faiss_params_config, training_params_config]): + raise ValueError("Missing configurations for LAET model training.") + + training_tsv_filename = f"{self.model_name_prefix}_gbdt_train.tsv" + full_training_tsv_path = self.training_data_output_dir / training_tsv_filename + + self._generate_training_data_tsv_internal( + full_training_tsv_path, + data_paths_config, + generation_params_config, + faiss_params_config + ) + self._train_gbdt_model_internal( + full_training_tsv_path, + path_to_check_or_create, + training_params_config + ) + if self.vector_dim is None and self.gbdt_model and hasattr(self.gbdt_model, 'num_feature'): + self.vector_dim = self.gbdt_model.num_feature() + + def _predict_base_search_param(self, query_vector_np): + if not self.gbdt_model: raise RuntimeError("LAET GBDT model not loaded.") + if self.vector_dim is None: self.vector_dim = query_vector_np.shape[0] + if query_vector_np.shape[0] != self.vector_dim: + raise ValueError(f"Query dim {query_vector_np.shape[0]} != model's {self.vector_dim}") + + features = query_vector_np.astype(np.float32).flatten() + pred_raw = self.gbdt_model.predict(features.reshape(1, -1))[0] + pred_val = np.power(2, pred_raw) if self.log_target_prediction else pred_raw + return max(1, int(round(pred_val))) + + def _compute_recall_single_q(self, res_ids_tensor, gt_set_top_k, k_val_for_recall): + if not gt_set_top_k or res_ids_tensor is None or res_ids_tensor.numel() == 0 or k_val_for_recall == 0: + return 0.0 + results_to_check = res_ids_tensor[:min(res_ids_tensor.numel(), k_val_for_recall)].tolist() + hits = 0 + for res_idx in results_to_check: + if res_idx in gt_set_top_k and res_idx >= 0: + hits += 1 + return float(hits) / k_val_for_recall + + + def run_inference_for_quake_experiment(self, quake_idx, queries_torch, gt_torch, + target_recall, k_search, base_quake_sp, + num_bs_steps=20, multiplier_range=(0.1, 50.0)): + if not self.gbdt_model: + print("LAET Inference Error: GBDT model not loaded.") + return np.nan, np.nan, np.nan + + nq = queries_torch.shape[0] + if self.vector_dim is None: self.vector_dim = queries_torch.shape[1] + + gt_sets = [set(gt_torch[i, :min(gt_torch.shape[1], k_search)].tolist()) for i in range(nq)] + queries_np = queries_torch.numpy() + + multi_low, multi_high = multiplier_range + best_mult = multi_high + quake_nlist_cap = quake_idx.nlist() if callable(getattr(quake_idx, 'nlist', None)) else getattr(quake_idx, 'nlist', 2048) + + # --- Log GBDT base_p predictions sample --- + sample_size_for_log = nq + sample_base_p_preds = [self._predict_base_search_param(queries_np[i]) for i in range(sample_size_for_log)] + if sample_base_p_preds: + print(f"LAET INFERENCE (TargetRecall={target_recall:.2f}): Sample GBDT base_p predictions (first {sample_size_for_log}): " + f"Min={np.min(sample_base_p_preds)}, Mean={np.mean(sample_base_p_preds):.2f}, Max={np.max(sample_base_p_preds)}") + # --- End Log --- + + for bs_run_idx in range(num_bs_steps): + curr_mult = (multi_low + multi_high) / 2.0 + if abs(multi_low - multi_high) < 1e-4: break + recall_sum = 0.0 + + tune_nq = nq + for i in range(tune_nq): + base_p = self._predict_base_search_param(queries_np[i]) + final_p = min(max(1, int(round(base_p * curr_mult))), quake_nlist_cap) + + sp = None + if hasattr(base_quake_sp, 'clone') and callable(base_quake_sp.clone): sp = base_quake_sp.clone() + else: sp = type(base_quake_sp)() + + sp.nprobe, sp.k = final_p, k_search + res = quake_idx.search(queries_torch[i:i+1], sp) + ids = res.ids[0] if res.ids is not None and len(res.ids)>0 and res.ids[0] is not None else torch.empty(0, dtype=torch.long) + recall_sum += self._compute_recall_single_q(ids, gt_sets[i], k_search) + avg_rec = recall_sum / tune_nq if tune_nq > 0 else 0.0 + if avg_rec >= target_recall: best_mult, multi_high = curr_mult, curr_mult + else: multi_low = curr_mult + + print(f"LAET INFERENCE (TargetRecall={target_recall:.2f}): Determined best_mult = {best_mult:.3f}") + + final_recall_sum, total_time_ns, sum_nprobes = 0.0, 0, 0 + final_p_values_sample = [] + + for i in range(nq): + base_p = self._predict_base_search_param(queries_np[i]) + final_p = min(max(1, int(round(base_p * best_mult))), quake_nlist_cap) + sum_nprobes += final_p + if i < sample_size_for_log: # Log sample of final_p + final_p_values_sample.append(final_p) + + sp_final = None + if hasattr(base_quake_sp, 'clone') and callable(base_quake_sp.clone): sp_final = base_quake_sp.clone() + else: sp_final = type(base_quake_sp)() + + sp_final.nprobe, sp_final.k = final_p, k_search + + t_s = time.perf_counter_ns() + res = quake_idx.search(queries_torch[i:i+1], sp_final) + total_time_ns += (time.perf_counter_ns() - t_s) + ids = res.ids[0] if res.ids is not None and len(res.ids)>0 and res.ids[0] is not None else torch.empty(0, dtype=torch.long) + final_recall_sum += self._compute_recall_single_q(ids, gt_sets[i], k_search) + + if final_p_values_sample: + print(f"LAET INFERENCE (TargetRecall={target_recall:.2f}): Sample final_p values used (first {len(final_p_values_sample)}): " + f"Min={np.min(final_p_values_sample)}, Mean={np.mean(final_p_values_sample):.2f}, Max={np.max(final_p_values_sample)}") + + + avg_rec_f = final_recall_sum / nq if nq > 0 else 0.0 + avg_time_ms_f = (total_time_ns / nq) / 1e6 if nq > 0 else 0.0 + avg_nprobe_f = sum_nprobes / nq if nq > 0 else 0.0 + + + return avg_nprobe_f, avg_rec_f, avg_time_ms_f diff --git a/test/experiments/osdi2025/early_termination/run.py b/test/experiments/osdi2025/early_termination/run.py new file mode 100644 index 00000000..c6538b95 --- /dev/null +++ b/test/experiments/osdi2025/early_termination/run.py @@ -0,0 +1,340 @@ +#!/usr/bin/env python3 +""" +APS vs. early-termination baselines (Simplified) +Generates CSV + Markdown table with: + RecallAchieved | nprobe | QueryLatencyMs | TuningTimeMs +for recall targets 0.80, 0.90, 0.99 +""" +from __future__ import annotations +import logging +import time +from pathlib import Path +from typing import List, Dict + +import numpy as np +import pandas as pd +import torch +import matplotlib.pyplot as plt + +import test.experiments.osdi2025.experiment_utils as common_utils + +from quake import QuakeIndex, SearchParams, IndexBuildParams +from quake.utils import compute_recall + +# LAET (mandatory for this experiment) +try: + from .laet import LAETPipeline +except ImportError as e: + raise ImportError( + "LAET is required for this experiment but could not be imported. " + "Install the laet package or fix PYTHONPATH." + ) from e + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s | %(levelname)7s | %(message)s", datefmt="%H:%M:%S" +) +logger = logging.getLogger("experiment") + +def tune_auncel_a( + idx: QuakeIndex, + queries: torch.Tensor, + gt: torch.Tensor, + k: int, + target_recall: float, + recompute_thr: float, + init_frac: float, + max_iters: int = 15, + a_min: float = 1e-5, + a_max: float = .5, +) -> tuple[float, list[tuple[float, float, float]]]: + best_a = a_max + lo, hi = a_min, a_max + best_stats: list[tuple[float, float, float]] = [] + + for _ in range(max_iters): + mid = (lo + hi) / 2.0 + current_per_query_stats: list[tuple[float, float, float]] = [] + + for i_q, q_vec in enumerate(queries): + sp = common_utils.create_search_params( + nprobe=-1, k=k, recall_target=target_recall, + use_auncel=True, auncel_a=mid, auncel_b=.5, + recompute_threshold=recompute_thr, initial_search_fraction=init_frac, + ) + partitions_scanned, rec, time_ms = common_utils.run_search_trial( + idx, q_vec, gt[i_q], k, sp + ) + current_per_query_stats.append((partitions_scanned, rec, time_ms)) + + mean_recall = np.mean([r for _, r, _ in current_per_query_stats]) + if mean_recall >= target_recall: + best_a, best_stats = mid, current_per_query_stats + hi = mid * 0.5 + else: + lo = mid * 1.5 + if abs(hi - lo) < 1e-4 or hi < lo : # Added small tolerance and check + break + return best_a, best_stats + + +def binary_search_best_nprobe( + idx: QuakeIndex, + queries_tensor: torch.Tensor, # Can be single or multiple queries + gt_tensor: torch.Tensor, # Corresponding ground truth + k_val: int, + target_recall_val: float, + max_nprobe_val: int, + use_spann_flag: bool = False, + spann_eps_val: float = 1.5, +) -> int: + lo, hi, best = 1, max_nprobe_val, max_nprobe_val + is_single_query = queries_tensor.shape[0] == 1 + + while lo <= hi: + mid = (lo + hi) // 2 + sp = common_utils.create_search_params( + k=k_val, nprobe=mid, use_spann=use_spann_flag, + spann_eps=spann_eps_val if use_spann_flag else 0.0 + ) + + if is_single_query: + # For single query, run_search_trial is fine for recall check + _, rec, _ = common_utils.run_search_trial(idx, queries_tensor[0], gt_tensor[0], k_val, sp) + mean_rec_val = rec + else: + # For multiple queries, need to aggregate recall + ids_all_list = [] + for i_q in range(queries_tensor.shape[0]): + # Directly call idx.search for id extraction for compute_recall over batch + res_search = idx.search(queries_tensor[i_q].unsqueeze(0), sp) + if res_search.ids.numel(): + ids_all_list.append(res_search.ids) + if not ids_all_list: + mean_rec_val = 0.0 + else: + mean_rec_val = compute_recall(torch.cat(ids_all_list, 0), gt_tensor, k_val).mean().item() + + if mean_rec_val >= target_recall_val: + best, hi = mid, mid - 1 + else: + lo = mid + 1 + return best + +def print_markdown(df: pd.DataFrame) -> None: + print("\n" + "-" * 80) + print(df.to_markdown(index=False, floatfmt=".4f")) + print("-" * 80 + "\n") + + +def run_experiment(cfg_path_str: str, output_dir_str: str) -> None: + cfg = common_utils.load_config(cfg_path_str) + out_dir = Path(output_dir_str) + out_dir.mkdir(parents=True, exist_ok=True) + + exp_cfg = cfg["experiment"] + dataset_cfg = cfg["dataset"] + idx_build_cfg = cfg["index"] + paths_cfg = cfg["paths"] + + k_val = exp_cfg["k"] + recall_targets_list: List[float] = exp_cfg["recall_targets"] + table_targets_set = {0.80, 0.90, 0.99} + + vecs, queries_all, gt_all = common_utils.load_data( + dataset_cfg["name"], dataset_cfg.get("path", ""), exp_cfg.get("nq") + ) + logger.info(f"Dataset loaded – {len(vecs):,} base vecs, {len(queries_all)} queries.") + + idx_file_path = Path(paths_cfg["index_dir"]) / f"{dataset_cfg['name']}_quake_ivf{idx_build_cfg['nlist']}.index" + quake_idx_instance = common_utils.prepare_quake_index( + vecs, + {"nlist": idx_build_cfg["nlist"], "metric": idx_build_cfg["metric"], "num_workers": exp_cfg.get("n_workers", 0)}, + idx_file_path, + force_rebuild=cfg["overwrite"].get("index", False), + num_workers_load=exp_cfg.get("n_workers",0) # Pass num_workers for loading context + ) + max_nprobe_val = quake_idx_instance.nlist() + + # LAET SPECIFIC CONFIG + laet_cfg_dict = cfg["laet_config"] + data_paths_cfg_laet = { + "base_vectors": laet_cfg_dict["base_vectors_path_for_laet_train"], + "train_query_vectors": laet_cfg_dict["train_query_vectors_path_for_laet_train"], + "gt_train": laet_cfg_dict["gt_train_path_for_laet_train"], + "gt_format": laet_cfg_dict.get("gt_format_for_laet_train", "ivecs"), + "training_vectors_for_faiss_index": laet_cfg_dict["faiss_training_vectors_path_for_laet_train"], + } + generation_params_cfg_laet = { + "num_train_queries_to_process": laet_cfg_dict.get("num_laet_training_samples", 10000), + "k_gt_load_for_target": laet_cfg_dict.get("laet_train_k_gt_load", 1), + "k_search_for_target": laet_cfg_dict.get("laet_train_k_search", 10), + "max_nprobe_scan": laet_cfg_dict.get("laet_train_max_nprobe", 128), + "param_scan_step": laet_cfg_dict.get("laet_train_param_scan_step", 1), + } + faiss_params_cfg_laet = { + "faiss_index_key_for_gen": laet_cfg_dict.get("faiss_index_key_for_laet_train_gen", f"IVF{max_nprobe_val},Flat") + } + training_params_cfg_laet = { + "log_target_training": laet_cfg_dict.get("train_log_target", False), + "test_size": laet_cfg_dict.get("train_gbdt_test_size", 0.1), + # ... other GBDT params from the LAET repo + "num_leaves": laet_cfg_dict.get("train_gbdt_num_leaves", 31), + "learning_rate": laet_cfg_dict.get("train_gbdt_lr", 0.05), + "n_estimators": laet_cfg_dict.get("train_gbdt_n_estimators", 100), + } + + laet_pipeline_instance = LAETPipeline( + vector_dim=queries_all.shape[1], + laet_artifacts_config={ + "base_dir": paths_cfg["laet_artifacts_base_dir"], + "dataset_name_for_files": laet_cfg_dict.get("dataset_name_for_files", dataset_cfg["name"].upper()), + "index_key_for_files": laet_cfg_dict.get("index_key_for_files", f"IVF{max_nprobe_val}_Flat"), + "model_subdir": laet_cfg_dict.get("model_subdir", "laet_gbdt_models"), + }, + ) + t_laet_train_start_ns = time.perf_counter_ns() + laet_pipeline_instance.get_or_train_model( + overwrite_model=False, # Or cfg["overwrite"].get("laet_model", False) + data_paths_config=data_paths_cfg_laet, + generation_params_config=generation_params_cfg_laet, + faiss_params_config=faiss_params_cfg_laet, + training_params_config=training_params_cfg_laet, + ) + laet_training_total_ns = time.perf_counter_ns() - t_laet_train_start_ns + logger.info(f"LAET model ready (train/load time: {laet_training_total_ns/1e6:.1f} ms).") + + experiment_records: List[Dict] = [] + + for method_name in cfg["methods"]: + for recall_target_val in recall_targets_list: + query_results_list = [] # list of (nprobe_val, recall_val, query_time_ms) + current_tuning_ns = 0 + + if method_name == "Oracle": + for i_q, q_vec in enumerate(queries_all): + gt_vec = gt_all[i_q] + t_tune_start_ns = time.perf_counter_ns() + # Pass single query and gt for Oracle's per-query tuning + best_np_val = binary_search_best_nprobe( + quake_idx_instance, q_vec.unsqueeze(0), gt_vec.unsqueeze(0), k_val, recall_target_val, max_nprobe_val + ) + current_tuning_ns += (time.perf_counter_ns() - t_tune_start_ns) + + sp_final = common_utils.create_search_params(k=k_val, nprobe=best_np_val) + _, rec_val, time_ms_val = common_utils.run_search_trial( + quake_idx_instance, q_vec, gt_vec, k_val, sp_final + ) + query_results_list.append((best_np_val, rec_val, time_ms_val)) + + elif method_name in ("FixedNProbe", "SPANN"): + t_tune_start_ns = time.perf_counter_ns() + best_np_val = binary_search_best_nprobe( + quake_idx_instance, queries_all, gt_all, k_val, recall_target_val, max_nprobe_val, + use_spann_flag=(method_name == "SPANN"), spann_eps_val=1.5 + ) + current_tuning_ns = time.perf_counter_ns() - t_tune_start_ns + + for i_q, q_vec in enumerate(queries_all): + sp_final = common_utils.create_search_params( + k=k_val, nprobe=best_np_val, + use_spann=(method_name == "SPANN"), + spann_eps=1.5 if method_name == "SPANN" else 0.0 + ) + _, rec_val, time_ms_val = common_utils.run_search_trial( + quake_idx_instance, q_vec, gt_all[i_q], k_val, sp_final + ) + query_results_list.append((best_np_val, rec_val, time_ms_val)) + + elif method_name == "APS": + for i_q, q_vec in enumerate(queries_all): + sp_aps = common_utils.create_search_params( + nprobe=-1, k=k_val, recall_target=recall_target_val, + recompute_threshold=exp_cfg["recompute_ratio"], + use_precomputed=exp_cfg["use_precompute"], + initial_search_fraction=exp_cfg["initial_search_fraction"] + ) + nprobe_scanned, rec_val, time_ms_val = common_utils.run_search_trial( + quake_idx_instance, q_vec, gt_all[i_q], k_val, sp_aps + ) + query_results_list.append((nprobe_scanned, rec_val, time_ms_val)) + + elif method_name == "Auncel": + t_tune_start_ns = time.perf_counter_ns() + best_a_val, query_results_list = tune_auncel_a( # tune_auncel_a returns the list directly + quake_idx_instance, queries_all, gt_all, k_val, recall_target_val, + recompute_thr=exp_cfg["recompute_ratio"], + init_frac=exp_cfg["initial_search_fraction"] + ) + current_tuning_ns = time.perf_counter_ns() - t_tune_start_ns + logger.info(f"Auncel @ RT={recall_target_val:.2f} → selected a = {best_a_val:.4f}") + + elif method_name == "LAET": + t_infer_start_ns = time.perf_counter_ns() + mean_np, mean_rec, mean_ms = laet_pipeline_instance.run_inference_for_quake_experiment( + quake_idx=quake_idx_instance, + queries_torch=queries_all, + gt_torch=gt_all, + target_recall=recall_target_val, + k_search=k_val, + base_quake_sp=common_utils.create_search_params(k=k_val), # Pass k here + ) + # For LAET, tuning_ns includes GBDT training + multiplier search + current_tuning_ns = laet_training_total_ns + (time.perf_counter_ns() - t_infer_start_ns) + query_results_list.append((mean_np, mean_rec, mean_ms)) # Store averages + + else: + logger.warning(f"Unknown method '{method_name}' – skipping.") + continue + + if not query_results_list: continue + + # logger.info(f"Method={method} RecallTarget={rt}, Recall={mean_recall:.4f} (std={std_recall:.4f}), " + # f"Time={mean_time_ms:.2f}ms (std={std_time_ms:.2f}ms), NProbe={mean_nprobe:.1f} (std={std_nprobe:.1f})") + + logger.info(f"Method={method_name} RecallTarget={recall_target_val:.2f}, " + f"Recall={np.mean([r for _, r, _ in query_results_list]):.4f}, " + f"Query Latency={np.mean([t for _, _, t in query_results_list]):.2f}ms, " + f"NProbe={np.mean([n for n, _, _ in query_results_list]):.1f}, " + f"Tuning Time={current_tuning_ns / 1e6:.1f}ms") + + arr = np.asarray(query_results_list, dtype=float) + if arr.shape[0] == 1 and method_name == "LAET": # LAET already returns mean values + mean_nprobe_val, mean_recall_val, mean_q_ms_val = arr[0,0], arr[0,1], arr[0,2] + std_nprobe_val, std_recall_val, std_q_ms_val = np.nan, np.nan, np.nan # Stdev not applicable for pre-averaged + else: + mean_nprobe_val = np.nanmean(arr[:,0]) + std_nprobe_val = np.nanstd(arr[:,0]) + mean_recall_val = np.nanmean(arr[:,1]) + std_recall_val = np.nanstd(arr[:,1]) + mean_q_ms_val = np.nanmean(arr[:,2]) + std_q_ms_val = np.nanstd(arr[:,2]) + + + experiment_records.append(dict( + Method=method_name, RecallTarget=recall_target_val, + RecallAchieved=mean_recall_val, nprobe=mean_nprobe_val, + QueryLatencyMs=mean_q_ms_val, TuningTimeMs=current_tuning_ns / 1e6, + StdRecall=std_recall_val, StdNprobe=std_nprobe_val, StdQueryLatencyMs=std_q_ms_val + )) + + df_results = pd.DataFrame(experiment_records) + df_output_final = df_results[df_results["RecallTarget"].isin(table_targets_set)].copy().sort_values( + ["Method", "RecallTarget"] + ) + + csv_output_path = out_dir / f"aps_et_results_{dataset_cfg['name']}.csv" + common_utils.save_results_csv(df_output_final, csv_output_path) # Use common save + print_markdown(df_output_final) + + if cfg.get("plot", {}).get("enable", False): + plot_path = out_dir / f"et_latency_recall_plot_{dataset_cfg['name']}.png" + common_utils.plot_recall_performance( + df_results, # Plot all results, not just table_targets + x_metric="RecallTarget", + y_metrics=["QueryLatencyMs", "RecallAchieved", "nprobe"], + y_labels=["Mean Query Latency (ms)", "Mean Recall Achieved", "Mean NProbe"], + y_scales=["log", "linear", "log"], # nprobe often benefits from log scale + plot_title_prefix=f"{dataset_cfg['name']} Early Termination", + output_path=plot_path, + recall_targets_for_line=recall_targets_list + ) \ No newline at end of file diff --git a/test/experiments/osdi2025/early_termination/util_laet.py b/test/experiments/osdi2025/early_termination/util_laet.py new file mode 100644 index 00000000..7a938c80 --- /dev/null +++ b/test/experiments/osdi2025/early_termination/util_laet.py @@ -0,0 +1,77 @@ +# File: util_laet.py +import numpy as np +import csv +import os +import sys + +def mmap_fvecs(fname): + """ Memory-maps an fvecs file. """ + x = np.memmap(fname, dtype='int32', mode='r') + d = x[0] + return x.view('float32').reshape(-1, d + 1)[:, 1:] + +def mmap_bvecs(fname): + """ Memory-maps a bvecs file. """ + x = np.memmap(fname, dtype='uint8', mode='r') + d = x[:4].view('int32')[0] + return x.reshape(-1, d + 4)[:, 4:] + +def read_tsv(filename, delimiter='\t', doInt=False, multi=1, rnd=-1): + """ + Reads a TSV file into a list of lists. + Handles potential errors during conversion and file access. + """ + ret = [] + if not os.path.exists(filename): + print(f"Warning: File not found {filename}", file=sys.stderr) + return ret + try: + with open(filename, 'r', newline='') as tsvfile: # Use newline='' for universal newlines + datareader = csv.reader(tsvfile, delimiter=delimiter) + for row_num, row in enumerate(datareader): + try: + l = [float(x) for x in row] + if multi != 1: + l = [multi * x for x in l] + if rnd != -1: + l = [round(x, rnd) for x in l] + if doInt: + l = [int(x) for x in l] + ret.append(l) + except ValueError as e: + print(f"Skipping malformed row #{row_num+1} in {filename}: {row}. Error: {e}", file=sys.stderr) + continue + except Exception as e: + print(f"Error opening or reading file {filename}: {e}", file=sys.stderr) + return ret + +def write_tsv(data, filename, delimiter='\t', append=False, doInt=False, multi=1, rnd=-1): + """ + Writes data to a TSV file. + """ + if not append: + try: + os.remove(filename) + except OSError: + pass # File doesn't exist, or other error (permissions etc.) + + processed_data = [] + if doInt or multi != 1 or rnd != -1: + for row_data in data: + new_row = list(row_data) # Make a copy to avoid modifying original list objects + if multi != 1: + new_row = [multi * x for x in new_row] + if rnd != -1: + new_row = [round(x, rnd) for x in new_row] + if doInt: + new_row = [int(x) for x in new_row] + processed_data.append(new_row) + else: + processed_data = data # Use data directly if no processing needed + + try: + with open(filename, 'a', newline='') as tsvfile: + writer = csv.writer(tsvfile, delimiter=delimiter) + writer.writerows(processed_data) + except Exception as e: + print(f"Error writing to file {filename}: {e}", file=sys.stderr) \ No newline at end of file diff --git a/test/experiments/osdi2025/experiment_runner.py b/test/experiments/osdi2025/experiment_runner.py new file mode 100644 index 00000000..fe2be901 --- /dev/null +++ b/test/experiments/osdi2025/experiment_runner.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +""" +Master runner for OSDI2025 vector‑search experiments. +""" +import argparse +from pathlib import Path + +from test.experiments.osdi2025.kick_the_tires.run import run_experiment as run_kick_the_tires +from test.experiments.osdi2025.numa_single_query.run import run_experiment as run_numa_single +from test.experiments.osdi2025.numa_multi_query.run import run_experiment as run_numa_multi +from test.experiments.osdi2025.aps_recall_targets.run import run_experiment as run_aps_recall +from test.experiments.osdi2025.early_termination.run import run_experiment as run_early_termination +from test.experiments.osdi2025.multi_level.run import run_experiment as run_multi_level +from test.experiments.osdi2025.vary_levels.run import run_experiment as run_vary_levels +from test.experiments.osdi2025.maintenance_ablation.run import run_experiment as run_ablation +from test.experiments.osdi2025.vary_batch_size.run import run_experiment as run_vary_batch_size +from test.experiments.osdi2025.wiki_workload.run import run_experiment as run_wiki_workload +from test.experiments.osdi2025.read_only_workload.run import run_experiment as run_read_only_workload +from test.experiments.osdi2025.open_images_workload.run import run_experiment as run_open_images_workload + +EXPERIMENTS = { + "kick_the_tires": run_kick_the_tires, + "numa_single_query": run_numa_single, + "numa_multi_query": run_numa_multi, + "aps_recall_targets": run_aps_recall, + "early_termination": run_early_termination, + "multi_level": run_multi_level, + "vary_levels": run_vary_levels, + "maintenance_ablation": run_ablation, + "vary_batch_size": run_vary_batch_size, + "wiki_workload": run_wiki_workload, + "read_only_workload": run_read_only_workload, + "open_images_workload": run_open_images_workload +} + +def main(): + parser = argparse.ArgumentParser(description="Run one OSDI2025 experiment") + parser.add_argument("-x","--experiment", choices=EXPERIMENTS.keys(), required=True) + parser.add_argument("-c","--config", required=True, + help="Config name (without .yaml) under configs/") + parser.add_argument("-o","--output-dir", default=None, + help="Where to write results (csv + plots)") + args = parser.parse_args() + + base = Path(__file__).resolve().parent + cfg_path = base / args.experiment / "configs" / f"{args.config}.yaml" + if not cfg_path.is_file(): + raise FileNotFoundError(f"Missing config: {cfg_path}") + + out_dir = Path(args.output_dir) if args.output_dir \ + else base / args.experiment / "results" / args.config + out_dir.mkdir(parents=True, exist_ok=True) + + # dispatch + EXPERIMENTS[args.experiment](str(cfg_path), str(out_dir)) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/test/experiments/osdi2025/experiment_utils.py b/test/experiments/osdi2025/experiment_utils.py new file mode 100644 index 00000000..7a2ef22b --- /dev/null +++ b/test/experiments/osdi2025/experiment_utils.py @@ -0,0 +1,530 @@ +# File: test/experiments/osdi2025/common/experiment_utils.py +import time +import yaml +import logging +from pathlib import Path +import contextlib +import sys +from typing import Dict, Any, List, Callable, Union, Optional, Tuple +import os +import multiprocessing +import subprocess +import threading +import json +import traceback +import signal +from datetime import datetime, timezone # For Glances metrics collection + +import torch +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt + +from quake import QuakeIndex, IndexBuildParams, SearchParams +from quake.datasets.ann_datasets import load_dataset as quake_load_dataset +from quake.utils import compute_recall as quake_compute_recall +from quake.workload_generator import DynamicWorkloadGenerator, WorkloadEvaluator +logger = logging.getLogger(__name__) + +def load_config(cfg_path: str) -> dict: + return yaml.safe_load(Path(cfg_path).read_text()) + +def load_data(dataset_name: str, dataset_path: str = "data/", nq_override: int = None): + vecs, queries, gt = quake_load_dataset(dataset_name, dataset_path) + if nq_override is not None and nq_override > 0 and nq_override < len(queries): + queries = queries[:nq_override] + if gt is not None: + gt = gt[:nq_override] + return vecs, queries, gt + +def prepare_quake_index( + vecs: torch.Tensor, + build_params_dict: dict, + index_file_path: Path, + force_rebuild: bool = False, + num_workers_load: int = 0, + num_parent_workers_load: int = 0, + use_numa: bool = False, + load: bool = True +) -> QuakeIndex: + idx = QuakeIndex() + + build_params = create_index_build_params(**build_params_dict) + + if index_file_path.exists() and not force_rebuild: + if load: + logger.info(f"Loading index from {index_file_path}") + idx.load(str(index_file_path), build_params) + else: + logger.info(f"Building index -> {index_file_path}") + idx.build(vecs, torch.arange(len(vecs)), build_params) + index_file_path.parent.mkdir(parents=True, exist_ok=True) + idx.save(str(index_file_path)) + logger.info(f"Index saved to {index_file_path}") + return idx + + +def prepare_wrapper_index( + IndexClass: type, + index_file_path: Path, + base_vectors: torch.Tensor, + build_params: Dict[str, Any], + force_rebuild: bool = False, + load: bool = True +) -> object: # Returns an instance of the IndexClass + + idx_instance = IndexClass() + index_file_path.parent.mkdir(parents=True, exist_ok=True) + build_params_copy = build_params.copy() + + if index_file_path.exists() and not force_rebuild: + logger.info(f"Index file {index_file_path} exists and force_rebuild is False.") + if load: + logger.info(f"Loading index from {index_file_path}") + load_kwargs = {} + if "num_workers" in build_params_copy: + load_kwargs["num_workers"] = build_params_copy.get("num_workers") + if "use_numa" in build_params_copy: + load_kwargs["use_numa"] = build_params_copy.get("use_numa") + + idx_instance.load(str(index_file_path), **load_kwargs) + logger.info(f"Loaded index from {index_file_path} with load_kwargs: {load_kwargs}") + else: + logger.info(f"Index file {index_file_path} exists, but load=False. Returning new, non-loaded instance.") + else: + if force_rebuild and index_file_path.exists(): + logger.info(f"Force rebuilding index at {index_file_path}") + else: + logger.info(f"Building index at {index_file_path} (file did not exist).") + + idx_instance.build(base_vectors, **build_params_copy) + idx_instance.save(str(index_file_path)) + logger.info(f"Built and saved index to {index_file_path}") + + if load: + logger.info(f"Index built. Instance is considered ready as per build_params (load=True).") + + return idx_instance + +def create_search_params(**attrs) -> SearchParams: + sp = SearchParams() + for k, v in attrs.items(): + if hasattr(sp, k): + setattr(sp, k, v) + return sp + +def run_search_trial( + idx: QuakeIndex, + query: torch.Tensor, + gt_vector: torch.Tensor, # Ground truth for this single query + k_val: int, + search_params: SearchParams +) -> tuple: + t0 = time.perf_counter_ns() + res = idx.search(query.unsqueeze(0), search_params) + elapsed_ns = time.perf_counter_ns() - t0 + rec = quake_compute_recall(res.ids, gt_vector.unsqueeze(0), k_val).item() + partitions_scanned = getattr(res.timing_info, "partitions_scanned", np.nan) + return partitions_scanned, rec, elapsed_ns / 1e6 # nprobe, recall, time_ms + +def generate_dynamic_workload( + dataset_main_cfg: Dict[str, Any], + workload_generator_cfg: Dict[str, Any], + global_output_dir: Path, + overwrite_workload: bool +) -> Path: + """ + Loads dataset and generates the dynamic workload. + Returns the path to the workload directory (which is the global_output_dir). + """ + log = logger # Use the logger defined in common_utils + + log.info(f"Preparing dataset '{dataset_main_cfg['name']}' for dynamic workload.") + # Assuming common_utils.load_data is defined elsewhere in experiment_utils.py + base_vectors, query_vectors, _ = load_data( + dataset_main_cfg["name"], dataset_main_cfg.get("path", "") + ) + + log.info("Setting up DynamicWorkloadGenerator.") + generator = DynamicWorkloadGenerator( + workload_dir=global_output_dir, + base_vectors=base_vectors, + metric=dataset_main_cfg["metric"], + insert_ratio=workload_generator_cfg["insert_ratio"], + delete_ratio=workload_generator_cfg["delete_ratio"], + query_ratio=workload_generator_cfg["query_ratio"], + update_batch_size=workload_generator_cfg["update_batch_size"], + query_batch_size=workload_generator_cfg["query_batch_size"], + number_of_operations=workload_generator_cfg["number_of_operations"], + initial_size=workload_generator_cfg["initial_size"], + cluster_size=workload_generator_cfg["cluster_size"], + cluster_sample_distribution=workload_generator_cfg["cluster_sample_distribution"], + queries=query_vectors, + query_cluster_sample_distribution=workload_generator_cfg["query_cluster_sample_distribution"], + seed=workload_generator_cfg["seed"] + ) + + if not generator.workload_exists() or overwrite_workload: + log.info(f"Generating workload in {global_output_dir}...") + generator.generate_workload() + else: + log.info(f"Existing workload found in {global_output_dir} – generation skipped.") + return global_output_dir + + +def evaluate_index_on_dynamic_workload( + index_config: Dict[str, Any], # Single entry from the 'indexes' list in YAML + index_class_mapping: Dict[str, type], # Maps index type string to class (e.g., {"Quake": QuakeWrapper}) + workload_data_dir: Path, # Path where DynamicWorkloadGenerator output (operations, groundtruth) is stored + experiment_main_output_dir: Path, # Base output dir for the whole experiment (e.g., .../maintenance_ablation/results/sift1m/) + overwrite_idx_results: bool, + do_maintenance_flag: bool = False # Specific to experiments that test maintenance +): + """ + Evaluates a single index configuration against a pre-generated dynamic workload. + Results are saved by WorkloadEvaluator in a subdirectory named after the index. + """ + log = logger # Use the logger defined in common_utils + idx_name = index_config["name"] + index_type_str = index_config["index"] + + # Determine the directory for this specific index's results + idx_specific_output_dir = experiment_main_output_dir / idx_name + idx_specific_output_dir.mkdir(parents=True, exist_ok=True) + + per_index_results_csv = idx_specific_output_dir / "results.csv" + + if per_index_results_csv.exists() and not overwrite_idx_results: + log.info(f"Results for {idx_name} exist at {per_index_results_csv} – skipping evaluation.") + return + + log.info(f"Evaluating index configuration: {idx_name} (Type: {index_type_str})") + + index_class_constructor = index_class_mapping.get(index_type_str) + if not index_class_constructor: + log.error(f"Unknown index type '{index_type_str}' in mapping for index '{idx_name}'. Skipping.") + return + + index_instance = index_class_constructor() + + evaluator = WorkloadEvaluator( + workload_dir=workload_data_dir, # Where operation files are located + output_dir=idx_specific_output_dir # Where this index's results.csv will be saved + ) + + evaluator.evaluate_workload( + name=idx_name, + index=index_instance, + build_params=index_config.get("build_params", {}), + search_params=index_config.get("search_params", {}), + m_params=index_config.get("maintenance_params", {}), # Pass maintenance_params + do_maintenance=do_maintenance_flag, + ) + log.info(f"Finished evaluating {idx_name}. Results should be in {idx_specific_output_dir}") + + +def save_results_csv(records: list, output_path: Path): + df = pd.DataFrame(records) + output_path.parent.mkdir(parents=True, exist_ok=True) + df.to_csv(output_path, index=False) + logger.info(f"Results saved to {output_path}") + +def plot_recall_performance( + df: pd.DataFrame, + x_metric: str, # e.g., "RecallTarget" + y_metrics: list, # e.g., ["mean_time_ms", "mean_recall", "mean_nprobe"] + y_labels: list, + y_scales: list = None, # e.g., ["linear", "linear", "log"] + plot_title_prefix: str = "", + output_path: Path = None, + recall_targets_for_line: list = None +): + num_plots = len(y_metrics) + fig, axes = plt.subplots(num_plots, 1, figsize=(6, 3 * num_plots + 1), sharex=True) + if num_plots == 1: + axes = [axes] + + for m_idx, method_name in enumerate(df["Method"].unique()): + grp = df[df["Method"] == method_name] + for i, y_col in enumerate(y_metrics): + y_err_col = df.columns[df.columns.get_loc(y_col) + 1] if f"std_{y_col.split('_')[1]}" in df.columns else None + yerr_values = grp[y_err_col] if y_err_col and y_err_col in grp.columns else None + axes[i].errorbar( + grp[x_metric], grp[y_col], + yerr=yerr_values, + marker="o", capsize=4, label=method_name, + linestyle='-' if m_idx == 0 else '--' # Differentiate methods + ) + + for i, y_col in enumerate(y_metrics): + axes[i].set_ylabel(y_labels[i]) + if y_scales and y_scales[i]: + axes[i].set_yscale(y_scales[i]) + if y_col == "mean_recall" and recall_targets_for_line: + axes[i].plot(recall_targets_for_line, recall_targets_for_line, "k--", label="Target") + axes[i].legend() + axes[i].grid(True, which="both", ls=":") + axes[i].set_title(f"{plot_title_prefix}: {x_metric} vs {y_labels[i]}") + + axes[-1].set_xlabel(x_metric) + plt.tight_layout(rect=[0, 0, 1, 0.97]) # Adjust for suptitle + fig.suptitle(f"{plot_title_prefix} Performance", fontsize=14) + + if output_path: + output_path.parent.mkdir(parents=True, exist_ok=True) + plt.savefig(output_path) + logger.info(f"Plot saved to {output_path}") + plt.close(fig) + +def expand_search_sweep(sweep_config: Dict[str, Any]) -> List[Dict[str, Any]]: + """Expands nprobes or recall_targets from sweep_config into individual search dicts.""" + base_params = {k: v for k, v in sweep_config.items() if k not in ["nprobes", "recall_targets"]} + expanded_list = [] + if "nprobes" in sweep_config: + for nprobe_val in sweep_config["nprobes"]: + expanded_list.append({**base_params, "nprobe": nprobe_val}) + elif "recall_targets" in sweep_config: + for rt_val in sweep_config["recall_targets"]: + expanded_list.append({**base_params, "recall_target": rt_val}) + else: # No sweep, just use base_params + expanded_list.append(base_params) + return expanded_list + +def create_index_build_params(**attrs) -> IndexBuildParams: + """Creates a Quake IndexBuildParams object from dictionary attributes.""" + bp = IndexBuildParams() + if 'nc' in attrs and 'nlist' not in attrs: # Allow 'nc' as an alias for 'nlist' + attrs['nlist'] = attrs.pop('nc') + + for k, v in attrs.items(): + if hasattr(bp, k): + setattr(bp, k, v) + else: + logger.warning(f"Attribute '{k}' not found in IndexBuildParams. Ignoring.") + return bp + +def save_results_df(records: list, output_path: Path): + df = pd.DataFrame(records) + output_path.parent.mkdir(parents=True, exist_ok=True) + df.to_csv(output_path, index=False) + logger.info(f"Results saved to {output_path}") # Already logs + +@contextlib.contextmanager +def redirect_all_stdout_to_file(filepath: Union[Path, str], mode: str = 'a'): + original_python_stdout = sys.stdout + original_c_stdout_fd = -1 + saved_c_stdout_fd = -1 + target_file = None + can_redirect_c_stdout = hasattr(sys.stdout, 'fileno') + + try: + if can_redirect_c_stdout: + original_c_stdout_fd = sys.stdout.fileno() + saved_c_stdout_fd = os.dup(original_c_stdout_fd) + target_file = open(filepath, mode) + if can_redirect_c_stdout: + os.dup2(target_file.fileno(), original_c_stdout_fd) + sys.stdout = target_file + yield + except Exception: + raise + finally: + if sys.stdout == target_file and target_file: + sys.stdout.flush() + if can_redirect_c_stdout and original_c_stdout_fd != -1 and saved_c_stdout_fd != -1: + os.dup2(saved_c_stdout_fd, original_c_stdout_fd) + sys.stdout = original_python_stdout + if target_file: + target_file.close() + if saved_c_stdout_fd != -1: + os.close(saved_c_stdout_fd) + +_GLANCES_REFRESH_SEC = 1 +_GLANCES_MB = 1_048_576.0 + +def start_glances_minimal(tmp_json_path: Path) -> subprocess.Popen: + cmd = [ + "glances", "-q", f"-t{_GLANCES_REFRESH_SEC}", "--export", "json", + "--export-json-file", str(tmp_json_path), "--disable-plugin", "all", + "--enable-plugin", "cpu,mem,diskio,gpu", + # Consider adding process monitoring if useful: "--enable-plugin", "processcount,processlist" + # and then filter by PID if possible with glances export, though might be complex. + # For now, system-wide A,B,C,D metrics. + ] + # print(f"Starting Glances: {' '.join(cmd)}") # Goes to main process stdout if called from there + # If called from child, it'll go to child's log via redirection. + # Use preexec_fn=os.setsid to create a new process group for Glances + glances_process = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True, preexec_fn=os.setsid) + time.sleep(2) # Allow Glances to start and write initial JSON + if glances_process.poll() is not None: # Check if Glances started successfully + stderr_output = glances_process.stderr.read() if glances_process.stderr else "No stderr." + raise RuntimeError(f"Glances failed to start. Exit code: {glances_process.returncode}. Stderr: {stderr_output}") + return glances_process + +def stop_glances_minimal(glances_process: Optional[subprocess.Popen]): + if glances_process and glances_process.poll() is None: + try: + os.killpg(os.getpgid(glances_process.pid), signal.SIGTERM) # Send SIGTERM to the entire process group + glances_process.wait(timeout=3) + except ProcessLookupError: pass # Process already dead + except subprocess.TimeoutExpired: + try: + os.killpg(os.getpgid(glances_process.pid), signal.SIGKILL) # Force kill + glances_process.wait(timeout=1) + except Exception: pass # Best effort + except Exception: pass # Other errors during stop + +def read_glances_json_minimal(tmp_json_path: Path) -> List[dict]: + if not tmp_json_path.exists(): return [] + text = tmp_json_path.read_text(encoding="utf-8").strip() + if not text: return [] + try: # Glances typically writes a list of JSON objects, or one object per file if snapshotting fast + data = json.loads(text) + return data if isinstance(data, list) else [data] # Ensure it's a list + except json.JSONDecodeError: # Handle cases where file might not be complete JSON list + records = [] + for line in text.splitlines(): # Try parsing line by line if not a single JSON doc + try: records.append(json.loads(line)) + except json.JSONDecodeError: pass # Ignore malformed lines + return records + +def extract_metrics_minimal(glances_record: Dict[str, Any]) -> Dict[str, Any]: + # Simplified extraction for A, B, C, D + cpu = glances_record.get("cpu", {}) + mem = glances_record.get("mem", {}) + diskio_list = glances_record.get("diskio", []) + gpu_list = glances_record.get("gpu", []) + + read_bytes = sum(d.get("read_bytes", 0) for d in diskio_list if isinstance(d, dict)) + write_bytes = sum(d.get("write_bytes", 0) for d in diskio_list if isinstance(d, dict)) + + # Assuming first GPU, if any. User script used 'proc' or 'utilization'. + gpu_util = np.nan + if gpu_list and isinstance(gpu_list[0], dict): + gpu_info = gpu_list[0] + gpu_util = gpu_info.get("proc", gpu_info.get("utilization", np.nan)) + + return { + "cpu_percent": cpu.get("total", cpu.get("user", 0) + cpu.get("system", 0)), # 'total' is often available + "mem_percent": mem.get("percent", np.nan), + "disk_read_mb_per_s": read_bytes / _GLANCES_MB / _GLANCES_REFRESH_SEC, # Approximate rate + "disk_write_mb_per_s": write_bytes / _GLANCES_MB / _GLANCES_REFRESH_SEC, # Approximate rate + "gpu_percent": gpu_util, + } +# --- End Glances Helpers --- + +def _glances_metric_collector_thread( + glances_tmp_json_path: Path, + metrics_output_ndjson_path: Path, + stop_event: threading.Event +): + # print(f"Glances metrics collector thread started. Output: {metrics_output_ndjson_path}") # Debug + with open(metrics_output_ndjson_path, "w") as mof: + while not stop_event.wait(timeout=_GLANCES_REFRESH_SEC): # Check event periodically + try: + glances_data_list = read_glances_json_minimal(glances_tmp_json_path) + # Glances --export-json-file overwrites the file. So we read the whole content. + # If it's a list, it might be multiple snapshots if glances is very fast relative to our read. + # Usually, it's one snapshot. + for record in glances_data_list: # Process each snapshot found + extracted = extract_metrics_minimal(record) + ts = datetime.now(timezone.utc).isoformat() + mof.write(json.dumps({"timestamp": ts, **extracted}) + "\n") + mof.flush() + except Exception as e: + # This print will go to the main process log for the child process + print(f"Glances metrics collector thread error: {e}", file=sys.stderr) # Use stderr + # print(f"Glances metrics collector thread stopped for {metrics_output_ndjson_path}") # Debug + +def _process_target_wrapper( + target_func: Callable, log_file_path: Path, env_vars: Optional[Dict[str, str]], + result_queue: multiprocessing.Queue, process_name_for_log: str, + enable_glances_monitoring: bool, args: Tuple, kwargs: Dict[str, Any] +): + current_process_name = multiprocessing.current_process().name + glances_proc: Optional[subprocess.Popen] = None + glances_tmp_json_file = log_file_path.with_suffix(".glances-temp.json") + metrics_output_file = log_file_path.with_suffix(".glances.ndjson") + metrics_thread: Optional[threading.Thread] = None + stop_glances_collection_event = threading.Event() + + try: + if env_vars: # Set environment variables first + for k, v in env_vars.items(): os.environ[str(k)] = str(v) + + if enable_glances_monitoring: + try: + print(f"[{current_process_name}] Starting Glances monitoring... Temp JSON: {glances_tmp_json_file}") + glances_proc = start_glances_minimal(glances_tmp_json_file) + metrics_thread = threading.Thread( + target=_glances_metric_collector_thread, + args=(glances_tmp_json_file, metrics_output_file, stop_glances_collection_event), + daemon=True + ) + metrics_thread.start() + print(f"[{current_process_name}] Glances monitoring and metrics collector thread started.") + except Exception as ge: + print(f"[{current_process_name}] WARNING: Failed to start Glances: {ge}", file=sys.stderr) + glances_proc = None # Ensure it's None if start failed + metrics_thread = None + + # with redirect_all_stdout_to_file(log_file_path): + print(f"--- Process {current_process_name} started. Main log: {log_file_path} ---") + if enable_glances_monitoring and glances_proc: + print(f"--- Glances metrics being collected to: {metrics_output_file} ---") + + result_data = target_func(*args, **kwargs) + result_queue.put({"status": "success", "data": result_data, "log_file": str(log_file_path), "metrics_file": str(metrics_output_file) if glances_proc else None}) + print(f"--- Process {current_process_name} finished successfully. ---") + + except Exception as e: + tb_str = traceback.format_exc() + error_message = f"Process {current_process_name} Exception: {e}\n{tb_str}" + try: + with open(log_file_path, 'a') as f_err: + f_err.write(f"\n--- PROCESS ERROR ({current_process_name}) ---\n{error_message}\n") + except: pass + result_queue.put({"status": "error", "message": str(e), "traceback": tb_str, "log_file": str(log_file_path), "metrics_file": str(metrics_output_file) if glances_proc else None}) + finally: + if metrics_thread and metrics_thread.is_alive(): + stop_glances_collection_event.set() + metrics_thread.join(timeout=5) # Wait for thread to finish + if glances_proc: + stop_glances_minimal(glances_proc) + if enable_glances_monitoring and glances_tmp_json_file.exists(): # Cleanup temp file + try: glances_tmp_json_file.unlink() + except OSError: pass + + +def run_operation_in_process( + target_func: Callable, args_tuple: Tuple = (), kwargs_dict: Optional[Dict[str, Any]] = None, + env_vars: Optional[Dict[str, str]] = None, log_file_path: Union[Path, str] = None, + process_name: Optional[str] = None, timeout_seconds: Optional[float] = None, + enable_glances: bool = False # New flag to control Glances +) -> Dict[str, Any]: + if kwargs_dict is None: kwargs_dict = {} + effective_process_name = process_name or target_func.__name__ + + if log_file_path is None: + log_file_path = Path(f"./{effective_process_name}_{os.getpid()}_process.log") + log_file_path = Path(log_file_path); log_file_path.parent.mkdir(parents=True, exist_ok=True) + + result_queue = multiprocessing.Queue(1) + process = multiprocessing.Process( + target=_process_target_wrapper, + args=(target_func, log_file_path, env_vars, result_queue, effective_process_name, enable_glances, args_tuple, kwargs_dict), + name=effective_process_name + ) + process.start() + result = None + try: + result = result_queue.get(timeout=timeout_seconds) + except multiprocessing.queues.Empty: # type: ignore + result = {"status": "error", "message": "Process timed out", "log_file": str(log_file_path)} + if process.is_alive(): + process.terminate(); process.join(5) + if process.is_alive(): process.kill() # type: ignore + except Exception as q_e: + result = {"status": "error", "message": f"Queue error: {q_e}", "log_file": str(log_file_path)} + process.join() + return result if result is not None else {"status": "error", "message": "Process ended without result", "log_file": str(log_file_path)} diff --git a/test/experiments/osdi2025/install.sh b/test/experiments/osdi2025/install.sh new file mode 100644 index 00000000..b114186b --- /dev/null +++ b/test/experiments/osdi2025/install.sh @@ -0,0 +1,247 @@ +#!/bin/bash +set -e # Exit immediately if a command exits with a non-zero status. +# set -x # Uncomment for debugging (print each command before execution) + +echo "Starting QUAKE Artifact and Baselines Setup Script." +echo "This script should be run with sudo (e.g., 'sudo ./install.sh')." +echo "Targeting Ubuntu 22.04 (Jammy) based system." + +# ----------------------------- +# Environment Variables +# ----------------------------- +export CONDA_DIR_PATH="/opt/miniconda" +# Ensure Miniconda's bin is in PATH for the root user running the script +export PATH="${CONDA_DIR_PATH}/bin:${PATH}" +export DEBIAN_FRONTEND=noninteractive + +# ----------------------------- +# Update System and Install Base APT Packages +# ----------------------------- +echo ">>> Updating system and installing base APT packages..." +apt-get update + +apt-get install -y --no-install-recommends \ + wget \ + curl \ + build-essential \ + ca-certificates \ + swig \ + git \ + libomp-dev \ + graphviz \ + numactl \ + libnuma-dev \ + htop \ + software-properties-common + +# ----------------------------- +# Install/Ensure GCC 11 and G++ 11 +# ----------------------------- +echo ">>> Ensuring GCC 11 and G++ 11 are set up correctly..." +add-apt-repository -y ppa:ubuntu-toolchain-r/test +apt-get update +apt-get install -y gcc-11 g++-11 + +echo ">>> Setting up GCC and G++ alternatives to point to version 11..." +update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \ + --slave /usr/bin/gcov gcc-gcov /usr/bin/gcov-11 \ + --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \ + --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-11 \ + --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 +update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 110 +update-alternatives --set gcc /usr/bin/gcc-11 +update-alternatives --set g++ /usr/bin/g++-11 + +echo "Verifying GCC version after update-alternatives:" +gcc --version +echo "Verifying G++ version after update-alternatives:" +g++ --version + +# ----------------------------- +# Install CMake 3.24.2 +# ----------------------------- +CMAKE_VERSION_EXPECTED="3.24.2" +CMAKE_INSTALL_PATH="/usr/local/bin/cmake" +echo ">>> Installing CMake ${CMAKE_VERSION_EXPECTED}..." +if command -v cmake &> /dev/null && [[ "$(cmake --version)" == *"cmake version ${CMAKE_VERSION_EXPECTED}"* ]]; then + echo "CMake version ${CMAKE_VERSION_EXPECTED} already installed at $(command -v cmake). Skipping." +else + echo "Installing CMake ${CMAKE_VERSION_EXPECTED}..." + cd /tmp + wget -qO cmake.sh https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION_EXPECTED}/cmake-${CMAKE_VERSION_EXPECTED}-linux-x86_64.sh + chmod +x cmake.sh + ./cmake.sh --skip-license --prefix=/usr/local # Runs as root + rm cmake.sh + cd - +fi +echo "Verifying CMake version:" +cmake --version + +# ----------------------------- +# Install Miniconda +# ----------------------------- +echo ">>> Installing Miniconda to ${CONDA_DIR_PATH}..." +if [ -x "${CONDA_DIR_PATH}/bin/conda" ]; then + echo "Miniconda already found in ${CONDA_DIR_PATH}. Skipping installation." +else + cd /tmp + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + bash miniconda.sh -b -p "${CONDA_DIR_PATH}" # Runs as root, installs to /opt/miniconda + rm miniconda.sh + cd - + echo ">>> Initializing Conda for bash (modifies /root/.bashrc as script runs with sudo)..." + "${CONDA_DIR_PATH}/bin/conda" init bash +fi + +# Source conda's profile script to make conda command available in this script session +echo ">>> Sourcing Conda profile script for current root session..." +if [ -f "${CONDA_DIR_PATH}/etc/profile.d/conda.sh" ]; then + source "${CONDA_DIR_PATH}/etc/profile.d/conda.sh" +else + echo "ERROR: Conda profile script not found at ${CONDA_DIR_PATH}/etc/profile.d/conda.sh after installation attempt." + echo "Conda commands might not be available. Exiting." + exit 1 +fi + +# ----------------------------- +# Create Conda Environment 'quake-env' and Install Dependencies +# ----------------------------- +CONDA_ENV_NAME="quake-env" +echo ">>> Setting up conda environment '${CONDA_ENV_NAME}'..." + +echo ">>> Creating conda.yaml file at /tmp/conda.yaml..." +# Ensure the file is removed first to prevent issues with existing file/permissions +rm -f /tmp/conda.yaml +cat > /tmp/conda.yaml << EOF +name: ${CONDA_ENV_NAME} +channels: + - pytorch + - defaults + - conda-forge +dependencies: + - python=3.11 + - numpy + - pandas + - faiss-cpu + - matplotlib + - pytest + - pip + - pip: + - sphinx + - sphinx_rtd_theme + - sphinxcontrib-mermaid + - graphviz + - pyyaml +EOF + +# If environment exists, remove it for a clean setup +if conda env list | grep -E "^${CONDA_ENV_NAME}\s+"; then + echo "Conda environment '${CONDA_ENV_NAME}' already exists. Removing for a clean setup..." + conda env remove -n "${CONDA_ENV_NAME}" --all -y +fi +echo ">>> Creating conda environment '${CONDA_ENV_NAME}' from /tmp/conda.yaml..." +conda env create -f /tmp/conda.yaml +rm /tmp/conda.yaml # Clean up +conda clean -afy + +echo ">>> Installing specific PyTorch (CPU) and other Python packages into '${CONDA_ENV_NAME}'..." +conda run -n "${CONDA_ENV_NAME}" pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + +conda run -n "${CONDA_ENV_NAME}" pip install \ + lightgbm \ + scikit-learn \ + scann==1.4.0 + +# Force numpy 1.25.0 for diskannpy compatibility +conda run -n "${CONDA_ENV_NAME}" pip install numpy==1.25.0 --force-reinstall +conda run -n "${CONDA_ENV_NAME}" pip install diskannpy==0.7.0 + +echo ">>> Verifying '${CONDA_ENV_NAME}'..." +conda env list +conda run -n "${CONDA_ENV_NAME}" python -c "import sys; print(f'OK in {sys.prefix}; python:', sys.executable); import torch; print('PyTorch version:', torch.__version__); import numpy; print('Numpy version:', numpy.__version__)" + +# ----------------------------- +# Install Intel oneAPI Base Toolkit +# ----------------------------- +ONEAPI_INSTALL_PATH="/opt/intel/oneapi" +echo ">>> Installing Intel oneAPI Base Toolkit to ${ONEAPI_INSTALL_PATH}..." +if [ -f "${ONEAPI_INSTALL_PATH}/setvars.sh" ]; then + echo "Intel oneAPI already found in ${ONEAPI_INSTALL_PATH}. Skipping installation." +else + cd /tmp + wget -O intel-oneapi-base-toolkit-offline.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/6bfca885-4156-491e-849b-1cd7da9cc760/intel-oneapi-base-toolkit-2025.1.1.36_offline.sh + chmod +x intel-oneapi-base-toolkit-offline.sh + ./intel-oneapi-base-toolkit-offline.sh -a --silent --cli --eula accept --install-dir "${ONEAPI_INSTALL_PATH}" # sudo implied + rm intel-oneapi-base-toolkit-offline.sh + cd - +fi +# Source oneAPI environment variables for the current session. +echo ">>> Sourcing Intel oneAPI setvars.sh for current root session (needed for SVS build)..." +if [ -f "${ONEAPI_INSTALL_PATH}/setvars.sh" ]; then + source "${ONEAPI_INSTALL_PATH}/setvars.sh" +else + echo "WARNING: oneAPI setvars.sh not found at ${ONEAPI_INSTALL_PATH}/setvars.sh. SVS build might fail." +fi + +# ----------------------------- +# Clone and Install QUAKE Repository +# ----------------------------- +QUAKE_DIR_BASE="/opt" +QUAKE_REPO_NAME="quake" +QUAKE_FULL_PATH="${QUAKE_DIR_BASE}/${QUAKE_REPO_NAME}" + +echo ">>> Cloning/Updating and installing QUAKE repository in ${QUAKE_FULL_PATH}..." +if [ -d "${QUAKE_FULL_PATH}" ]; then + echo "QUAKE directory ${QUAKE_FULL_PATH} already exists. Updating..." + cd "${QUAKE_FULL_PATH}" + git pull +else + git clone https://github.com/marius-team/quake.git "${QUAKE_FULL_PATH}" + cd "${QUAKE_FULL_PATH}" +fi +git checkout osdi2025 + +git config --global --add safe.directory "${QUAKE_FULL_PATH}" # For root's global config +SUBMODULE_PATHS=("src/cpp/third_party/concurrentqueue" "src/cpp/third_party/faiss" "src/cpp/third_party/pybind11") +for SUBMODULE_PATH in "${SUBMODULE_PATHS[@]}"; do + git config --global --add safe.directory "${QUAKE_FULL_PATH}/${SUBMODULE_PATH}" +done +git submodule update --init --recursive + +echo ">>> Building and installing QUAKE Python package..." +conda run -n "${CONDA_ENV_NAME}" pip install . --no-use-pep517 +cd / + +# ----------------------------- +# Clone and Install ScalableVectorSearch (SVS) +# ----------------------------- +SVS_DIR_BASE="/opt" +SVS_REPO_NAME="ScalableVectorSearch" +SVS_FULL_PATH="${SVS_DIR_BASE}/${SVS_REPO_NAME}" + +echo ">>> Cloning/Updating ScalableVectorSearch repository in ${SVS_FULL_PATH}..." +if [ -d "${SVS_FULL_PATH}" ]; then + echo "ScalableVectorSearch directory ${SVS_FULL_PATH} already exists. Updating..." + cd "${SVS_FULL_PATH}" + git pull +else + git clone https://github.com/intel/ScalableVectorSearch.git "${SVS_FULL_PATH}" + cd "${SVS_FULL_PATH}" +fi +git config --global --add safe.directory "${SVS_FULL_PATH}" + +echo ">>> Attempting to build and install SVS Python bindings..." +echo "This step requires GCC/G++ 11 and Intel oneAPI tools (which should be sourced)." +echo "Current compilers: GCC=$(which gcc || echo not_found), G++=$(which g++ || echo not_found)" +# The oneAPI setvars.sh should have configured the environment for Intel compilers if available and preferred by SVS build. +# Otherwise, it should use the g++-11 from update-alternatives. +conda run -n "${CONDA_ENV_NAME}" pip install ./bindings/python +cd / + +echo "--------------------------------------------------------------------" +echo "Setup script finished successfully." +echo "To activate the conda environment (in a new shell, as root or user depending on .bashrc): conda activate ${CONDA_ENV_NAME}" +echo "QUAKE repository is in ${QUAKE_FULL_PATH}" +echo "Intel oneAPI (if installed) is in ${ONEAPI_INSTALL_PATH}. Source with: source ${ONEAPI_INSTALL_PATH}/setvars.sh" +echo "SVS repository (if cloned) is in ${SVS_FULL_PATH}" +echo "--------------------------------------------------------------------" diff --git a/test/experiments/osdi2025/kick_the_tires/configs/sift1m.yaml b/test/experiments/osdi2025/kick_the_tires/configs/sift1m.yaml new file mode 100644 index 00000000..352627ec --- /dev/null +++ b/test/experiments/osdi2025/kick_the_tires/configs/sift1m.yaml @@ -0,0 +1,59 @@ +# configs/experiment.yaml +mode: run # options: build, run, plot +dataset: + name: sift1m + metric: l2 + path: data +workload_generator: + recall_target: 0.9 + insert_ratio: 0.5 + delete_ratio: 0.0 + query_ratio: 0.5 + number_of_operations: 1000 + initial_size: 100000 + cluster_size: 1000 + update_batch_size: 10000 + query_batch_size: 100 + cluster_sample_distribution: skewed + query_cluster_sample_distribution: uniform + seed: 9299 +indexes: + - name: Quake10000 + index: Quake + build_params: + nc: 10000 + num_workers: 0 + search_params: + k: 10 +# nprobe: 10 + recall_target: .9 + initial_search_fraction: 0.25 + - name: Quake10000_sample + index: Quake + build_params: + nc: 10000 + num_workers: 0 + search_params: + k: 10 + recall_target: .9 + initial_search_fraction: 0.1 +plot: + x_axis: operation_number # any column from the results CSV + metrics: [latency_ms, recall] # any subset of stored metrics + styles: # one entry per index name + Quake10000: + color: "#1f77b4" # default Matplotlib hex’s are fine + linestyle: "-" # see plt.Line2D docs + marker: "o" + Quake10000_sample: + color: "#ff7f0e" # default Matplotlib hex’s are fine + linestyle: "-" # see plt.Line2D docs + marker: "o" + IVF: + color: "#d62728" + linestyle: "--" + marker: "s" +overwrite: + workload: false + index: false + results: false \ No newline at end of file diff --git a/test/experiments/osdi2025/kick_the_tires/run.py b/test/experiments/osdi2025/kick_the_tires/run.py new file mode 100644 index 00000000..13a08248 --- /dev/null +++ b/test/experiments/osdi2025/kick_the_tires/run.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +""" +Unified experiment runner + +* Reads a single YAML config (`configs/experiment.yaml`). +* Modes + - **build** –‑ only (re‑)generate the workload. + - **run** –‑ generate workload (if needed), evaluate every index that + does not already have a CSV of prior results (unless + `overwrite.results: true`), then emit the unified plot. + - **plot** –‑ skip evaluation; just regenerate the unified plot. +* A results CSV (`//results.csv`) is written after every + evaluation; its presence is what lets us skip reruns. +* Plot styling (colour / marker / linestyle) is entirely configurable from + the YAML under `plot.styles`. +""" + +import logging +import sys +import time +from pathlib import Path +from typing import Dict, List + +import matplotlib.pyplot as plt +from matplotlib.lines import Line2D +import numpy as np +import pandas as pd +import torch +import yaml + +# ── quake imports ────────────────────────────────────────────────────────────── +from quake.datasets.ann_datasets import load_dataset +from quake.index_wrappers.faiss_ivf import FaissIVF +from quake.index_wrappers.quake import QuakeWrapper +from quake.utils import compute_recall +from quake.workload_generator import DynamicWorkloadGenerator, WorkloadEvaluator + +logger = logging.getLogger(__name__) + + +LAT_STYLES = {"insert": "--", "delete": "-.", "query": "-"} + +def unified_plot(cfg: Dict, output_dir: Path) -> None: + """Produce a 4‑panel plot with: + • one figure‑level legend (colour → index) + • latency‑panel legend (linestyle → op‑type)""" + styles = cfg["plot"].get("styles", {}) + + # ── collect colours / markers per index for global legend ──────────────── + index_handles = [] + for idx_cfg in cfg["indexes"]: + name = idx_cfg["name"] + st = styles.get(name, {}) + index_handles.append( + Line2D( + [0], + [0], + color=st.get("color", None), + marker=None, + linestyle="-", + label=name, + ) + ) + + fig, axs = plt.subplots(2, 2, figsize=(12, 10)) + + # ── panel references ───────────────────────────────────────────────────── + ax_lat, ax_part = axs[0] + ax_res, ax_rec = axs[1] + + # ── plot per index ─────────────────────────────────────────────────────── + for idx_cfg in cfg["indexes"]: + name = idx_cfg["name"] + csv = output_dir / name / "results.csv" + if not csv.exists(): + logger.warning("Skipping %s (no CSV).", name); continue + + df = pd.read_csv(csv) + st = styles.get(name, {}) + color, marker = st.get("color", None), st.get("marker", None) + + # A – latency + for op in ("insert", "delete", "query"): + sub = df[df["operation_type"] == op] + if not sub.empty: + ax_lat.plot( + sub["operation_number"], sub["latency_ms"], + color=color, linestyle=LAT_STYLES[op], marker=None) + + ax_lat.set_yscale("log") + + # B – partitions + sub = df[df["n_list"].notna()] + if not sub.empty: + ax_part.plot(sub["operation_number"], sub["n_list"], + color=color, marker=marker) + + # C – resident vectors + sub = df[df["n_resident"].notna()] + if not sub.empty: + ax_res.plot(sub["operation_number"], sub["n_resident"], + color=color, marker=marker) + + # D – query recall + sub = df[(df["operation_type"] == "query") & df["recall"].notna()] + if not sub.empty: + ax_rec.plot(sub["operation_number"], sub["recall"], + color=color, marker=marker) + + # ── format panels (titles, axes) ───────────────────────────────────────── + ax_lat.set_title("Operation Latency") + ax_lat.set_xlabel("Operation Number"); ax_lat.set_ylabel("Latency (ms)") + + ax_part.set_title("Partitions per Operation") + ax_part.set_xlabel("Operation Number"); ax_part.set_ylabel("Number of Partitions") + + ax_res.set_title("Resident Set Size") + ax_res.set_xlabel("Operation Number"); ax_res.set_ylabel("Resident Vectors") + + ax_rec.set_title("Query Recall") + ax_rec.set_xlabel("Operation Number"); ax_rec.set_ylabel("Recall") + + # legends + # latency‑style legend (insert/delete/query) + style_handles = [ + Line2D([0], [0], color="k", linestyle=LAT_STYLES[op], label=op.capitalize()) + for op in ("insert", "delete", "query") + ] + ax_lat.legend(handles=style_handles, title="Operation Type", fontsize=8) + + # global colour legend (index names) + fig.legend( + handles=index_handles, + loc="upper center", + bbox_to_anchor=(0.5, 1.02), + ncol=max(1, len(index_handles)), + title="Index", + ) + + plt.tight_layout() + out_path = output_dir / "unified_plot.png" + plt.savefig(out_path, bbox_inches="tight") + plt.close() + print(f"Saved unified plot → {out_path}") + + +# ─────────────────────────────────────────────────────────────────────────────── +# main execution +# ─────────────────────────────────────────────────────────────────────────────── +def run_experiment(cfg_path: str, output_dir: str) -> None: + cfg = yaml.safe_load(Path(cfg_path).read_text()) + out = Path(output_dir) + out.mkdir(parents=True, exist_ok=True) + + mode = cfg.get("mode", "run") + overwrite = cfg.get("overwrite", {}) + overwrite_workload = overwrite.get("workload", False) + overwrite_results = overwrite.get("results", False) + + # ── build / run both need the dataset loaded ─────────────────────────────── + if mode in {"build", "run"}: + data_cfg = cfg["dataset"] + vectors, queries, _ = load_dataset(data_cfg["name"], data_cfg.get("path", "")) + metric = data_cfg["metric"] + print(f"Dataset loaded: vectors {vectors.shape}, queries {queries.shape}") + + # ── workload generation (build or run) ───────────────────────────────────── + if mode in {"build", "run"}: + wg_cfg = cfg["workload_generator"] + generator = DynamicWorkloadGenerator( + workload_dir=out, + base_vectors=vectors, + metric=metric, + insert_ratio=wg_cfg["insert_ratio"], + delete_ratio=wg_cfg["delete_ratio"], + query_ratio=wg_cfg["query_ratio"], + update_batch_size=wg_cfg["update_batch_size"], + query_batch_size=wg_cfg["query_batch_size"], + number_of_operations=wg_cfg["number_of_operations"], + initial_size=wg_cfg["initial_size"], + cluster_size=wg_cfg["cluster_size"], + cluster_sample_distribution=wg_cfg["cluster_sample_distribution"], + queries=queries, + query_cluster_sample_distribution=wg_cfg["query_cluster_sample_distribution"], + seed=wg_cfg["seed"], + ) + + if generator.workload_exists() and not overwrite_workload: + print("Workload already exists – skipping generation.") + else: + print("Generating workload …") + generator.generate_workload() + + # ── evaluation pass (run mode only) ──────────────────────────────────────── + if mode == "run": + for idx_cfg in cfg["indexes"]: + name = idx_cfg["name"] + index_type = idx_cfg["index"] + res_dir = out / name + csv_path = res_dir / "results.csv" + need_eval = overwrite_results or not csv_path.exists() + + if need_eval: + print(f"\nIndex {name}: running evaluation.") + res_dir.mkdir(parents=True, exist_ok=True) + evaluator = WorkloadEvaluator(workload_dir=out, output_dir=res_dir) + + index = {"Quake": QuakeWrapper, "IVF": FaissIVF}.get(index_type) + if index is None: + raise ValueError(f"Unknown index type: {index_type}") + + results = evaluator.evaluate_workload( + name=name, + index=index(), + build_params=idx_cfg["build_params"], + search_params=idx_cfg["search_params"], + do_maintenance=True, + ) + else: + print(f"\nIndex {name}: using cached results ({csv_path}).") + + # ── generate / regenerate the unified plot ───────────────────────────────── + if mode in {"run", "plot"}: + unified_plot(cfg, out) \ No newline at end of file diff --git a/test/experiments/osdi2025/maintenance_ablation/configs/sift1m.yaml b/test/experiments/osdi2025/maintenance_ablation/configs/sift1m.yaml new file mode 100644 index 00000000..8b4895eb --- /dev/null +++ b/test/experiments/osdi2025/maintenance_ablation/configs/sift1m.yaml @@ -0,0 +1,102 @@ +# Ablation experiment on Quake maintenance policy +# ─────────────────────────────────────────────── +mode: run # {build, run, plot} +dataset: + name: sift1m + metric: l2 + path: data + +workload_generator: + recall_target: 0.9 + insert_ratio: 0.3 + delete_ratio: 0.2 + query_ratio: 0.5 + number_of_operations: 1000 + initial_size: 100000 + cluster_size: 1000 + update_batch_size: 10000 + query_batch_size: 1000 + cluster_sample_distribution: skewed + query_cluster_sample_distribution: uniform + seed: 9299 + +# ── six index configurations ───────────────────────────────────────────────── +indexes: + - name: Quake + index: Quake + build_params: {nc: 1000, num_workers: 0} + search_params: {k: 10, recall_target: .9} + maintenance_params: + split_threshold_ns: 50 + delete_threshold_ns: 50 + + - name: Quake-NoReject + index: Quake + build_params: {nc: 1000, num_workers: 0} + search_params: {k: 10, recall_target: .9} + maintenance_params: + enable_split_rejection: false + enable_delete_rejection: false + split_threshold_ns: 50 + delete_threshold_ns: 50 + + - name: Quake-NoRefine + index: Quake + build_params: {nc: 1000, num_workers: 0} + search_params: {k: 10, recall_target: .9} + maintenance_params: + refinement_radius: 0 + split_threshold_ns: 50 + delete_threshold_ns: 50 + + - name: Quake-NoRefineNoReject + index: Quake + build_params: {nc: 1000, num_workers: 0} + search_params: {k: 10, recall_target: .9} + maintenance_params: + refinement_radius: 0 + enable_split_rejection: false + enable_delete_rejection: false + split_threshold_ns: 50 + delete_threshold_ns: 50 + + - name: Quake-NoCost + index: Quake + build_params: {nc: 1000, num_workers: 0} + search_params: {k: 10, recall_target: .9} + maintenance_params: + max_partition_size: 2000 + + - name: Quake-NoCostNoRefine + index: Quake + build_params: {nc: 1000, num_workers: 0} + search_params: {k: 10, recall_target: .9} + maintenance_params: + refinement_radius: 0 + max_partition_size: 2000 + + - name: LIRE + index: Quake + build_params: {nc: 1000, num_workers: 0} + search_params: {k: 10, recall_target: .9} + maintenance_params: + refinement_iterations: 0 + max_partition_size: 2000 + +plot: + x_axis: operation_number + metrics: [latency_ms, recall] + styles: + Quake: {color: "#1f77b4", linestyle: "-", marker: "o"} + Quake-NoReject: {color: "#2ca02c", linestyle: "-", marker: "v"} + Quake-NoRefine: {color: "#9467bd", linestyle: "-", marker: "^"} + Quake-NoRefineNoReject: {color: "#8c564b", linestyle: "-", marker: "s"} + Quake-NoCost: {color: "#d627bb", linestyle: "-", marker: "D"} + Quake-NoCostNoRefine: {color: "#ff7f0e", linestyle: "-", marker: "+"} + + LIRE: {color: "#ffbb78", linestyle: "-", marker: "P"} + +overwrite: + workload: false + index: true + results: true \ No newline at end of file diff --git a/test/experiments/osdi2025/maintenance_ablation/run.py b/test/experiments/osdi2025/maintenance_ablation/run.py new file mode 100644 index 00000000..4667d458 --- /dev/null +++ b/test/experiments/osdi2025/maintenance_ablation/run.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +""" +Quake Maintenance Policy Ablation Study Runner +─────────────────────────────────────────────── +This experiment evaluates different maintenance policy configurations within Quake +under a dynamic workload involving inserts, deletes, and queries. +It generates detailed reports including: +1. Per-index CSV logs of operations. +2. A 9-panel unified plot comparing various metrics across configurations. +3. A stacked bar chart breaking down cumulative time per operation type. +4. A summary table (CSV and Markdown) of key performance indicators. +""" +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Dict, List, Any # Added Any for type hinting + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from tabulate import tabulate +from matplotlib.cm import get_cmap +from matplotlib.lines import Line2D + +# Common utilities +import test.experiments.osdi2025.experiment_utils as common_utils + +# Quake specific imports +from quake.index_wrappers.quake import QuakeWrapper # This experiment only uses QuakeWrapper + +logging.basicConfig(level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s") +log = logging.getLogger(__name__) + +OP_STYLE = { + "query": dict(ls="-", marker="o", mfc="none", ms=4, lw=1.2), + "insert": dict(ls="-", marker="s", mfc="none", ms=4, lw=1.2), + "delete": dict(ls="-", marker="^", mfc="none", ms=4, lw=1.2), + "maintain": dict(ls="None", marker="X", mfc="black", ms=5), +} +LAT_OPS = ["query", "insert", "delete", "maintain"] +IDX_LAT_Q, IDX_LAT_I, IDX_LAT_D, IDX_LAT_M = 0, 1, 2, 3 +IDX_PART, IDX_RES, IDX_REC, IDX_TOT, IDX_SPL = 4, 5, 6, 7, 8 + + +def unified_plot(cfg: Dict[str, Any], out_dir: Path) -> None: + styles = cfg["plot"].get("styles", {}) + fig, axs2d = plt.subplots(3, 3, figsize=(18, 12), sharex="col") + axs = axs2d.flatten() + + idx_handles: List[Line2D] = [] + for j, idx_cfg in enumerate(cfg.get("indexes", [])): # Added .get for safety + nm, st = idx_cfg["name"], styles.get(idx_cfg["name"], {}) + idx_handles.append(Line2D([0], [0], + color=st.get("color", f"C{j % 10}"), + marker=st.get("marker", "o"), + ls="", markersize=6, label=nm)) + + max_ops_overall = 0 + + for j, idx_cfg in enumerate(cfg.get("indexes", [])): + name = idx_cfg["name"] + st = styles.get(name, {}) + colour = st.get("color", f"C{j % 10}") + marker = st.get("marker", "o") + + csv_path = out_dir / name / "results.csv" + if not csv_path.exists(): + log.warning("[unified_plot] Results CSV %s missing – skipped for this index.", csv_path) + continue + try: + df = pd.read_csv(csv_path) + if df.empty: + log.warning(f"[unified_plot] Empty CSV: {csv_path}") + continue + if 'operation_number' in df.columns and not df['operation_number'].empty: + max_ops_overall = max(max_ops_overall, df.operation_number.max()) + except pd.errors.EmptyDataError: + log.warning(f"[unified_plot] Could not read or empty CSV: {csv_path}") + continue + + # Latency plots + for op, ax_idx_val in zip(LAT_OPS, [IDX_LAT_Q, IDX_LAT_I, IDX_LAT_D, IDX_LAT_M]): + ax = axs[ax_idx_val] + y_val_series = None + if op == "maintain": + if 'maintenance_time_ms' in df.columns: + sub = df[df.maintenance_time_ms.fillna(0) > 0] + y_val_series = sub.maintenance_time_ms + elif 'operation_type' in df.columns and 'latency_ms' in df.columns: + sub = df[df.operation_type == op] + y_val_series = sub.latency_ms + + if y_val_series is not None and not y_val_series.empty: + ax.plot(sub.operation_number, y_val_series, color=colour, **OP_STYLE[op]) + + # Other metric plots + plot_specs = [ + (IDX_PART, 'n_list'), (IDX_RES, 'n_resident'), + (IDX_REC, 'recall', lambda d: (d.operation_type == "query") & d.recall.notna()), + (IDX_TOT, 'total_cumulative_time', + lambda d: pd.Series(np.cumsum(d.latency_ms.fillna(0) + d.maintenance_time_ms.fillna(0)), name='total_cumulative_time')), + (IDX_SPL, ['n_splits', 'n_deletes']) # Special handling for splits/deletes + ] + + for spec_item in plot_specs: + ax_idx_val = spec_item[0] + metric_name_or_list = spec_item[1] + condition_func = spec_item[2] if len(spec_item) > 2 else lambda d: d[metric_name_or_list].notna() + + ax = axs[ax_idx_val] + + if ax_idx_val == IDX_TOT: # Cumulative time calculation + if 'latency_ms' in df.columns and 'maintenance_time_ms' in df.columns: + total_ms_series = np.cumsum(df.latency_ms.fillna(0) + df.maintenance_time_ms.fillna(0)) + if not total_ms_series.empty: + ax.plot(df.operation_number, total_ms_series, color=colour, marker=marker, lw=1.2) + continue + + if ax_idx_val == IDX_SPL: # Splits and deletes + if 'operation_number' in df.columns: + spl_series = df.n_splits.fillna(0) if 'n_splits' in df.columns else pd.Series(0, index=df.index) + del_series = df.n_deletes.fillna(0) if 'n_deletes' in df.columns else pd.Series(0, index=df.index) + if not df.empty: + if (spl_series > 0).any(): ax.step(df.operation_number, np.cumsum(spl_series), where="post", color=colour, ls="--", lw=1.2) + if (del_series > 0).any(): ax.step(df.operation_number, np.cumsum(del_series), where="post", color=colour, ls=":", lw=1.2) + continue + + # General case for other plots + if isinstance(metric_name_or_list, str) and metric_name_or_list in df.columns: + sub_df = df[condition_func(df)] + if not sub_df.empty: + ax.plot(sub_df.operation_number, sub_df[metric_name_or_list], color=colour, marker=marker, lw=1.2) + + titles = { + IDX_LAT_Q: "Latency – Query", IDX_LAT_I: "Latency – Insert", + IDX_LAT_D: "Latency – Delete", IDX_LAT_M: "Latency – Maintain", + IDX_PART: "# Partitions", IDX_RES: "Resident Vectors", + IDX_REC: "Recall", IDX_TOT: "Running Total Time (ms)", + IDX_SPL: "Cumulative Splits / Deletes", + } + y_labels_map = { + IDX_PART: "Count", IDX_RES: "# Vectors", IDX_REC: "Recall", + IDX_TOT: "Cumulative Time (ms)", IDX_SPL: "Count" + } + for i, ax_val in enumerate(axs): + ax_val.set_title(titles[i], fontsize=11) + ax_val.grid(True, which="both", ls=":", alpha=0.7) + ax_val.set_axisbelow(True) + if i in y_labels_map: ax_val.set_ylabel(y_labels_map[i]) + # Set x-label only for the bottom row of plots + if i // 3 == 2 : ax_val.set_xlabel("Operation #") + + is_latency_plot = i in [IDX_LAT_Q, IDX_LAT_I, IDX_LAT_D, IDX_LAT_M] + if is_latency_plot: ax_val.set_ylabel("Latency (ms)") + + + if max_ops_overall > 0: + for ax_val in axs: + ax_val.set_xlim(left=0, right=max_ops_overall) + + fig.legend(idx_handles, [h.get_label() for h in idx_handles], + loc="upper center", bbox_to_anchor=(0.5, 1.035), + ncol=min(4, len(idx_handles)), + fontsize=9, title="Index Configuration", frameon=False) + + plt.tight_layout(rect=[0, 0.03, 1, 0.97]) + out_plot_path = out_dir / "unified_plot.png" + plt.savefig(out_plot_path, dpi=150) + log.info("Unified plot saved to %s", out_plot_path) + plt.close(fig) + + +def make_time_breakdown(cfg: Dict[str, Any], out_dir: Path) -> None: + categories = ["Search", "Insert", "Delete", "Maintain", "Total"] + totals_data, recall_data = {}, {} + cmap = get_cmap("tab10") + + for j, idx_cfg in enumerate(cfg.get("indexes", [])): + name = idx_cfg["name"] + csv_path = out_dir / name / "results.csv" + if not csv_path.exists(): + log.warning("[time_breakdown] %s missing – skipped", csv_path) + continue + try: + df = pd.read_csv(csv_path) + if df.empty: raise pd.errors.EmptyDataError + except pd.errors.EmptyDataError: + log.warning(f"[time_breakdown] Empty or unreadable CSV: {csv_path}") + continue + + totals_data[name] = [ + df[df.operation_type == "query"].latency_ms.sum() if 'operation_type' in df.columns else 0, + df[df.operation_type == "insert"].latency_ms.sum() if 'operation_type' in df.columns else 0, + df[df.operation_type == "delete"].latency_ms.sum() if 'operation_type' in df.columns else 0, + df.maintenance_time_ms.fillna(0).sum() if 'maintenance_time_ms' in df.columns else 0, + ] + totals_data[name].append(sum(totals_data[name])) + + query_df = df[df.operation_type == "query"] if 'operation_type' in df.columns else pd.DataFrame() + recall_data[name] = query_df.recall.mean() if not query_df.empty and 'recall' in query_df else np.nan + + + if not totals_data: + log.info("[time_breakdown] No data available for time breakdown plot.") + return + + n_idx, n_cats = len(totals_data), len(categories) + x_indices = np.arange(n_cats) + bar_width = 0.8 / max(1, n_idx) + + fig, ax = plt.subplots(figsize=(max(10, n_idx * 1.5 + 2), 6)) + for j, (name, values) in enumerate(totals_data.items()): + style = cfg.get("plot", {}).get("styles", {}).get(name, {}) + color = style.get("color", cmap(j % cmap.N)) + x_offset = x_indices + (j - (n_idx - 1) / 2) * bar_width + ax.bar(x_offset, values, width=bar_width, label=name, color=color, edgecolor="black") + current_recall = recall_data.get(name) + if pd.notna(current_recall) and values[0] > 0: + ax.text(x_offset[0], values[0] * 1.01, f"R={current_recall:.3f}", + ha="center", va="bottom", fontsize=8, rotation=90, color='dimgrey') # Standard color for text + + ax.set_xticks(x_indices) + ax.set_xticklabels(categories, fontsize=10) + ax.set_ylabel("Cumulative Time (ms)", fontsize=10) + ax.set_title("Time Breakdown per Index Configuration", fontsize=12) + ax.legend(title="Index Configuration", frameon=False, fontsize=9) + ax.grid(True, axis='y', linestyle=':', alpha=0.7) + plt.tight_layout() + + out_plot_path = out_dir / "time_breakdown.png" + plt.savefig(out_plot_path, dpi=150) + log.info("Time breakdown plot saved to %s", out_plot_path) + plt.close(fig) + + +def produce_summary_table(cfg: Dict[str, Any], out_dir: Path) -> None: + rows = [] + for idx_cfg in cfg.get("indexes", []): + name = idx_cfg["name"] + csv_path = out_dir / name / "results.csv" + if not csv_path.exists(): + log.warning("[summary_table] %s missing – skipped", csv_path) + continue + try: + df = pd.read_csv(csv_path) + if df.empty: raise pd.errors.EmptyDataError + except pd.errors.EmptyDataError: + log.warning(f"[summary_table] Empty or unreadable CSV: {csv_path}") + continue + + search_time = df[df.operation_type == "query"].latency_ms.sum() if 'operation_type' in df.columns else 0 + insert_time = df[df.operation_type == "insert"].latency_ms.sum() if 'operation_type' in df.columns else 0 + delete_time = df[df.operation_type == "delete"].latency_ms.sum() if 'operation_type' in df.columns else 0 + maintain_time = df.maintenance_time_ms.fillna(0).sum() if 'maintenance_time_ms' in df.columns else 0 + total_time = search_time + insert_time + delete_time + maintain_time + + query_df = df[df.operation_type == "query"] if 'operation_type' in df.columns else pd.DataFrame() + recall_val = query_df.recall.mean() if not query_df.empty and 'recall' in query_df else np.nan + + n_list_series = df.n_list.dropna() if 'n_list' in df.columns else pd.Series(dtype=float) # ensure series exists + n_partitions_val = n_list_series.iloc[-1] if not n_list_series.empty else np.nan + + rows.append(dict( + Index=name, Search=int(search_time), Insert=int(insert_time), + Delete=int(delete_time), Maintain=int(maintain_time), Total=int(total_time), + Recall=f"{recall_val:.4f}" if pd.notna(recall_val) else "—", + Partitions=int(n_partitions_val) if pd.notna(n_partitions_val) else "—" + )) + + if not rows: + log.info("[summary_table] No data for summary table.") + return + + summary_df = pd.DataFrame(rows).sort_values("Index") + summary_csv_path = out_dir / "summary_table.csv" + common_utils.save_results_csv(summary_df, summary_csv_path) # Use common util + + summary_md_path = out_dir / "summary_table.md" + md_content = tabulate(summary_df, headers="keys", tablefmt="github", showindex=False) + summary_md_path.write_text(md_content + "\n") + log.info("Summary table (Markdown) saved to %s", summary_md_path) + log.info("\n%s", md_content) + + +def run_experiment(cfg_path_str: str, output_dir_str: str) -> None: + cfg = common_utils.load_config(cfg_path_str) + main_output_dir = Path(output_dir_str).expanduser() + main_output_dir.mkdir(parents=True, exist_ok=True) + + current_mode = cfg.get("mode", "run") + log.info(f"Running Maintenance Ablation experiment in mode: {current_mode}") + + workload_actual_dir = main_output_dir # Workload files are stored at the top level of main_output_dir + + # --- Phase 1: Dataset and Workload Generation --- + if current_mode in {"build", "run"}: + workload_actual_dir = common_utils.generate_dynamic_workload( + dataset_main_cfg=cfg["dataset"], + workload_generator_cfg=cfg["workload_generator"], + global_output_dir=main_output_dir, # Pass main_output_dir as the place to store workload files + overwrite_workload=cfg["overwrite"].get("workload", False) + ) + + # --- Phase 2: Index Evaluation --- + if current_mode == "run": + log.info("Starting index evaluation phase...") + # For this specific experiment, the index class is always QuakeWrapper + index_class_map = {"Quake": QuakeWrapper} + + for index_conf in cfg.get("indexes", []): + common_utils.evaluate_index_on_dynamic_workload( + index_config=index_conf, + index_class_mapping=index_class_map, + workload_data_dir=workload_actual_dir, # Use the returned/set workload_actual_dir + experiment_main_output_dir=main_output_dir, + overwrite_idx_results=cfg["overwrite"].get("results", False), + do_maintenance_flag=True # This experiment always does maintenance + ) + + # --- Phase 3: Global Artifact Generation --- + if current_mode in {"run", "plot"}: + log.info("Generating global plots and summary tables...") + any_results_exist = any( + (main_output_dir / index_conf["name"] / "results.csv").exists() + for index_conf in cfg.get("indexes", []) + ) + + if any_results_exist: + unified_plot(cfg, main_output_dir) + make_time_breakdown(cfg, main_output_dir) + produce_summary_table(cfg, main_output_dir) + else: + log.warning("No results found to generate plots or summary tables. Skipping this step.") + + log.info("Maintenance Ablation experiment finished for mode: %s", current_mode) \ No newline at end of file diff --git a/test/experiments/osdi2025/measure_wiki_skew_and_perf/run.py b/test/experiments/osdi2025/measure_wiki_skew_and_perf/run.py new file mode 100644 index 00000000..139597f9 --- /dev/null +++ b/test/experiments/osdi2025/measure_wiki_skew_and_perf/run.py @@ -0,0 +1,2 @@ + + diff --git a/test/experiments/osdi2025/multi_level/configs/sift10m.yaml b/test/experiments/osdi2025/multi_level/configs/sift10m.yaml new file mode 100644 index 00000000..384ffd13 --- /dev/null +++ b/test/experiments/osdi2025/multi_level/configs/sift10m.yaml @@ -0,0 +1,75 @@ +# General settings +overwrite: + index_files: false # If true, rebuilds all L0 index files + results_csv: true # If true, re-runs searches and overwrites result CSVs + +dataset: + name: sift10m + metric: l2 + nq: 100 + +# --- L0 (Base Level) Definitions --- +l0_definitions: + "L0_40k_config": # Unique reference name for this L0 build configuration + type: Quake + build_params: + nlist: 40000 + num_workers: 4 + "L0_10k_config": + type: Quake + build_params: + nlist: 10000 + num_workers: 4 + +# --- Global L0 Search Sweep --- +global_l0_search_sweep: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256] # List of nprobe values for L0 + # recall_targets: [0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 0.99] # Alternative L0 sweep + # initial_search_fraction: 0.01 # Required if using recall_targets + k: 1 # k for L0 search (and overall recall calculation) + batch_size: 1 # Query batch_size for L0 search + +# --- Index Hierarchy Definitions --- +index_hierarchies: + - name: "Flat_L0_40k" # Unique name for this specific hierarchy/curve + l0_ref: "L0_40k_config" # Referenlistes an L0 from 'l0_definitions' + # parent_levels: (empty or omitted) -> This is a flat index using the referenlisted L0. + + - name: "Flat_L0_10k" # Unique name for this specific hierarchy/curve + l0_ref: "L0_10k_config" # Referenlistes an L0 from 'l0_definitions' + # parent_levels: (empty or omitted) -> This is a flat index using the referenlisted L0. + + + - name: "Hier_L0_40k_L1_1k_P50" # Name implies: L0 is 40k, L1 is 1k centroids, L1 nprobe is 50 + l0_ref: "L0_40k_config" + parent_levels: # List of parent levels (L1, L2, ...) built on top of L0 + - type: Quake # L1 configuration + build_params: {nlist: 1000, num_workers: 8} + search_params: {nprobe: 50} # Fixed search parameters for this L1 scan + + - name: "Hier_L0_10k_L1_0.5k_P20" + l0_ref: "L0_10k_config" + parent_levels: + - type: Quake # L1 + build_params: {nlist: 500, num_workers: 8} + search_params: {nprobe: 20} + + - name: "Custom_Flat_L0_5k" # Defines its L0 ad-hoc (not from l0_definitions) + custom_l0: # L0 definition specific to this hierarchy + type: Quake + build_params: {nlist: 5000, num_workers: 8} + # parent_levels: (empty or omitted) -> This is a custom flat L0. + +# --- General Run Settings --- +run_settings: + trials: 3 + warmup: 1 + default_batch_size: 1 + omp_threads_build: "16" # Or "MAX" as a placeholder for max available + omp_threads_search: "1" + +plot_styles: + "Flat_L0_40k": {color: "#1f77b4", marker: "o"} + "Hier_L0_40k_L1_1k_P50": {color: "#ff7f0e", marker: "s"} + "Hier_L0_10k_L1_0.5k_P20": {color: "#2ca02c", marker: "D"} + "Custom_Flat_L0_5k": {color: "#d62728", marker: "^"} \ No newline at end of file diff --git a/test/experiments/osdi2025/multi_level/configs/sift1m.yaml b/test/experiments/osdi2025/multi_level/configs/sift1m.yaml new file mode 100644 index 00000000..2cdda0aa --- /dev/null +++ b/test/experiments/osdi2025/multi_level/configs/sift1m.yaml @@ -0,0 +1,61 @@ +# General settings +overwrite: + index_files: false # Set to true to force rebuild L0s on first run of this config + results_csv: true # Re-runs searches and overwrites this config's output CSVs + +dataset: + name: sift1m # Using sift1m for a quicker test + metric: l2 + nq: 100 # Reduced number of queries for faster testing + +# --- L0 (Base Level) Definitions --- +l0_definitions: + "L0_1k_config": # L0 with 1000 centroids, suitable for sift1m + type: Quake + build_params: + nlist: 1000 + num_workers: 0 # Adjust based on your test machine + "L0_0.5k_config": # A smaller L0 + type: Quake + build_params: + nlist: 500 + num_workers: 0 + +# --- Global L0 Search Sweep --- +# Applies to the L0 of ALL index_hierarchies +global_l0_search_sweep: + nprobes: [1, 5, 10, 20] # Reduced list of nprobes for L0 + k: 10 + batch_size: 1 + +# --- Index Hierarchy Definitions --- +index_hierarchies: +# - name: "SingleLevel_L0_1k" # A single-level index using the 1k L0 +# l0_ref: "L0_1k_config" +# # No parent_levels + + - name: "Hier_L0_1k_L1_30_P5" # L0=1k, L1 groups L0's centroids, L1 nprobe=5 + l0_ref: "L0_1k_config" + parent_levels: + - type: Quake # L1 configuration + build_params: {nlist: 30, num_workers: 0} # L1 nc groups centroids of L0 + search_params: {nprobe: 5} # Fixed search params for L1 scan + + - name: "Custom_SingleLevel_L0_0.5k" # Ad-hoc custom L0, used as single-level + custom_l0: + type: Quake + build_params: {nlist: 500, num_workers: 0} # nc should be custom_l0.build_params.nc + +# --- General Run Settings --- +run_settings: + trials: 1 # Reduced trials for speed + warmup: 0 # No warmup for a quick test + default_batch_size: 1 + omp_threads_build: "16" # Adjust as needed + omp_threads_search: "1" + +# --- Plotting Styles --- +plot_styles: + "SingleLevel_L0_1k": {color: "#1f77b4", marker: "o"} + "Hier_L0_1k_L1_30_P5": {color: "#ff7f0e", marker: "s"} + "Custom_SingleLevel_L0_0.5k": {color: "#2ca02c", marker: "D"} \ No newline at end of file diff --git a/test/experiments/osdi2025/multi_level/run.py b/test/experiments/osdi2025/multi_level/run.py new file mode 100644 index 00000000..2788889b --- /dev/null +++ b/test/experiments/osdi2025/multi_level/run.py @@ -0,0 +1,399 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import logging +import time +from pathlib import Path +from typing import Dict, List, Any, Tuple +import copy +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import torch +import os +import sys +import yaml +from quake import QuakeIndex, IndexBuildParams, SearchParams, SearchTimingInfo +from test.experiments.osdi2025 import experiment_utils as common_utils +from quake.utils import compute_recall + +logger = logging.getLogger("multi_level_debug_exp") +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)-8s | %(name)-25s | %(message)s", + datefmt="%H:%M:%S", + handlers=[logging.StreamHandler(sys.stdout)] +) + +def build_or_verify_l0( + l0_key_name: str, + l0_build_config: Dict[str, Any], + base_vectors: torch.Tensor, + dataset_metric: str, + output_dir: Path, + force_rebuild: bool, + quake_verbose_log_path: Path # Added for redirection +) -> Path: + index_file_path = output_dir / f"l0_index_{l0_key_name}.bin" + l0_type = l0_build_config.get("type", "Quake") + if l0_type != "Quake": + raise NotImplementedError(f"L0 index type '{l0_type}' not yet supported.") + + build_params_dict = copy.deepcopy(l0_build_config.get("build_params", {})) + if 'metric' not in build_params_dict: + build_params_dict['metric'] = dataset_metric + + if not index_file_path.exists() or force_rebuild: + logger.info(f"Building L0 '{l0_key_name}' (nlist={build_params_dict.get('nlist')}) -> {index_file_path}") + with common_utils.redirect_all_stdout_to_file(quake_verbose_log_path, mode='a'): + _ = common_utils.prepare_quake_index( + vecs=base_vectors, + build_params_dict=build_params_dict, + index_file_path=index_file_path, + force_rebuild=True, + load=False + ) + else: + logger.info(f"Using existing L0 '{l0_key_name}' from {index_file_path}") + return index_file_path + +def construct_full_search_params( + l0_variant_params: Dict[str, Any], + parent_levels_config: List[Dict[str, Any]] +) -> SearchParams: + current_sp = common_utils.create_search_params(**l0_variant_params) + previous_level_sp = current_sp + for parent_cfg in parent_levels_config: + parent_search_dict = parent_cfg.get("search_params", {"nprobe": 1}) + parent_level_sp = common_utils.create_search_params(**parent_search_dict) + previous_level_sp.parent_params = parent_level_sp + previous_level_sp = parent_level_sp + return current_sp + +def extract_recursive_timing_stats( + timing_info: SearchTimingInfo | None, max_levels: int +) -> Dict[str, Any]: + stats = {} + current_ti = timing_info + for i in range(max_levels): + level_prefix = f"L{i}" + if current_ti: + stats[f"{level_prefix}_total_scan_ms"] = current_ti.total_time_ns / 1e6 + stats[f"{level_prefix}_partitions_scanned"] = getattr(current_ti, "partitions_scanned", np.nan) + current_ti = current_ti.parent_info + else: + stats[f"{level_prefix}_total_scan_ms"] = np.nan + stats[f"{level_prefix}_partitions_scanned"] = np.nan + return stats + +def benchmark_hierarchy_search_point( + index_instance: QuakeIndex, + queries: torch.Tensor, + ground_truth: np.ndarray | None, + full_hierarchical_sp: SearchParams, + num_actual_levels: int, + trials: int, + warmup: int, + quake_verbose_log_path: Path # Added for redirection +) -> Dict[str, Any]: + trial_latencies_ns = [] + trial_recalls = [] + trial_per_level_stats_list = [] + k_val_for_recall = full_hierarchical_sp.k + + with common_utils.redirect_all_stdout_to_file(quake_verbose_log_path, mode='a'): + for _ in range(warmup): + index_instance.search(queries, full_hierarchical_sp) + for _ in range(trials): + t_start_ns = time.perf_counter_ns() + search_result = index_instance.search(queries, full_hierarchical_sp) + t_elapsed_ns = time.perf_counter_ns() - t_start_ns + trial_latencies_ns.append(t_elapsed_ns) + + if ground_truth is not None and search_result.ids is not None: + max_k_recall = min(k_val_for_recall, ground_truth.shape[1], search_result.ids.shape[1]) + recall_val = compute_recall( + search_result.ids[:, :max_k_recall], + ground_truth[:, :max_k_recall],max_k_recall + ).mean().item() if max_k_recall > 0 else np.nan + trial_recalls.append(recall_val) + else: + trial_recalls.append(np.nan) + trial_per_level_stats_list.append( + extract_recursive_timing_stats(search_result.timing_info, num_actual_levels) + ) + + mean_latency_ms = np.mean(trial_latencies_ns) / 1e6 if trial_latencies_ns else np.nan + qps_mean = len(queries) / (mean_latency_ms / 1000) if mean_latency_ms > 0 else np.inf + results = { + "qps_mean": qps_mean, + "recall_mean": np.nanmean(trial_recalls) if trial_recalls else np.nan, + "overall_latency_ms_mean": mean_latency_ms, + } + if trial_per_level_stats_list: + df_level_stats = pd.DataFrame(trial_per_level_stats_list) + for col in df_level_stats.columns: + results[f"{col}_mean"] = df_level_stats[col].mean() + return results + +def run_experiment(config_path_str: str, output_dir_str: str): + config = common_utils.load_config(config_path_str) + output_dir = Path(output_dir_str) + output_dir.mkdir(parents=True, exist_ok=True) + + quake_verbose_log = output_dir / "quake_internal_details.log" + if quake_verbose_log.exists(): quake_verbose_log.unlink() # Clear log at start of run + + logger.info(f"Loaded configuration from: {config_path_str}") + dataset_cfg = config["dataset"] + overwrite_cfg = config["overwrite"] + l0_definitions = config.get("l0_definitions", {}) + global_l0_sweep_cfg = config["global_l0_search_sweep"] + index_hierarchies_cfg = config["index_hierarchies"] + run_settings = config["run_settings"] + original_omp_threads = os.environ.get("OMP_NUM_THREADS") + plot_styles = config.get("plot_styles", {}) + + logger.info(f"Dataset: {dataset_cfg['name']}, Metric: {dataset_cfg['metric']}, NQ: {dataset_cfg['nq']}") + logger.info(f"Overwrite index_files: {overwrite_cfg['index_files']}, Overwrite results_csv: {overwrite_cfg['results_csv']}") + logger.info(f"Verbose Quake output will be logged to: {quake_verbose_log}") + + + base_vectors, queries, gt_numpy = common_utils.load_data( + dataset_cfg["name"], dataset_cfg.get("path", "data/"), nq_override=dataset_cfg.get("nq") + ) + logger.info(f"Loaded data: Base vectors {base_vectors.shape}, Queries {queries.shape}, GT {gt_numpy.shape if gt_numpy is not None else 'None'}") + + built_l0_files = {} + all_experiment_records = [] + + omp_build_threads = str(run_settings.get("omp_threads_build", os.cpu_count())) + os.environ["OMP_NUM_THREADS"] = omp_build_threads + logger.info(f"BUILD PHASE: Set OMP_NUM_THREADS={os.environ['OMP_NUM_THREADS']}") + unique_l0s_to_process = {} + for l0_ref_name, l0_conf in l0_definitions.items(): + unique_l0s_to_process[l0_ref_name] = l0_conf + for hierarchy_cfg in index_hierarchies_cfg: + if "custom_l0" in hierarchy_cfg: + unique_l0s_to_process[f"custom_for_{hierarchy_cfg['name']}"] = hierarchy_cfg["custom_l0"] + logger.info(f"Found {len(unique_l0s_to_process)} unique L0 configurations.") + for l0_key, l0_build_conf in unique_l0s_to_process.items(): + built_l0_files[l0_key] = build_or_verify_l0( + l0_key, l0_build_conf, base_vectors, dataset_cfg["metric"], + output_dir, overwrite_cfg["index_files"], quake_verbose_log # Pass log path + ) + + l0_search_variants = common_utils.expand_search_sweep(global_l0_sweep_cfg) + global_k_val = global_l0_sweep_cfg["k"] + global_batch_size = global_l0_sweep_cfg.get("batch_size", run_settings.get("default_batch_size", 1)) + for variant in l0_search_variants: + if "k" not in variant: variant["k"] = global_k_val + if "batch_size" not in variant: variant["batch_size"] = global_batch_size + logger.info(f"Global L0 search sweep: {len(l0_search_variants)} variants. k={global_k_val}, batch_size={global_batch_size}.") + + omp_search_threads = str(run_settings.get("omp_threads_search", "1")) + os.environ["OMP_NUM_THREADS"] = omp_search_threads + logger.info(f"SEARCH PHASE: Set OMP_NUM_THREADS={os.environ['OMP_NUM_THREADS']}") + + for hierarchy_cfg in index_hierarchies_cfg: + hierarchy_name = hierarchy_cfg["name"] + logger.info(f"===== Processing Hierarchy: {hierarchy_name} =====") + l0_file_to_load, l0_build_details_for_reporting, l0_load_params_dict = None, {}, {} + + if "l0_ref" in hierarchy_cfg: + ref_name = hierarchy_cfg["l0_ref"] + l0_file_to_load = built_l0_files.get(ref_name) + l0_build_details_for_reporting = l0_definitions.get(ref_name, {}).get("build_params", {}) + elif "custom_l0" in hierarchy_cfg: + custom_l0_key = f"custom_for_{hierarchy_name}" + l0_file_to_load = built_l0_files.get(custom_l0_key) + l0_build_details_for_reporting = hierarchy_cfg["custom_l0"].get("build_params", {}) + if not l0_file_to_load: + logger.error(f"L0 file path not found for {hierarchy_name}. Skipping.") + continue + + l0_load_params_dict = { + "num_workers": l0_build_details_for_reporting.get("num_workers",0), + "use_numa": l0_build_details_for_reporting.get("use_numa",False), + "parent_num_workers": 0 # For L0 load itself + } + + num_actual_levels = 1 + len(hierarchy_cfg.get("parent_levels", [])) + + current_index = QuakeIndex() + with common_utils.redirect_all_stdout_to_file(quake_verbose_log, mode='a'): + current_index.load(str(l0_file_to_load), l0_load_params_dict["num_workers"], l0_load_params_dict["use_numa"], l0_load_params_dict["parent_num_workers"]) + + parent_levels_config_for_sp = [] + for i, parent_cfg in enumerate(hierarchy_cfg.get("parent_levels", [])): + parent_bp = common_utils.create_index_build_params(**parent_cfg["build_params"]) + if not parent_bp.metric: parent_bp.metric = dataset_cfg["metric"] + with common_utils.redirect_all_stdout_to_file(quake_verbose_log, mode='a'): + current_index.add_level(parent_bp) + parent_levels_config_for_sp.append(parent_cfg) + + for l0_variant in l0_search_variants: + l0_nprobe_val = l0_variant.get("nprobe") + l0_rt_val = l0_variant.get("recall_target") + sweep_param_log = f"nprobe={l0_nprobe_val}" if l0_nprobe_val is not None else f"RT={l0_rt_val}" + logger.info(f" Testing L0 Sweep: {sweep_param_log} for '{hierarchy_name}'") + + full_sp = construct_full_search_params(l0_variant, parent_levels_config_for_sp) + metrics = benchmark_hierarchy_search_point( + current_index, queries, gt_numpy, full_sp, + num_actual_levels, run_settings["trials"], run_settings["warmup"], + quake_verbose_log + ) + + record = { + "hierarchy_name": hierarchy_name, + "L0_nlist": l0_build_details_for_reporting.get("nlist"), + "L0_nprobe": l0_nprobe_val, "L0_recall_target": l0_rt_val, + "L0_k": l0_variant["k"], "L0_batch_size": l0_variant["batch_size"], + } + for i, p_cfg in enumerate(hierarchy_cfg.get("parent_levels", [])): + level_num = i + 1 + record[f"L{level_num}_nlist"] = p_cfg.get("build_params", {}).get("nlist") + record[f"L{level_num}_search_nprobe"] = p_cfg.get("search_params", {}).get("nprobe") + record.update(metrics) + all_experiment_records.append(record) + logger.info(f" Done: Recall={metrics.get('recall_mean',0):.4f}, QPS={metrics.get('qps_mean',0):.1f}, Latency={metrics.get('overall_latency_ms_mean',0):.2f}ms") + del current_index + + if original_omp_threads is not None: os.environ["OMP_NUM_THREADS"] = original_omp_threads + elif "OMP_NUM_THREADS" in os.environ: del os.environ["OMP_NUM_THREADS"] + logger.info(f"OMP_NUM_THREADS settings reverted.") + + if not all_experiment_records: + logger.warning("No results collected.") + return + + final_df = pd.DataFrame(all_experiment_records) + unified_csv_path = output_dir / f"{dataset_cfg['name']}_summary.csv" # Simplified name + common_utils.save_results_df(final_df, unified_csv_path) + + plot_qps_vs_recall(final_df, plot_styles, dataset_cfg, output_dir) + plot_level_scan_time_breakdown(final_df, plot_styles, dataset_cfg, global_l0_sweep_cfg, output_dir) + + logger.info(f"===== Experiment Finished: {config_path_str} =====") + logger.info(f"Output directory: {output_dir.resolve()}") + + logger.info("\n--- Per-Hierarchy Per-L0-Parameter Summary ---") + for name, group in final_df.groupby('hierarchy_name'): + logger.info(f"\nResults for Hierarchy: {name}") + sweep_param = "L0_nprobe" if "L0_nprobe" in group.columns and group["L0_nprobe"].notna().any() else "L0_recall_target" + summary_cols = [sweep_param, 'recall_mean', 'overall_latency_ms_mean', 'qps_mean'] + # Filter out columns that might be all NaN if a sweep param wasn't used + summary_cols = [col for col in summary_cols if col in group.columns and group[col].notna().any()] + if not summary_cols or sweep_param not in summary_cols : # if sweep_param is all NaN + logger.info(group[['recall_mean', 'overall_latency_ms_mean', 'qps_mean']].to_string(float_format="%.4f")) + else: + logger.info(group[summary_cols].sort_values(by=sweep_param).to_string(float_format="%.4f")) + logger.info("-----------------------------------------------\n") + + +def plot_qps_vs_recall(results_df: pd.DataFrame, plot_styles: Dict[str, Any], dataset_cfg: dict, output_dir: Path): + if results_df.empty: return + plt.figure(figsize=(10, 6)) + for hierarchy_name, group in results_df.groupby("hierarchy_name"): + style = plot_styles.get(hierarchy_name, {}) + sorted_group = group.sort_values(by="recall_mean").dropna(subset=["qps_mean", "recall_mean"]) + if sorted_group.empty: continue + plt.plot( + sorted_group["recall_mean"], sorted_group["qps_mean"], label=hierarchy_name, + marker=style.get("marker", "o"), color=style.get("color"), linestyle=style.get("linestyle", "-") + ) + plt.xlabel(f"Mean Recall@{results_df['L0_k'].iloc[0] if 'L0_k' in results_df.columns and results_df['L0_k'].notna().any() else 'k'}") + plt.ylabel("Mean QPS") + plt.title(f"{dataset_cfg['name']}: QPS vs. Recall") + plt.legend(title="Hierarchy Config", bbox_to_anchor=(1.05, 1), loc='upper left') + plt.grid(True, which="both", ls=":", alpha=0.7) + plt.xlim(left=max(0.0, results_df["recall_mean"].min() - 0.05 if results_df["recall_mean"].notna().any() else 0.0), right=1.05) + if results_df["qps_mean"].notna().any() and results_df["qps_mean"].max() > 10 * results_df["qps_mean"].min() and results_df["qps_mean"].min() > 0: + plt.yscale("log") + plt.tight_layout(rect=[0, 0, 0.82, 1]) + plt.savefig(output_dir / f"{dataset_cfg['name']}_qps_vs_recall.png", dpi=150) + plt.close() + + +def find_best_op_point_for_recall_plot( + hierarchy_data: pd.DataFrame, + target_recall: float, + l0_sweep_key: str +): + # Find points meeting or exceeding target recall + valid_points = hierarchy_data[hierarchy_data['recall_mean'] >= target_recall] + + if valid_points.empty: + # If none meet, pick the one with highest recall + if not hierarchy_data.empty: + best_effort_point = hierarchy_data.loc[hierarchy_data['recall_mean'].idxmax()] + logger.warning( + f"For breakdown plot of '{best_effort_point['hierarchy_name']}', no L0 param met target recall {target_recall:.3f}. " + f"Using best achieved: R={best_effort_point['recall_mean']:.3f} " + f"(L0 {l0_sweep_key.split('_')[-1]}={best_effort_point[l0_sweep_key]})" + ) + return best_effort_point + return None + + # Among valid points, pick one with highest QPS, then lowest latency + best_point = valid_points.sort_values(by=['qps_mean', 'overall_latency_ms_mean'], ascending=[False, True]).iloc[0] + return best_point + +def plot_level_scan_time_breakdown( + results_df: pd.DataFrame, + plot_styles: Dict[str, Any], + dataset_cfg: dict, + l0_sweep_cfg: Dict[str, Any], + output_dir: Path, + target_recall_for_plot: float = 0.9 # New parameter +): + if results_df.empty: + logger.warning("Level scan time breakdown plot: No data.") + return + + l0_sweep_key = "L0_nprobe" if "nprobes" in l0_sweep_cfg else "L0_recall_target" + + points_for_plot = [] + for hierarchy_name, group in results_df.groupby("hierarchy_name"): + op_point = find_best_op_point_for_recall_plot(group, target_recall_for_plot, l0_sweep_key) + if op_point is not None: + points_for_plot.append(op_point) + + if not points_for_plot: + logger.warning(f"No suitable data points found for any hierarchy at target recall ~{target_recall_for_plot} for breakdown plot.") + return + + df_subset = pd.DataFrame(points_for_plot) + plot_title_suffix = f"(at or above R≈{target_recall_for_plot:.2f})" + + max_level = max([-1] + [int(c.split('_')[0][1:]) for c in df_subset.columns if c.startswith("L") and "_total_scan_ms_mean" in c]) + if max_level == -1: + logger.warning("No per-level scan time columns found for breakdown plot.") + return + + level_cols = [f"L{i}_total_scan_ms_mean" for i in range(max_level + 1)] + for col in level_cols: + if col not in df_subset.columns: df_subset[col] = 0.0 + + plot_data = df_subset.set_index("hierarchy_name")[level_cols].fillna(0) + plot_data.columns = [f"L{i} Scan (ms)" for i in range(max_level + 1)] + + ax = plot_data.plot(kind='bar', stacked=True, figsize=(12, 7), colormap='viridis_r') + + # Add text for achieved recall on top of each bar + for i, hierarchy_name in enumerate(df_subset["hierarchy_name"]): + row_data = df_subset.iloc[i] + total_height = plot_data.loc[hierarchy_name].sum() + actual_recall = row_data.get('recall_mean', np.nan) + if pd.notna(actual_recall): + ax.text(i, total_height * 1.01, f"R={actual_recall:.3f}", ha='center', va='bottom', fontsize=8, color='dimgrey') + + plt.xlabel("Hierarchy Configuration") + plt.ylabel("Mean Scan Time per Level (ms)") + plt.title(f"{dataset_cfg['name']}: Scan Time Breakdown {plot_title_suffix}") + plt.xticks(rotation=25, ha='right') + plt.legend(title="Level", bbox_to_anchor=(1.05, 1), loc='upper left') + plt.tight_layout(rect=[0, 0, 0.82, 1]) + plt.savefig(output_dir / f"{dataset_cfg['name']}_level_scan_breakdown_at_recall_target.png", dpi=150) + plt.close() + logger.info(f"Level scan time breakdown (at R~{target_recall_for_plot:.2f}) plot saved.") \ No newline at end of file diff --git a/test/experiments/osdi2025/numa_multi_query/__init__.py b/test/experiments/osdi2025/numa_multi_query/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/experiments/osdi2025/numa_multi_query/configs/msturing10m.yaml b/test/experiments/osdi2025/numa_multi_query/configs/msturing10m.yaml new file mode 100644 index 00000000..2f7f9cc5 --- /dev/null +++ b/test/experiments/osdi2025/numa_multi_query/configs/msturing10m.yaml @@ -0,0 +1,157 @@ +# numa_latency_experiment.yaml +dataset: + name: msturing10m + num_queries: 1000 + k: 100 + +trials: 3 +warmup: 1 +overwrite: true # Set to true to force all indexes to rerun + +indexes: + - name: Quake_0 + index: Quake + build_params: + nc: 20000 + metric: l2 + num_workers: 0 + use_numa: false + search_params: + nprobe: 300 + batched_scan: true + + - name: Quake_1 + index: Quake + build_params: + nc: 20000 + metric: l2 + num_workers: 1 + use_numa: false + search_params: + nprobe: 300 + batched_scan: true + + - name: Quake_4 + index: Quake + build_params: + nc: 20000 + metric: l2 + num_workers: 4 + use_numa: false + search_params: + nprobe: 300 + batched_scan: true + + - name: Quake_16 + index: Quake + build_params: + nc: 20000 + metric: l2 + num_workers: 16 + use_numa: false + search_params: + nprobe: 300 + batched_scan: true + + - name: Quake_0_numa + index: Quake + build_params: + nc: 20000 + metric: l2 + num_workers: 0 + use_numa: true + search_params: + nprobe: 300 + batched_scan: true + + - name: Quake_1_numa + index: Quake + build_params: + nc: 20000 + metric: l2 + num_workers: 1 + use_numa: true + search_params: + nprobe: 300 + batched_scan: true + + - name: Quake_4_numa + index: Quake + build_params: + nc: 20000 + metric: l2 + num_workers: 4 + use_numa: true + search_params: + nprobe: 300 + batched_scan: true + + - name: Quake_16_numa + index: Quake + build_params: + nc: 20000 + metric: l2 + num_workers: 16 + use_numa: true + search_params: + nprobe: 300 + batched_scan: true + + - name: Quake_unbatched + index: Quake + build_params: + nc: 20000 + metric: l2 + num_workers: 16 + use_numa: true + search_params: + nprobe: 300 + batched_scan: false + search_threads: 16 + +# - name: FaissIVF +# index: IVF +# build_params: +# nc: 1000 +# metric: l2 +# search_params: +# nprobe: 15 +# +# - name: SCANN +# index: SCANN +# build_params: +# num_leaves: 1000 +# metric: squared_l2 +# search_params: +# leaves_to_search: 15 +# +# - name: FaissHNSW +# index: HNSW +# build_params: +# m: 64 +# ef_construction: 128 +# metric: l2 +# search_params: +# ef_search: 24 +# +# - name: DiskANN +# index: DiskANN +# build_params: +# complexity: 128 +# graph_degree: 64 +# metric: l2 +# search_params: +# complexity: 16 +# +# - name: SVS +# index: SVS +# build_params: +# graph_max_degree: 64 +# alpha: 1.1 +# window_size: 128 +# metric: l2 +# search_params: +# search_window_size: 24 + +output: + results_csv: numa_single_query_results.csv \ No newline at end of file diff --git a/test/experiments/osdi2025/numa_multi_query/configs/sift1m.yaml b/test/experiments/osdi2025/numa_multi_query/configs/sift1m.yaml new file mode 100644 index 00000000..cd56ed38 --- /dev/null +++ b/test/experiments/osdi2025/numa_multi_query/configs/sift1m.yaml @@ -0,0 +1,117 @@ +# numa_latency_experiment.yaml +dataset: + name: sift1m + num_queries: 10000 + k: 1 + +trials: 3 +warmup: 1 +overwrite: true # Set to true to force all indexes to rerun +force_rebuild: false # Set to true to force all indexes to rebuild + +indexes: + - name: Quake_16 + index: Quake + build_params: + nc: 1000 + metric: l2 + num_workers: 1 + num_merge_workers: 1 + use_numa: true + parent: + num_workers: 1 + num_merge_workers: 1 + use_numa: true + search_params: + nprobe: 100 + batch_size: 64 + parent: + batched_scan: true + batch_size: 128 + batched_scan: true +# +# - name: Quake_16_numa +# index: Quake +# build_params: +# nc: 1000 +# metric: l2 +# num_workers: 16 +# use_numa: true +# parent_num_workers: 4 +# search_params: +# nprobe: 20 +# batched_scan: true +# +# - name: Quake_16_numa_no_batch +# index: Quake +# build_params: +# nc: 1000 +# metric: l2 +# num_workers: 16 +# use_numa: true +# parent_num_workers: 4 +# search_params: +# nprobe: 20 +# batched_scan: false + +# - name: FaissIVF +# index: IVF +# build_params: +# nc: 1000 +# metric: l2 +# search_params: +# nprobe: 20 +# omp_num_threads_search: 16 +# +# +# - name: FaissHNSW +# index: HNSW +# build_params: +# m: 64 +# ef_construction: 128 +# metric: l2 +# omp_num_threads_build: 1 +# search_params: +# ef_search: 16 +# omp_num_threads_search: 1 + +# +# - name: SCANN +# index: SCANN +# build_params: +# num_leaves: 1000 +# metric: squared_l2 +# search_params: +# leaves_to_search: 15 +# +# - name: FaissHNSW +# index: HNSW +# build_params: +# m: 64 +# ef_construction: 128 +# metric: l2 +## omp_num_threads_build: 16 +# search_params: +# ef_search: 24 +# +# - name: DiskANN +# index: DiskANN +# build_params: +# complexity: 128 +# graph_degree: 64 +# metric: l2 +# search_params: +# complexity: 16 +# +# - name: SVS +# index: SVS +# build_params: +# graph_max_degree: 64 +# alpha: 1.1 +# window_size: 128 +# metric: l2 +# search_params: +# search_window_size: 24 + +output: + results_csv: numa_single_query_results.csv \ No newline at end of file diff --git a/test/experiments/osdi2025/numa_multi_query/run.py b/test/experiments/osdi2025/numa_multi_query/run.py new file mode 100644 index 00000000..c424b04b --- /dev/null +++ b/test/experiments/osdi2025/numa_multi_query/run.py @@ -0,0 +1,263 @@ +# test/experiments/osdi2025/numa_multi_query/run.py + +import logging +from pathlib import Path +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import torch +import os +import faiss # Import faiss at the top level as it's now a hard dependency for thread setting + +import test.experiments.osdi2025.experiment_utils as common_utils +from quake.utils import compute_recall +from quake.index_wrappers.faiss_ivf import FaissIVF +from quake.index_wrappers.faiss_hnsw import FaissHNSW +from quake.index_wrappers.quake import QuakeWrapper # type: ignore + +try: from quake.index_wrappers.scann import Scann # type: ignore +except ImportError: Scann = None +try: from quake.index_wrappers.diskann import DiskANNDynamic # type: ignore +except ImportError: DiskANNDynamic = None +try: from quake.index_wrappers.vamana import Vamana # type: ignore +except ImportError: Vamana = None + +INDEX_CLASSES = { + "Quake": QuakeWrapper, "IVF": FaissIVF, "SCANN": Scann, + "HNSW": FaissHNSW, "DiskANN": DiskANNDynamic, "SVS": Vamana, +} +logger = logging.getLogger("numa_multi_query_orchestrator") + +def _apply_faiss_thread_settings(config_threads_str: str, task_name_info: str): + num_threads = int(config_threads_str) + faiss.omp_set_num_threads(num_threads) + print(f"[{task_name_info}] Called faiss.omp_set_num_threads({num_threads}).") + + +def task_build_index( + index_config: dict, dataset_cfg: dict, global_run_params: dict, + index_file_path: Path +): + index_name = index_config["name"] + + print(f"[{index_name} BUILD_TASK] Process ID: {os.getpid()}. Target file: {index_file_path}") + current_build_params = dict(index_config.get("build_params", {})) + omp_build_threads_config_val = str(current_build_params.pop("omp_num_threads_build", "1")) + + _apply_faiss_thread_settings(omp_build_threads_config_val, f"{index_name} BUILD_TASK") + + base_vectors, _, _ = common_utils.load_data( + dataset_cfg["name"], nq_override=1) + + IndexCls = INDEX_CLASSES.get(index_config["index"]) + if IndexCls is None or (IndexCls == Scann and Scann is None): # type: ignore + return {"error": f"Index type '{index_config['index']}' unavailable.", "index_file_path": str(index_file_path)} + + current_build_params.pop("num_parent_workers", None) + + idx_instance_for_build = common_utils.prepare_wrapper_index( + IndexCls, index_file_path, base_vectors, current_build_params, + global_run_params["force_rebuild_indices"], load=False + ) + del base_vectors + + if idx_instance_for_build is None : + return {"error": f"Index build failed for {index_name}.", "index_file_path": str(index_file_path)} + + print(f"[{index_name} BUILD_TASK] Build/Save completed for {index_file_path}.") + return {"status": "success", "index_file_path": str(index_file_path)} + + +def task_search_index( + index_config: dict, dataset_cfg: dict, global_run_params: dict, + index_file_path: Path +): + index_name = index_config["name"] + index_type_str = index_config["index"] + + print(f"[{index_name} SEARCH_TASK] Process ID: {os.getpid()}. Loading index from: {index_file_path}") + current_build_params_for_load = dict(index_config.get("build_params", {})) + current_search_params_for_method = dict(index_config.get("search_params", {})) + + omp_search_threads_config_val = str(current_search_params_for_method.pop("omp_num_threads_search", "1")) + _apply_faiss_thread_settings(omp_search_threads_config_val, f"{index_name} SEARCH_TASK") + + _, query_vectors, gt_vectors = common_utils.load_data( + dataset_cfg["name"], nq_override=dataset_cfg.get("num_queries")) + + IndexCls = INDEX_CLASSES.get(index_type_str) + if IndexCls is None or (IndexCls == Scann and Scann is None): # type: ignore + return {"error": f"Index type '{index_type_str}' unavailable for search."} + + if not Path(index_file_path).exists(): + return {"error": f"Index file {index_file_path} not found for search."} + + idx_instance = IndexCls() + load_kwargs = {} + if "num_workers" in current_build_params_for_load: load_kwargs["num_workers"] = current_build_params_for_load.get("num_workers") + if "use_numa" in current_build_params_for_load: load_kwargs["use_numa"] = current_build_params_for_load.get("use_numa") + if "num_merge_workers" in current_build_params_for_load: load_kwargs["num_merge_workers"] = current_build_params_for_load.get("num_merge_workers") + if "parent" in current_build_params_for_load: load_kwargs["parent"] = current_build_params_for_load.get("parent") + + idx_instance.load(str(index_file_path), **load_kwargs) + print(f"[{index_name} SEARCH_TASK] Index loaded with kwargs: {load_kwargs}") + + k_val = global_run_params["k_val"] + print(f"[{index_name} SEARCH_TASK] Warmup ({global_run_params['num_warmup']} iterations)...") + for _ in range(global_run_params['num_warmup']): + _ = idx_instance.search(query_vectors, k_val, **current_search_params_for_method) + + trial_latencies_ms = [] + trial_recalls = [] + print(f"[{index_name} SEARCH_TASK] Benchmarking ({global_run_params['num_trials']} trials)...") + for i in range(global_run_params['num_trials']): + search_result = idx_instance.search(query_vectors, k_val, **current_search_params_for_method) + latency_ms = np.nan + timing_info = getattr(search_result, "timing_info", None) + if timing_info and hasattr(timing_info, "total_time_ns"): latency_ms = getattr(timing_info, "total_time_ns") / 1e6 + elif timing_info and hasattr(timing_info, "child_total_time_ns"): latency_ms = getattr(timing_info, "child_total_time_ns") / 1e6 + trial_latencies_ms.append(latency_ms) + + recall_val = np.nan + if gt_vectors is not None and hasattr(search_result, "ids"): + try: recall_val = float(compute_recall(search_result.ids, gt_vectors, k_val).mean()) + except Exception: pass + trial_recalls.append(recall_val) + + print(f"[{index_name} SEARCH_TASK] Individual trial results (Latency, Recall@K={k_val}):") + for i in range(len(trial_latencies_ms)): + lat_str = f"{trial_latencies_ms[i]:.3f}ms" if not np.isnan(trial_latencies_ms[i]) else "N/A" + rec_str = f"{trial_recalls[i]:.4f}" if not np.isnan(trial_recalls[i]) else "N/A" + print(f" Trial {i+1}: Latency={lat_str}, Recall={rec_str}") + + final_row = {"index_name_from_process": index_name} + valid_latencies = [l for l in trial_latencies_ms if not np.isnan(l)] + valid_recalls = [r for r in trial_recalls if not np.isnan(r)] + final_row["mean_latency_ms"] = float(np.mean(valid_latencies)) if valid_latencies else np.nan + final_row["std_latency_ms"] = float(np.std(valid_latencies)) if valid_latencies else np.nan + final_row[f"mean_recall_at_{k_val}"] = float(np.mean(valid_recalls)) if valid_recalls else np.nan + final_row[f"std_recall_at_{k_val}"] = float(np.std(valid_recalls)) if valid_recalls else np.nan + + n_concurrency_param = int(omp_search_threads_config_val) + if index_type_str == "Quake" and "num_workers" in current_build_params_for_load : # type: ignore + n_concurrency_param = current_build_params_for_load.get("num_workers", n_concurrency_param) # type: ignore + final_row["n_config_concurrency"] = n_concurrency_param + + print(f"[{index_name} SEARCH_TASK] Benchmark task finished.") + return final_row + +def run_experiment(cfg_path_str: str, output_dir_str: str): + logging.basicConfig(level=logging.INFO, format="%(asctime)s|%(levelname)7s|%(name)s| %(message)s", datefmt="%H:%M:%S") + logger.info(f"Orchestrator: Config: {cfg_path_str}, Output: {output_dir_str}") + + cfg = common_utils.load_config(cfg_path_str) + main_output_dir = Path(output_dir_str); main_output_dir.mkdir(parents=True, exist_ok=True) + process_logs_dir = main_output_dir / "process_logs"; process_logs_dir.mkdir(parents=True, exist_ok=True) + individual_index_files_dir = main_output_dir / "individual_indices"; + individual_index_files_dir.mkdir(parents=True, exist_ok=True) + + dataset_cfg = cfg["dataset"] + global_run_params = { + "num_trials": cfg.get("trials", 3), "num_warmup": cfg.get("warmup", 1), "k_val": dataset_cfg["k"], + "force_overwrite_results": cfg.get("overwrite", False), + "force_rebuild_indices": cfg.get("force_rebuild", False), + "enable_glances": cfg.get("enable_glances_monitoring", False) + } + indexes_config_list = cfg["indexes"] + output_csv_name = cfg.get("output", {}).get("results_csv", "numa_multi_query_results.csv") + + all_experiment_rows = [] + for idx_conf in indexes_config_list: + index_name = idx_conf["name"] + index_file_path = individual_index_files_dir / f"{index_name}_index.bin" + per_index_final_results_csv = main_output_dir / f"{index_name}_final_results.csv" + + if per_index_final_results_csv.exists() and not global_run_params["force_overwrite_results"]: + try: + all_experiment_rows.append(pd.read_csv(per_index_final_results_csv).iloc[0].to_dict()) + logger.info(f"Loaded cached final results for {index_name}.") + continue + except Exception as e: logger.warning(f"Could not load cached final results for {index_name}: {e}. Re-running.") + + logger.info(f"Orchestrator: Submitting BUILD task for index: {index_name}") + build_log_file = process_logs_dir / f"{index_name}_build_process.log" + # OMP settings are now handled inside the tasks by _apply_faiss_thread_settings + # Other env_vars can still be passed if needed. + build_env_vars = idx_conf.get("build_env_vars", {}) + process_timeout = idx_conf.get("process_timeout", cfg.get("default_process_timeout", 7200)) + + build_result = common_utils.run_operation_in_process( + task_build_index, (idx_conf, dataset_cfg, global_run_params, index_file_path), + env_vars=build_env_vars, log_file_path=build_log_file, process_name=f"BuildTask_{index_name}", + timeout_seconds=process_timeout, enable_glances=global_run_params["enable_glances"] + ) + + if not (build_result["status"] == "success" and build_result.get("data", {}).get("status") == "success"): + err_msg = build_result.get('message', build_result.get('data', {}).get('error', 'Build task failed')) + logger.error(f"BUILD task for {index_name} failed. Log: {build_result.get('log_file')}. Error: {err_msg}") + all_experiment_rows.append({"index": index_name, "error": f"Build Failed: {err_msg}", "mean_latency_ms": np.nan, f"mean_recall_at_{global_run_params['k_val']}": np.nan}) + continue + + logger.info(f"BUILD task for {index_name} succeeded. Index file: {build_result.get('data',{}).get('index_file_path')}") + built_index_file_path = Path(build_result.get("data",{}).get("index_file_path", index_file_path)) + + logger.info(f"Orchestrator: Submitting SEARCH task for index: {index_name}") + search_log_file = process_logs_dir / f"{index_name}_search_process.log" + search_env_vars = idx_conf.get("search_env_vars", {}) + + search_process_result = common_utils.run_operation_in_process( + task_search_index, (idx_conf, dataset_cfg, global_run_params, built_index_file_path), + env_vars=search_env_vars, log_file_path=search_log_file, process_name=f"SearchTask_{index_name}", + timeout_seconds=process_timeout, enable_glances=global_run_params["enable_glances"] + ) + + if search_process_result["status"] == "success" and isinstance(search_process_result["data"], dict) and "error" not in search_process_result["data"]: + logger.info(f"Search task {index_name} succeeded. Log: {search_process_result.get('log_file')}. Metrics: {search_process_result.get('metrics_file')}") + result_data_row = search_process_result["data"]; result_data_row["index"] = index_name + all_experiment_rows.append(result_data_row) + common_utils.save_results_csv(pd.DataFrame([result_data_row]), per_index_final_results_csv) + else: + err_msg = search_process_result.get('message', search_process_result.get('data', {}).get('error','Search task failed')) + logger.error(f"Search task {index_name} failed. Log: {search_process_result.get('log_file')}. Metrics: {search_process_result.get('metrics_file')}. Error: {err_msg}") + all_experiment_rows.append({"index": index_name, "error": f"Search Failed: {err_msg}", "mean_latency_ms": np.nan, f"mean_recall_at_{global_run_params['k_val']}": np.nan}) + + if not all_experiment_rows: logger.warning("No results. Skipping CSV/plots."); return + final_df = pd.DataFrame(all_experiment_rows) + if 'index' not in final_df.columns and 'index_name_from_process' in final_df.columns: + final_df.rename(columns={'index_name_from_process': 'index'}, inplace=True) + final_df['index'] = final_df['index'].fillna(final_df.get('index_name_from_process', "unknown_index")) + + unified_csv_path = main_output_dir / output_csv_name + common_utils.save_results_csv(final_df, unified_csv_path); logger.info(f"Unified results: {unified_csv_path}") + + plot_suffix = Path(output_csv_name).stem + plot_x_param = "n_config_concurrency" + plot_df = final_df.dropna(subset=['index', plot_x_param]).copy() + if not plot_df.empty: + plot_df.loc[:, plot_x_param] = pd.to_numeric(plot_df[plot_x_param], errors='coerce').fillna(-1).astype(int) + k_val_plot = global_run_params['k_val'] + + if "mean_latency_ms" in plot_df.columns: + try: + plt.figure(figsize=(8,6)); + for name, group in plot_df.groupby("index"): + group = group.sort_values(by=plot_x_param) + plt.errorbar(group[plot_x_param], group["mean_latency_ms"], yerr=group.get("std_latency_ms"), marker="o", label=name) + plt.xlabel("Configured Concurrency"); plt.ylabel("Mean Latency (ms)"); plt.title(f"Latency ({dataset_cfg['name']})") + plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left'); plt.grid(True); plt.tight_layout(rect=[0,0,0.8,1]) + plt.savefig(main_output_dir / f"{plot_suffix}_latency.png"); plt.close(); logger.info(f"Latency plot saved.") + except Exception as e: logger.error(f"Latency plot error: {e}") + + recall_col = f"mean_recall_at_{k_val_plot}" + if recall_col in plot_df.columns: + try: + plt.figure(figsize=(8,6)); + for name, group in plot_df.groupby("index"): + group = group.sort_values(by=plot_x_param) + plt.errorbar(group[plot_x_param], group[recall_col], yerr=group.get(f"std_recall_at_{k_val_plot}"), marker="o", label=name) + plt.xlabel("Configured Concurrency"); plt.ylabel(f"Mean Recall@{k_val_plot}"); plt.title(f"Recall ({dataset_cfg['name']})") + plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left'); plt.grid(True); plt.tight_layout(rect=[0,0,0.8,1]); plt.ylim(0, 1.05) + plt.savefig(main_output_dir / f"{plot_suffix}_recall.png"); plt.close(); logger.info(f"Recall plot saved.") + except Exception as e: logger.error(f"Recall plot error: {e}") + else: logger.warning("Not enough data for plotting after filtering.") + logger.info("Experiment orchestration finished.") \ No newline at end of file diff --git a/test/experiments/osdi2025/numa_single_query/__init__.py b/test/experiments/osdi2025/numa_single_query/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/experiments/osdi2025/numa_single_query/configs/msturing10m.yaml b/test/experiments/osdi2025/numa_single_query/configs/msturing10m.yaml new file mode 100644 index 00000000..8c671df9 --- /dev/null +++ b/test/experiments/osdi2025/numa_single_query/configs/msturing10m.yaml @@ -0,0 +1,152 @@ +dataset: + name: msturing10m + num_queries: 100 + k: 1 + +trials: 2 +warmup: 1 +overwrite: true # Set to true to force all indexes to rerun + +indexes: + - name: Quake_1_NUMA + index: Quake + build_params: + nc: 10000 + metric: l2 + num_workers: 1 + use_numa: true + search_params: + nprobe: 2000 + + - name: Quake_2_NUMA + index: Quake + build_params: + nc: 10000 + metric: l2 + num_workers: 2 + use_numa: true + search_params: + nprobe: 2000 + + - name: Quake_4_NUMA + index: Quake + build_params: + nc: 10000 + metric: l2 + num_workers: 4 + use_numa: true + search_params: + nprobe: 2000 + + - name: Quake_8_NUMA + index: Quake + build_params: + nc: 10000 + metric: l2 + num_workers: 8 + use_numa: true + search_params: + nprobe: 2000 + + - name: Quake_16_NUMA + index: Quake + build_params: + nc: 10000 + metric: l2 + num_workers: 16 + use_numa: true + search_params: + nprobe: 2000 + + - name: Quake_32_NUMA + index: Quake + build_params: + nc: 10000 + metric: l2 + num_workers: 32 + use_numa: true + search_params: + nprobe: 2000 + + - name: Quake_64_NUMA + index: Quake + build_params: + nc: 10000 + metric: l2 + num_workers: 63 + use_numa: true + search_params: + nprobe: 2000 + + - name: Quake_1_NONUMA + index: Quake + build_params: + nc: 10000 + metric: l2 + num_workers: 1 + use_numa: false + search_params: + nprobe: 2000 + + - name: Quake_2_NONUMA + index: Quake + build_params: + nc: 10000 + metric: l2 + num_workers: 2 + use_numa: false + search_params: + nprobe: 2000 + + - name: Quake_4_NONUMA + index: Quake + build_params: + nc: 10000 + metric: l2 + num_workers: 4 + use_numa: false + search_params: + nprobe: 2000 + + - name: Quake_8_NONUMA + index: Quake + build_params: + nc: 10000 + metric: l2 + num_workers: 8 + use_numa: false + search_params: + nprobe: 2000 + + - name: Quake_16_NONUMA + index: Quake + build_params: + nc: 10000 + metric: l2 + num_workers: 16 + use_numa: false + search_params: + nprobe: 2000 + + - name: Quake_32_NONUMA + index: Quake + build_params: + nc: 10000 + metric: l2 + num_workers: 32 + use_numa: false + search_params: + nprobe: 2000 + + - name: Quake_64_NONUMA + index: Quake + build_params: + nc: 10000 + metric: l2 + num_workers: 64 + use_numa: false + search_params: + nprobe: 2000 + +output: + results_csv: numa_single_query_results.csv \ No newline at end of file diff --git a/test/experiments/osdi2025/numa_single_query/configs/sift1m.yaml b/test/experiments/osdi2025/numa_single_query/configs/sift1m.yaml new file mode 100644 index 00000000..54c1711f --- /dev/null +++ b/test/experiments/osdi2025/numa_single_query/configs/sift1m.yaml @@ -0,0 +1,152 @@ +dataset: + name: sift1m + num_queries: 100 + k: 1 + +trials: 3 +warmup: 1 +overwrite: true + +indexes: + - name: Quake_1_NUMA + index: Quake + build_params: + nc: 1000 + metric: l2 + num_workers: 1 + use_numa: true + search_params: + nprobe: 20 + + - name: Quake_2_NUMA + index: Quake + build_params: + nc: 1000 + metric: l2 + num_workers: 2 + use_numa: true + search_params: + nprobe: 20 + + - name: Quake_4_NUMA + index: Quake + build_params: + nc: 1000 + metric: l2 + num_workers: 4 + use_numa: true + search_params: + nprobe: 20 + + - name: Quake_8_NUMA + index: Quake + build_params: + nc: 1000 + metric: l2 + num_workers: 8 + use_numa: true + search_params: + nprobe: 20 + + - name: Quake_16_NUMA + index: Quake + build_params: + nc: 1000 + metric: l2 + num_workers: 16 + use_numa: true + search_params: + nprobe: 20 + + - name: Quake_32_NUMA + index: Quake + build_params: + nc: 1000 + metric: l2 + num_workers: 32 + use_numa: true + search_params: + nprobe: 20 + + - name: Quake_64_NUMA + index: Quake + build_params: + nc: 1000 + metric: l2 + num_workers: 64 + use_numa: true + search_params: + nprobe: 20 + + - name: Quake_1_NONUMA + index: Quake + build_params: + nc: 1000 + metric: l2 + num_workers: 1 + use_numa: false + search_params: + nprobe: 20 + + - name: Quake_2_NONUMA + index: Quake + build_params: + nc: 1000 + metric: l2 + num_workers: 2 + use_numa: false + search_params: + nprobe: 20 + + - name: Quake_4_NONUMA + index: Quake + build_params: + nc: 1000 + metric: l2 + num_workers: 4 + use_numa: false + search_params: + nprobe: 20 + + - name: Quake_8_NONUMA + index: Quake + build_params: + nc: 1000 + metric: l2 + num_workers: 8 + use_numa: false + search_params: + nprobe: 20 + + - name: Quake_16_NONUMA + index: Quake + build_params: + nc: 1000 + metric: l2 + num_workers: 16 + use_numa: false + search_params: + nprobe: 20 + + - name: Quake_32_NONUMA + index: Quake + build_params: + nc: 1000 + metric: l2 + num_workers: 32 + use_numa: false + search_params: + nprobe: 20 + + - name: Quake_64_NONUMA + index: Quake + build_params: + nc: 1000 + metric: l2 + num_workers: 64 + use_numa: false + search_params: + nprobe: 20 + +output: + results_csv: numa_single_query_results.csv \ No newline at end of file diff --git a/test/experiments/osdi2025/numa_single_query/run.py b/test/experiments/osdi2025/numa_single_query/run.py new file mode 100644 index 00000000..5682f89c --- /dev/null +++ b/test/experiments/osdi2025/numa_single_query/run.py @@ -0,0 +1,379 @@ +import sys +from pathlib import Path +import logging +import yaml +import numpy as np +import torch +import pandas as pd +import matplotlib.pyplot as plt + +from quake import SearchParams, SearchTimingInfo # Quake specific +from test.experiments.osdi2025 import experiment_utils as common_utils + +from quake.index_wrappers.faiss_ivf import FaissIVF +from quake.index_wrappers.faiss_hnsw import FaissHNSW +from quake.index_wrappers.quake import QuakeWrapper +try: + from quake.index_wrappers.scann import Scann +except ImportError: + Scann = None +try: + from quake.index_wrappers.diskann import DiskANNDynamic +except ImportError: + DiskANNDynamic = None +try: + from quake.index_wrappers.vamana import Vamana +except ImportError: + Vamana = None + +INDEX_CLASSES = { + "Quake": QuakeWrapper, + "IVF": FaissIVF, + "SCANN": Scann, + "HNSW": FaissHNSW, + "DiskANN": DiskANNDynamic, + "SVS": Vamana, +} + +logging.basicConfig(level=logging.INFO, + format="%(asctime)s | %(levelname)7s | %(name)s | %(message)s", + datefmt="%H:%M:%S") +logger = logging.getLogger("numa_single_query_exp") + + +def build_and_save_generic_index(index_cls, build_params_dict, base_vectors, index_file_path: Path): + if index_file_path.exists(): + logger.info(f"Index {index_file_path} already exists. Skipping build.") + return + logger.info(f"Building index {index_file_path}...") + idx_instance = index_cls() + # FaissIVF and FaissHNSW expect metric at build, not in build_params dict necessarily + metric = build_params_dict.pop("metric", "l2") + idx_instance.build(base_vectors, metric=metric, **build_params_dict) + index_file_path.parent.mkdir(parents=True, exist_ok=True) + idx_instance.save(str(index_file_path)) + logger.info(f"Index saved to {index_file_path}") + +def get_timing_details(timing_info_obj: SearchTimingInfo | None, is_parent: bool = False) -> dict: + prefix = "parent_" if is_parent else "child_" + details = { + f"{prefix}buffer_init_ns": np.nan, + f"{prefix}enqueue_ns": np.nan, + f"{prefix}wait_ns": np.nan, + f"{prefix}aggregate_ns": np.nan, + f"{prefix}total_time_ns": np.nan, + } + if timing_info_obj: + details[f"{prefix}buffer_init_ns"] = getattr(timing_info_obj, "buffer_init_time_ns", 0) + details[f"{prefix}enqueue_ns"] = getattr(timing_info_obj, "job_enqueue_time_ns", 0) + details[f"{prefix}wait_ns"] = getattr(timing_info_obj, "job_wait_time_ns", 0) + details[f"{prefix}aggregate_ns"] = getattr(timing_info_obj, "result_aggregate_time_ns", 0) + details[f"{prefix}total_time_ns"] = getattr(timing_info_obj, "total_time_ns", 0) + return details + + +def benchmark_single_query_detailed( + idx, + queries_tensor: torch.Tensor, + gt_tensor: torch.Tensor | None, + k_val: int, + search_params_dict: dict, + index_type: str, + num_trials: int, + warmup_runs: int +) -> dict: + + search_kwargs_final = {} + if index_type == "Quake": + # For Quake, convert dict to SearchParams object + # And pop non-SearchParams like 'nprobe' if they are handled differently + # For single query, SearchParams is directly constructed. + pass # SearchParams will be constructed per query for Quake + elif index_type == "SCANN": + search_kwargs_final = {"leaves_to_search": search_params_dict.get("leaves_to_search", 100)} + elif index_type == "HNSW": + search_kwargs_final = {"ef_search": search_params_dict.get("ef_search", 32)} + elif index_type == "DiskANN": + search_kwargs_final = {"complexity": search_params_dict.get("complexity", 32)} + elif index_type == "SVS": + search_kwargs_final = {"search_window_size": search_params_dict.get("search_window_size", 32)} + else: # IVF or other Faiss based that might use nprobe directly + search_kwargs_final = {"nprobe": search_params_dict.get("nprobe", 100)} + + + logger.info(f"Warmup ({warmup_runs} queries)...") + for i in range(min(warmup_runs, len(queries_tensor))): + q_vec = queries_tensor[i].unsqueeze(0).float() + if index_type == "Quake": + idx.search(q_vec, k_val, **search_params_dict) + else: + idx.search(q_vec, k_val, **search_kwargs_final) + + trial_aggregated_metrics = [] + + for t_idx in range(num_trials): + per_query_timings_in_trial = [] + recalls_in_trial = [] + + for q_idx, q_vec_single in enumerate(queries_tensor): + q_vec_single = q_vec_single.unsqueeze(0).float() + + search_result = None + if index_type == "Quake": + search_result = idx.search(q_vec_single, k_val, **search_params_dict) + else: + search_result = idx.search(q_vec_single, k_val, **search_kwargs_final) + + timing_info_child = getattr(search_result, "timing_info", None) + query_metrics = get_timing_details(timing_info_child, is_parent=False) + + if index_type == "Quake" and timing_info_child: # Quake specific parent info + timing_info_parent = getattr(timing_info_child, "parent_info", None) + query_metrics.update(get_timing_details(timing_info_parent, is_parent=True)) + else: # For non-Quake or if no parent_info + query_metrics.update(get_timing_details(None, is_parent=True)) + + + if gt_tensor is not None: + ids_this_query = search_result.ids + if ids_this_query.ndim == 2 and ids_this_query.shape[0] == 1: # Ensure it's [1, k_found] + recall_val = common_utils.quake_compute_recall( + ids_this_query, + gt_tensor[q_idx].unsqueeze(0), + k_val + ).item() + recalls_in_trial.append(recall_val) + + per_query_timings_in_trial.append(query_metrics) + + # Aggregate for this trial + trial_df = pd.DataFrame(per_query_timings_in_trial) + mean_metrics_for_trial = trial_df.mean().to_dict() + if recalls_in_trial: + mean_metrics_for_trial["recall"] = np.mean(recalls_in_trial) + + trial_aggregated_metrics.append(mean_metrics_for_trial) + logger.info(f" Trial {t_idx+1}/{num_trials}: " + f"child_total_ms={mean_metrics_for_trial.get('child_total_time_ns', np.nan)/1e6:.2f}, " + f"parent_total_ms={mean_metrics_for_trial.get('parent_total_time_ns', np.nan)/1e6:.2f}" + f"recall@{k_val}={mean_metrics_for_trial.get('recall', np.nan):.4f}") + + + # Aggregate across trials + final_metrics_df = pd.DataFrame(trial_aggregated_metrics) + results_summary = {} + for col in final_metrics_df.columns: + # Convert times from ns to ms for mean, keep std in ns then convert if needed, or just report mean in ms. + if col.endswith("_ns"): + results_summary[f"{col.replace('_ns', '_ms')}_mean"] = final_metrics_df[col].mean() / 1e6 + results_summary[f"{col.replace('_ns', '_ms')}_std"] = final_metrics_df[col].std() / 1e6 + elif col == "recall": + results_summary[f"recall_at_{k_val}_mean"] = final_metrics_df[col].mean() + results_summary[f"recall_at_{k_val}_std"] = final_metrics_df[col].std() + else: # other metrics if any + results_summary[f"{col}_mean"] = final_metrics_df[col].mean() + results_summary[f"{col}_std"] = final_metrics_df[col].std() + + return results_summary + + +def run_experiment(cfg_path_str: str, output_dir_str: str) -> None: + config = common_utils.load_config(cfg_path_str) + dataset_cfg = config["dataset"] + k_val = dataset_cfg["k"] + num_queries = dataset_cfg["num_queries"] + + exp_params_cfg = config # Top level items like trials, warmup + num_trials = exp_params_cfg.get("trials", 5) + warmup_runs = exp_params_cfg.get("warmup", 10) + + indexes_config_list = config["indexes"] + output_csv_name = config.get("output", {}).get("results_csv", "numa_single_query_summary.csv") + force_overwrite_results = config.get("overwrite", False) + # force_rebuild_indexes = config.get("force_rebuild", False) # If we add a global force_rebuild + + main_output_dir = Path(output_dir_str).expanduser().absolute() + main_output_dir.mkdir(parents=True, exist_ok=True) + + logger.info(f"Loading dataset '{dataset_cfg['name']}' for {num_queries} queries.") + base_vectors, query_vectors, gt_vectors = common_utils.load_data( + dataset_cfg["name"], + nq_override=num_queries + ) + + # Shared Quake base index build logic (specific to this experiment script) + quake_base_build_params_shared = None + quake_base_index_file = main_output_dir / "Quake_base_index_single_query.bin" # Distinct name + + for idx_c in indexes_config_list: + if idx_c["index"] == "Quake": + temp_params = dict(idx_c.get("build_params", {})) + # These are stripped for the *base shared* build + temp_params.pop("num_workers", None) + temp_params.pop("use_numa", None) + temp_params.pop("parent_num_workers", None) # Not relevant for single level base + quake_base_build_params_shared = temp_params + break # Assume all Quake configs share the same base parameters for nc, metric etc. + + if quake_base_build_params_shared: + # Check if force_rebuild_indexes is globally set, or if file doesn't exist + # For now, let's assume force_rebuild_indexes comes from a potential global config key + # if 'force_rebuild_indexes' not in config: config['force_rebuild_indexes'] = False # Default + needs_shared_build = not quake_base_index_file.exists() # or config['force_rebuild_indexes'] + if needs_shared_build: + logger.info(f"Building shared Quake base index at {quake_base_index_file}...") + # For Quake, build_params_dict for prepare_quake_index needs 'metric' and 'nlist' (nc) + # Ensure these are present in quake_base_build_params_shared + if 'metric' not in quake_base_build_params_shared: + quake_base_build_params_shared['metric'] = dataset_cfg.get('metric', 'l2') + if 'nc' not in quake_base_build_params_shared and 'nlist' not in quake_base_build_params_shared : # common mistake + logger.error("Missing 'nc' (nlist) in Quake base build_params.") + # Fallback or error, for now assume it is there if this block is reached + else: + if 'nc' in quake_base_build_params_shared and 'nlist' not in quake_base_build_params_shared: + quake_base_build_params_shared['nlist'] = quake_base_build_params_shared.pop('nc') + + common_utils.prepare_quake_index( + vecs=base_vectors, + build_params_dict=quake_base_build_params_shared, + index_file_path=quake_base_index_file, + force_rebuild=needs_shared_build, # force if it doesn't exist + load=False + ) + logger.info(f"Shared Quake base index preparation complete at {quake_base_index_file}") + else: + logger.info(f"Using existing shared Quake base index from {quake_base_index_file}") + + + all_experiment_rows = [] + + for idx_conf in indexes_config_list: + index_name = idx_conf["name"] + index_type_str = idx_conf["index"] + current_build_params = dict(idx_conf.get("build_params", {})) # Copy for modification + current_search_params = dict(idx_conf.get("search_params", {})) + + IndexCls = INDEX_CLASSES.get(index_type_str) + if IndexCls is None or (IndexCls == Scann and Scann is None): + logger.warning(f"Index type '{index_type_str}' for '{index_name}' is unavailable. Skipping.") + continue + + per_index_csv_path = main_output_dir / f"{index_name}_detailed_results.csv" + + if per_index_csv_path.exists() and not force_overwrite_results: + logger.info(f"Results for {index_name} exist at {per_index_csv_path}. Loading cached.") + try: + # Assuming one row per config in these detailed CSVs + cached_df = pd.read_csv(per_index_csv_path) + if not cached_df.empty: + all_experiment_rows.append(cached_df.iloc[0].to_dict()) + except Exception as e: + logger.error(f"Could not load cached results for {index_name}: {e}") + continue + + logger.info(f"--- Processing index: {index_name} (Type: {index_type_str}) ---") + + idx_instance = None + if index_type_str == "Quake": + if not quake_base_index_file.exists(): + logger.error(f"Shared Quake base index {quake_base_index_file} not found for {index_name}. Skipping.") + continue + idx_instance = QuakeWrapper() + # Load-time parameters for this specific Quake configuration + load_time_workers = current_build_params.get("num_workers", 0) + load_time_use_numa = current_build_params.get("use_numa", False) + load_time_parent_workers = current_build_params.get("parent_num_workers", 0) # If applicable + logger.info(f"Loading Quake index '{index_name}' from shared base with: " + f"workers={load_time_workers}, numa={load_time_use_numa}, parent_workers={load_time_parent_workers}") + idx_instance.load(str(quake_base_index_file), + num_workers=load_time_workers, + use_numa=load_time_use_numa, + parent_num_workers=load_time_parent_workers) + else: # For non-Quake indexes + generic_idx_file_path = main_output_dir / f"{index_name}_index.bin" + # Assume force_rebuild_indexes applies here too, or build if not exists + # build_if_needed_flag = not generic_idx_file_path.exists() # or config['force_rebuild_indexes'] + # if build_if_needed_flag: + build_and_save_generic_index(IndexCls, current_build_params.copy(), base_vectors, generic_idx_file_path) + + idx_instance = IndexCls() + load_params_non_quake = current_build_params.copy() + if index_type_str == "DiskANN": load_params_non_quake.pop("metric", None) # DiskANN doesn't take metric at load + idx_instance.load(str(generic_idx_file_path), **load_params_non_quake) + + + if idx_instance is None: + logger.error(f"Failed to load or build index {index_name}. Skipping.") + continue + + benchmark_results = benchmark_single_query_detailed( + idx_instance, query_vectors, gt_vectors, k_val, + current_search_params, index_type_str, num_trials, warmup_runs + ) + + # Combine config with results + final_row_data = {"index_name": index_name, "index_type": index_type_str} + final_row_data.update({f"build_{k}": v for k,v in current_build_params.items()}) + final_row_data.update({f"search_{k}": v for k,v in current_search_params.items()}) + final_row_data.update(benchmark_results) + + all_experiment_rows.append(final_row_data) + common_utils.save_results_csv(pd.DataFrame([final_row_data]), per_index_csv_path) + + if not all_experiment_rows: + logger.warning("No results collected. Skipping final CSV and plots.") + return + + final_summary_df = pd.DataFrame(all_experiment_rows) + unified_csv_path = main_output_dir / output_csv_name + common_utils.save_results_csv(final_summary_df, unified_csv_path) + logger.info(f"Unified results for NUMA single-query written to {unified_csv_path}") + + # Plotting (using the structure from the original script) + plot_suffix = Path(output_csv_name).stem + + # Mean Latency (child_total_time_ms_mean) vs. build_num_workers + if "child_total_time_ms_mean" in final_summary_df.columns and "build_num_workers" in final_summary_df.columns: + plt.figure(figsize=(10, 6)) + for name_plot, group_data in final_summary_df.groupby("index_name"): + sorted_group = group_data.sort_values(by="build_num_workers") + yerr_val = sorted_group["child_total_time_ms_std"] if "child_total_time_ms_std" in sorted_group else None + plt.errorbar( + sorted_group["build_num_workers"], # Treat as categorical for distinct points + sorted_group["child_total_time_ms_mean"], + yerr=yerr_val, + marker="o", capsize=5, label=name_plot, linestyle='-' + ) + # plt.xscale("symlog", base=2) # May not be ideal if num_workers are not powers of 2 or include 0 + plt.xlabel("Build Param: Num Workers (Configuration)") + plt.ylabel("Mean Single-Query Latency (ms) - Child Total") + plt.title(f"NUMA Single-Query: Latency vs. Num Workers ({dataset_cfg['name']})") + plt.legend(title="Index Configuration", bbox_to_anchor=(1.05, 1), loc='upper left') + plt.grid(True, linestyle=':', alpha=0.7) + plt.tight_layout(rect=[0,0,0.8,1]) # Adjust for legend + plt.savefig(main_output_dir / f"{plot_suffix}_latency_vs_workers.png", dpi=150) + logger.info(f"Latency plot saved to {main_output_dir / f'{plot_suffix}_latency_vs_workers.png'}") + plt.close() + + recall_col_name = f"recall_at_{k_val}_mean" + if recall_col_name in final_summary_df.columns and "build_num_workers" in final_summary_df.columns: + plt.figure(figsize=(10, 6)) + for name_plot, group_data in final_summary_df.groupby("index_name"): + sorted_group = group_data.sort_values(by="build_num_workers") + plt.plot( + sorted_group["build_num_workers"], + sorted_group[recall_col_name], + marker="o", label=name_plot, linestyle='-' + ) + plt.xlabel("Build Param: Num Workers (Configuration)") + plt.ylabel(f"Mean Recall@{k_val}") + plt.title(f"NUMA Single-Query: Recall vs. Num Workers ({dataset_cfg['name']})") + plt.legend(title="Index Configuration", bbox_to_anchor=(1.05, 1), loc='upper left') + plt.grid(True, linestyle=':', alpha=0.7) + plt.ylim(0, 1.05) # Typical recall range + plt.tight_layout(rect=[0,0,0.8,1]) + plt.savefig(main_output_dir / f"{plot_suffix}_recall_vs_workers.png", dpi=150) + logger.info(f"Recall plot saved to {main_output_dir / f'{plot_suffix}_recall_vs_workers.png'}") + plt.close() + + logger.info("NUMA Single-Query experiment finished.") \ No newline at end of file diff --git a/test/experiments/osdi2025/open_images_workload/configs/openimages.yaml b/test/experiments/osdi2025/open_images_workload/configs/openimages.yaml new file mode 100644 index 00000000..2be55e61 --- /dev/null +++ b/test/experiments/osdi2025/open_images_workload/configs/openimages.yaml @@ -0,0 +1,186 @@ +workload_dir: data/prepped_openimages/openimages +dataset_dir: data/openimages13m +overwrite: True +batch: False + +indexes: +# - name: Quake_Serial_NoAPS_NoMaintenance +# index: Quake +# build_params: +# nc: 1000 +# metric: ip +# num_workers: 1 +# use_numa: true +# parent: +# num_workers: 1 +# search_params: +# k: 100 +# nprobe: 10 +# initial_search_fraction: .05 +# batched_scan: false +# parent: +# batched_scan: false +# batch_size: 128 +# maintenance_params: +# refinement_radius: 50 +# split_threshold_ns: 500000 +# delete_threshold_ns: 500000 +# enable_split_rejection: true +# enable_delete_rejection: false +# window_size: 1000 + +# - name: Quake_Serial_NoAPS +# index: Quake +# build_params: +# nc: 1000 +# metric: ip +# num_workers: 1 +# use_numa: true +# parent: +# num_workers: 1 +# search_params: +# k: 100 +# nprobe: 30 +# initial_search_fraction: .05 +# batched_scan: false +# parent: +# batched_scan: true +# batch_size: 128 +# maintenance_params: +# refinement_radius: 50 +# split_threshold_ns: 250 +# delete_threshold_ns: 500000 +# enable_split_rejection: true +# enable_delete_rejection: false +# window_size: 10000 + + - name: Quake_Serial + index: Quake + build_params: + nc: 1000 + metric: ip + num_workers: 1 + use_numa: true + parent: + num_workers: 1 + search_params: + k: 100 + recall_target: .9 + initial_search_fraction: .05 + batched_scan: false + parent: + batched_scan: true + batch_size: 128 + maintenance_params: + refinement_radius: 50 + split_threshold_ns: 250 + delete_threshold_ns: 500000 + enable_split_rejection: true + enable_delete_rejection: false + window_size: 10000 + +# - name: QuakeNoMaintenanceNoAPS +# index: Quake +# build_params: +# nc: 1000 +# metric: ip +# num_workers: 8 +# use_numa: true +# parent: +# num_workers: 2 +# search_params: +# k: 100 +# nprobe: 30 +# batched_scan: false +# aps_flush_period_us: 10 +# parent: +# batched_scan: true +# batch_size: 128 +# maintenance_params: +# refinement_radius: 50 +# split_threshold_ns: 500000 +# delete_threshold_ns: 500000 +# enable_split_rejection: true +# enable_delete_rejection: false +# window_size: 10000 +# +# - name: QuakeNoMaintenance +# index: Quake +# build_params: +# nc: 1000 +# metric: ip +# num_workers: 8 +# use_numa: true +# parent: +# num_workers: 2 +# search_params: +# k: 100 +# recall_target: .9 +# batched_scan: false +# recompute_threshold: 0.01 +# use_precomputed: true +# initial_search_fraction: .05 +# aps_flush_period_us: 10 +# parent: +# batched_scan: true +# batch_size: 128 +# maintenance_params: +# refinement_radius: 50 +# split_threshold_ns: 500000 +# delete_threshold_ns: 500000 +# enable_split_rejection: true +# enable_delete_rejection: false +# window_size: 10000 +# +# - name: QuakeNoAPS +# index: Quake +# build_params: +# nc: 1000 +# metric: ip +# num_workers: 8 +# use_numa: true +# parent: +# num_workers: 2 +# search_params: +# k: 100 +# nprobe: 30 +# batched_scan: false +# aps_flush_period_us: 10 +# parent: +# batched_scan: true +# batch_size: 128 +# maintenance_params: +# refinement_radius: 50 +# split_threshold_ns: 250 +# delete_threshold_ns: 500000 +# enable_split_rejection: true +# enable_delete_rejection: false +# window_size: 10000 + + - name: Quake + index: Quake + build_params: + nc: 1000 + metric: ip + num_workers: 8 + use_numa: true + parent: + num_workers: 2 + search_params: + k: 100 + recall_target: .9 + batched_scan: false + recompute_threshold: 0.01 + use_precomputed: true + initial_search_fraction: .05 + aps_flush_period_us: 10 + parent: + batched_scan: true + batch_size: 128 + maintenance_params: + refinement_radius: 50 + split_threshold_ns: 250 + delete_threshold_ns: 500000 + enable_split_rejection: true + enable_delete_rejection: false + window_size: 10000 \ No newline at end of file diff --git a/test/experiments/osdi2025/open_images_workload/run.py b/test/experiments/osdi2025/open_images_workload/run.py new file mode 100644 index 00000000..e353c23e --- /dev/null +++ b/test/experiments/osdi2025/open_images_workload/run.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import json, logging, time, shutil +from pathlib import Path +from typing import Dict, List, Any, Optional, Union + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import torch +import quake +import test.experiments.osdi2025.experiment_utils as common_utils +from quake.utils import to_path, compute_recall +from quake.index_wrappers.quake import QuakeWrapper +from quake.index_wrappers.faiss_ivf import FaissIVF +from quake.index_wrappers.faiss_hnsw import FaissHNSW +try: from quake.index_wrappers.scann import Scann +except ImportError: Scann = None # type: ignore +try: from quake.index_wrappers.diskann import DiskANNDynamic +except ImportError: DiskANNDynamic = None # type: ignore +try: from quake.index_wrappers.vamana import Vamana +except ImportError: Vamana = None # type: ignore + +INDEX_CLASSES: Dict[str, Any] = { + "Quake": QuakeWrapper, + "IVF": FaissIVF, + "HNSW": FaissHNSW, + "SCANN": Scann, + "DiskANN": DiskANNDynamic, + "SVS": Vamana, +} + +log = logging.getLogger("openimages") +logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") + +WINDOW_SIZE = 2_000_000 +K = 100 + +STYLE = { + "query": dict(ls="-", marker="o", ms=4, lw=1.1, mfc="none"), + "insert": dict(ls="-", marker="s", ms=4, lw=1.1, mfc="none"), + "delete": dict(ls="-", marker="^", ms=4, lw=1.1, mfc="none"), +} + +class OpenImagesEvaluator: + def __init__(self, workload_dir: Union[str, Path], out_dir: Union[str, Path], + *, dataset_dir: Union[str, Path]): + self.workload_dir = to_path(workload_dir) + self.runbook_path = self.workload_dir / "runbook.json" + self.ops_dir = self.workload_dir / "operations" + self.out_dir = to_path(out_dir); self.out_dir.mkdir(parents=True, exist_ok=True) + + ds = to_path(dataset_dir) + self._BASE = torch.load(ds / "BASE.pt") + self._IDS = torch.load(ds / "IDS.pt") + self._QUERIES = torch.load(ds / "QUERIES.pt") + self._BASE /= self._BASE.norm(dim=1, keepdim=True) + self._QUERIES /= self._QUERIES.norm(dim=1, keepdim=True) + + self.current_start, self.current_end = 0, WINDOW_SIZE + self.current_ids = self._IDS[:WINDOW_SIZE] + self.current_base = self._BASE[:WINDOW_SIZE] + + max_id = int(self._IDS.max()) + self.inv_map = torch.full((max_id + 1,), -1, dtype=torch.long) + self.inv_map[self.current_ids] = torch.arange(len(self.current_ids)) + + # ------------------------------------------------------------------ helpers + def _init_index(self, name: str, wrapper, build_p: Dict[str, Any], + m_p: Optional[Dict[str, Any]]): + idx_dir = self.workload_dir / "init_indexes" + idx_file = idx_dir / f"{name}.index" + idx_dir.mkdir(parents=True, exist_ok=True) + + if not idx_file.exists(): + wrapper.build(self.current_base, ids=self.current_ids, **build_p) + wrapper.save(idx_file) + else: + wrapper.load(idx_file, + num_workers=build_p.get("num_workers", 0), + use_numa =build_p.get("use_numa", True), + parent =build_p.get("parent")) + + if isinstance(wrapper, QuakeWrapper) and m_p: + mp = quake.MaintenancePolicyParams() + for k, v in m_p.items(): setattr(mp, k, v) + wrapper.index.initialize_maintenance_policy(mp) + return wrapper + + # ------------------------------------------------------------------ core + def evaluate(self, *, name: str, wrapper, build_params: Dict[str, Any], + search_params: Dict[str, Any], do_maintenance: bool, + m_params: Optional[Dict[str, Any]], batch: bool, + max_q: int = 1000) -> None: + + wrapper = self._init_index(name, wrapper, build_params, m_params) + runbook = json.load(open(self.runbook_path)) + totals = dict(query=0.0, insert=0.0, delete=0.0, maintenance=0.0) + rows: List[Dict[str, Any]] = [] + + recall = None + for key, op in runbook["operations"].items(): + op_id = int(key); typ = op["type"]; t0 = time.perf_counter() + + if typ == "delete": + n = op["size"]; ids_del = self.current_ids[:n] + wrapper.remove(ids_del) + self.current_start += n + self.current_ids = self._IDS[self.current_start:self.current_end] + self.current_base = self._BASE[self.current_start:self.current_end] + self.inv_map[ids_del] = -1 + + elif typ == "insert": + n = op["size"]; old_end = self.current_end; self.current_end += n + vecs = self._BASE[old_end:self.current_end]; ids = self._IDS[old_end:self.current_end] + wrapper.add(vecs, ids=ids) + self.current_ids = self._IDS[self.current_start:self.current_end] + self.current_base = self._BASE[self.current_start:self.current_end] + if int(ids.max()) >= self.inv_map.size(0): + bigger = torch.full((int(ids.max()) + 1,), -1, dtype=torch.long) + bigger[:len(self.inv_map)] = self.inv_map; self.inv_map = bigger + self.inv_map[ids] = torch.arange(len(self.current_ids)-n, len(self.current_ids)) + + elif typ == "query": + q_idx = torch.load(self.ops_dir / f"{op_id}.pt") + queries = self._QUERIES[q_idx] + if len(queries) > max_q: + perm = torch.randperm(len(queries))[:max_q] + queries, q_idx = queries[perm], q_idx[perm] + if batch: + pred_ids = wrapper.search(queries, **search_params).ids + else: + parts = [wrapper.search(q.unsqueeze(0), **search_params).ids for q in queries] + pred_ids = torch.cat(parts) + gt_ids = torch.load(self.ops_dir / f"{op_id}_gt_ids.pt")[:len(pred_ids)] + recall = compute_recall(pred_ids, gt_ids, search_params.get("k", K)).mean().item() + op["recall"] = recall + else: + raise ValueError(typ) + + lat_ms = 1e3 * (time.perf_counter() - t0) + maint_ms = spl = dele = spl_ms = del_ms = ref_ms = 0.0 + if do_maintenance: + t1 = time.perf_counter(); info = wrapper.maintenance() + maint_ms = 1e3 * (time.perf_counter() - t1) + spl, dele = info.n_splits, info.n_deletes + spl_ms, del_ms = info.split_time_us/1000.0, info.delete_time_us/1000.0 + ref_ms = info.refinement_time_us/1000.0 + + totals[typ] += lat_ms; totals["maintenance"] += maint_ms + state = wrapper.index_state() + + row = dict( + operation_number = op_id, + operation_type = typ, + latency_ms = lat_ms, + maintenance_latency_ms= maint_ms, + n_resident = state.get("n_total", len(self.current_ids)), + nlist = state.get("n_list", np.nan), + n_splits = spl, + n_deletes = dele, + split_time_ms = spl_ms, + delete_time_ms = del_ms, + refinement_time_ms = ref_ms, + recall = recall, + ) + print(row) + rows.append(row) + df = pd.DataFrame(rows) + df.to_csv(self.out_dir / "results.csv", index=False) + self._plots(df, totals, name) + + # ------------------------------------------------------------------ plots + def _plots(self, df: pd.DataFrame, tot: Dict[str,float], title: str): + fig, axs = plt.subplots(2,2, figsize=(12,9)) + for t in ("query","insert","delete"): + sub = df[df.operation_type==t] + if not sub.empty: + axs[0,0].plot(sub.operation_number, sub.latency_ms, label=t, **STYLE[t]) + axs[0,0].set_title("Latency"); axs[0,0].legend(); axs[0,0].grid(ls=":") + if "nlist" in df: + axs[0,1].plot(df.operation_number, df.nlist, marker="o", lw=1.1) + axs[0,1].set_title("#Partitions"); axs[0,1].grid(ls=":") + axs[1,0].plot(df.operation_number, df.n_resident, marker="o", lw=1.1) + axs[1,0].set_title("Resident"); axs[1,0].grid(ls=":") + rec = df[(df.operation_type=="query") & df.recall.notna()] + if not rec.empty: + axs[1,1].plot(rec.operation_number, rec.recall, marker="o", lw=1.1) + axs[1,1].set_title("Recall"); axs[1,1].set_ylim(0,1); axs[1,1].grid(ls=":") + fig.suptitle(title); fig.tight_layout() + fig.savefig(self.out_dir / "four_panel.png", dpi=150); plt.close(fig) + + lbl = ["Query","Insert","Delete","Maintenance","Total"] + val = [tot.get("query",0), tot.get("insert",0), + tot.get("delete",0), tot.get("maintenance",0)] + val.append(sum(val)) + plt.bar(lbl, val); plt.title(f"Time breakdown – {title}") + plt.ylabel("ms"); plt.tight_layout() + plt.savefig(self.out_dir / "time_breakdown.png", dpi=150); plt.close() + +# ────────────────────────────────────────────────────────────────────────── +def run_experiment(cfg_path_str: Dict[str, Any], output_root: Union[str, Path]) -> None: + cfg = common_utils.load_config(cfg_path_str) + wdir = Path(cfg["workload_dir"]).expanduser() + dset = Path(cfg["dataset_dir"]).expanduser() + out = Path(output_root).expanduser() + batch = cfg.get("batch", False) + out.mkdir(parents=True, exist_ok=True) + + for idx in cfg["indexes"]: + name = idx["name"]; wrapper_key = idx["index"] + out_dir = out / name + + if out_dir.exists() and not cfg.get("overwrite", False): + log.info("Skip %s – results exist (overwrite=False)", name); continue + if out_dir.exists() and cfg.get("overwrite", False): + shutil.rmtree(out_dir) + + cls = INDEX_CLASSES.get(wrapper_key) + if cls is None: + log.warning("Unknown wrapper %s – skip %s", wrapper_key, name); continue + + ev = OpenImagesEvaluator(wdir, out_dir, dataset_dir=dset) + ev.evaluate( + name = name, + wrapper= cls(), + build_params = idx.get("build_params", {}), + search_params= idx.get("search_params", {}), + do_maintenance = idx.get("maintenance_params") is not None, + m_params = idx.get("maintenance_params"), + batch = batch, + ) diff --git a/test/experiments/osdi2025/paper/Quake-Revision.pdf b/test/experiments/osdi2025/paper/Quake-Revision.pdf new file mode 100644 index 00000000..2adb6883 Binary files /dev/null and b/test/experiments/osdi2025/paper/Quake-Revision.pdf differ diff --git a/test/experiments/osdi2025/paper/Quake.pdf b/test/experiments/osdi2025/paper/Quake.pdf new file mode 100644 index 00000000..de92f349 Binary files /dev/null and b/test/experiments/osdi2025/paper/Quake.pdf differ diff --git a/test/experiments/osdi2025/read_only_workload/configs/msturing10m.yaml b/test/experiments/osdi2025/read_only_workload/configs/msturing10m.yaml new file mode 100644 index 00000000..df50e43a --- /dev/null +++ b/test/experiments/osdi2025/read_only_workload/configs/msturing10m.yaml @@ -0,0 +1,55 @@ +# configs/experiment.yaml +mode: run # options: build, run, plot +dataset: + name: msturing10m + metric: l2 + path: data +workload_generator: + recall_target: 0.9 + insert_ratio: 0.0 + delete_ratio: 0.0 + query_ratio: 1.0 + number_of_operations: 100 + initial_size: 10000000 + cluster_size: 1000 # N/A for read-only workload + update_batch_size: 10000 # N/A for read-only workload + query_batch_size: 1000 + cluster_sample_distribution: uniform # N/A for read-only workload + query_cluster_sample_distribution: uniform + seed: 9299 + +indexes: + - name: FaissIVF + index: IVF + build_params: + nc: 4000 + metric: l2 + search_params: + k: 100 + nprobe: 250 + + - name: Quake + index: Quake + build_params: + nc: 4000 + metric: ip + num_workers: + num_merge_workers: 1 + use_numa: true + parent: + num_workers: 1 + search_params: + k: 100 + recall_target: .9 + initial_search_fraction: .05 + batched_scan: false + parent: + batched_scan: false + batch_size: 128 + maintenance_params: + refinement_radius: 50 + split_threshold_ns: 250 + delete_threshold_ns: 5000000 + enable_split_rejection: true + enable_delete_rejection: false + window_size: 1000 diff --git a/test/experiments/osdi2025/read_only_workload/run.py b/test/experiments/osdi2025/read_only_workload/run.py new file mode 100644 index 00000000..560a8887 --- /dev/null +++ b/test/experiments/osdi2025/read_only_workload/run.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +""" +Unified experiment runner + +* Reads a single YAML config (`configs/experiment.yaml`). +* Modes + - **build** –‑ only (re‑)generate the workload. + - **run** –‑ generate workload (if needed), evaluate every index that + does not already have a CSV of prior results (unless + `overwrite.results: true`), then emit the unified plot. + - **plot** –‑ skip evaluation; just regenerate the unified plot. +* A results CSV (`//results.csv`) is written after every + evaluation; its presence is what lets us skip reruns. +* Plot styling (colour / marker / linestyle) is entirely configurable from + the YAML under `plot.styles`. +""" + +import logging, shutil, time, yaml +from pathlib import Path +from typing import Dict, Any, List, Optional + +import matplotlib.pyplot as plt +from matplotlib.lines import Line2D +import numpy as np +import pandas as pd +import torch + +# ── quake imports ────────────────────────────────────────────────────────── +from quake.datasets.ann_datasets import load_dataset +from quake.index_wrappers.quake import QuakeWrapper +from quake.index_wrappers.faiss_ivf import FaissIVF +from quake.utils import compute_recall +from quake.workload_generator import DynamicWorkloadGenerator, WorkloadEvaluator + +logging.basicConfig(level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s") +log = logging.getLogger("runner") + +INDEX_CLASSES = { + "Quake": QuakeWrapper, + "IVF": FaissIVF, +} + +LAT_STYLE = {"insert": "--", "delete": "-.", "query": "-"} + + +# ────────────────────────────────────────────────────────────────────────── +def unified_plot(cfg: Dict[str, Any], out_dir: Path) -> None: + styles = cfg.get("plot", {}).get("styles", {}) + fig, axs = plt.subplots(2, 2, figsize=(12, 10)) + ax_lat, ax_part = axs[0] + ax_res, ax_rec = axs[1] + + legend_idx: List[Line2D] = [] + + for idx in cfg["indexes"]: + name = idx["name"] + csv = out_dir / name / "results.csv" + if not csv.exists(): + log.warning("Skipping plot for %s (no CSV).", name) + continue + df = pd.read_csv(csv) + styl = styles.get(name, {}) + col, mark = styl.get("color"), styl.get("marker") + + # latency + for op in ("insert", "delete", "query"): + sub = df[df.operation_type == op] + if not sub.empty: + ax_lat.plot(sub.operation_number, sub.latency_ms, + color=col, linestyle=LAT_STYLE[op]) + # partitions + if "n_list" in df.columns: + ax_part.plot(df.operation_number, df.n_list, color=col, marker=mark) + # resident + if "n_resident" in df.columns: + ax_res.plot(df.operation_number, df.n_resident, color=col, marker=mark) + # recall + q = df[(df.operation_type == "query") & df.recall.notna()] + if not q.empty: + ax_rec.plot(q.operation_number, q.recall, color=col, marker=mark) + + legend_idx.append(Line2D([0], [0], color=col, label=name)) + + ax_lat.set_yscale("log"); ax_lat.set_title("Latency"); ax_lat.set_xlabel("Op #") + ax_part.set_title("#Partitions"); ax_part.set_xlabel("Op #") + ax_res.set_title("Resident"); ax_res.set_xlabel("Op #") + ax_rec.set_title("Recall"); ax_rec.set_xlabel("Op #"); ax_rec.set_ylim(0, 1) + + style_handles = [Line2D([0], [0], + color="k", ls=LAT_STYLE[o], label=o.title()) + for o in ("insert", "delete", "query")] + ax_lat.legend(handles=style_handles, title="Op type", fontsize=8) + fig.legend(handles=legend_idx, loc="upper center", ncol=max(1, len(legend_idx)), + title="Index", bbox_to_anchor=(0.5, 1.02)) + fig.tight_layout() + path = out_dir / "unified_plot.png" + fig.savefig(path, bbox_inches="tight"); plt.close(fig) + print(f"Unified plot → {path}") + + +# ────────────────────────────────────────────────────────────────────────── +def _should(overwrite: bool, path: Path) -> bool: + return overwrite or not path.exists() + + +# ────────────────────────────────────────────────────────────────────────── +def _evaluate_index(cfg: Dict[str, Any], out_root: Path, batch: bool, + overwrite_results: bool) -> None: + workload_dir = out_root # workload lives in the output root + for idx in cfg["indexes"]: + name = idx["name"] + res_dir = out_root / name + csv = res_dir / "results.csv" + if not _should(overwrite_results, csv): + print(f"{name}: cached results -> skip evaluation"); continue + + cls_key = idx["index"] + cls = INDEX_CLASSES.get(cls_key) + if cls is None: + log.error("Unknown index wrapper %s – skip %s", cls_key, name); continue + res_dir.mkdir(parents=True, exist_ok=True) + + evaluator = WorkloadEvaluator(workload_dir=workload_dir, output_dir=res_dir) + evaluator.evaluate_workload( + name = name, + index = cls(), + build_params = idx.get("build_params", {}), + search_params = idx.get("search_params", {}), + do_maintenance= idx.get("maintenance_params") is not None, + m_params = idx.get("maintenance_params"), + batch = batch, + ) + + +# ────────────────────────────────────────────────────────────────────────── +def _build_workload(cfg: Dict[str, Any], out_root: Path, + overwrite_workload: bool) -> None: + + print("overwrite_workload:", overwrite_workload, "out_root:", out_root) + + if not _should(overwrite_workload, out_root / "runbook.json"): + print("Workload exists – skip generation"); return + + ds_cfg = cfg["dataset"] + vecs, qvecs, _ = load_dataset(ds_cfg["name"], ds_cfg.get("path", "")) + if ds_cfg["metric"] == "l2": # ensure normalisation for L2 if desired + pass + wg_cfg = cfg["workload_generator"] + gen = DynamicWorkloadGenerator( + workload_dir=out_root, + base_vectors=vecs, + metric=ds_cfg["metric"], + insert_ratio=wg_cfg["insert_ratio"], + delete_ratio=wg_cfg["delete_ratio"], + query_ratio =wg_cfg["query_ratio"], + number_of_operations=wg_cfg["number_of_operations"], + initial_size=wg_cfg["initial_size"], + cluster_size=wg_cfg["cluster_size"], + update_batch_size=wg_cfg["update_batch_size"], + query_batch_size =wg_cfg["query_batch_size"], + cluster_sample_distribution=wg_cfg["cluster_sample_distribution"], + queries=qvecs, + query_cluster_sample_distribution=wg_cfg["query_cluster_sample_distribution"], + seed=wg_cfg["seed"], + ) + print("Generating workload …"); gen.generate_workload() + + +# ────────────────────────────────────────────────────────────────────────── +def run_experiment(cfg_src: str | Dict[str, Any], output_dir: str | Path) -> None: + cfg = yaml.safe_load(Path(cfg_src).read_text()) if isinstance(cfg_src, (str, Path)) else cfg_src + out_root = Path(output_dir).expanduser() + out_root.mkdir(parents=True, exist_ok=True) + + mode = cfg.get("mode", "run") + overwrite_results = cfg.get("overwrite_results", False) + overwrite_workload = cfg.get("overwrite_workload", False) + batch = cfg.get("batch", False) + + if mode in {"build", "run"}: + _build_workload(cfg, out_root, overwrite_workload) + if mode == "run": + _evaluate_index(cfg, out_root, batch, overwrite_results) + if mode in {"run", "plot"}: + unified_plot(cfg, out_root) \ No newline at end of file diff --git a/test/experiments/osdi2025/vary_batch_size/configs/wiki_single_thread.yaml b/test/experiments/osdi2025/vary_batch_size/configs/wiki_single_thread.yaml new file mode 100644 index 00000000..95d9f255 --- /dev/null +++ b/test/experiments/osdi2025/vary_batch_size/configs/wiki_single_thread.yaml @@ -0,0 +1,119 @@ +dataset: + name: wiki13m + # Instead of a single num_queries, we sweep over these batch sizes: +# query_batch_sizes: [1, 8, 16, 32, 64, 128, 256, 512] + num_queries: 8192 + query_batch_sizes: [1, 4, 16, 64, 256, 1024, 4096, 8192] + k: 100 + +trials: 3 +warmup: 1 +overwrite: true +force_rebuild: false + +indexes: + - name: Quake_Unbatched + index: Quake + index_file: /opt/quake/data/vary_batch_sizes/indices/Quake_3500.bin + build_params: + nc: 3500 + metric: ip + num_workers: 16 + use_numa: true + omp_num_threads_build: 8 + parent: + num_workers: 4 + search_params: + nprobe: 4 + batch_size: 128 + parent: + batched_scan: true + batch_size: 128 + batched_scan: false + omp_num_threads_search: 1 + + - name: Quake_Batched + index: Quake + index_file: /opt/quake/data/vary_batch_sizes/indices/Quake_3500.bin + build_params: + nc: 3500 + metric: ip + num_workers: 16 + omp_num_threads_build: 8 + use_numa: true + parent: + num_workers: 4 + search_params: + nprobe: 4 + batch_size: 128 + parent: + batched_scan: true + batch_size: 128 + batched_scan: true + omp_num_threads_search: 1 + + - name: FaissIVF + index: IVF + index_file: /opt/quake/data/vary_batch_sizes/indices/FaissIVF.bin + build_params: + nc: 3500 + metric: ip + omp_num_threads_build: 1 + search_params: + nprobe: 4 + omp_num_threads_search: 16 + + - name: FaissHNSW + index: HNSW + index_file: /opt/quake/data/vary_batch_sizes/indices/FaissHNSW.bin + build_params: + m: 64 + ef_construction: 128 + metric: ip + omp_num_threads_build: 1 + search_params: + ef_search: 16 + omp_num_threads_search: 16 + + - name: SCANN + index: SCANN + index_file: /opt/quake/data/vary_batch_sizes/indices/SCANN.bin + build_params: + num_leaves: 3500 + metric: dot_product + omp_num_threads_build: 16 + search_params: + leaves_to_search: 4 + omp_num_threads_search: 1 + n_threads: 16 + + - name: DiskANN + index: DiskANN + index_file: /opt/quake/data/vary_batch_sizes/indices/DiskANN.bin + build_params: + complexity: 128 + graph_degree: 64 + metric: mips + num_threads: 16 + omp_num_threads_build: 1 + search_params: + complexity: 128 + num_threads: 16 + omp_num_threads_search: 1 + + - name: SVS + index: SVS + index_file: /opt/quake/data/vary_batch_sizes/indices/SVS.bin + build_params: + graph_max_degree: 64 + alpha: 1.1 + window_size: 128 + metric: ip + num_threads: 16 + omp_num_threads_build: 1 + search_params: + search_window_size: 24 + omp_num_threads_search: 16 + +output: + results_csv: numa_query_batch_results.csv \ No newline at end of file diff --git a/test/experiments/osdi2025/vary_batch_size/run.py b/test/experiments/osdi2025/vary_batch_size/run.py new file mode 100644 index 00000000..ff32951e --- /dev/null +++ b/test/experiments/osdi2025/vary_batch_size/run.py @@ -0,0 +1,360 @@ +import logging +from pathlib import Path +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import os +import faiss +import torch + +import test.experiments.osdi2025.experiment_utils as common_utils +from quake.utils import compute_recall +from quake.index_wrappers.faiss_ivf import FaissIVF +from quake.index_wrappers.faiss_hnsw import FaissHNSW +from quake.index_wrappers.quake import QuakeWrapper # type: ignore + +# optional wrappers +try: from quake.index_wrappers.scann import Scann +except ImportError: Scann = None +try: from quake.index_wrappers.diskann import DiskANNDynamic +except ImportError: DiskANNDynamic = None +try: from quake.index_wrappers.vamana import Vamana +except ImportError: Vamana = None + +INDEX_CLASSES = { + "Quake": QuakeWrapper, + "IVF": FaissIVF, + "HNSW": FaissHNSW, + "SCANN": Scann, + "DiskANN": DiskANNDynamic, + "SVS": Vamana, +} + +logger = logging.getLogger("numa_query_batch_orchestrator") + +def _set_faiss_threads(val: str, tag: str): + t = int(val) + faiss.omp_set_num_threads(t) + print(f"[{tag}] faiss.omp_set_num_threads({t})") + +def task_build_index(idx_cfg, ds_cfg, run_params, idx_file: Path): + name = idx_cfg["name"] + print(f"[{name} BUILD] PID={os.getpid()} -> {idx_file}") + bp = dict(idx_cfg.get("build_params", {})) + omp_build = str(bp.pop("omp_num_threads_build", "1")) + _set_faiss_threads(omp_build, f"{name} BUILD") + + if idx_file.exists() and not run_params["force_rebuild_indices"]: + return {"status": "skipped", "index_file_path": str(idx_file)} + + base_vecs, _, _ = common_utils.load_data(ds_cfg["name"], nq_override=1) + Cls = INDEX_CLASSES[idx_cfg["index"]] + idx = common_utils.prepare_wrapper_index( + Cls, idx_file, base_vecs, bp, run_params["force_rebuild_indices"] + ) + del base_vecs + if idx is None: + return {"error": "build failed"} + return {"status": "success", "index_file_path": str(idx_file)} + +def task_search_index(idx_cfg, ds_cfg, run_params, idx_file: Path, batch_size: int): + name = idx_cfg["name"] + itype = idx_cfg["index"] + print(f"[{name} SEARCH bs={batch_size}] PID={os.getpid()} Loading index") + bp = dict(idx_cfg.get("build_params", {})) + sp = dict(idx_cfg.get("search_params", {})) + omp_search = str(sp.pop("omp_num_threads_search", "1")) + _set_faiss_threads(omp_search, f"{name} SEARCH") + + # load all nq queries & gt + _, all_qvecs, all_gt = common_utils.load_data( + ds_cfg["name"], nq_override=ds_cfg["num_queries"] + ) + nq = all_qvecs.shape[0] + + # randomize queries for fair comparison + + Cls = INDEX_CLASSES[itype] + inst = Cls() + load_kwargs = { + k: bp[k] + for k in ("num_workers", "use_numa", "num_merge_workers", "parent") + if k in bp + } + + if itype == "DiskANN": + # DiskANN requires special handling for loading, all build params are passed as kwargs + load_kwargs = bp.copy() + bp.pop("omp_num_threads_build", None) # remove omp threads from load params + + + inst.load(str(idx_file), **load_kwargs) + + # warmup on one chunk using random vectors + warmup = torch.randn(batch_size, all_qvecs.shape[1]) + for _ in range(run_params["num_warmup"]): + _ = inst.search(warmup, run_params["k_val"], **sp) + + trial_latencies = [] + trial_recalls = [] + + for _ in range(run_params["num_trials"]): + total_ns = 0 + ids_list = [] + rand_perm = torch.randperm(nq) + all_gt = all_gt[rand_perm] + all_qvecs = all_qvecs[rand_perm] + for i in range(0, nq, batch_size): + chunk = all_qvecs[i : min(i + batch_size, nq)] + res = inst.search(chunk, run_params["k_val"], **sp) + ti = getattr(res, "timing_info", None) + ns = getattr(ti, "total_time_ns", None) or getattr(ti, "child_total_time_ns", 0) + total_ns += ns + ids_list.append(res.ids) + + total_ms = total_ns / 1e6 + trial_latencies.append(total_ms) + + all_ids = np.vstack(ids_list) + rec = float(compute_recall(all_ids, all_gt, run_params["k_val"]).mean()) + trial_recalls.append(rec) + + return { + "mean_total_latency_ms": float(np.mean(trial_latencies)), + "std_total_latency_ms": float(np.std(trial_latencies)), + "mean_recall": float(np.mean(trial_recalls)), + "std_recall": float(np.std(trial_recalls)), + "nq": nq, + "query_batch_size": batch_size, + } + + +def bench_scann(cfg_path: str, out_dir: Path, force_rebuild: bool, force_overwrite: bool): + """ + Build and benchmark SCANN across all batch sizes defined in the config. SCANNs serialization is broken when run in a subprocess, so it is handled separately. + """ + if Scann is None: + raise ImportError("SCANN wrapper not installed; please install quake.index_wrappers.scann") + + # load config and dataset settings + cfg = common_utils.load_config(cfg_path) + ds_cfg = cfg["dataset"] + batch_sizes = ds_cfg["query_batch_sizes"] + k_val = ds_cfg["k"] + num_trials = cfg.get("trials", 3) + num_warmup = cfg.get("warmup", 1) + + # prepare paths + out_dir = Path(out_dir) + results_csv = out_dir / cfg["output"]["results_csv"] + results_csv.parent.mkdir(parents=True, exist_ok=True) + + # find SCANN entry in indexes + scann_cfg = next((ic for ic in cfg["indexes"] if ic["index"] == "SCANN"), None) + if scann_cfg is None: + return # no SCANN to run + + idx_name = scann_cfg["name"] + # check existing results + if results_csv.exists() and not force_overwrite: + df_existing = pd.read_csv(results_csv) + if idx_name in df_existing["index"].unique() and not force_rebuild: + logger.info(f"[SCANN] results already exist and no force overwrite → skipping") + return + + # BUILD SCANN index + logger.info(f"[{idx_name} BUILD] building SCANN index") + base_vecs, _, _ = common_utils.load_data(ds_cfg["name"], nq_override=1) + scann_idx = Scann() + scann_idx.build(base_vecs, **scann_cfg.get("build_params", {})) + idx_file = Path(scann_cfg.get("index_file", out_dir / "indices" / f"{idx_name}.bin")) + idx_file.parent.mkdir(parents=True, exist_ok=True) + del base_vecs + + # LOAD queries & GT once + _, all_qvecs, all_gt = common_utils.load_data( + ds_cfg["name"], nq_override=ds_cfg["num_queries"] + ) + nq = all_qvecs.shape[0] + + # BENCHMARK SCANN over batch sizes + records = [] + for bs in batch_sizes: + logger.info(f"[{idx_name} SEARCH] batch_size={bs}") + + # warmup runs + warmup_q = all_qvecs[:bs] + for _ in range(num_warmup): + scann_idx.search(warmup_q, k_val, **scann_cfg.get("search_params", {})) + + latencies = [] + recalls = [] + for _ in range(num_trials): + total_ns = 0 + ids_batches = [] + for i in range(0, nq, bs): + chunk = all_qvecs[i : min(i + bs, nq)] + res = scann_idx.search(chunk, k_val, **scann_cfg.get("search_params", {})) + ti = getattr(res, "timing_info", None) + ns = getattr(ti, "total_time_ns", None) or getattr(ti, "child_total_time_ns", 0) + total_ns += ns + ids_batches.append(res.ids) + + latencies.append(total_ns / 1e6) + all_ids = np.vstack(ids_batches) + recalls.append(float(compute_recall(all_ids, all_gt, k_val).mean())) + + records.append({ + "index": idx_name, + "query_batch_size": bs, + "nq": nq, + "mean_total_latency_ms": float(np.mean(latencies)), + "std_total_latency_ms": float(np.std(latencies)), + "mean_recall": float(np.mean(recalls)), + "std_recall": float(np.std(recalls)), + "QPS": float(nq * 1000.0 / np.mean(latencies)), + }) + + # save (or overwrite) SCANN results + df = pd.DataFrame(records) + + # append to existing results if available + if results_csv.exists() and not force_overwrite: + existing_df = pd.read_csv(results_csv) + df = pd.concat([existing_df, df], ignore_index=True) + + df.to_csv(results_csv, index=False) + logger.info(f"[{idx_name}] SCANN benchmark results saved to {results_csv}") + + +def run_experiment(cfg_path, out_dir): + """ + Main entrypoint: first runs SCANN via bench_scann, then + proceeds with build/search for all other indexes. + """ + logging.basicConfig(level=logging.INFO) + cfg = common_utils.load_config(cfg_path) + out = Path(out_dir); out.mkdir(parents=True, exist_ok=True) + + # run SCANN special path + bench_scann( + cfg_path=cfg_path, + out_dir=out, + force_rebuild=cfg.get("force_rebuild", False), + force_overwrite=cfg.get("overwrite", False) + ) + + ds_cfg = cfg["dataset"] + batch_sizes = ds_cfg["query_batch_sizes"] + run_params = { + "num_trials": cfg.get("trials", 3), + "num_warmup": cfg.get("warmup", 1), + "k_val": ds_cfg["k"], + "force_rebuild_indices": cfg.get("force_rebuild", False), + "force_overwrite": cfg.get("overwrite", False), + "enable_glances": cfg.get("enable_glances_monitoring", False), + } + + # prepare directories + process_logs_dir = out / "process_logs"; process_logs_dir.mkdir(exist_ok=True) + index_store_dir = out / "indices"; index_store_dir.mkdir(exist_ok=True) + + # load existing results + results_csv = out / cfg["output"]["results_csv"] + if results_csv.exists() and not run_params["force_overwrite"]: + existing_df = pd.read_csv(results_csv) + done_pairs = set(zip(existing_df["index"], existing_df["query_batch_size"])) + else: + existing_df = None + done_pairs = set() + + all_rows = [] + + # === BUILD OTHER INDEXES === + for idx_cfg in cfg["indexes"]: + if idx_cfg["index"] == "SCANN": + continue # already handled + name = idx_cfg["name"] + idx_path = Path(idx_cfg.get("index_file", index_store_dir/f"{name}.bin")) + idx_path.parent.mkdir(parents=True, exist_ok=True) + + build_log = process_logs_dir / f"{name}_build.log" + res = common_utils.run_operation_in_process( + task_build_index, + (idx_cfg, ds_cfg, run_params, idx_path), + env_vars=idx_cfg.get("build_env_vars", {}), + log_file_path=str(build_log), + process_name=f"BuildTask_{name}", + timeout_seconds=idx_cfg.get("process_timeout", cfg.get("default_process_timeout", 7200)), + enable_glances=run_params["enable_glances"] + ) + data = res.get("data", {}) + if res.get("status")!="success" or data.get("error"): + logger.error(f"[BUILD FAILED] {name}: {data.get('error','unknown')}") + else: + logger.info(f"[BUILD OK] {name} → {data['index_file_path']}") + + # === SEARCH OTHER INDEXES === + for bs in batch_sizes: + for idx_cfg in cfg["indexes"]: + if idx_cfg["index"] == "SCANN": + continue # skip SCANN in normal loop + name = idx_cfg["name"] + if (name, bs) in done_pairs: + logger.info(f"Skipping existing result for {name}, batch={bs}") + continue + + idx_path = Path(idx_cfg.get("index_file", index_store_dir/f"{name}.bin")) + search_log = process_logs_dir / f"{name}_search_bs{bs}.log" + res = common_utils.run_operation_in_process( + task_search_index, + (idx_cfg, ds_cfg, run_params, idx_path, bs), + env_vars=idx_cfg.get("search_env_vars", {}), + log_file_path=str(search_log), + process_name=f"SearchTask_{name}_bs{bs}", + timeout_seconds=idx_cfg.get("process_timeout", cfg.get("default_process_timeout", 7200)), + enable_glances=run_params["enable_glances"] + ) + + if res.get("status")=="success" and not res.get("data",{}).get("error"): + row = {"index": name, **res["data"]} + logger.info(f"[SEARCH OK] {name}, batch={bs}: {row}") + else: + err = res.get("data",{}).get("error","proc failed") + row = {"index": name, "query_batch_size": bs, "error": err} + logger.error(f"[SEARCH ERR] {name}, batch={bs}: {err}") + + all_rows.append(row) + + # === MERGE & SAVE ALL RESULTS === + new_df = pd.DataFrame(all_rows) + if existing_df is not None and not run_params["force_overwrite"]: + df = pd.concat([existing_df, new_df], ignore_index=True) + else: + df = new_df + + df["QPS"] = df["nq"] * 1000.0 / df["mean_total_latency_ms"] + df.to_csv(results_csv, index=False) + logger.info(f"Wrote results to {results_csv}") + + # === PLOT === + plt.figure(figsize=(8,6)) + for name, grp in df.groupby("index"): + grp = grp.sort_values("query_batch_size") + plt.errorbar( + grp["query_batch_size"], + grp["QPS"], + marker="o", + label=name + ) + plt.xscale("log", base=2) + plt.xlabel("Query Batch Size") + plt.ylabel("QPS") + plt.title(f"QPS vs Batch Size ({ds_cfg['name']}, k={ds_cfg['k']})") + plt.legend(bbox_to_anchor=(1.05,1), loc="upper left") + plt.grid(True, which="both", ls="--", alpha=0.5) + plt.tight_layout(rect=[0,0,0.8,1]) + plot_path = out / f"qps_vs_batch_{ds_cfg['name']}.png" + plt.savefig(plot_path) + plt.close() + logger.info(f"Saved plot to {plot_path}") \ No newline at end of file diff --git a/test/experiments/osdi2025/vary_levels/configs/sift10m.yaml b/test/experiments/osdi2025/vary_levels/configs/sift10m.yaml new file mode 100644 index 00000000..728a35a7 --- /dev/null +++ b/test/experiments/osdi2025/vary_levels/configs/sift10m.yaml @@ -0,0 +1,77 @@ +# ─── top-level controls ─────────────────────────────────── +overwrite: + index: false # Force rebuild of the base child index (L0) + results: true # Force rerun of searches for all experiment setups + +# ─── dataset ─────────────────────────────────────────────── +dataset: + name: sift10m + metric: l2 + path: data # Path to dataset if not packaged locally + nq: 100 # Number of queries to use for benchmarking + +# ─── base child index (L0 - innermost data layer) ────────── +# This index is built once and used as the foundation for all experiment_setups. +child_index_config: + type: Quake # Index type, currently supports Quake + build_params: + nc: 40000 # Number of centroids for the L0 index + metric: l2 # Metric for L0 (inherited by parents unless overridden, though Quake parents don't typically change metric) + num_workers: 8 # Build workers for L0 index + +# ─── global innermost search sweep (applies to L0 search phase) ─── +# Define EITHER nprobes OR recall_targets for the L0 search sweep. +# These parameters are applied to the SearchParams of the L0 layer. +global_innermost_search_sweep: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] # Option 1: nprobe sweep for L0 +# recall_targets: [0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 0.99] # Option 2: recall_target sweep for L0 +# initial_search_fraction: 0.01 # Required if using recall_targets for L0 + k: 1 # k for the L0 search + batch_size: 1 # Query batch size for L0 search + # Other common SearchParams for L0 can be added here. + +# ─── experiment setups (different hierarchical configurations) ───── +experiment_setups: + - name: "ChildOnly_L0" + # This setup uses only the base child_index_config. + # additional_levels is empty or omitted. + # Search on L0 is governed by global_innermost_search_sweep. + + - name: "L0_plus_L1_Parent1k_PScan50" + additional_levels: # Levels added ON TOP of the base child_index (L0) + - type: Quake # Describes L1 (parent of L0) + build_params: # How to build this L1 layer + nc: 1000 + num_workers: 4 # Build workers for this L1 layer + search_params: # How to search this L1 layer (when it's acting as parent of L0) + nprobe: 50 + num_workers: 4 # Search workers for this L1 layer's scan + use_numa: true # NUMA setting for this L1 layer's scan + + - name: "L0_plus_L1_Parent5k_PScan100" + additional_levels: + - type: Quake # L1 + build_params: {nc: 5000, num_workers: 8} + search_params: {nprobe: 100, num_workers: 8, use_numa: false} + + - name: "L0_L1_Parent1k_L2_Parent100_PScan10" + additional_levels: + - type: Quake # L1 (parent of L0) + build_params: {nc: 1000, num_workers: 4} + search_params: {nprobe: 50, num_workers: 4} # Search params for L1 when it's parent of L0 + - type: Quake # L2 (parent of L1) + build_params: {nc: 100, num_workers: 2} + search_params: {nprobe: 10, num_workers: 2} # Search params for L2 when it's parent of L1 + +# ─── common experiment parameters ────────────────────────── +experiment_params: + trials: 3 # Number of timed trials for each search configuration + warmup: 1 # Number of warmup runs before timed trials + +# ─── plotting (optional) ────────────────────────────────── +plot: + styles: # Keyed by experiment_setups.name + ChildOnly_L0: {color: "#1f77b4", marker: "o", linestyle: "-"} + L0_plus_L1_Parent1k_PScan50: {color: "#ff7f0e", marker: "s", linestyle: "--"} + L0_plus_L1_Parent5k_PScan100: {color: "#2ca02c", marker: "D", linestyle: "-."} + L0_L1_Parent1k_L2_Parent100_PScan10: {color: "#d62728", marker: "^", linestyle: ":"} diff --git a/test/experiments/osdi2025/vary_levels/run.py b/test/experiments/osdi2025/vary_levels/run.py new file mode 100644 index 00000000..5c63ba47 --- /dev/null +++ b/test/experiments/osdi2025/vary_levels/run.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +""" +vary_levels.py - Benchmarks Quake with varying hierarchical structures. + +This script evaluates Quake's performance with an arbitrary number of index levels. +It builds a base child index (L0) once. Then, for each defined 'experiment_setup', +it loads the base index and adds any specified additional parent levels (L1, L2, etc.) +on top. + +Each complete multi-level index is then benchmarked against a sweep of search +parameters applied to the innermost (L0) level. Search parameters for higher +levels (L1, L2, ...) are fixed for each experiment_setup. + +Key metrics, including overall QPS, latency, recall, and per-level scan times +and statistics (e.g., partitions scanned), are recorded. +""" + +from __future__ import annotations +import logging +import time +from pathlib import Path +from typing import Dict, List, Any, Tuple + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import torch +import yaml + +from quake import QuakeIndex, IndexBuildParams, SearchParams, SearchTimingInfo +from test.experiments.osdi2025 import experiment_utils as common_utils + +logger = logging.getLogger("vary_levels_exp") +logging.basicConfig( + format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + datefmt="%H:%M:%S", + level=logging.INFO, +) + +def build_hierarchical_index( + base_vectors: torch.Tensor, + child_index_config: Dict[str, Any], + additional_levels_config: List[Dict[str, Any]], + base_child_index_path: Path, + force_rebuild_base: bool, + exp_setup_name: str +) -> Tuple[QuakeIndex | None, float]: + """ + Builds or loads the base child index (L0), then adds additional levels on top. + Returns the fully constructed QuakeIndex instance and total build time for L0. + Build time for additional levels is not explicitly tracked here but happens during add_level. + """ + # Prepare L0 (base child index) + # Note: build_params_dict for prepare_quake_index should be flat, matching IndexBuildParams attributes + l0_build_params_dict = child_index_config.get("build_params", {}) + + # Ensure metric is in the build_params_dict for prepare_quake_index if it expects it + if 'metric' not in l0_build_params_dict and 'metric' in child_index_config: + l0_build_params_dict['metric'] = child_index_config['metric'] + if 'metric' not in l0_build_params_dict and 'metric' in dataset_cfg: # Fallback to dataset metric + l0_build_params_dict['metric'] = dataset_cfg['metric'] + + + # Use common_utils to prepare the base L0 index + # This function handles building if not exists or force_rebuild is true, and saving. + # It returns the loaded/built index instance. + idx = common_utils.prepare_quake_index( + vecs=base_vectors, + build_params_dict=l0_build_params_dict, + index_file_path=base_child_index_path, + force_rebuild=force_rebuild_base, + # Pass load-time worker/NUMA settings for L0 if they were part of its build_params + # For Quake, these are typically search-time. If prepare_quake_index needs them for load: + num_workers_load=l0_build_params_dict.get("num_workers", 0), # Assuming build num_workers is also load num_workers + use_numa=l0_build_params_dict.get("use_numa", False) # Assuming build use_numa is also load use_numa + ) + # Build time for L0 is implicitly handled by prepare_quake_index if it were to return it. + # For now, we'll assume build_time_l0 is primarily for the initial creation. + # If the file existed, build_time_l0 would be 0. + build_time_l0 = 0.0 # Placeholder, as prepare_quake_index doesn't return build time directly. + + if idx is None: + logger.error(f"Failed to prepare base child index for {exp_setup_name}.") + return None, build_time_l0 + + # Add additional levels (L1, L2, ...) + for i, level_cfg in enumerate(additional_levels_config): + level_num = i + 1 # L1, L2, ... + logger.info(f"For '{exp_setup_name}', adding L{level_num}...") + + add_level_build_params_dict = level_cfg.get("build_params", {}) + if not add_level_build_params_dict.get("nc"): + logger.warning(f"Skipping L{level_num} for '{exp_setup_name}' due to missing 'nc' in build_params.") + continue + + # Create IndexBuildParams for adding the level + # The metric for parent levels is typically inherited or fixed by Quake internally. + # We primarily need to set 'nc' and 'num_workers' for building this parent level. + parent_bp = IndexBuildParams() + parent_bp.nlist = add_level_build_params_dict["nc"] + parent_bp.num_workers = add_level_build_params_dict.get("num_workers", 0) + parent_bp.metric = l0_build_params_dict.get("metric", "l2") # Parent inherits metric + + try: + idx.add_level(parent_bp) # Assumes QuakeIndex has .index attribute pointing to C++ object + logger.info(f"L{level_num} (NC={parent_bp.nlist}) added successfully for '{exp_setup_name}'.") + except Exception as e: + logger.error(f"Failed to add L{level_num} for '{exp_setup_name}': {e}") + return None, build_time_l0 + + return idx, build_time_l0 + + +def construct_hierarchical_search_params( + innermost_level_sp_item: Dict[str, Any], + additional_levels_config: List[Dict[str, Any]] +) -> SearchParams: + """ + Constructs a nested SearchParams object for the entire hierarchy. + """ + # Innermost level (L0) search parameters + l0_sp = common_utils.create_search_params(**innermost_level_sp_item) + + current_parent_sp = l0_sp + for level_cfg in additional_levels_config: # L1, L2, ... + level_search_params_dict = level_cfg.get("search_params", {}) + if not level_search_params_dict: # If no specific search params, create default + level_search_params_dict = {"nprobe": 1} # Minimal valid SearchParams + + # These are search-time worker/NUMA settings for this specific parent level's scan + # common_utils.create_search_params will set these if present in level_search_params_dict + higher_level_sp = common_utils.create_search_params(**level_search_params_dict) + + current_parent_sp.parent_params = higher_level_sp + current_parent_sp = higher_level_sp + + return l0_sp + + +def _extract_recursive_timings(innermost_timing_info: SearchTimingInfo | None, num_levels: int) -> Dict[str, Any]: + """ + Extracts scan time and other stats for each level from the TimingInfo hierarchy. + The innermost_timing_info is for L0. Its .parent_info is for L1, and so on. + """ + timings = {} + current_ti = innermost_timing_info + + for i in range(num_levels): + level_prefix = f"level_{i}" + if current_ti: + timings[f"{level_prefix}_scan_time_ms"] = current_ti.total_time_ns / 1e6 + timings[f"{level_prefix}_buffer_init_ms"] = current_ti.buffer_init_time_ns / 1e6 + timings[f"{level_prefix}_enqueue_ms"] = current_ti.job_enqueue_time_ns / 1e6 + timings[f"{level_prefix}_wait_ms"] = current_ti.job_wait_time_ns / 1e6 + timings[f"{level_prefix}_aggregate_ms"] = current_ti.result_aggregate_time_ns / 1e6 + timings[f"{level_prefix}_partitions_scanned"] = getattr(current_ti, "partitions_scanned", np.nan) + current_ti = current_ti.parent_info + else: + timings[f"{level_prefix}_scan_time_ms"] = np.nan + timings[f"{level_prefix}_buffer_init_ms"] = np.nan + timings[f"{level_prefix}_enqueue_ms"] = np.nan + timings[f"{level_prefix}_wait_ms"] = np.nan + timings[f"{level_prefix}_aggregate_ms"] = np.nan + timings[f"{level_prefix}_partitions_scanned"] = np.nan + return timings + + +def benchmark_search_config( + idx: QuakeIndex, + queries: torch.Tensor, + ground_truth: np.ndarray | None, + hierarchical_sp: SearchParams, + num_levels_in_hierarchy: int, + num_trials: int, + warmup_trials: int +) -> Dict[str, Any]: + """ + Benchmarks a specific hierarchical search configuration. + Returns aggregated metrics including per-level timings. + """ + k_val = hierarchical_sp.k # k from the L0 SearchParams + + # Warm-up + for _ in range(warmup_trials): + idx.search(queries, hierarchical_sp) + + trial_metrics_list = [] + for _ in range(num_trials): + t_start_ns = time.perf_counter_ns() + search_result = idx.search(queries, hierarchical_sp) + t_elapsed_ns = time.perf_counter_ns() - t_start_ns + + current_recall = np.nan + if ground_truth is not None and search_result.ids is not None: + # Ensure k_val does not exceed columns in ground_truth or search_result.ids + max_k_recall = min(k_val, ground_truth.shape[1], search_result.ids.shape[1]) + if max_k_recall > 0 : + current_recall = common_utils.quake_compute_recall( + search_result.ids[:, :max_k_recall], + ground_truth[:, :max_k_recall], + max_k_recall + ).mean().item() + + + per_level_timings = _extract_recursive_timings(search_result.timing_info, num_levels_in_hierarchy) + + trial_metrics_list.append({ + "overall_latency_ns": t_elapsed_ns, + "recall": current_recall, + **per_level_timings + }) + + # Aggregate results from trials + num_queries_total = len(queries) + aggregated_results = { + "qps_mean": np.mean([num_queries_total / (m["overall_latency_ns"] / 1e9) for m in trial_metrics_list]), + "overall_latency_ms_mean": np.mean([m["overall_latency_ns"] / 1e6 for m in trial_metrics_list]), + "recall_mean": np.nanmean([m["recall"] for m in trial_metrics_list]), + } + + # Aggregate per-level timings and stats + for i in range(num_levels_in_hierarchy): + level_prefix = f"level_{i}" + for metric_suffix in ["scan_time_ms", "buffer_init_ms", "enqueue_ms", "wait_ms", "aggregate_ms", "partitions_scanned"]: + key = f"{level_prefix}_{metric_suffix}" + valid_values = [m[key] for m in trial_metrics_list if key in m and pd.notna(m[key])] + if valid_values: + aggregated_results[f"{key}_mean"] = np.mean(valid_values) + else: + aggregated_results[f"{key}_mean"] = np.nan + + return aggregated_results + + +def run_experiment(config_path_str: str, output_dir_str: str) -> None: + global dataset_cfg # Make dataset_cfg accessible to build_hierarchical_index if needed for metric fallback + config = common_utils.load_config(config_path_str) + output_dir = Path(output_dir_str) + output_dir.mkdir(parents=True, exist_ok=True) + + dataset_cfg = config["dataset"] + child_index_cfg = config["child_index_config"] # L0 build config + innermost_search_sweep_cfg = config["global_innermost_search_sweep"] # L0 search sweep + experiment_setups_cfg = config["experiment_setups"] + exp_params_cfg = config["experiment_params"] + overwrite_cfg = config["overwrite"] + + logger.info(f"Loading dataset: {dataset_cfg['name']}") + base_vecs, queries, gt_numpy = common_utils.load_data( + dataset_cfg["name"], dataset_cfg.get("path", ""), nq_override=dataset_cfg.get("nq") + ) + + # Path for the L0 index binary + base_child_index_path = output_dir / "base_index.bin" + + l0_build_params_dict = child_index_cfg.get("build_params", {}) + if 'metric' not in l0_build_params_dict: # Ensure metric is set for L0 build + l0_build_params_dict['metric'] = dataset_cfg.get('metric', 'l2') + + _ = common_utils.prepare_quake_index( # Build L0 and save it + vecs=base_vecs, + build_params_dict=l0_build_params_dict, + index_file_path=base_child_index_path, + force_rebuild=overwrite_cfg.get("index", False), + load=False + ) + logger.info(f"Base L0 index prepared at {base_child_index_path}") + + all_experiment_run_records = [] + innermost_sweep_list = common_utils.expand_search_sweep(innermost_search_sweep_cfg) # Adapted from common_utils or defined here + + for exp_setup_item in experiment_setups_cfg: + exp_setup_name = exp_setup_item["name"] + additional_levels_config = exp_setup_item.get("additional_levels", []) + num_actual_levels = 1 + len(additional_levels_config) # L0 + additional levels + + exp_setup_results_path = output_dir / f"{exp_setup_name}_results.csv" + if exp_setup_results_path.exists() and not overwrite_cfg.get("results", True): + logger.info(f"Results for setup '{exp_setup_name}' exist, loading cached.") + try: + cached_df = pd.read_csv(exp_setup_results_path) + all_experiment_run_records.extend(cached_df.to_dict('records')) + except Exception as e: + logger.error(f"Could not load cached results for {exp_setup_name}: {e}") + continue + + logger.info(f"--- Processing Experiment Setup: {exp_setup_name} ({num_actual_levels} levels) ---") + + # Assemble the full hierarchical index for this setup + # This loads L0 and adds L1, L2... on top. + current_hier_idx, _ = build_hierarchical_index( + base_vecs, child_index_cfg, additional_levels_config, + base_child_index_path, False, # False: don't force rebuild L0 if already done + exp_setup_name + ) + if not current_hier_idx: + logger.error(f"Failed to build index for setup '{exp_setup_name}'. Skipping.") + continue + + setup_specific_records = [] + for l0_sp_item in innermost_sweep_list: # Iterate L0 search params (nprobe/recall_target) + param_key = "nprobe" if "nprobe" in l0_sp_item else "recall_target" + param_val = l0_sp_item.get(param_key, "N/A") + logger.info(f"Benchmarking '{exp_setup_name}' with L0 {param_key}={param_val}...") + + # Construct the full stack of SearchParams for this run + hierarchical_sp = construct_hierarchical_search_params(l0_sp_item, additional_levels_config) + + benchmark_stats = benchmark_search_config( + current_hier_idx, queries, gt_numpy, hierarchical_sp, + num_levels_in_hierarchy=num_actual_levels, + num_trials=exp_params_cfg.get("trials", 3), + warmup_trials=exp_params_cfg.get("warmup", 1) + ) + + # Record configuration and results + record = {"exp_setup_name": exp_setup_name} + record[f"L0_{param_key}"] = param_val # L0 sweep parameter + record["L0_k"] = l0_sp_item.get("k") + record["L0_batch_size"] = l0_sp_item.get("batch_size") + record["L0_nc"] = child_index_cfg.get("build_params", {}).get("nc") + + + for i, add_level_cfg in enumerate(additional_levels_config): + level_num = i + 1 # L1, L2... + record[f"L{level_num}_nc"] = add_level_cfg.get("build_params",{}).get("nc") + record[f"L{level_num}_build_workers"] = add_level_cfg.get("build_params",{}).get("num_workers") + record[f"L{level_num}_search_nprobe"] = add_level_cfg.get("search_params",{}).get("nprobe") + record[f"L{level_num}_search_recall_target"] = add_level_cfg.get("search_params",{}).get("recall_target") + record[f"L{level_num}_search_workers"] = add_level_cfg.get("search_params",{}).get("num_workers") + record[f"L{level_num}_search_use_numa"] = add_level_cfg.get("search_params",{}).get("use_numa") + + record.update(benchmark_stats) + setup_specific_records.append(record) + all_experiment_run_records.append(record) + + if setup_specific_records: + setup_df = pd.DataFrame(setup_specific_records) + setup_df.to_csv(exp_setup_results_path, index=False) + logger.info(f"Results for setup '{exp_setup_name}' saved to {exp_setup_results_path}") + + if all_experiment_run_records: + final_df = pd.DataFrame(all_experiment_run_records) + unified_csv_path = output_dir / f"{dataset_cfg['name']}_vary_levels_all_setups_summary.csv" + final_df.to_csv(unified_csv_path, index=False) + logger.info(f"Unified results for all setups saved to {unified_csv_path}") + + + if not final_df.empty: + plot_metric_vs_l0_param(final_df, config, output_dir, metric_to_plot="recall_mean", ylabel="Mean Recall") + plot_metric_vs_l0_param(final_df, config, output_dir, metric_to_plot="overall_latency_ms_mean", ylabel="Mean Overall Latency (ms)") + plot_level_scan_times(final_df, config, output_dir) + + else: + logger.info("No results generated across all setups.") + + logger.info("Vary Levels (Generalized) experiment finished.") + +def plot_metric_vs_l0_param(results_df: pd.DataFrame, config: Dict[str, Any], output_dir: Path, metric_to_plot: str, ylabel: str): + """Plots a given metric against the L0 sweep parameter for different experiment setups.""" + if results_df.empty or metric_to_plot not in results_df.columns: + logger.warning(f"Cannot plot '{metric_to_plot}': Data missing or empty.") + return + + fig, ax = plt.subplots(figsize=(10, 7)) + plot_styles = config.get("plot", {}).get("styles", {}) + + # Determine L0 sweep key (nprobe or recall_target) + l0_sweep_key = None + if "L0_nprobe" in results_df.columns and results_df["L0_nprobe"].notna().any(): + l0_sweep_key = "L0_nprobe" + elif "L0_recall_target" in results_df.columns and results_df["L0_recall_target"].notna().any(): + l0_sweep_key = "L0_recall_target" + + if not l0_sweep_key: + logger.warning("L0 sweep parameter (nprobe/recall_target) not found in results. Skipping plot.") + plt.close(fig) + return + + for exp_name, group in results_df.groupby("exp_setup_name"): + style = plot_styles.get(exp_name, {}) + sorted_group = group.sort_values(by=l0_sweep_key) + ax.plot( + sorted_group[l0_sweep_key], + sorted_group[metric_to_plot], + label=exp_name, + marker=style.get("marker", "o"), + color=style.get("color"), + linestyle=style.get("linestyle", "-") + ) + + ax.set_xlabel(f"Innermost Level (L0) Search: {l0_sweep_key.replace('_', ' ').title()}") + ax.set_ylabel(ylabel) + ax.set_title(f"{config['dataset']['name']}: {ylabel} vs. L0 Search Parameter") + ax.legend(title="Experiment Setup", bbox_to_anchor=(1.05, 1), loc='upper left') + ax.grid(True, which="both", ls=":", alpha=0.7) + plt.tight_layout(rect=[0, 0, 0.78, 1]) # Adjust for legend + + plot_filename = f"{config['dataset']['name']}_vary_levels_{metric_to_plot}_vs_L0param.png" + plt.savefig(output_dir / plot_filename, dpi=150) + logger.info(f"Plot saved: {output_dir / plot_filename}") + plt.close(fig) + + +def plot_level_scan_times(results_df: pd.DataFrame, config: Dict[str, Any], output_dir: Path): + """Plots the breakdown of scan times per level for each experiment setup, at a chosen L0 parameter value.""" + if results_df.empty: + logger.warning("Cannot plot level scan times: Data missing or empty.") + return + + l0_sweep_key = None + if "L0_nprobe" in results_df.columns and results_df["L0_nprobe"].notna().any(): + l0_sweep_key = "L0_nprobe" + elif "L0_recall_target" in results_df.columns and results_df["L0_recall_target"].notna().any(): + l0_sweep_key = "L0_recall_target" + if not l0_sweep_key: return + + + # Select a representative L0 parameter value for the bar chart (e.g., median or a common high-recall point) + if results_df[l0_sweep_key].nunique() > 1: + target_l0_val = np.percentile(results_df[l0_sweep_key].dropna(), 75) # Example: 75th percentile + # Find closest available value in the df + available_l0_vals = results_df[l0_sweep_key].dropna().unique() + closest_l0_val = available_l0_vals[np.abs(available_l0_vals - target_l0_val).argmin()] + + df_subset = results_df[results_df[l0_sweep_key] == closest_l0_val].copy() + plot_title_suffix = f"(at L0 {l0_sweep_key.split('_')[-1]} ≈ {closest_l0_val:.2f})" + else: + df_subset = results_df.copy() + plot_title_suffix = f"(at L0 {l0_sweep_key.split('_')[-1]} = {df_subset[l0_sweep_key].iloc[0]:.2f})" + + + if df_subset.empty: + logger.warning("No data for selected L0 parameter for level scan time plot.") + return + + level_time_cols = sorted([col for col in df_subset.columns if col.startswith("level_") and col.endswith("_scan_time_ms_mean")]) + if not level_time_cols: + logger.warning("No per-level scan time columns found for plotting.") + return + + max_level_idx = max([int(c.split('_')[1]) for c in level_time_cols]) + num_levels_to_plot = max_level_idx + 1 + + plot_data = df_subset.set_index("exp_setup_name")[level_time_cols] + + # Ensure all level columns up to max_level_idx are present, fill with 0 if not (for consistent plotting) + for i in range(num_levels_to_plot): + col_name = f"level_{i}_scan_time_ms_mean" + if col_name not in plot_data.columns: + plot_data[col_name] = 0 + # Reorder columns for stacked bar plot + ordered_level_time_cols = [f"level_{i}_scan_time_ms_mean" for i in range(num_levels_to_plot)] + plot_data = plot_data[ordered_level_time_cols] + + + plot_data.plot(kind='bar', stacked=True, figsize=(12, 8), colormap='viridis') + + plt.xlabel("Experiment Setup") + plt.ylabel("Mean Scan Time per Level (ms)") + plt.title(f"{config['dataset']['name']}: Scan Time Breakdown by Level {plot_title_suffix}") + plt.xticks(rotation=30, ha='right') + plt.legend(title="Index Level (L0=innermost)", bbox_to_anchor=(1.05, 1), loc='upper left') + plt.tight_layout(rect=[0, 0, 0.8, 1]) # Adjust for legend + + plot_filename = f"{config['dataset']['name']}_vary_levels_scan_time_breakdown.png" + plt.savefig(output_dir / plot_filename, dpi=150) + logger.info(f"Plot saved: {output_dir / plot_filename}") + plt.close() \ No newline at end of file diff --git a/test/experiments/osdi2025/wiki_workload/configs/wiki.yaml b/test/experiments/osdi2025/wiki_workload/configs/wiki.yaml new file mode 100644 index 00000000..143d5f34 --- /dev/null +++ b/test/experiments/osdi2025/wiki_workload/configs/wiki.yaml @@ -0,0 +1,186 @@ +workload_dir: data/wikidata/workload +dataset_dir: /Users/jason/projects/compass/data/wikidata/wikidata +overwrite: False +batch: True + +indexes: + - name: Quake_Serial_NoAPS_NoMaintenance + index: Quake + build_params: + nc: 1000 + metric: ip + num_workers: 1 + use_numa: true + parent: + num_workers: 1 + search_params: + k: 100 + nprobe: 10 + initial_search_fraction: .05 + batched_scan: false + parent: + batched_scan: true + batch_size: 128 + maintenance_params: + refinement_radius: 50 + split_threshold_ns: 500000 + delete_threshold_ns: 500000 + enable_split_rejection: true + enable_delete_rejection: false + window_size: 10000 + + - name: Quake_Serial_NoAPS + index: Quake + build_params: + nc: 1000 + metric: ip + num_workers: 1 + use_numa: true + parent: + num_workers: 1 + search_params: + k: 100 + nprobe: 30 + initial_search_fraction: .05 + batched_scan: false + parent: + batched_scan: true + batch_size: 128 + maintenance_params: + refinement_radius: 50 + split_threshold_ns: 250 + delete_threshold_ns: 500000 + enable_split_rejection: true + enable_delete_rejection: false + window_size: 10000 + + - name: Quake_Serial + index: Quake + build_params: + nc: 1000 + metric: ip + num_workers: 1 + use_numa: true + parent: + num_workers: 1 + search_params: + k: 100 + recall_target: .9 + initial_search_fraction: .05 + batched_scan: false + parent: + batched_scan: true + batch_size: 128 + maintenance_params: + refinement_radius: 50 + split_threshold_ns: 250 + delete_threshold_ns: 500000 + enable_split_rejection: true + enable_delete_rejection: false + window_size: 10000 + + - name: QuakeNoMaintenanceNoAPS + index: Quake + build_params: + nc: 1000 + metric: ip + num_workers: 8 + use_numa: true + parent: + num_workers: 2 + search_params: + k: 100 + nprobe: 30 + batched_scan: false + aps_flush_period_us: 10 + parent: + batched_scan: true + batch_size: 128 + maintenance_params: + refinement_radius: 50 + split_threshold_ns: 500000 + delete_threshold_ns: 500000 + enable_split_rejection: true + enable_delete_rejection: false + window_size: 10000 + + - name: QuakeNoMaintenance + index: Quake + build_params: + nc: 1000 + metric: ip + num_workers: 8 + use_numa: true + parent: + num_workers: 2 + search_params: + k: 100 + recall_target: .9 + batched_scan: false + recompute_threshold: 0.01 + use_precomputed: true + initial_search_fraction: .05 + aps_flush_period_us: 10 + parent: + batched_scan: true + batch_size: 128 + maintenance_params: + refinement_radius: 50 + split_threshold_ns: 500000 + delete_threshold_ns: 500000 + enable_split_rejection: true + enable_delete_rejection: false + window_size: 10000 + + - name: QuakeNoAPS + index: Quake + build_params: + nc: 1000 + metric: ip + num_workers: 8 + use_numa: true + parent: + num_workers: 2 + search_params: + k: 100 + nprobe: 30 + batched_scan: false + aps_flush_period_us: 10 + parent: + batched_scan: true + batch_size: 128 + maintenance_params: + refinement_radius: 50 + split_threshold_ns: 250 + delete_threshold_ns: 500000 + enable_split_rejection: true + enable_delete_rejection: false + window_size: 10000 + + - name: Quake + index: Quake + build_params: + nc: 1000 + metric: ip + num_workers: 8 + use_numa: true + parent: + num_workers: 2 + search_params: + k: 100 + recall_target: .9 + batched_scan: false + recompute_threshold: 0.01 + use_precomputed: true + initial_search_fraction: .05 + aps_flush_period_us: 10 + parent: + batched_scan: true + batch_size: 128 + maintenance_params: + refinement_radius: 50 + split_threshold_ns: 250 + delete_threshold_ns: 500000 + enable_split_rejection: true + enable_delete_rejection: false + window_size: 10000 \ No newline at end of file diff --git a/test/experiments/osdi2025/wiki_workload/run.py b/test/experiments/osdi2025/wiki_workload/run.py new file mode 100644 index 00000000..eacdbe0f --- /dev/null +++ b/test/experiments/osdi2025/wiki_workload/run.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +""" +Wikidata Workload Experiment Runner +───────────────────────────────────── +This experiment runs a Wikidata workload across multiple index configurations. +It generates detailed reports including: +1. Per-index CSV logs of operations. +2. A 9-panel unified plot comparing various metrics across configurations. +3. A stacked bar chart breaking down cumulative time per operation type. +4. A summary table (CSV and Markdown) of key performance indicators. +""" +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Dict, List, Any, Optional + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from tabulate import tabulate +from matplotlib.cm import get_cmap +from matplotlib.lines import Line2D + +# Common utilities +import test.experiments.osdi2025.experiment_utils as common_utils + +# evaluator and wrappers +from .wiki_workload import WikidataWorkloadEvaluator +from quake.index_wrappers.quake import QuakeWrapper +from quake.index_wrappers.faiss_ivf import FaissIVF +from quake.index_wrappers.faiss_hnsw import FaissHNSW +try: from quake.index_wrappers.scann import Scann +except ImportError: Scann = None +try: from quake.index_wrappers.diskann import DiskANNDynamic +except ImportError: DiskANNDynamic = None +try: from quake.index_wrappers.vamana import Vamana +except ImportError: Vamana = None + +INDEX_CLASSES: Dict[str, Any] = { + "Quake": QuakeWrapper, + "IVF": FaissIVF, + "HNSW": FaissHNSW, + "SCANN": Scann, + "DiskANN": DiskANNDynamic, + "SVS": Vamana, +} + +logging.basicConfig(level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s") +log = logging.getLogger(__name__) + +OP_STYLE = { + "query": dict(ls="-", marker="o", mfc="none", ms=4, lw=1.2), + "insert": dict(ls="-", marker="s", mfc="none", ms=4, lw=1.2), + "delete": dict(ls="-", marker="^", mfc="none", ms=4, lw=1.2), + "maintain": dict(ls="None", marker="X", mfc="black", ms=5), +} +LAT_OPS = ["query", "insert", "delete", "maintain"] +IDX_LAT_Q, IDX_LAT_I, IDX_LAT_D, IDX_LAT_M = 0, 1, 2, 3 +IDX_PART, IDX_RES, IDX_REC, IDX_TOT, IDX_SPL = 4, 5, 6, 7, 8 + + +def unified_plot(cfg: Dict[str, Any], out_dir: Path) -> None: + styles = cfg.get("plot", {}).get("styles", {}) + fig, axs2d = plt.subplots(3, 3, figsize=(18, 12), sharex="col") + axs = axs2d.flatten() + + idx_handles: List[Line2D] = [] + for j, idx_cfg in enumerate(cfg.get("indexes", [])): + nm = idx_cfg["name"] + st = styles.get(nm, {}) + idx_handles.append(Line2D([0], [0], + color=st.get("color", f"C{j % 10}"), + marker=st.get("marker", "o"), + ls="", markersize=6, label=nm)) + + max_ops_overall = 0 + for j, idx_cfg in enumerate(cfg.get("indexes", [])): + name = idx_cfg["name"] + colour = styles.get(name, {}).get("color", f"C{j % 10}") + marker = styles.get(name, {}).get("marker", "o") + + csv_path = out_dir / name / "results.csv" + if not csv_path.exists(): + log.warning("[unified_plot] missing %s", csv_path) + continue + try: + df = pd.read_csv(csv_path) + if df.empty: continue + max_ops_overall = max(max_ops_overall, df.operation_number.max()) + except Exception: + log.warning("[unified_plot] failed to read %s", csv_path) + continue + + # latency + for op, ax_idx in zip(LAT_OPS, [IDX_LAT_Q, IDX_LAT_I, IDX_LAT_D, IDX_LAT_M]): + ax = axs[ax_idx] + if op != "maintain": + sub = df[df.operation_type == op] + if not sub.empty: + ax.plot(sub.operation_number, sub.latency_ms, color=colour, **OP_STYLE[op]) + + # partitions + part_ix = IDX_PART + if "nlist" in df: + axs[part_ix].plot(df.operation_number, df.nlist, color=colour, marker=marker, lw=1.2) + # resident + res_ix = IDX_RES + if "n_resident" in df: + axs[res_ix].plot(df.operation_number, df.n_resident, color=colour, marker=marker, lw=1.2) + + # splits/deletes + spl_ix = IDX_SPL + if "n_splits" in df and "n_deletes" in df: + axs[spl_ix].plot(df.operation_number, df.n_splits, color=colour, marker="x", lw=1.2, label="Splits") + axs[spl_ix].plot(df.operation_number, df.n_deletes, color=colour, marker="o", lw=1.2, label="Deletes") + # axs[spl_ix].legend(loc="upper left", frameon=False) + + # maintenance time + if "maintenance_latency_ms" in df: + m_ix = IDX_LAT_M + axs[m_ix].plot(df.operation_number, df.maintenance_latency_ms, color=colour, marker=marker, lw=1.2, + label="Maintenance") + + # recall + rec_ix = IDX_REC + sub = df[df.operation_type=="query"] + if "recall" in sub: + axs[rec_ix].plot(sub.operation_number, sub.recall, color=colour, marker=marker, lw=1.2) + # cumulative time + tot_ix = IDX_TOT + cum = np.cumsum(df.latency_ms.fillna(0)) + axs[tot_ix].plot(df.operation_number, cum, color=colour, marker=marker, lw=1.2) + + # titles and labels + titles = { + IDX_LAT_Q: "Latency – Query", IDX_LAT_I: "Insert", IDX_LAT_D: "Delete", IDX_LAT_M: "Maintain", + IDX_PART: "# Partitions", IDX_RES: "Resident", IDX_REC: "Recall", + IDX_TOT: "Cum. Time (ms)", IDX_SPL: "Splits/Deletes" + } + for i, ax in enumerate(axs): + ax.set_title(titles.get(i, "")) + ax.grid(True, ls=":", alpha=0.7) + if i//3==2: ax.set_xlabel("Op #") + if max_ops_overall>0: + for ax in axs: ax.set_xlim(0, max_ops_overall) + + fig.legend(idx_handles, [h.get_label() for h in idx_handles], + loc="upper center", ncol=min(4,len(idx_handles)), frameon=False) + plt.tight_layout(rect=[0,0.03,1,0.97]) + path = out_dir / "unified_plot.png" + plt.savefig(path, dpi=150) + log.info("Saved unified_plot to %s", path) + plt.close(fig) + + +def make_time_breakdown(cfg: Dict[str, Any], out_dir: Path) -> None: + categories = ["Query","Insert","Maintenance", "Total"] + data = {} + for idx_cfg in cfg.get("indexes", []): + name = idx_cfg["name"] + df = pd.read_csv(out_dir/name/"results.csv") + data[name] = [ + df[df.operation_type=="query"].latency_ms.sum(), + df[df.operation_type=="insert"].latency_ms.sum(), + df.maintenance_latency_ms.sum() if "maintenance_latency_ms" in df else 0, + ] + data[name].append(sum(data[name])) + fig, ax = plt.subplots(figsize=(8,6)) + cmap = get_cmap("tab10") + n = len(data) + x = np.arange(len(categories)) + w = 0.8/n + for j,(name,vals) in enumerate(data.items()): + ax.bar(x + j*w, vals, width=w, label=name, color=cmap(j)) + ax.set_xticks(x); ax.set_xticklabels(categories) + ax.set_ylabel("Cumulative ms") + ax.legend(frameon=False) + path = out_dir/"time_breakdown.png" + plt.savefig(path, dpi=150) + log.info("Saved time_breakdown to %s", path) + plt.close(fig) + + +def produce_summary_table(cfg: Dict[str, Any], out_dir: Path) -> None: + rows = [] + for idx_cfg in cfg.get("indexes", []): + name = idx_cfg["name"] + df = pd.read_csv(out_dir/name/"results.csv") + q = df[df.operation_type=="query"] + rows.append({ + "Index": name, + "MeanLatency": float(q.latency_ms.mean()), + "MeanRecall": float(q.recall.mean()) if "recall" in q else None, + }) + if not rows: return + sdf = pd.DataFrame(rows) + common_utils.save_results_csv(sdf, out_dir/"summary_table.csv") + md = tabulate(sdf, headers="keys", tablefmt="github", showindex=False) + print(md) + + +def run_experiment(cfg_path_str: str, output_dir_str: str) -> None: + cfg = common_utils.load_config(cfg_path_str) + out = Path(output_dir_str).expanduser(); out.mkdir(parents=True, exist_ok=True) + mode = cfg.get("mode","run") + log.info("Mode: %s", mode) + + workload_dir = cfg["workload_dir"] + dataset_dir = cfg["dataset_dir"] + + if mode in {"run"}: # Phase 2: evaluation + for idx_cfg in cfg.get("indexes", []): + + # check if results already exist + out_name = out / idx_cfg["name"] + if out_name.is_dir() and (out_name / "results.csv").is_file() and not cfg.get("overwrite", False): + log.info("Skipping %s, results already exist", idx_cfg["name"]) + continue + + name = idx_cfg["name"] + key = idx_cfg["index"] + Cls = INDEX_CLASSES.get(key) + if Cls is None: + log.warning("Unknown index %s", key); continue + wrapper = Cls() + maint = idx_cfg.get("maintenance_params") + do_maint = maint is not None + ev = WikidataWorkloadEvaluator(workload_dir, out/name, wiki_dataset_dir=dataset_dir) + ev.evaluate_workload( + name=name, + index=wrapper, + build_params=idx_cfg.get("build_params",{}), + search_params=idx_cfg.get("search_params",{}), + do_maintenance=do_maint, + m_params=maint, + batch=cfg.get("batch",False), + ) + if mode in {"run","plot"}: # Phase 3 + unified_plot(cfg, out) + make_time_breakdown(cfg, out) + produce_summary_table(cfg, out) \ No newline at end of file diff --git a/test/experiments/osdi2025/wiki_workload/wiki_workload.py b/test/experiments/osdi2025/wiki_workload/wiki_workload.py new file mode 100644 index 00000000..b2236f9f --- /dev/null +++ b/test/experiments/osdi2025/wiki_workload/wiki_workload.py @@ -0,0 +1,640 @@ +import json +import time +from pathlib import Path +from typing import Union, Dict, List, Optional + +import numpy as np +import torch + +import quake +from quake.utils import to_path, compute_recall +import pandas as pd +import matplotlib.pyplot as plt +from quake.index_wrappers.quake import QuakeWrapper + +TOTAL_NUM_UPDATES = 104 +TOTAL_NUM_EMBEDDINGS = 11_790_930 +K = 100 + +wikidata_dir = Path("/Users/jason/projects/compass/data/wikidata") +metric = "ip" + +def get_node_mappings() -> dict: + """ + This function retrieves the node mappings from the S3 bucket. + """ + node_mappings = None + + with open(wikidata_dir / "node_mappings.json", "r") as f: + node_mappings = json.load(f) + + return node_mappings + + +def update_id_to_month(n) -> str: + """ + This function converts the update id to the corresponding month and year. + """ + # starting point: August 2015 corresponds to update 27 + start_month = 8 + start_year = 2015 + + # calculate the number of months to add + months_to_add = n - 27 + + # calculate the target year and month + target_year = start_year + (start_month + months_to_add - 1) // 12 + target_month = (start_month + months_to_add - 1) % 12 + 1 + + # get the month and year as a string + return f"{target_year}{target_month:02d}" + + +def compute_ip_ground_truth( + queries: torch.Tensor, + base : torch.Tensor, + # ids : torch.Tensor, + k: int, +) -> np.ndarray: + + + flat_quake_index = quake.QuakeIndex() + build_params = quake.IndexBuildParams() + build_params.metric = "ip" + build_params.nlist = 1 + build_params.num_workers = 6 + ids = torch.arange(base.shape[0], dtype=torch.int64) + flat_quake_index.build(base, ids, build_params) + + search_params = quake.SearchParams() + search_params.k = k + search_params.batched_scan = True + res = flat_quake_index.search(queries, search_params) + return res.ids.numpy(), res.distances.numpy() + + +def wiki_sampler(current_ids: torch.tensor, query_size: int, update_id: int) -> torch.tensor: + """ + This function samples query ids from the current_ids. + + :param current_ids: The current ids in the index. + :param query_size: The size of the query. + :param update_id: The id of the update. + + :return: The sampled query ids. + """ + assert update_id >= -1 and update_id <= 103 + + month_str = update_id_to_month(update_id) + if month_str == "201412": + month_str = "201312" + elif month_str == "201310": + month_str = "201210" + + # for each of the current ids, get the corresponding pageviews for the given update_id + df_id = pd.DataFrame(current_ids.numpy(), columns=["id"]) + df_pageviews = pd.read_parquet(wikidata_dir / f"pageviews/pageviews_{month_str}.parquet") + # df_pageviews = pd.read_parquet(Path(f"data/wikidata/pageviews/pageviews_{month_str}.parquet")) + + df = pd.merge(df_id, df_pageviews, on="id", how="left") + df["pageviews"] = df["pageviews"].fillna(0).astype(int) + + pageviews = df["pageviews"].values + assert len(pageviews) == len(current_ids) + + # sample the query ids based on the pageviews + np.random.seed(42) + probs = pageviews / pageviews.sum() + query_ids = np.random.choice(current_ids.numpy(), size=query_size, replace=True, p=probs) + + assert len(query_ids) == query_size + return torch.tensor(query_ids) + +class WikidataWorkloadGenerator: + """ + Generates a dynamic workload from the Wikipedia data for updates + and sample embeddings for queries (for now). + + The workflow is as follows: + 1. Takes the embeddings from the Wikipedia dataset for the initial base embeddings and updates. + 2. After each update, samples a set of resident embeddings for queries. + 3. The workload is saved to a directory as a set of operations with a corresponding runbook. + """ + + def __init__( + self, + workload_dir: Union[str, Path], + num_updates: int, + query_batch_size: int, + metric: str = "ip", + seed: int = 1874, + with_patterns: bool = True, + overwrite: bool = False, + ): + # download the wikipedia dataset if not already downloaded + self.num_updates = num_updates + + assert num_updates <= TOTAL_NUM_UPDATES + + self.workload_dir = Path(workload_dir) + self.query_batch_size = query_batch_size + self.metric = metric.lower() + self.seed = seed + self.with_patterns = with_patterns + + assert self.metric == "ip" + assert self.query_batch_size > 0 + + torch.manual_seed(self.seed) + np.random.seed(self.seed) + + self.workload_dir.mkdir(parents=True, exist_ok=True) + + # delete the runbook if it exists and overwrite is True + runbook_path = self.workload_dir / "runbook.json" + if runbook_path.exists() and overwrite: + runbook_path.unlink() + + self.operations_dir = self.workload_dir / "operations" + if self.operations_dir.exists() and overwrite: + for file in self.operations_dir.iterdir(): + file.unlink() + + self.operations_dir.mkdir(parents=True, exist_ok=True) + + # set generator state + self.current_base = None + self.current_ids = None + + self.runbook = {} + + def workload_exists(self): + return (self.workload_dir / "runbook.json").exists() + + def generate_workload(self): + """ + Generate the workload. + """ + + self.runbook["parameters"] = { + "query_batch_size": self.query_batch_size, + "num_updates": self.num_updates, + "metric": self.metric, + "pattern_sampling": self.with_patterns, + } + + initial_embeddings = torch.from_numpy(np.load(wikidata_dir / "wikidata" / "embeddings_initial.npy")) + initial_embeddings /= torch.norm(initial_embeddings, dim=1, keepdim=True) + self.current_base = initial_embeddings + self.current_ids = torch.from_numpy(np.load(wikidata_dir / "wikidata" / "ids_initial.npy")) + + self.runbook["initialize"] = { + "size": self.current_ids.shape[0], + } + self.runbook["operations"] = {} + + n_inserts = 0 + n_queries = 0 + n_operations = 0 + + # the first operation is a query operation + n_resident = self.current_ids.shape[0] + assert n_resident == self.current_base.shape[0] + + if not self.with_patterns: + rand_perm = torch.randperm(n_resident)[: self.query_batch_size] + sample_ids = self.current_ids[rand_perm] + queries = self.current_base[rand_perm] + else: + sample_ids = wiki_sampler(self.current_ids, self.query_batch_size, -1) + indices = torch.tensor([torch.where(self.current_ids == s_id)[0][0] for s_id in sample_ids]) + queries = self.current_base[indices] + + entry = {"type": "query", "sample_size": len(sample_ids), "n_resident": n_resident} + torch.save(sample_ids, self.operations_dir / "0.pt") + + # compute the ground truth based on the resident set of vectors + start_time = time.time() + ids, dists = compute_ip_ground_truth(queries, self.current_base, K) + ids = self.current_ids[ids] + gt_time = time.time() - start_time + + entry["gt_time"] = gt_time + + # save the ground truth + torch.save(ids, self.operations_dir / "0_gt_ids.pt") + torch.save(dists, self.operations_dir / "0_gt_dists.pt") + + print("Operation 0", entry) + self.runbook["operations"][0] = entry + + n_queries += 1 + n_operations += 1 + + # for each of the updates, do one insert and one query operation + for i in range(self.num_updates): + print(f"Processing update {i}...") + new_embeddings = torch.from_numpy(np.load(wikidata_dir / "wikidata" / f"embeddings_update_{i}.npy")) + new_embeddings /= torch.norm(new_embeddings, dim=1, keepdim=True) + new_ids = torch.from_numpy(np.load(wikidata_dir / "wikidata" / f"ids_update_{i}.npy")) + + # insert the new embeddings + self.current_base = torch.cat([self.current_base, new_embeddings], dim=0) + self.current_ids = torch.cat([self.current_ids, new_ids], dim=0) + + n_resident = self.current_ids.shape[0] + assert n_resident == self.current_base.shape[0] + + entry = {"type": "insert", "size": new_ids.shape[0], "n_resident": n_resident} + # torch.save(new_ids, self.operations_dir / f"{n_operations}.pt") + + print(f"Operation {n_operations} (insert)", entry) + self.runbook["operations"][n_operations] = entry + + n_operations += 1 + n_inserts += 1 + + # the next operation is a query operation + if not self.with_patterns: + # randomly sample a set of resident embeddings for queries + rand_perm = torch.randperm(n_resident)[: self.query_batch_size] + sample_ids = self.current_ids[rand_perm] + queries = self.current_base[rand_perm] + else: + sample_ids = wiki_sampler(self.current_ids, self.query_batch_size, i) + indices = torch.tensor([torch.where(self.current_ids == s_id)[0][0] for s_id in sample_ids]) + queries = self.current_base[indices] + + entry = {"type": "query", "sample_size": len(sample_ids), "n_resident": n_resident} + torch.save(sample_ids, self.operations_dir / f"{n_operations}.pt") + + # compute the ground truth based on the resident set of vectors + start_time = time.time() + ids, dists = compute_ip_ground_truth(queries, self.current_base, K) + ids = self.current_ids[ids] + gt_time = time.time() - start_time + + entry["gt_time"] = gt_time + + # save the ground truth + torch.save(ids, self.operations_dir / f"{n_operations}_gt_ids.pt") + torch.save(dists, self.operations_dir / f"{n_operations}_gt_dists.pt") + + print(f"Operation {n_operations} (query)", entry) + self.runbook["operations"][n_operations] = entry + + n_operations += 1 + n_queries += 1 + + self.runbook["summary"] = { + "n_inserts": n_inserts, + "n_queries": n_queries, + "n_operations": n_operations, + } + + # save the runbook + with open(self.workload_dir / "runbook.json", "w") as f: + json.dump(self.runbook, f, indent=4) + + +class WikidataWorkloadEvaluator: + """ + Evaluates a Wikidata workload using the same interface as WorkloadEvaluator, + but with explicit NumPy-based embedding handling. + """ + + def __init__( + self, + workload_dir: Union[str, Path], + output_dir: Union[str, Path], + wiki_dataset_dir: Union[str, Path], + ): + self.workload_dir = to_path(workload_dir) + self.wiki_dataset_dir = to_path(wiki_dataset_dir) + self.output_dir = to_path(output_dir) + self.runbook_path = self.workload_dir / "runbook.json" + self.ops_dir = self.workload_dir / "operations" + + # will hold embeddings and ids + self.current_base: Optional[torch.Tensor] = None + self.current_ids: Optional[torch.Tensor] = None + self.index_initialized = False + + def _init_index(self, name: str, wrapper, build_params: Dict, + m_params: Optional[Dict]): + + idx_dir = self.workload_dir / "init_indexes" + idx_dir.mkdir(parents=True, exist_ok=True) + idx_file = idx_dir / f"{name}.index" + + vecs_init = torch.from_numpy(np.load(self.wiki_dataset_dir / "embeddings_initial.npy")) + init_ids = torch.from_numpy(np.load(self.wiki_dataset_dir / "ids_initial.npy")) + + self.current_base = vecs_init + self.current_ids = init_ids + + max_id = int(self.current_ids.max().item()) + inv_map = torch.full((max_id + 1,), -1, dtype=torch.long) + inv_map[self.current_ids] = torch.arange(len(self.current_ids), dtype=torch.long) + self.inv_map = inv_map + + if not idx_file.exists(): + print(f"[{name}] building base index …") + wrapper.build(vecs_init, ids=init_ids, **build_params) + wrapper.save(idx_file) + print(f"[{name}] stored → {idx_file}") + else: + wrapper.load(idx_file, + num_workers=build_params.get("num_workers", 0), use_numa=build_params.get("use_numa", True), parent=build_params.get("parent", None)) + print(f"[{name}] loaded ← {idx_file}") + + if isinstance(wrapper, QuakeWrapper) and m_params: + mp = quake.MaintenancePolicyParams() + for k, v in m_params.items(): + setattr(mp, k, v) + wrapper.index.initialize_maintenance_policy(mp) + print(f"[{name}] maintenance policy: {m_params}") + + + + return wrapper + + def evaluate_workload( + self, + *, + name: str, + index, + build_params: Dict, + search_params: Dict, + do_maintenance: bool = False, + m_params: Optional[Dict] = None, + batch: bool = False, + max_q: int = 1000, + ) -> List[Dict]: + """ + Evaluate inserts, deletes, and queries by loading NumPy embeddings directly. + """ + self.output_dir.mkdir(parents=True, exist_ok=True) + index = self._init_index(name, index, build_params, m_params) + + runbook = json.load(open(self.runbook_path)) + results: List[Dict] = [] + totals = dict(query=0.0, insert=0.0, delete=0.0, maintenance=0.0) + + print(f"─ Evaluating workload on {name} ({len(runbook['operations'])} ops) ─") + for op_id, op in runbook["operations"].items(): + op_no = int(op_id) + typ = op["type"] + print(f"[{name}] op {op_no:4d} | {typ:<6}", end="", flush=True) + + if typ == "insert": + upd_idx = (op_no - 1) // 2 + emb_up = np.load(self.wiki_dataset_dir / f"embeddings_update_{upd_idx}.npy") + id_up = np.load(self.wiki_dataset_dir / f"ids_update_{upd_idx}.npy") + new_emb = torch.from_numpy(emb_up).float() + new_emb /= new_emb.norm(dim=1, keepdim=True) + new_ids = torch.from_numpy(id_up).long() + + t0 = time.perf_counter() + index.add(new_emb, ids=new_ids) + latency_ms = (time.perf_counter() - t0) * 1e3 + recall = None + + # update current state + old_len = len(self.current_ids) + self.current_base = torch.cat([self.current_base, new_emb], dim=0) + self.current_ids = torch.cat([self.current_ids, new_ids], dim=0) + + max_new_id = int(self.current_ids.max().item()) + if max_new_id >= self.inv_map.size(0): + # need a larger map + new_map = torch.full((max_new_id + 1,), -1, dtype=torch.long) + new_map[: self.inv_map.size(0)] = self.inv_map + self.inv_map = new_map + + # positions of the just‐inserted IDs + new_positions = torch.arange(old_len, old_len + len(new_ids), dtype=torch.long) + self.inv_map[new_ids] = new_positions + + elif typ == "delete": + # assume ops_dir contains .pt of ids to delete + ids = torch.load(self.ops_dir / f"{op_id}.pt") + t0 = time.perf_counter() + index.remove(ids) + latency_ms = (time.perf_counter() - t0) * 1e3 + recall = None + + elif typ == "query": + # load sample IDs and map + samp_ids = torch.load(self.ops_dir / f"{op_id}.pt", weights_only=True) + indices = self.inv_map[samp_ids] + queries = self.current_base[indices] + queries /= queries.norm(dim=1, keepdim=True) + + t0 = time.perf_counter() + + randperm = torch.randperm(len(queries)) + queries = queries[randperm[:max_q]] + t_infos = [] + if batch: + sr = index.search(queries, **search_params) + pred_ids = sr.ids + t_infos.append(sr.timing_info) + else: + parts = [] + for q in queries: + out = index.search(q.unsqueeze(0), **search_params) + parts.append(out.ids) + t_infos.append(out.timing_info) + pred_ids = torch.cat(parts) + latency_ms = (time.perf_counter() - t0) * 1e3 + + gt_ids = torch.load(self.ops_dir / f"{op_id}_gt_ids.pt")[randperm[:max_q]] + recall = compute_recall(pred_ids, gt_ids, search_params.get("k")).mean().item() + op["recall"] = recall + + total_parent_time = 0 + total_time = 0 + total_boundary_time = 0 + total_aps_time = 0 + + total_buffer_init_time = 0 + total_copy_query_time = 0 + total_job_enqueue_time = 0 + total_job_wait_time = 0 + total_result_aggregate_time = 0 + +# res->timing_info->buffer_init_time_ns = +# duration_cast(s2 - s1).count(); +# res->timing_info->copy_query_time_ns = +# duration_cast(s3 - s2).count(); +# res->timing_info->job_enqueue_time_ns = +# duration_cast(s4 - s3).count(); +# res->timing_info->job_wait_time_ns = +# duration_cast(s5 - s4).count(); +# res->timing_info->result_aggregate_time_ns = +# duration_cast(s6 - s5).count(); + + for t_info in t_infos: + total_parent_time += t_info.parent_info.total_time_ns / 1e6 + total_time += t_info.total_time_ns / 1e6 + total_boundary_time += t_info.boundary_distance_time_ns / 1e6 + total_aps_time += t_info.aps_time_ns / 1e6 + total_buffer_init_time += t_info.buffer_init_time_ns / 1e6 + total_copy_query_time += t_info.copy_query_time_ns / 1e6 + total_job_enqueue_time += t_info.job_enqueue_time_ns / 1e6 + total_job_wait_time += t_info.job_wait_time_ns / 1e6 + total_result_aggregate_time += t_info.result_aggregate_time_ns / 1e6 + + + + print(f" | parent {total_parent_time:.2f} ms" + f" | total {total_time:.2f} ms" + f" | boundary {total_boundary_time:.2f} ms" + f" | aps {total_aps_time:.2f} ms" + f" | buffer init {total_buffer_init_time:.2f} ms" + f" | copy query {total_copy_query_time:.2f} ms" + f" | job enqueue {total_job_enqueue_time:.2f} ms" + f" | job wait {total_job_wait_time:.2f} ms" + f" | result aggregate {total_result_aggregate_time:.2f} ms") + + + else: + raise ValueError(f"Unknown op type {typ}") + + n_splits = 0 + n_deletes = 0 + split_time_ms = 0.0 + delete_time_ms = 0.0 + refinement_time_ms = 0.0 + maintenance_latency_ms = 0.0 + + if do_maintenance: + t0 = time.perf_counter() + nlist_before = index.index_state()["n_list"] + info = index.maintenance() + nlist_after = index.index_state()["n_list"] + maintenance_latency_ms = (time.perf_counter() - t0) * 1e3 + n_splits = info.n_splits + n_deletes = info.n_deletes + delete_time_ms = info.delete_time_us / 1000.0 + split_time_ms = info.split_time_us / 1000.0 + refinement_time_ms = info.refinement_time_us / 1000.0 + + + + + n_resident = index.index_state()["n_total"] + nlist = index.index_state()["n_list"] + + totals[typ] += latency_ms + totals["maintenance"] += maintenance_latency_ms + print(f" | lat {latency_ms:8.2f} ms" + (f" | rec {recall:.3f}" if recall is not None else "")) + + row = { + "operation_number": op_no, + "operation_type": typ, + "latency_ms": latency_ms, + "n_resident": n_resident, + "n_splits": n_splits, + "n_deletes": n_deletes, + "nlist": nlist, + "split_time_ms": split_time_ms, + "delete_time_ms": delete_time_ms, + "maintenance_latency_ms": maintenance_latency_ms, + "refinement_time_ms": refinement_time_ms, + "recall": recall, + } + print(row) + + results.append(row) + + # four-panel + df = pd.DataFrame(results) + self._four_panel_plot(df) + + # time breakdown + self._time_breakdown_plot(totals, name) + + # CSV output + df.to_csv(self.output_dir / "results.csv", index=False) + print(f"Results → {self.output_dir / 'results.csv'}") + return results + + # --------------------------------------------------------------------- + def _four_panel_plot(self, df: pd.DataFrame): + lat_ins = df[df.operation_type == "insert"] + lat_del = df[df.operation_type == "delete"] + lat_q = df[df.operation_type == "query"] + + fig, axs = plt.subplots(2, 2, figsize=(12, 10)) + ax = axs[0, 0] + if not lat_ins.empty: + ax.plot(lat_ins.operation_number, lat_ins.latency_ms, marker="o", label="Insert") + if not lat_del.empty: + ax.plot(lat_del.operation_number, lat_del.latency_ms, marker="s", label="Delete") + if not lat_q.empty: + ax.plot(lat_q.operation_number, lat_q.latency_ms, marker="^", label="Query") + ax.set(xlabel="Op #", ylabel="Latency (ms)", title="Latency"); ax.legend() + # other panels blank except recall + ax = axs[0, 1] + ax.text(0.5,0.5,"Partitions not tracked",ha="center") + ax.axis("off") + ax = axs[1, 0] + ax.text(0.5,0.5,"Resident not tracked",ha="center") + ax.axis("off") + rec = df[(df.operation_type=="query") & df.recall.notna()] + ax = axs[1, 1] + if not rec.empty: + ax.plot(rec.operation_number, rec.recall, marker="o") + ax.set(xlabel="Op #", ylabel="Recall", title="Recall") + else: + ax.text(0.5,0.5,"No recall",ha="center") + ax.axis("off") + plt.tight_layout() + plt.savefig(self.output_dir / "evaluation_plots.png") + plt.close() + + # --------------------------------------------------------------------- + def _time_breakdown_plot(self, totals: Dict[str, float], title: str): + plt.figure(figsize=(6,4)) + bars = [totals.get("query",0), totals.get("insert",0), totals.get("delete",0), sum(totals.values())] + labels = ["Query","Insert","Delete","Total"] + plt.bar(labels, bars) + plt.ylabel("Cumulative ms") + plt.title(f"Time breakdown – {title}") + plt.tight_layout() + plt.savefig(self.output_dir / "time_breakdown.png", dpi=150) + plt.close() + + +if __name__ == "__main__": + + # parse arguments + import argparse + parser = argparse.ArgumentParser(description="Generate a dynamic workload for Wikidata.") + parser.add_argument( + "--workload_dir", + type=str, + default="data/wikidata/workload", + help="Directory to save the workload.", + ) + parser.add_argument( + "--query_batch_size", + type=int, + default=1000, + help="Batch size for queries between updates.", + ) + + # Initialize the workload generator + args = parser.parse_args() + workload_generator = WikidataWorkloadGenerator( + workload_dir=args.workload_dir, + num_updates=TOTAL_NUM_UPDATES, + query_batch_size=args.query_batch_size, + metric=metric, + with_patterns=True, + overwrite=True, + ) + # Generate the workload + workload_generator.generate_workload() \ No newline at end of file