diff --git a/docs/hub/experimaestro-ir.md b/docs/hub/experimaestro-ir.md new file mode 100644 index 000000000..973adcb0b --- /dev/null +++ b/docs/hub/experimaestro-ir.md @@ -0,0 +1,68 @@ +# Using experimaestro-IR at Hugging Face + +`experimaestro-IR` is an open-source toolkit for neural information retrieval models. It allows using and building experiments around those models, with a focus on reusable components. More up-to-date documentation can be found on the [experimaestro-IR pre-trained model documentation page](https://experimaestro-ir.readthedocs.io/en/latest/pretrained.html). + +## Exploring experimaestro-IR in the Hub + +You can find `experimaestro-IR` models by filtering at the left of the [models page](https://huggingface.co/models?library=xpmir). + +All models on the Hub come up with useful features: +1. An automatically generated model card with a description and metrics on IR datasets; +2. Metadata tags that help for discoverability. + + +## Install the library +To install the `experimaestro-IR` library, you can use pip: + +```sh +pip install experimaestro-ir +``` + +## Using existing models + +You can simply download a model from the Hub using `xpmir.models.AutoModel`. +Thanks to the [experimaestro framework](https://github.com/experimaestro/experimaestro-python), +you can either use models in your own experiments or in pure inference mode. + +### As experimental models + +In this mode, you can reuse the model in your experiments -- e.g. to compare this model +with your own, or using it in a complex IR pipeline (e.g. distillation). Please +refer to the [experimaestro-IR documentation](https://experimaestro-ir.readthedocs.io/) +for more details. + +```py +from xpmir.models import AutoModel + +# Model that can be re-used in experiments +model = AutoModel.load_from_hf_hub("xpmir/monobert") +``` + +### Pure inference mode + +In this mode, the model can be used right away to score documents + +```py +from xpmir.models import AutoModel + +# Use this if you want to actually use the model +model = AutoModel.load_from_hf_hub("xpmir/monobert", as_instance=True) +model.initialize(None) +model.rsv("walgreens store sales average", "The average Walgreens salary ranges...") +``` + + +## Sharing your models + +You can easily upload your models using `AutoModel.push_to_hf_hub`: + +``` +from xpmir.models import AutoModel + +AutoModel.push_to_hf_hub(model, readme=readme_md) +``` + +## Additional resources + +* Experimaestro-IR [documentation](https://experimaestro-ir.readthedocs.io/en/latest/pretrained.html) +* Experimaestro-IR [huggingface integration documentation](https://experimaestro-ir.readthedocs.io/en/latest/pretrained.html) diff --git a/docs/hub/models-libraries.md b/docs/hub/models-libraries.md index 2b6335473..ea056bd79 100644 --- a/docs/hub/models-libraries.md +++ b/docs/hub/models-libraries.md @@ -13,6 +13,7 @@ The table below summarizes the supported libraries and their level of integratio | [Asteroid](https://github.com/asteroid-team/asteroid) | Pytorch-based audio source separation toolkit | ✅ | ✅ | ✅ | ❌ | | [docTR](https://github.com/mindee/doctr) | Models and datasets for OCR-related tasks in PyTorch & TensorFlow | ✅ | ✅ | ✅ | ❌ | | [ESPnet](https://github.com/espnet/espnet) | End-to-end speech processing toolkit (e.g. TTS) | ✅ | ✅ | ✅ | ❌ | +| [experimaestro IR](https://github.com/experimaestro/experimaestro-ir) | Library for (neural) information retrieval | ❌ | ❌ | ✅ | ✅ | | [fastai](https://github.com/fastai/fastai) | Library to train fast and accurate models with state-of-the-art outputs. | ✅ | ✅ | ✅ | ✅ | | [Keras](https://huggingface.co/docs/hub/keras) | Library that uses a consistent and simple API to build models leveraging TensorFlow and its ecosystem. | ❌ | ❌ | ✅ | ✅ | | [Flair](https://github.com/flairNLP/flair) | Very simple framework for state-of-the-art NLP. | ✅ | ✅ | ✅ | ❌ | diff --git a/js/src/lib/interfaces/Libraries.ts b/js/src/lib/interfaces/Libraries.ts index 8729a8ca0..c88c781c9 100644 --- a/js/src/lib/interfaces/Libraries.ts +++ b/js/src/lib/interfaces/Libraries.ts @@ -33,6 +33,7 @@ export enum ModelLibrary { "stable-baselines3" = "Stable-Baselines3", "ml-agents" = "ML-Agents", "pythae" = "Pythae", + "xpmir" = "Experimaestro IR" } export type ModelLibraryKey = keyof typeof ModelLibrary; @@ -425,6 +426,21 @@ const pythae = (model: ModelData) => model = AutoModel.load_from_hf_hub("${model.id}")`; +const xpmir = (model: ModelData) => { + if (model.config?.variants?.length > 0) { + return `from xpmir.models import AutoModel + +// Use a variant among: +// ${model.config.variants.join(", ")} +model = AutoModel.load_from_hf_hub("${model.id}", variant)`; + } + + return `from xpmir.models import AutoModel + +model = AutoModel.load_from_hf_hub("${model.id}")`; + +} + //#endregion @@ -587,5 +603,11 @@ export const MODEL_LIBRARIES_UI_ELEMENTS: Partial