IntelCPU_for_allHF_models.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "installiong required libraries"
      ],
      "metadata": {
        "id": "U55pHUq-cpJP"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "js-hLKRRceo8"
      },
      "outputs": [],
      "source": [
        "%pip install intel-extension-for-pytorch"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%pip install sentence-transformers"
      ],
      "metadata": {
        "id": "jDZXaDa8chUl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%pip install intel-extension-for-pytorch"
      ],
      "metadata": {
        "id": "681imohZcn9r"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import intel_extension_for_pytorch as ipex"
      ],
      "metadata": {
        "id": "d9EDyNKPcsa_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "loading models from sentence-transformers"
      ],
      "metadata": {
        "id": "mi4a-MH9cv6w"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sentence_transformers import SentenceTransformer, util\n",
        "\n",
        "model_embed = SentenceTransformer(\"BAAI/bge-m3\").eval()"
      ],
      "metadata": {
        "id": "-uy1MrqZcykI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We have put the model on eval model because Intel extension supports this method to load on Intel architecture"
      ],
      "metadata": {
        "id": "1xgTUeTPc6Au"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "we're using ipex to optimize the embedding model for intel architecture (CPU only)"
      ],
      "metadata": {
        "id": "zIiX0_p_dEYm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_embed = ipex.optimize(model_embed)"
      ],
      "metadata": {
        "id": "cLK_0TGTc0-o"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "query_embedding = model_embed.encode('How big is London')\n",
        "passage_embedding = model_embed.encode(['London has 9,787,426 inhabitants at the 2011 census', 'London is known for its finacial district'])\n",
        "\n",
        "print(\"Similarity:\", util.dot_score(query_embedding, passage_embedding))"
      ],
      "metadata": {
        "id": "oBUm8vV2c2cB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now, loading any random HF model of your choice and let's get started with Intel hardware"
      ],
      "metadata": {
        "id": "VXDhZgANdPcg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
        "\n",
        "model_path = \"gpt2\"\n",
        "\n",
        "model = AutoModelForCausalLM.from_pretrained(model_path).eval()\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_path)"
      ],
      "metadata": {
        "id": "xpXV8EnVdo1L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dtype = torch.float\n",
        "model = ipex.llm.optimize(model, dtype=dtype)"
      ],
      "metadata": {
        "id": "HdQ1Kttqejmk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "input_prompt = \"Once upon a time\"\n",
        "input_ids = tokenizer.encode(input_prompt, return_tensors='pt')"
      ],
      "metadata": {
        "id": "9Whu4_8kelL2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with torch.no_grad():\n",
        "    output_ids = model.generate(input_ids, max_length=50, num_return_sequences=1)"
      ],
      "metadata": {
        "id": "bQeNCN0qem5e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "generated_text = tokenizer.decode(output_ids[0], skip_special_tokens=True)\n",
        "\n",
        "print(f\"Input prompt: {input_prompt}\")\n",
        "print(f\"Generated text: {generated_text}\")"
      ],
      "metadata": {
        "id": "_NyOr7yBepDw"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}