Merge pull request #211 from Exabyte-io/update/SOF-7324

Update/SOF-7324 Fix API-notebooks

Author: VsevolodX

config.yml

@@ -70,3 +70,7 @@ notebooks:
   - name: specific_examples
     packages_common:
       - mat3ra-standata
+  - name: api_examples
+    packages_common:
+      - mat3ra-standata
+      - exabyte-api-client

examples/job/create_and_submit_job.ipynb

@@ -90,10 +90,9 @@
     "    import micropip\n",
     "\n",
     "    await micropip.install(\"mat3ra-api-examples\", deps=False)\n",
-    "    await micropip.install(\"exabyte-api-client\")\n",
     "    from utils.jupyterlite import install_packages\n",
     "\n",
-    "    await install_packages(\"\")"
+    "    await install_packages(\"api_examples\")"
    ]
   },
   {
@@ -181,8 +180,7 @@
     "default_workflow = workflow_endpoints.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0]\n",
     "\n",
     "material_id = default_material[\"_id\"]\n",
-    "workflow_id = default_workflow[\"_id\"]\n",
-    "owner_id = default_material[\"owner\"][\"_id\"]"
+    "workflow_id = default_workflow[\"_id\"]"
    ]
   },
   {
@@ -206,7 +204,7 @@
    "outputs": [],
    "source": [
     "config = {\n",
-    "    \"owner\": {\"_id\": owner_id},\n",
+    "    \"owner\": {\"_id\": OWNER_ID},\n",
     "    \"_material\": {\"_id\": material_id},\n",
     "    \"workflow\": {\"_id\": workflow_id},\n",
     "    \"name\": JOB_NAME,\n",

examples/job/get-file-from-job.ipynb

@@ -109,7 +109,6 @@
     "    import micropip\n",
     "\n",
     "    await micropip.install(\"mat3ra-api-examples\", deps=False)\n",
-    "    await micropip.install(\"exabyte-api-client\")\n",
     "    from utils.jupyterlite import install_packages\n",
     "\n",
     "    await install_packages(\"\")"
@@ -320,14 +319,9 @@
     "outputId": "e4e2e6b1-8431-439a-8606-ae500534e601"
    },
    "source": [
-    "import urllib\n",
+    "from mat3ra.utils.jupyterlite.url import read_from_url\n",
     "\n",
-    "server_response = urllib.request.urlopen(output_file_metadata[\"signedUrl\"])\n",
-    "output_file_bytes = server_response.read()\n",
-    "\n",
-    "# The server returns us a bytes-string. That's useful for things like binaries or other non-human-readable data, but this should be decoded if we're planning to write to console.\n",
-    "# Because this is a human-readable text file, we'll decode it to UTF-8.\n",
-    "output_file = output_file_bytes.decode(encoding=\"UTF-8\")\n",
+    "output_file = await read_from_url(output_file_metadata[\"signedUrl\"])\n",
     "\n",
     "# Tail the last 90 lines\n",
     "lines = output_file.split(\"\\n\")\n",
@@ -344,50 +338,20 @@
     "id": "-z7VqM3K8GIx"
    },
    "source": [
-    "### Save the input file and output file to disk.\n",
+    "### Save the output file to disk.\n",
     "\n",
     "Now that we've verified the job is done, let's go ahead and save it and its input to disk."
    ]
   },
-  {
-   "cell_type": "code",
-   "metadata": {
-    "id": "osy2JKpB8GIx"
-   },
-   "source": [
-    "# We've already got an output file, so let's grab the input file we sent to Quantum Espresso\n",
-    "for file in files:\n",
-    "    if \"pw_scf.in\" == file[\"name\"]:\n",
-    "        input_file_metadata = file\n",
-    "server_response = urllib.request.urlopen(input_file_metadata[\"signedUrl\"])\n",
-    "input_file_bytes = server_response.read()"
-   ],
-   "outputs": [],
-   "execution_count": null
-  },
   {
    "cell_type": "code",
    "metadata": {
     "id": "COPinBrO8GIx"
    },
    "source": [
-    "# Let's write the input file to disk. Note that we get files as a bytes string from the server, which is convenient for binaries, images, and other non-human-readable data.\n",
-    "# Although we could decode before writing to disk, we can just write it directly with the \"wb\" (write bytes) file mode.\n",
-    "with open(input_file_metadata[\"name\"], \"wb\") as file_descriptor:\n",
-    "    file_descriptor.write(input_file_bytes)"
-   ],
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "cell_type": "code",
-   "metadata": {
-    "id": "rzDcWyrL8GIx"
-   },
-   "source": [
-    "# Now, let's write our output file to the disk. Note that because we already decoded it, we can just use the 'w' file mode.\n",
-    "with open(output_file_metadata[\"name\"], \"w\") as file_descriptor:\n",
-    "    file_descriptor.write(output_file)"
+    "from utils.jupyterlite import download_content_to_file\n",
+    "\n",
+    "download_content_to_file(output_file_metadata[\"name\"], output_file)"
    ],
    "outputs": [],
    "execution_count": null

examples/job/ml-train-model-predict-properties.ipynb

@@ -110,10 +110,9 @@
     "    import micropip\n",
     "\n",
     "    await micropip.install(\"mat3ra-api-examples\", deps=False)\n",
-    "    await micropip.install(\"exabyte-api-client\")\n",
     "    from utils.jupyterlite import install_packages\n",
     "\n",
-    "    await install_packages(\"\")"
+    "    await install_packages(\"api_examples\")"
    ],
    "outputs": [],
    "execution_count": null
@@ -168,11 +167,11 @@
    "source": [
     "#### Materials\n",
     "\n",
-    "Set queries to import materials from Materials Bank:\n",
+    "Set parameters for the materials to be imported:\n",
     "\n",
-    "- **TRAIN_MATERIAL_SI_QUERY**: Query to import Si materials.\n",
-    "- **TRAIN_MATERIAL_SIGE_QUERY**: Query to import SiGe materials.\n",
-    "- **TARGET_MATERIAL_GE_QUERY**: Query to import Ge materials."
+    "- **TRAIN_MATERIAL_SI**: 1st material to be used for training the model (\"Si\").\n",
+    "- **TRAIN_MATERIAL_GE**: 2nd material to be used for training the model (\"Ge\").\n",
+    "- **TARGET_MATERIAL_SIGE**: Material to be used for predicting the properties (\"SiGe\")."
    ]
   },
   {
@@ -181,9 +180,9 @@
     "id": "qeZIEenS2Qxb"
    },
    "source": [
-    "TRAIN_MATERIAL_SI_QUERY = {\"formula\": \"Si\"}\n",
-    "TRAIN_MATERIAL_SIGE_QUERY = {\"formula\": \"SiGe\"}\n",
-    "TARGET_MATERIAL_GE_QUERY = {\"formula\": \"Ge\"}"
+    "TRAIN_MATERIAL_SI = \"Si\"\n",
+    "TRAIN_MATERIAL_GE = \"Ge\"\n",
+    "TARGET_MATERIAL_SIGE = \"SiGe\""
    ],
    "outputs": [],
    "execution_count": null
@@ -314,8 +313,10 @@
     "id": "_JUT4f9E2Qxe"
    },
    "source": [
-    "band_gap_workflow_id = copy_bank_workflow_by_system_name(bank_workflow_endpoints, \"espresso-band-gap\", owner_id)\n",
-    "ml_train_workflow_id = copy_bank_workflow_by_system_name(bank_workflow_endpoints, \"exabyteml-ml-train-model\", owner_id)"
+    "band_gap_workflow_id = copy_bank_workflow_by_system_name(bank_workflow_endpoints, \"espresso-band-gap\", OWNER_ID)\n",
+    "ml_train_workflow_id = copy_bank_workflow_by_system_name(\n",
+    "    bank_workflow_endpoints, \"exabyteml-ml-linear-least-squares-train-model\", OWNER_ID\n",
+    ")"
    ],
    "outputs": [],
    "execution_count": null
@@ -329,7 +330,7 @@
    "source": [
     "### Import materials\n",
     "\n",
-    "Import materials from Materials Bank."
+    "Import materials from Standata and create them in the account's materials collection."
    ]
   },
   {
@@ -338,16 +339,22 @@
     "id": "t1dMm9ke2Qxe"
    },
    "source": [
-    "bank_train_materials_id_si = bank_material_endpoints.list(TRAIN_MATERIAL_SI_QUERY)[0][\"_id\"]\n",
-    "bank_train_materials_id_sige = bank_material_endpoints.list(TRAIN_MATERIAL_SIGE_QUERY)[0][\"_id\"]\n",
-    "bank_target_materials_id_ge = bank_material_endpoints.list(TARGET_MATERIAL_GE_QUERY)[0][\"_id\"]\n",
+    "from mat3ra.standata.materials import Materials\n",
+    "\n",
+    "train_material_json_si = Materials.get_by_name_first_match(TRAIN_MATERIAL_SI)\n",
+    "train_material_json_ge = Materials.get_by_name_first_match(TRAIN_MATERIAL_GE)\n",
+    "target_material_json_sige = Materials.get_by_name_first_match(TARGET_MATERIAL_SIGE)\n",
+    "\n",
+    "train_material_id_si = material_endpoints.create(train_material_json_si, OWNER_ID)[\"_id\"]\n",
+    "train_material_id_ge = material_endpoints.create(train_material_json_ge, OWNER_ID)[\"_id\"]\n",
+    "target_material_id_sige = material_endpoints.create(target_material_json_sige, OWNER_ID)[\"_id\"]\n",
     "\n",
-    "train_materials_id_si = bank_material_endpoints.copy(bank_train_materials_id_si, owner_id)[\"_id\"]\n",
-    "train_materials_id_sige = bank_material_endpoints.copy(bank_train_materials_id_sige, owner_id)[\"_id\"]\n",
-    "target_materials_id_ge = bank_material_endpoints.copy(bank_target_materials_id_ge, owner_id)[\"_id\"]\n",
+    "train_material_si = material_endpoints.list({\"_id\": train_material_id_si})[0]\n",
+    "train_material_ge = material_endpoints.list({\"_id\": train_material_id_ge})[0]\n",
+    "target_material_sige = material_endpoints.list({\"_id\": target_material_id_sige})[0]\n",
     "\n",
-    "train_materials = material_endpoints.list({\"_id\": {\"$in\": [train_materials_id_si, train_materials_id_sige]}})\n",
-    "target_materials = material_endpoints.list({\"_id\": target_materials_id_ge})"
+    "train_materials = [train_material_ge, train_material_si]\n",
+    "target_materials = [target_material_sige]"
    ],
    "outputs": [],
    "execution_count": null
@@ -372,7 +379,7 @@
    "source": [
     "compute = job_endpoints.get_compute(CLUSTER, PPN, NODES, QUEUE, TIME_LIMIT)\n",
     "jobs = job_endpoints.create_by_ids(\n",
-    "    train_materials, band_gap_workflow_id, project_id, JOB_NAME_PREFIX, owner_id, compute\n",
+    "    train_materials, band_gap_workflow_id, project_id, JOB_NAME_PREFIX, OWNER_ID, compute\n",
     ")"
    ],
    "outputs": [],