Auto build cambridge library - add dependencies and new probes (#39)

* debug

* Add dependencies

* Add dependencies 2

* Fix args

* Fix script

* Unify check_for_new_probes and add to generate CN action

* Fix check new probes script

Author: alejoe91

.github/workflows/check_for_new_NP_probes.yml

@@ -32,16 +32,14 @@ jobs:
       - name: Install probeinterface and matplotlib
         run: pip install ./probeinterface matplotlib
 
-      - name: Generate full NP library
+      - name: Generate full NP library and check for new probes
         run: |
           cd scripts/
           python ../probeinterface/resources/generate_neuropixels_library.py
+          # check for new probes
+          python check_for_new_probes.py ../imec/ ./neuropixels_library_generated/
 
-      # Check for any new probes
-      - name: Run local script
-        run: |
-          cd scripts/
-          python check_for_new_NP_probes.py
+          # clean up
           rm -r neuropixels_library_generated/
           cd ..
           rm -r ./probeinterface

.github/workflows/generate_cambridge_neurotech_library.yml

@@ -33,16 +33,19 @@ jobs:
           # cambridge neurotech probe maps
           git clone https://github.com/cambridge-neurotech/probe_maps.git ./probe_maps
 
-      - name: Install probeinterface and matplotlib
-        run: pip install ./probeinterface matplotlib
+      - name: Install probeinterface and dependencies
+        run: pip install ./probeinterface matplotlib pandas shapely openpyxl tqdm
 
       - name: Generate full cambridge library and check for changes
         run: |
           cd scripts/
-          python ../probeinterface/resources/generate_cambridgeneurotech_library.py ../probe_maps/ ./cambridge_library_generated/
+          python ../probeinterface/resources/generate_cambridgeneurotech_library.py ../probe_maps/ --output-folder ./cambridge_library_generated/
+
+          # check for new probes
+          python check_for_new_probes.py ../cambridgeneurotech/ ./cambridge_library_generated/
 
           # check for json changes (except probeinterface version)
-          python check_for_json_changes_in_probes.py ../cambridgeneurotech/ ./cambridge_library_generated/
+          python check_for_json_changes_in_probes.py ../cambridgeneurotech/ ./cambridge_library_generated/ --copy-figures
           # clean up
           rm -r cambridge_library_generated/
           cd ..
@@ -61,7 +64,7 @@ jobs:
             echo "No changes to commit"
             echo "changes=false" >> $GITHUB_OUTPUT
           else
-            git commit -m "Update json files for NP probes"
+            git commit -m "Update json files for cambridge neurotech probes"
             echo "changes=true" >> $GITHUB_OUTPUT
           fi
 
@@ -70,6 +73,6 @@ jobs:
         uses: peter-evans/create-pull-request@v7
         with:
           title: "Update Cambridge Neurotech json files"
-          body: "This PR updates the Neuropixel probes in probeinterace library, based on new data from the ProbeTable repository or because of a new ProbeInterface release."
+          body: "This PR updates the Cambridge Neurotech probes in probeinterace library, based on new data from the cambridge-neurotech/probe_maps repository or because of a new ProbeInterface release."
           branch-suffix: short-commit-hash
           base: "main"

scripts/check_for_json_changes_in_probes.py

@@ -17,6 +17,12 @@
     help="Path to the new probes directory",
 )
 
+parser.add_argument(
+    "--copy-figures",
+    action="store_true",
+    help="If set, copies figures as well when JSON files are different.",
+)
+
 
 if __name__ == "__main__":
     args = parser.parse_args()
@@ -40,5 +46,11 @@
             if lines1[3:] == lines2[3:]:
                 continue
             else:
-                shutil.copy(f"{temp_probe_json_path}", f"../imec/{probe_name}")
+                shutil.copy(f"{temp_probe_json_path}", old_dir / probe_name)
+                if args.copy_figures:
+                    temp_figure_path = temp_probe_directory / (probe_name + '.png')
+                    shutil.copy(f"{temp_figure_path}", old_dir / probe_name)
+                
+                
+                            
 

scripts/check_for_new_NP_probes.py

@@ -0,0 +1,27 @@
+from argparse import ArgumentParser
+from pathlib import Path
+import shutil
+
+
+parser = ArgumentParser(description="Check for new probes from manufacturer")
+parser.add_argument(
+    "old_dir",
+    type=str,
+    help="Path to the old probes directory",
+)
+parser.add_argument(
+    "new_dir",
+    type=str,
+    help="Path to the new probes directory",
+)
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    old_dir = Path(args.old_dir)
+    new_dir = Path(args.new_dir)
+
+    existing_probes = list(probe_path.name for probe_path in old_dir.iterdir())
+
+    for temp_probe_path in new_dir.iterdir():
+        if temp_probe_path.name not in existing_probes:
+            shutil.copytree(temp_probe_path, old_dir / temp_probe_path.name)