Merge pull request #457 from fmihpc/master

dev up-to-date

Author: alhom

.github/workflows/make_sbatch_job.sh

@@ -8,7 +8,7 @@ fi
 #   first with srun (in case the file has not yet updated on the front end)
 #   second if srun fails (in case communication failure) it tries to cat it on the frontend (may be empty if file has not been updated pyproject)
 
-sbatch -W --array=1-$ARRAY_SIZE -o "$1" ./testpackage/$1.sh $2 > jobid_$1 || (srun cat $1 || cat $1 || echo "cat failed exit code $?" && exit 1)
+sbatch -W --array=1-$ARRAY_SIZE -o "$1" ./testpackage/$1.sh $2 > jobid_$1 || true
 
 #in case we do exit 0 successfully
 LOG=$(srun cat $1 || cat $1)

analysator/vlsvfile/vlsvcache.py

@@ -55,10 +55,10 @@ def __init__(self, *args, **kwargs):
 
 class VariableCache:
     ''' Class for handling in-memory variable/reducer caching.
+    TODO: Add some maximum memory use guards.
     '''
-    def __init__(self, reader):
+    def __init__(self):
         self.__varcache = {} # {(varname, operator):data}
-        self.__reader = reader
 
     def keys(self):
         return self.__varcache.keys()
@@ -70,7 +70,7 @@ def __setitem__(self, key, value):
        self.__varcache[key] = value
 
 
-    def read_variable_from_cache(self, name, cellids, operator):
+    def read_variable_from_cache(self, reader, name, cellids, operator):
       ''' Read variable from cache instead of the vlsv file.
          :param name: Name of the variable
          :param cellids: a value of -1 reads all data
@@ -90,10 +90,10 @@ def read_variable_from_cache(self, name, cellids, operator):
          if cellids == -1:
             return var_data
          else:
-            return var_data[self.__reader.get_cellid_locations()[cellids]]
+            return var_data[reader.get_cellid_locations()[cellids]]
       else:
          if(len(cellids) > 0):
-            indices = np.array(itemgetter(*cellids)(self.__reader.get_cellid_locations()),dtype=np.int64)
+            indices = np.array(itemgetter(*cellids)(reader.get_cellid_locations()),dtype=np.int64)
          else:
             indices = np.array([],dtype=np.int64)
          if value_len == 1:
@@ -106,22 +106,21 @@ class FileCache:
    '''
 
    def __init__(self, reader) -> None:
-      self.__reader = reader
       self.__metadata_dict = {}
       self.__metadata_read = False
 
       self.__rtree_index_files = []
       self.__rtree_index = None
-      self.__rtree_idxfile = os.path.join(self.get_cache_folder(),"rtree.idx")
-      self.__rtree_datfile = os.path.join(self.get_cache_folder(),"rtree.dat")               
+      self.__rtree_idxfile = os.path.join(self.get_cache_folder(reader),"rtree.idx")
+      self.__rtree_datfile = os.path.join(self.get_cache_folder(reader),"rtree.dat")               
 
       self.__rtree_properties = rtree.index.Property()
       self.__rtree_properties.dimension = 3
       self.__rtree_properties.overwrite=True
 
 
-   def get_cache_folder(self):
-      fn = self.__reader.file_name
+   def get_cache_folder(self, reader):
+      fn = reader.file_name
 
       head,tail = os.path.split(fn)
       path = head
@@ -147,14 +146,14 @@ def get_cache_folder(self):
 
       return path
 
-   def clear_cache_folder(self):
-      path = self.get_cache_folder()
+   def clear_cache_folder(self, reader):
+      path = self.get_cache_folder(reader)
       import shutil
       shutil.rmtree(path)
 
-   def set_cellid_spatial_index(self, force = False):
-      if not os.path.exists(self.get_cache_folder()):
-         os.makedirs(self.get_cache_folder())
+   def set_cellid_spatial_index(self, reader, force = False):
+      if not os.path.exists(self.get_cache_folder(reader)):
+         os.makedirs(self.get_cache_folder(reader))
 
       if force:
          if os.path.exists(self.__rtree_idxfile):
@@ -164,7 +163,7 @@ def set_cellid_spatial_index(self, force = False):
       if(not os.path.isfile(self.__rtree_idxfile) or not os.path.isfile(self.__rtree_datfile)):
          t0 = time.time()
 
-         bboxes = self.__reader.get_mesh_domain_extents("SpatialGrid")
+         bboxes = reader.get_mesh_domain_extents("SpatialGrid")
          bboxes = bboxes.reshape((-1,6), order='C')
 
          self.__rtree_index = rtree.index.Index(self.__rtree_idxfile[:-4],properties=self.__rtree_properties, interleaved=False)
@@ -175,39 +174,39 @@ def set_cellid_spatial_index(self, force = False):
       else:
          pass
 
-   def get_cellid_spatial_index(self, force = False):
+   def get_cellid_spatial_index(self, reader, force = False):
       if self.__rtree_index == None:
          if(force):
-            self.set_cellid_spatial_index(force)
+            self.set_cellid_spatial_index(reader, force)
          elif not os.path.isfile(self.__rtree_idxfile) or not os.path.isfile(self.__rtree_datfile):
             self.__rtree_index = None
          else:
             self.__rtree_index = rtree.index.Index(self.__rtree_idxfile[:-4], properties=self.__rtree_properties, interleaved=False)
 
       return self.__rtree_index
 
-   def add_metadata(self, key, value):
+   def add_metadata(self, reader, key, value):
       self.__metadata_dict[key] = value
-      self.save_metadata()
+      self.save_metadata(reader)
 
-   def get_metadata_filename(self):
-      pth, base = os.path.split(self.__reader.file_name)
-      path = self.get_cache_folder()
+   def get_metadata_filename(self, reader):
+      pth, base = os.path.split(reader.file_name)
+      path = self.get_cache_folder(reader)
 
       s = os.path.join(path,"metadata.pkl")
       return s
 
-   def save_metadata(self):
-      fn = self.get_metadata_filename()
-      if not os.path.exists(self.get_cache_folder()):
-         os.makedirs(self.get_cache_folder())
+   def save_metadata(self, reader):
+      fn = self.get_metadata_filename(reader)
+      if not os.path.exists(self.get_cache_folder(reader)):
+         os.makedirs(self.get_cache_folder(reader))
       try:
          with open(fn,'wb') as f:
             pickle.dump(self.__metadata_dict,f)
       except Exception as e:
          logging.warning("Could not save metadata file, error: "+str(e))
 
-   def get_metadata(self, key, default):
+   def get_metadata(self, reader, key, default):
       ''' Read metadata from metadata file/memory, and if not available,
       return the given default value.
 
@@ -217,11 +216,11 @@ def get_metadata(self, key, default):
 
       if not self.__metadata_read:
          try:
-            fn = self.get_metadata_filename()
+            fn = self.get_metadata_filename(reader)
             with open(fn,'rb') as f:
                self.__metadata_dict = pickle.load(f)
          except Exception as e:
-            logging.debug("No metadata file found at "+self.get_metadata_filename()+":\n"+str(e))
+            logging.debug("No metadata file found at "+self.get_metadata_filename(reader)+":\n"+str(e))
 
          self.__metadata_read = True
 

analysator/vlsvfile/vlsvreader.py

@@ -202,7 +202,7 @@ def __init__(self, file_name, fsGridDecomposition=None, file_cache = 0):
       self.__order_for_cellid_blocks = {} # per-pop
       self.__vg_indexes_on_fg = np.array([]) # SEE: map_vg_onto_fg(self)
 
-      self.__variable_cache = vlsvcache.VariableCache(self) # {(varname, operator):data}
+      self.__variable_cache = vlsvcache.VariableCache() # {(varname, operator):data}
       self.__params_cache = {} # {name:data}
 
       self.__pops_init = False
@@ -323,11 +323,10 @@ def get_linked_readers_filename(self):
       '''Need to go to a consolidated metadata handler - keeping human-readable for now'''
       pth, base = os.path.split(self.file_name)
 
-      s = os.path.join(self.__metadata_cache.get_cache_folder(),"linked_readers.txt")
+      s = os.path.join(self.__metadata_cache.get_cache_folder(self),"linked_readers.txt")
       return s
 
    def get_linked_readers(self, reload=False):
-      # self.__linked_files = self.__metadata_cache.get_metadata("linked_reader_files", set())
       if len(self.__linked_files)==0 or reload:
          if(os.path.isfile(self.get_linked_readers_filename())):
             with open(self.get_linked_readers_filename(), 'r') as f:
@@ -549,7 +548,7 @@ def __read_fileindex_for_cellid(self):
       # print("fileindex!")
       cellids=self.read(mesh="SpatialGrid",name="CellID", tag="VARIABLE")
 
-      #Check if it is not iterable. If it is a scale then make it a list
+      #Check if it is not iterable. If it is a scalar then make it a list (single-cell runs?)
       if(not isinstance(cellids, Iterable)):
          cellids=[ cellids ]
       # self.__fileindex_for_cellid = {cellid:index for index,cellid in enumerate(cellids)}
@@ -1430,7 +1429,12 @@ def read(self, name="", tag="", mesh="", operator="pass", cellids=-1):
                data=data.reshape(result_size, vector_size)
 
             if not isinstance(cellids, numbers.Number):
-               data_out = np.full_like(data, np.nan, shape=(len(cellids),*data.shape[1:]))
+               if np.issubdtype(data.dtype, np.floating):
+                  data_out = np.full_like(data, np.nan, shape=(len(cellids),*data.shape[1:]))
+               elif np.issubdtype(data.dtype, np.integer):
+                  data_out = np.full_like(data, np.iinfo(data.dtype).min, shape=(len(cellids),*data.shape[1:]))
+               else:
+                  raise ValueError("unexpected dtype encountered in read ("+str(data.dtype)+")")
                data_out[cellids!=0,...] = data
                data = data_out
 
@@ -2074,7 +2078,7 @@ def read_fsgrid_variable_cellid(self, name, cellids=-1, operator="pass"):
    def get_fsgrid_decomposition(self):
       # Try if in metadata
       if(self.__fsGridDecomposition is not None):
-         print("read ",self.__fsGridDecomposition)
+         logging.info("read " + str(self.__fsGridDecomposition))
 
       if self.__fsGridDecomposition is None:
          self.__fsGridDecomposition = self.read(tag="MESH_DECOMPOSITION",mesh='fsgrid')
@@ -2085,7 +2089,7 @@ def get_fsgrid_decomposition(self):
             logging.info("Did not find FsGrid decomposition from vlsv file.")
 
       if self.__fsGridDecomposition is None:
-         self.__fsGridDecomposition = self.__metadata_cache.get_metadata(("MESH_DECOMPOSITION","fsgrid"),None)
+         self.__fsGridDecomposition = self.__metadata_cache.get_metadata(self,("MESH_DECOMPOSITION","fsgrid"),None)
          if self.__fsGridDecomposition is not None:
             logging.info("Found FsGrid decomposition from metadata file: " + str(self.__fsGridDecomposition))
             return self.__fsGridDecomposition
@@ -2098,7 +2102,7 @@ def get_fsgrid_decomposition(self):
          logging.info("Calculating fsGrid decomposition from the file")
          self.__fsGridDecomposition = fsDecompositionFromGlobalIds(self)
          logging.info("Computed FsGrid decomposition to be: " + str(self.__fsGridDecomposition))
-         self.__metadata_cache.add_metadata(("MESH_DECOMPOSITION","fsgrid"), self.__fsGridDecomposition)
+         self.__metadata_cache.add_metadata(self,("MESH_DECOMPOSITION","fsgrid"), self.__fsGridDecomposition)
          return self.__fsGridDecomposition
       else:
          # Decomposition is a list (or fail assertions below) - use it instead
@@ -2245,8 +2249,8 @@ def print_metadata_cache(self):
       ''' Prints the contents of the metadata cache file.
       '''
 
-      print("Metadata cache at "+self.__metadata_cache.get_metadata_filename()+":")
-      self.__metadata_cache.get_metadata("dummy",None) # Dummy call to read in the metadata file
+      print("Metadata cache at "+self.__metadata_cache.get_metadata_filename(self)+":")
+      self.__metadata_cache.get_metadata(self,"dummy",None) # Dummy call to read in the metadata file
       for k,v in self.__metadata_cache._FileCache__metadata_dict.items():
          print(k, v)
 
@@ -2690,8 +2694,11 @@ def do_partial_fileindex_update(self, coords):
       if coords.shape[0] == 0:
          return
 
+      # We already know everything, do nothing and return
+      if self.__full_fileindex_for_cellid:
+         return
+
       if self.get_cellid_spatial_index() == None:
-         
          self.__read_fileindex_for_cellid()
          return
 
@@ -4347,17 +4354,17 @@ def load_neighbor_stencils_from_filecache(self):
             self.__neighbors_cache_loaded = True
 
    def set_cellid_spatial_index(self, force=False):
-      self.__cellid_spatial_index =  self.__metadata_cache.set_cellid_spatial_index(force)
+      self.__cellid_spatial_index =  self.__metadata_cache.set_cellid_spatial_index(self, force)
 
    def get_cellid_spatial_index(self, force=False):
       return None
       if not force:
          if self.__cellid_spatial_index is None:   
-            self.__cellid_spatial_index = self.__metadata_cache.get_cellid_spatial_index(force)
+            self.__cellid_spatial_index = self.__metadata_cache.get_cellid_spatial_index(self, force)
          else:
             pass
       else:
-         self.__cellid_spatial_index =  self.__metadata_cache.set_cellid_spatial_index(force)
+         self.__cellid_spatial_index =  self.__metadata_cache.set_cellid_spatial_index(self, force)
 
       return self.__cellid_spatial_index
 
@@ -4370,4 +4377,4 @@ def cache_optimization_files(self, force=False):
       ''' Create cached optimization files for this reader object (e.g. spatial index)
       
       '''
-      self.__metadata_cache.set_cellid_spatial_index(force)
+      self.__metadata_cache.set_cellid_spatial_index(self, force)