Feature/gloo spark lightfm wrapper 0.2 #63
Conversation
| features_columns = features_np[:, 1:] | ||
| number_of_features = features_columns.shape[1] | ||
| # Scale down dense features | ||
| scaler_name = f"{entity}_feat_scaler" |
There was a problem hiding this comment.
Explicit is better than implicit.
def init(....)
self.__item_feat_scaler: Optional[Scaler] = None
self.__user_feat_scaler: Optional[Scaler] = None
def __get_scaler(entity: str) -> Scaler:
# create or choose from __item_feat_scaler or __user_feat_scaler
...
if self.__feat_scaler is None:
pass
There was a problem hiding this comment.
let's leave for now as it is. The logic why you do what you do is clear now.
But add an extensive comments to justify why you do it this way
| self.__setattr__(scaler_name, MinMaxScaler().fit(features_columns)) | ||
|
|
||
| if features_columns.size: | ||
| features_dense = self.__getattribute__(scaler_name).transform( |
| def _convert_features_to_csr( | ||
| self, entity_ids: DataFrame, features: Optional[DataFrame] = None | ||
| ) -> Optional[sp.csr_matrix]: | ||
| """ |
| "user_bias_momentum", | ||
| ) | ||
| losses = ["warp", "bpr", "warp-kos", "logistic"] | ||
| user_feat_scaler: Optional[MinMaxScaler] = None |
There was a problem hiding this comment.
remove from here and move to init (filed of the instance, not field of the class)
| losses = ["warp", "bpr", "warp-kos", "logistic"] | ||
| user_feat_scaler: Optional[MinMaxScaler] = None | ||
| item_feat_scaler: Optional[MinMaxScaler] = None | ||
| logger = logging.getLogger("replay") |
There was a problem hiding this comment.
remove from here
Better options right in the module file:
logger = logging.getLogger(name)
| self._initialize_model() | ||
| self._initialize_world_size_and_threads() | ||
|
|
||
| if user_features is not None: |
There was a problem hiding this comment.
if user_features is not None:
self.can_predict_cold_users = True
- >
self.can_predict_cold_users = user_features is not None
|
|
||
| if user_features is not None: | ||
| self.can_predict_cold_users = True | ||
| if item_features is not None: |
| _num_epochs = self._num_epochs | ||
| _pygloo_timeout = self._pygloo_timeout_sec | ||
|
|
||
| def udf_to_map_on_interactions_with_index( |
There was a problem hiding this comment.
Not a comment, but rather TODO for future: need to reconsider whole this function
| """Trainer used in distributed LightFM.""" | ||
|
|
||
| def __init__(self, model: LightFM, world_size: int, num_threads: int): | ||
| self.model = model |
|
|
||
| # pylint: disable=too-many-instance-attributes, too-many-arguments | ||
| class LightFMTraining: | ||
| """Trainer used in distributed LightFM.""" |
There was a problem hiding this comment.
add more important details to this docstring
No description provided.