set log_level and dump_logs flag

Signed-off-by: Mamta Singh <[email protected]>

Author: mamtsing

QEfficient/cloud/finetune.py

@@ -5,7 +5,6 @@
 #
 # -----------------------------------------------------------------------------
 
-import logging
 import random
 import warnings
 from typing import Any, Dict, Optional, Union
@@ -41,8 +40,6 @@
 from QEfficient.utils._utils import login_and_download_hf_lm
 from QEfficient.utils.logging_utils import logger
 
-logger.setLevel(logging.INFO)
-
 # Try importing QAIC-specific module, proceed without it if unavailable
 try:
     import torch_qaic  # noqa: F401
@@ -330,6 +327,9 @@ def main(peft_config_file: str = None, **kwargs) -> None:
     dataset_config = generate_dataset_config(train_config.dataset)
     update_config(dataset_config, **kwargs)
 
+    logger.prepare_dump_logs(train_config.dump_logs)
+    logger.setLevel(train_config.log_level)
+
     setup_distributed_training(train_config)
     setup_seeds(train_config.seed)
     model, tokenizer = load_model_and_tokenizer(train_config, dataset_config, peft_config_file, **kwargs)

QEfficient/cloud/infer.py

@@ -311,6 +311,13 @@ def main(
         action="store_true",
         help="pass to print info logs",
     )
+    parser.add_argument(
+        "--log_level",
+        "--log-level",
+        type=int,
+        default=20,
+        help="set the Log level {NOTSET:0, DEBUG:10, INFO:20, WARNING:30, ERROR:40, CRITICAL:50}",
+    )
     parser.add_argument(
         "--full_batch_size",
         "--full-batch-size",
@@ -353,6 +360,11 @@ def main(
             )
             compiler_options_dict[key] = value
     if args.verbose:
-        logger.setLevel(logging.INFO)
+        logger.prepare_dump_logs(args.verbose)
+        if args.log_level:
+            logger.setLevel(args.log_level)
+        else:
+            logger.setLevel(logging.INFO)
     del args.verbose  # type: ignore
+    del args.log_level  # type: ignore
     main(**args.__dict__, **compiler_options_dict)

QEfficient/finetune/configs/training.py

@@ -4,6 +4,8 @@
 # SPDX-License-Identifier: BSD-3-Clause
 #
 # -----------------------------------------------------------------------------
+
+import logging
 from dataclasses import dataclass
 
 
@@ -107,3 +109,4 @@ class TrainConfig:
     opByOpVerifier: bool = False
 
     dump_logs: bool = True
+    log_level: str = logging.INFO

QEfficient/utils/logging_utils.py

@@ -56,21 +56,6 @@ def create_logger() -> logging.Logger:
     ch.setFormatter(QEffFormatter())
     logger.addHandler(ch)
 
-    dump_logs = True
-    if dump_logs:
-        logs_path = os.path.join(ROOT_DIR, "logs")
-        if not os.path.exists(logs_path):
-            os.makedirs(logs_path, exist_ok=True)
-        file_name = f"log-file-{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}" + ".txt"
-        log_file = os.path.join(logs_path, file_name)
-
-        # create file handler and set level
-        fh = logging.FileHandler(log_file)
-        fh.setLevel(logging.INFO)
-        formatter = logging.Formatter("%(levelname)s - %(name)s - %(message)s")
-        fh.setFormatter(formatter)
-        logger.addHandler(fh)
-
     return logger
 
 
@@ -85,8 +70,7 @@ def log_rank_zero(self, msg: str, level: int = logging.INFO) -> None:
             return
         self.log(level, msg, stacklevel=2)
 
-
-"""    def dump_logs(self, dump_logs=True):
+    def prepare_dump_logs(self, dump_logs=False):
         if dump_logs:
             logs_path = os.path.join(ROOT_DIR, "logs")
             if not os.path.exists(logs_path):
@@ -100,7 +84,6 @@ def log_rank_zero(self, msg: str, level: int = logging.INFO) -> None:
             formatter = logging.Formatter("%(levelname)s - %(name)s - %(message)s")
             fh.setFormatter(formatter)
             logger.addHandler(fh)
-"""
 
 
 logging.setLoggerClass(CustomLogger)